From 6b03e84807424eb31ae7610ac84749ace9a120c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 24 Sep 2026 13:08:53 -0300 Subject: [PATCH 001/132] refactor(beacon): name every fork at each fork gate A catch-all arm compiles unchanged when a fork is added and quietly applies an older fork's rule; with every variant named, the next fork fails to build at each site that has to decide. --- crates/blockchain/src/beacon_engine.rs | 7 +- crates/blockchain/src/lib.rs | 11 ++- .../src/beacon/fork_choice.rs | 6 +- .../src/beacon/helpers/accessors.rs | 29 +++++--- .../src/beacon/helpers/altair.rs | 8 ++- .../src/beacon/helpers/mutators.rs | 8 ++- .../src/beacon/stf/epoch/mod.rs | 8 ++- .../src/beacon/stf/operations.rs | 71 +++++++++++++++---- .../tests/beacon_spec/epoch_processing.rs | 31 ++++++-- .../tests/beacon_spec/fork_choice.rs | 14 +++- .../tests/beacon_spec/operations.rs | 10 ++- crates/common/types/src/beacon/fork.rs | 8 ++- crates/net/p2p/src/beacon/decode.rs | 37 +++++++--- 13 files changed, 200 insertions(+), 48 deletions(-) diff --git a/crates/blockchain/src/beacon_engine.rs b/crates/blockchain/src/beacon_engine.rs index 7943603a1..b33f6e9ad 100644 --- a/crates/blockchain/src/beacon_engine.rs +++ b/crates/blockchain/src/beacon_engine.rs @@ -44,7 +44,12 @@ pub struct NewPayloadRequest<'a> { pub fn new_payload_request(block: &SignedBeaconBlock) -> Option> { let inner = match block { SignedBeaconBlock::Electra(inner) | SignedBeaconBlock::Fulu(inner) => inner, - _ => return None, + SignedBeaconBlock::Phase0(_) + | SignedBeaconBlock::Altair(_) + | SignedBeaconBlock::Bellatrix(_) + | SignedBeaconBlock::Capella(_) + | SignedBeaconBlock::Deneb(_) + | SignedBeaconBlock::Lean(_) => return None, }; let versioned_hashes = inner diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 0044889d6..06f255128 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -836,7 +836,16 @@ fn data_availability_for( Some(fork_choice::DataAvailability::Columns(sidecars)) } - _ => Some(fork_choice::DataAvailability::NotRequired), + SignedBeaconBlock::Phase0(_) + | SignedBeaconBlock::Altair(_) + | SignedBeaconBlock::Bellatrix(_) + | SignedBeaconBlock::Capella(_) => Some(fork_choice::DataAvailability::NotRequired), + // `process_block` dispatches a lean block to `store::on_block` before + // this function is ever reached, so this arm is never observed for + // one; named on its own rather than folded into the group above so + // this value is not read as a claim about lean blocks' own + // availability. + SignedBeaconBlock::Lean(_) => Some(fork_choice::DataAvailability::NotRequired), } } diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 5560782b8..7332a2327 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -2353,7 +2353,11 @@ pub fn on_block( "is_data_available(hash_tree_root(block))", )?; } - _ => {} + SignedBeaconBlock::Phase0(_) + | SignedBeaconBlock::Altair(_) + | SignedBeaconBlock::Bellatrix(_) + | SignedBeaconBlock::Capella(_) => {} + SignedBeaconBlock::Lean(_) => lean_block_unreachable("fork_choice::on_block"), } // Check the block is valid and compute the post-state. The engine's answer diff --git a/crates/blockchain/state_transition/src/beacon/helpers/accessors.rs b/crates/blockchain/state_transition/src/beacon/helpers/accessors.rs index 677e895e8..6624360bf 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/accessors.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/accessors.rs @@ -19,6 +19,7 @@ use crate::beacon::containers::BeaconState; use crate::beacon::error::Result; use crate::beacon::fork::ForkName; use crate::beacon::hash::hash; +use crate::beacon::lean_state_unreachable; use crate::beacon::preset; use crate::beacon::primitives::{ Bytes32, CommitteeIndex, Domain, DomainType, Epoch, Gwei, Root, Slot, ValidatorIndex, @@ -358,8 +359,15 @@ pub fn get_beacon_proposer_index(state: &BeaconState) -> Result // than a shuffle run now. See `crate::beacon::helpers::fulu`'s own module docs for // why a seed, and therefore a proposer, is only ever knowable that far // ahead of time in the first place. - if state.fork_name() == ForkName::Fulu { - return super::fulu::get_beacon_proposer_index(state); + match state.fork_name() { + ForkName::Fulu => return super::fulu::get_beacon_proposer_index(state), + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra => {} + ForkName::Lean => lean_state_unreachable("get_beacon_proposer_index"), } let epoch = get_current_epoch(state); @@ -371,17 +379,22 @@ pub fn get_beacon_proposer_index(state: &BeaconState) -> Result let seed = hash(&input); let indices = get_active_validator_indices(state, epoch); - if state.fork_name() == ForkName::Electra { - super::electra::compute_proposer_index(&indices, seed, |index| { + match state.fork_name() { + ForkName::Electra => super::electra::compute_proposer_index(&indices, seed, |index| { Ok(state.validator(index)?.effective_balance) - }) - } else { - super::shuffling::compute_proposer_index( + }), + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb => super::shuffling::compute_proposer_index( &indices, seed, preset::MAX_EFFECTIVE_BALANCE, |index| Ok(state.validator(index)?.effective_balance), - ) + ), + ForkName::Fulu => unreachable!("Fulu returns via the dispatch match above"), + ForkName::Lean => lean_state_unreachable("get_beacon_proposer_index"), } } diff --git a/crates/blockchain/state_transition/src/beacon/helpers/altair.rs b/crates/blockchain/state_transition/src/beacon/helpers/altair.rs index eaa008c2e..5c315582e 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/altair.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/altair.rs @@ -40,6 +40,7 @@ use crate::beacon::containers::{BeaconState, altair}; use crate::beacon::error::{Error, Result}; use crate::beacon::fork::ForkName; use crate::beacon::hash::hash; +use crate::beacon::lean_state_unreachable; use crate::beacon::preset; use crate::beacon::primitives::{Epoch, Gwei, ParticipationFlags, ValidatorIndex}; @@ -157,7 +158,12 @@ pub fn get_next_sync_committee(state: &BeaconState) -> Result { crate::beacon::helpers::electra::get_next_sync_committee_indices(state)? } - _ => get_next_sync_committee_indices(state)?, + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb => get_next_sync_committee_indices(state)?, + ForkName::Lean => lean_state_unreachable("get_next_sync_committee"), }; let mut pubkeys = Vec::with_capacity(indices.len()); for index in &indices { diff --git a/crates/blockchain/state_transition/src/beacon/helpers/mutators.rs b/crates/blockchain/state_transition/src/beacon/helpers/mutators.rs index 6f37c7360..21dffed57 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/mutators.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/mutators.rs @@ -8,6 +8,7 @@ use crate::beacon::config::Config; use crate::beacon::constants::FAR_FUTURE_EPOCH; use crate::beacon::containers::BeaconState; use crate::beacon::error::{Error, Result}; +use crate::beacon::lean_state_unreachable; use crate::beacon::preset; use crate::beacon::primitives::{Epoch, Gwei, ValidatorIndex}; @@ -142,7 +143,12 @@ fn initiate_validator_exit_for_fork( ForkName::Electra | ForkName::Fulu => { crate::beacon::helpers::electra::initiate_validator_exit(state, index, config) } - _ => initiate_validator_exit(state, index, config), + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb => initiate_validator_exit(state, index, config), + ForkName::Lean => lean_state_unreachable("initiate_validator_exit_for_fork"), } } diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs index 026953803..f0fb05521 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs @@ -110,7 +110,13 @@ pub fn process_justification_and_finalization( ) -> Result<()> { match state.fork_name() { ForkName::Phase0 => justification::process_justification_and_finalization(state, config), - _ => altair::process_justification_and_finalization(state), + ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Fulu => altair::process_justification_and_finalization(state), + ForkName::Lean => lean_state_unreachable("process_justification_and_finalization"), } } diff --git a/crates/blockchain/state_transition/src/beacon/stf/operations.rs b/crates/blockchain/state_transition/src/beacon/stf/operations.rs index c6fd62293..efe0d154a 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/operations.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/operations.rs @@ -36,13 +36,14 @@ use crate::beacon::helpers::mutators::{ use crate::beacon::helpers::predicates::{ is_active_validator, is_slashable_attestation_data, is_slashable_validator, }; +use crate::beacon::lean_state_unreachable; use crate::beacon::preset; use crate::beacon::primitives::{Gwei, HashTreeRoot as _, ValidatorIndex}; /// Runs every operation in a block, in the specification's order. /// /// Takes each operation list as a slice rather than a whole body, which is -/// what lets this one function serve every fork through deneb even though +/// what lets this one function serve every fork through capella even though /// their body types are all distinct: nothing here needs to know what else a /// fork's body carries alongside these five lists. Capella's body adds a sixth /// list (BLS-to-execution changes), and electra reshapes the attestation types @@ -73,6 +74,28 @@ pub fn process_operations( config: &Config, committees: &CommitteeCache, ) -> Result<()> { + // Checked before anything below can mutate `state`, and unconditionally + // rather than only when `attestations` is non-empty: a refused fork must + // not be able to slip an all-empty attestation list past this check and + // have its proposer- and attester-slashings run anyway. + // + // Deneb never reaches here: it has its own `process_operations` + // (`deneb.rs`), which calls its own `process_attestation` for this loop + // instead of this shared one. Electra and fulu have their own + // `process_operations` too (`electra.rs`, which fulu reuses), so all + // three are refused rather than guessed at. + let altair_attestations = match state.fork_name() { + ForkName::Phase0 => false, + ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => true, + fork @ (ForkName::Deneb | ForkName::Electra | ForkName::Fulu) => { + return Err(Error::UnsupportedForFork { + function: "process_operations", + fork, + }); + } + ForkName::Lean => lean_state_unreachable("process_operations"), + }; + // `eth1_deposit_index` only ever advances by one per processed deposit, // and `deposit_count` only ever grows, so in a correctly-derived state the // index never exceeds the count. Nothing here re-derives that invariant, @@ -98,18 +121,18 @@ pub fn process_operations( } for attestation in attestations { // Phase0 defers an attestation's reward to the epoch boundary, so it - // needs its own version of this step (below); altair scores one the - // moment it is processed instead, and nothing about that changed - // through deneb, so every later fork this signature serves shares - // altair's version rather than getting one of its own. This is the - // same coexisting-by-fork pattern `crate::beacon::helpers::altair` and + // needs its own version of this step; altair scores one the moment + // it is processed instead, and nothing about that changed through + // capella, so every later fork this signature serves shares altair's + // version rather than getting one of its own. This is the same + // coexisting-by-fork pattern `crate::beacon::helpers::altair` and // `crate::beacon::stf::epoch::rewards` already use for the two // `get_base_reward` implementations: neither is renamed, and the call // site picks between them by fully-qualified path. - if state.fork_name() == ForkName::Phase0 { - process_attestation(state, attestation, config, committees)?; - } else { + if altair_attestations { crate::beacon::stf::altair::process_attestation(state, attestation, committees)?; + } else { + process_attestation(state, attestation, config, committees)?; } } for deposit in deposits { @@ -394,16 +417,36 @@ pub fn get_validator_from_deposit( /// symptom. /// /// Electra replaces this function outright, since a deposit there is queued -/// rather than credited, so `crate::beacon::stf::electra` has its own; this one serves -/// phase0 through deneb. The altair branch still covers every later fork -/// anyway, because being conservative here costs nothing and a silent -/// length mismatch costs a great deal. +/// rather than credited, so `crate::beacon::stf::electra` has its own +/// (`electra::add_validator_to_registry`); this one serves phase0 through +/// deneb (deneb reaches it through this module's own [`process_deposit`], +/// which every fork through deneb calls, even the fork that no longer calls +/// [`process_operations`] for the rest of a block's operations). Electra and +/// fulu are refused rather than folded into the altair branch: pushing the +/// three altair-onward lists the way altair through deneb do would be the +/// wrong answer for either even setting reachability aside, since electra +/// queues a deposit instead of crediting it through this function at all. pub fn add_validator_to_registry( state: &mut BeaconState, pubkey: crate::beacon::primitives::BlsPubkey, withdrawal_credentials: crate::beacon::primitives::Bytes32, amount: Gwei, ) -> Result<()> { + // Checked before either list is touched, so a refused fork leaves the + // state exactly as it found it rather than half-applying a mutation the + // caller's `?` then discards. + let grows_altair_lists = match state.fork_name() { + ForkName::Phase0 => false, + ForkName::Altair | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb => true, + fork @ (ForkName::Electra | ForkName::Fulu) => { + return Err(Error::UnsupportedForFork { + function: "add_validator_to_registry", + fork, + }); + } + ForkName::Lean => lean_state_unreachable("add_validator_to_registry"), + }; + state.validators_mut().push(get_validator_from_deposit( pubkey, withdrawal_credentials, @@ -411,7 +454,7 @@ pub fn add_validator_to_registry( ))?; state.balances_mut().push(amount)?; - if state.fork_name() >= ForkName::Altair { + if grows_altair_lists { let (previous, current, scores) = state.altair_validator_lists_mut()?; previous.push(0)?; current.push(0)?; diff --git a/crates/blockchain/state_transition/tests/beacon_spec/epoch_processing.rs b/crates/blockchain/state_transition/tests/beacon_spec/epoch_processing.rs index 2df06c9b8..a3ab100bb 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/epoch_processing.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/epoch_processing.rs @@ -48,7 +48,7 @@ use ethlambda_state_transition::beacon::containers::BeaconState; use ethlambda_state_transition::beacon::stf::epoch; use libtest_mimic::{Failed, Trial}; -use super::{Case, PRESET, collect_all_handlers}; +use super::{Case, PRESET, collect_all_handlers, lean_is_not_a_fixture_fork}; /// Runs the single epoch-processing step the handler names. fn apply( @@ -77,7 +77,13 @@ fn apply( // its own step. "rewards_and_penalties" => match fork { ForkName::Phase0 => epoch::rewards::process_rewards_and_penalties(state, config), - _ => epoch::altair::process_rewards_and_penalties(state, config), + ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Fulu => epoch::altair::process_rewards_and_penalties(state, config), + ForkName::Lean => lean_is_not_a_fixture_fork("rewards_and_penalties"), }, // Deneb's own change (EIP-7514's activation-churn cap) is selected // internally, by fork, inside `process_registry_updates` itself, so @@ -92,7 +98,12 @@ fn apply( ForkName::Electra | ForkName::Fulu => { epoch::electra::process_registry_updates(state, config) } - _ => epoch::registry::process_registry_updates(state, config), + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb => epoch::registry::process_registry_updates(state, config), + ForkName::Lean => lean_is_not_a_fixture_fork("registry_updates"), }, // Altair's and bellatrix's own changes here are scoped to the // proportional multiplier `registry::process_slashings` already selects @@ -105,7 +116,12 @@ fn apply( // own constant. "slashings" => match fork { ForkName::Electra | ForkName::Fulu => epoch::electra::process_slashings(state, config), - _ => epoch::registry::process_slashings(state, config), + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb => epoch::registry::process_slashings(state, config), + ForkName::Lean => lean_is_not_a_fixture_fork("slashings"), }, "eth1_data_reset" => epoch::process_eth1_data_reset(state), // Phase0 through deneb round each validator toward the single ceiling @@ -117,7 +133,12 @@ fn apply( ForkName::Electra | ForkName::Fulu => { epoch::electra::process_effective_balance_updates(state) } - _ => epoch::process_effective_balance_updates(state), + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb => epoch::process_effective_balance_updates(state), + ForkName::Lean => lean_is_not_a_fixture_fork("effective_balance_updates"), }, "slashings_reset" => epoch::process_slashings_reset(state), "randao_mixes_reset" => epoch::process_randao_mixes_reset(state), diff --git a/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs b/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs index 87eb3ff97..fbbe69533 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs @@ -309,7 +309,12 @@ fn decode_attestation(case: &Case, name: &str) -> Result { Ok(fork_choice::Attestation::Electra(decode(case, name)?)) } - _ => Ok(fork_choice::Attestation::Phase0(decode(case, name)?)), + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb => Ok(fork_choice::Attestation::Phase0(decode(case, name)?)), + ForkName::Lean => lean_is_not_a_fixture_fork("fork_choice"), } } @@ -323,7 +328,12 @@ fn decode_attester_slashing( ForkName::Electra | ForkName::Fulu => { Ok(fork_choice::AttesterSlashing::Electra(decode(case, name)?)) } - _ => Ok(fork_choice::AttesterSlashing::Phase0(decode(case, name)?)), + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb => Ok(fork_choice::AttesterSlashing::Phase0(decode(case, name)?)), + ForkName::Lean => lean_is_not_a_fixture_fork("fork_choice"), } } diff --git a/crates/blockchain/state_transition/tests/beacon_spec/operations.rs b/crates/blockchain/state_transition/tests/beacon_spec/operations.rs index e79d29f2e..dbddf6d79 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/operations.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/operations.rs @@ -371,11 +371,12 @@ fn apply( let body: electra::BeaconBlockBody = case.ssz("body"); fulu_stf::process_execution_payload(state, &body, config, &engine) } - other => { + fork @ (ForkName::Phase0 | ForkName::Altair) => { return Err(format!( - "execution_payload has no handler for fork `{other}`" + "execution_payload has no handler for fork `{fork}`" )); } + ForkName::Lean => lean_is_not_a_fixture_fork("execution_payload"), } } // New in capella, named `address_change` in the fixture tree even @@ -412,7 +413,10 @@ fn apply( let payload: deneb::ExecutionPayload = case.ssz("execution_payload"); electra_stf::process_withdrawals(state, &payload) } - other => return Err(format!("withdrawals has no handler for fork `{other}`")), + fork @ (ForkName::Phase0 | ForkName::Altair | ForkName::Bellatrix) => { + return Err(format!("withdrawals has no handler for fork `{fork}`")); + } + ForkName::Lean => lean_is_not_a_fixture_fork("withdrawals"), }, // All three are new in electra, alongside `withdrawals`; fulu's // specification lists none of them as modified, so electra's own diff --git a/crates/common/types/src/beacon/fork.rs b/crates/common/types/src/beacon/fork.rs index b9c7fc4ae..9e6192b4e 100644 --- a/crates/common/types/src/beacon/fork.rs +++ b/crates/common/types/src/beacon/fork.rs @@ -142,7 +142,13 @@ impl ForkName { pub const fn snapshot_interval(self) -> u64 { match self { ForkName::Lean => LEAN_SNAPSHOT_INTERVAL, - _ => BEACON_SNAPSHOT_INTERVAL, + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Fulu => BEACON_SNAPSHOT_INTERVAL, } } diff --git a/crates/net/p2p/src/beacon/decode.rs b/crates/net/p2p/src/beacon/decode.rs index 79c0c4655..0be81c439 100644 --- a/crates/net/p2p/src/beacon/decode.rs +++ b/crates/net/p2p/src/beacon/decode.rs @@ -200,11 +200,20 @@ pub fn decode_aggregate_and_proof( bytes: &[u8], ) -> Result { let fork = fork_at_slot(config, aggregate_slot(bytes)?); - if fork >= ForkName::Electra { - electra::SignedAggregateAndProof::from_ssz_bytes(bytes) - .map(SignedAggregateAndProof::Electra) - } else { - phase0::SignedAggregateAndProof::from_ssz_bytes(bytes).map(SignedAggregateAndProof::Phase0) + match fork { + ForkName::Electra | ForkName::Fulu => { + electra::SignedAggregateAndProof::from_ssz_bytes(bytes) + .map(SignedAggregateAndProof::Electra) + } + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb => phase0::SignedAggregateAndProof::from_ssz_bytes(bytes) + .map(SignedAggregateAndProof::Phase0), + ForkName::Lean => { + unreachable!("fork_at_slot never returns Lean: it is absent from ForkName::ALL") + } } .map_err(|_| DecodeError::Ssz) } @@ -270,10 +279,20 @@ pub fn decode_gossip( .map(|value| BeaconGossip::AggregateAndProof(Box::new(value))), topics::ATTESTER_SLASHING => { let fork = fork_at_slot(config, attester_slashing_slot(bytes)?); - let decoded = if fork >= ForkName::Electra { - electra::AttesterSlashing::from_ssz_bytes(bytes).map(AttesterSlashing::Electra) - } else { - phase0::AttesterSlashing::from_ssz_bytes(bytes).map(AttesterSlashing::Phase0) + let decoded = match fork { + ForkName::Electra | ForkName::Fulu => { + electra::AttesterSlashing::from_ssz_bytes(bytes).map(AttesterSlashing::Electra) + } + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb => { + phase0::AttesterSlashing::from_ssz_bytes(bytes).map(AttesterSlashing::Phase0) + } + ForkName::Lean => { + unreachable!("fork_at_slot never returns Lean: it is absent from ForkName::ALL") + } }; decoded .map(|value| BeaconGossip::AttesterSlashing(Box::new(value))) From 10661aced5bf3dcf49443e41bc21bcfced3936ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 24 Sep 2026 14:04:43 -0300 Subject: [PATCH 002/132] build: bump libssz for in-place ProgressiveList writes Gloas's progressive state lists are written element by element, the same way the Beacon Chain validator registry already is through `SszList`. lambdaclass/libssz#37 gives `ProgressiveList` the `DerefMut`/`Index`/ `IndexMut` that lambdaclass/libssz#33 gave `SszList`, so both containers now support the same in-place access. Moves the workspace's libssz git pin from 36802dd (0.3.0 plus lambdaclass/libssz#33) to 5cb1437, the head of lambdaclass/libssz#37's PR branch. That rev also carries lambdaclass/libssz#35 (`as_chunks` in the merkleize fold loop) and lambdaclass/libssz#36 (CI only) from libssz's main. Every beacon spec fixture case still passes on both presets with the same pass/ignore counts as before the bump, so lambdaclass/libssz#35 changed no hash_tree_root. The pin points at an unmerged PR branch head for now. It must move to lambdaclass/libssz#37's merge commit before this branch merges, since a PR branch ref stops being fetchable once the branch is deleted. --- CLAUDE.md | 8 +++--- Cargo.lock | 76 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 22 +++++++++------- 3 files changed, 56 insertions(+), 50 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d4cfc073f..6130d323f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -682,10 +682,12 @@ transitions are in `ethlambda-types`, per the section above. Nothing above catch-all `_`, so a real new fork still breaks every match that must grow one. - **Needs mutable element access on `SszList`/`SszVector`**, which no published libssz release has yet. Nothing extra is required here: the workspace already - tracks all four libssz crates from git at `36802dd` for the beacon containers + tracks all four libssz crates from git at `5cb1437` for the beacon containers in `ethlambda-types` (see the section above), and that rev is the `0.3.0` - release plus the single commit adding `DerefMut`/`IndexMut` - (lambdaclass/libssz#33). This module needs that commit for the same reason. + release plus `DerefMut`/`IndexMut` on `SszList` (lambdaclass/libssz#33), + `as_chunks` in the merkleize fold loop (#35), and the same `DerefMut`/ + `IndexMut` for `ProgressiveList` (#37, an unmerged PR branch head for now). + This module needs the `SszList` commit (#33) for the same reason. - **Status:** all seven forks (phase0 through fulu) have containers, fork upgrades, state transitions, and epoch processing. Every fixture case passes on both presets: mainnet is 5705 cases and minimal 40009. The crate's lib diff --git a/Cargo.lock b/Cargo.lock index fcf2acfcd..d0b5404ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1953,8 +1953,8 @@ dependencies = [ "eyre", "hex", "libc", - "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", + "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", "rayon", "reqwest", "secp256k1 0.30.0", @@ -1987,8 +1987,8 @@ dependencies = [ "ethlambda-test-fixtures", "ethlambda-types", "hex", - "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", + "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", "serde", "spawned-concurrency", "thiserror 2.0.18", @@ -2067,10 +2067,10 @@ dependencies = [ "hex", "libp2p", "libp2p-mplex", - "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-derive 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-merkle 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", + "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-derive 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-merkle 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", "rand 0.8.6", "secp256k1 0.30.0", "sha2", @@ -2101,9 +2101,9 @@ dependencies = [ "hex", "http-body-util", "jemalloc_pprof", - "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-derive 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", + "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-derive 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", "serde", "serde_json", "spawned-concurrency", @@ -2117,10 +2117,10 @@ dependencies = [ name = "ethlambda-ssz-tree" version = "0.1.0" dependencies = [ - "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-derive 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-merkle 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", + "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-derive 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-merkle 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", "proptest", "rayon", ] @@ -2137,10 +2137,10 @@ dependencies = [ "ethlambda-test-fixtures", "ethlambda-types", "hex", - "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-derive 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-merkle 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", + "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-derive 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-merkle 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", "libtest-mimic", "lru", "num-bigint", @@ -2161,9 +2161,9 @@ dependencies = [ "ethlambda-crypto", "ethlambda-metrics", "ethlambda-types", - "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-derive 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", + "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-derive 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", "lru", "proptest", "rocksdb", @@ -2180,8 +2180,8 @@ dependencies = [ "ethlambda-state-transition", "ethlambda-types", "hex", - "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", + "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", "serde", "serde_json", ] @@ -2194,10 +2194,10 @@ dependencies = [ "ethlambda-ssz-tree", "ethlambda-test-fixtures", "hex", - "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-derive 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-merkle 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", + "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-derive 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-merkle 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", "rand 0.10.1", "serde", "serde_json", @@ -2221,9 +2221,9 @@ dependencies = [ "hmac", "http-body-util", "libc", - "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-derive 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", + "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-derive 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", "pbkdf2", "reqwest", "scrypt", @@ -4615,7 +4615,7 @@ dependencies = [ [[package]] name = "libssz" version = "0.3.0" -source = "git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7#36802dd1d3e3a83d95d2ac552647539cbe3f7fd7" +source = "git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073#5cb14378ffdadcb51b387c003567e2630d0c7073" dependencies = [ "smallvec", ] @@ -4634,7 +4634,7 @@ dependencies = [ [[package]] name = "libssz-derive" version = "0.3.0" -source = "git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7#36802dd1d3e3a83d95d2ac552647539cbe3f7fd7" +source = "git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073#5cb14378ffdadcb51b387c003567e2630d0c7073" dependencies = [ "proc-macro2", "quote", @@ -4654,9 +4654,9 @@ dependencies = [ [[package]] name = "libssz-merkle" version = "0.3.0" -source = "git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7#36802dd1d3e3a83d95d2ac552647539cbe3f7fd7" +source = "git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073#5cb14378ffdadcb51b387c003567e2630d0c7073" dependencies = [ - "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", + "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", "sha2", ] @@ -4674,10 +4674,10 @@ dependencies = [ [[package]] name = "libssz-types" version = "0.3.0" -source = "git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7#36802dd1d3e3a83d95d2ac552647539cbe3f7fd7" +source = "git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073#5cb14378ffdadcb51b387c003567e2630d0c7073" dependencies = [ - "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", - "libssz-merkle 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=36802dd1d3e3a83d95d2ac552647539cbe3f7fd7)", + "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", + "libssz-merkle 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", "smallvec", ] diff --git a/Cargo.toml b/Cargo.toml index c641d6696..d2b7d0784 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -134,17 +134,21 @@ postcard = { version = "1.1.3", features = ["alloc"] } # hand out `&mut` into an `SszList` (the validator registry, through # `altair_validator_lists_mut`), which published libssz-types 0.3.0 cannot do: # it exposes only `Deref` and `Index`. `DerefMut` and `IndexMut` landed after -# that release, in lambdaclass/libssz#33. +# that release, in lambdaclass/libssz#33. Gloas's progressive state lists need +# the same in-place access on `ProgressiveList`, which lambdaclass/libssz#37 +# adds. # # Pinned by `rev` rather than tracking `main`: the requirement is a commit that -# carries #33, not whatever `main` holds today, and `main` here is the SSZ encoder -# and merkleizer that decides every `hash_tree_root`, so a routine `cargo update` -# must not be able to move it. Return these to a crates.io version once a release -# carrying #33 exists. -libssz = { git = "https://github.com/lambdaclass/libssz", rev = "36802dd1d3e3a83d95d2ac552647539cbe3f7fd7" } -libssz-derive = { git = "https://github.com/lambdaclass/libssz", rev = "36802dd1d3e3a83d95d2ac552647539cbe3f7fd7" } -libssz-merkle = { git = "https://github.com/lambdaclass/libssz", rev = "36802dd1d3e3a83d95d2ac552647539cbe3f7fd7" } -libssz-types = { git = "https://github.com/lambdaclass/libssz", rev = "36802dd1d3e3a83d95d2ac552647539cbe3f7fd7" } +# carries #33 and #37, not whatever `main` holds today, and `main` here is the +# SSZ encoder and merkleizer that decides every `hash_tree_root`, so a routine +# `cargo update` must not be able to move it. This rev is #37's PR branch head, +# not yet merged: it must move to #37's merge commit before this branch merges, +# since a PR branch ref stops being fetchable once the branch is deleted. Return +# these to a crates.io version once a release carrying #33 and #37 exists. +libssz = { git = "https://github.com/lambdaclass/libssz", rev = "5cb14378ffdadcb51b387c003567e2630d0c7073" } +libssz-derive = { git = "https://github.com/lambdaclass/libssz", rev = "5cb14378ffdadcb51b387c003567e2630d0c7073" } +libssz-merkle = { git = "https://github.com/lambdaclass/libssz", rev = "5cb14378ffdadcb51b387c003567e2630d0c7073" } +libssz-types = { git = "https://github.com/lambdaclass/libssz", rev = "5cb14378ffdadcb51b387c003567e2630d0c7073" } # VCDIFF delta compression for beacon state diffs. # From 14dfe49730e953caf1f688db4952932961c909d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 24 Sep 2026 12:18:47 -0300 Subject: [PATCH 003/132] feat(ssz-tree): add a tree-backed ProgressiveList Gloas (EIP-7688) turns the validator registry into an EIP-7916 progressive SSZ list: no fixed limit, merkleized as a chain of balanced subtrees holding 1, 4, 16, ... chunks. The registry needs the same persistent-tree properties List already gives Validators and Balances: buffered writes, per-subtree hash caching, and cheap rebase onto a resident state. ProgressiveList gets there by keeping one existing Interface per subtree, at heights 0, 2, 4, ..., rather than inventing new tree machinery. Every List property (buffered writes via UpdateMap, OnceLock hash caching, Arc-shared rebase) holds per subtree unchanged; only the O(log n) chain above them, one hash_nodes per subtree, is new. This is a pure data-structure addition: nothing constructs a Gloas registry yet. Verified against libssz_types::ProgressiveList (the Vec-backed reference) by a proptest model over random push/set/apply sequences for packed (u64), composite (Item), and variable-size (Blob) elements; boundary cases at every subtree edge, including one element past each newly-opened subtree; rebase proptests mirroring List's (a base with fewer subtrees, one with more, and equal-shaped lists sharing a common prefix); a deterministic case for a base whose pending push has just opened a subtree with nothing committed yet (rebase must not adopt it, and must reclaim it once applied); iter_from at a subtree boundary, mid-subtree, and past the end; and decode-error parity with the reference, including one that pins a fixed-size decode loop to reporting the first bad element rather than a later one. Mainnet-scale bench (2.4M validators, ProgressiveList> vs. a validators-only List baseline built in the same test for a like-for-like comparison, release-fast, one sequential run): build 224.4 ms (List: 177.3 ms) memory for one resident state 696 MiB (List: 696 MiB) cold hash (first root) 1233.3 ms (List: 1226.2 ms) 1000 scattered writes + apply + root 3.4 ms independent copy of the same data 110.6 ms rebase that copy onto the written state 169.9 ms (cargo test -p ethlambda-types --test tree_bench --profile release-fast -- --ignored --nocapture --test-threads=1; the two #[ignore]d tests in that file share a process-wide allocation counter, so --test-threads=1 is required for either one's memory figure to mean anything.) --- crates/common/ssz-tree/src/lib.rs | 10 + .../common/ssz-tree/src/progressive_list.rs | 369 +++++++++++++++++ crates/common/ssz-tree/tests/model.rs | 372 +++++++++++++++++- crates/common/types/tests/tree_bench.rs | 82 ++++ 4 files changed, 830 insertions(+), 3 deletions(-) create mode 100644 crates/common/ssz-tree/src/progressive_list.rs diff --git a/crates/common/ssz-tree/src/lib.rs b/crates/common/ssz-tree/src/lib.rs index 561be2a9b..34b8efae2 100644 --- a/crates/common/ssz-tree/src/lib.rs +++ b/crates/common/ssz-tree/src/lib.rs @@ -52,10 +52,19 @@ //! always use SHA-256 (`libssz_merkle::Sha2Hasher`). A cached hash is only //! valid for the function that produced it, and SHA-256 is the only one the //! consensus specs use. +//! +//! # Progressive lists +//! +//! [`ProgressiveList`] is an unbounded SSZ list (EIP-7916): instead of one +//! tree padded to a fixed limit, it is a chain of balanced subtrees holding +//! 1, 4, 16, ... chunks, each an ordinary [`List`]-style tree in its own +//! right, so writes, hash caching and rebasing all work per subtree exactly +//! as they do for [`List`]. mod interface; mod iter; mod list; +mod progressive_list; mod rebase; mod tree; mod update_map; @@ -63,6 +72,7 @@ mod vector; pub use iter::Iter; pub use list::List; +pub use progressive_list::{ProgressiveIter, ProgressiveList}; pub use update_map::{UpdateMap, VecMap}; pub use vector::Vector; diff --git a/crates/common/ssz-tree/src/progressive_list.rs b/crates/common/ssz-tree/src/progressive_list.rs new file mode 100644 index 000000000..081dd1206 --- /dev/null +++ b/crates/common/ssz-tree/src/progressive_list.rs @@ -0,0 +1,369 @@ +//! [`ProgressiveList`]: an SSZ `ProgressiveList[T]` (EIP-7916) kept in +//! persistent Merkle trees. + +use std::fmt; +use std::iter::FusedIterator; +use std::ops::{Index, IndexMut}; + +use libssz::{BYTES_PER_LENGTH_OFFSET, DecodeError, SszDecode, SszEncode}; +use libssz_merkle::{HashTreeRoot, Sha2Hasher, Sha256Hasher, hash_nodes, mix_in_length}; +use libssz_types::TypeError; + +use crate::interface::Interface; +use crate::iter::Iter; +use crate::update_map::{UpdateMap, VecMap}; +use crate::{Hash256, Value, packing_factor}; + +/// An SSZ progressive list: no length limit, merkleized as a chain of +/// balanced subtrees holding 1, 4, 16, ... chunks (EIP-7916). +/// +/// Stands in for `libssz_types::ProgressiveList` in a derived container the +/// way [`List`](crate::List) stands in for `SszList`: same encoding, same +/// root, same element API, and a mirror of `List`'s method set (including +/// `push -> Result`, which never fails here) so code written against one +/// reads the same against the other. +/// +/// Subtree `k` is a tree of height `2k`, so every write, cache and rebase +/// rule `List` has holds per subtree. The chain above them, one hash per +/// subtree, is recomputed on every root. +#[derive(Clone)] +pub struct ProgressiveList> { + /// Subtree `k` holds the next `capacity(k)` elements. Every subtree but + /// the last is full, and none is empty. + subtrees: Vec>, +} + +impl> ProgressiveList { + /// Elements subtree `k` holds when full: `4^k` chunks. + fn capacity(k: usize) -> usize { + packing_factor::() << (2 * k) + } + + /// Height of subtree `k`, which has `4^k` leaves. + fn height(k: usize) -> usize { + 2 * k + } + + /// Streams `values` into subtrees, filling each before opening the next. + fn from_values(values: impl IntoIterator) -> Self { + let mut values = values.into_iter().peekable(); + let mut subtrees = Vec::new(); + while values.peek().is_some() { + let k = subtrees.len(); + let chunk = values.by_ref().take(Self::capacity(k)); + subtrees.push(Interface::from_values(chunk, Self::height(k))); + } + Self { subtrees } + } + + /// The subtree holding element `index`, and the index within it. + fn locate(&self, mut index: usize) -> Option<(usize, usize)> { + for (k, subtree) in self.subtrees.iter().enumerate() { + if index < subtree.len() { + return Some((k, index)); + } + index -= subtree.len(); + } + None + } + + /// An empty list. + pub fn empty() -> Self { + Self { + subtrees: Vec::new(), + } + } + + /// The number of elements, pending pushes included. + pub fn len(&self) -> usize { + self.subtrees.iter().map(Interface::len).sum() + } + + /// Whether the list holds no elements. + pub fn is_empty(&self) -> bool { + self.subtrees.is_empty() + } + + /// The element at `index`, or `None` past the end. + pub fn get(&self, index: usize) -> Option<&T> { + let (k, offset) = self.locate(index)?; + self.subtrees[k].get(offset) + } + + /// A pending copy of the element at `index`, or `None` past the end. + pub fn get_mut(&mut self, index: usize) -> Option<&mut T> { + let (k, offset) = self.locate(index)?; + self.subtrees[k].get_mut(offset) + } + + /// Buffers `value` as the new last element. + /// + /// Returns a `Result` only to mirror [`List::push`](crate::List::push); + /// a progressive list has no limit, so this never fails. + pub fn push(&mut self, value: T) -> Result<(), TypeError> { + let k = self.subtrees.len(); + match self.subtrees.last_mut() { + Some(last) if last.len() < Self::capacity(k - 1) => last.push(value), + _ => { + let mut subtree = Interface::from_values(std::iter::empty(), Self::height(k)); + subtree.push(value); + self.subtrees.push(subtree); + } + } + Ok(()) + } + + /// Every element, in order, pending writes included. + pub fn iter(&self) -> ProgressiveIter<'_, T, U> { + self.iter_from(0) + } + + /// The elements from `start` on. + pub fn iter_from(&self, start: usize) -> ProgressiveIter<'_, T, U> { + let len = self.len(); + let start = start.min(len); + let (k, offset) = self.locate(start).unwrap_or((self.subtrees.len(), 0)); + let mut subtrees = self.subtrees[k..].iter(); + let current = subtrees.next().map(|subtree| subtree.iter_from(offset)); + ProgressiveIter { + subtrees, + current, + remaining: len - start, + } + } + + /// The elements, copied out. + pub fn to_vec(&self) -> Vec { + self.iter().cloned().collect() + } + + /// Folds every pending write into the trees. + pub fn apply_updates(&mut self) { + self.subtrees.iter_mut().for_each(Interface::apply_updates); + } + + /// Whether any write is still buffered. + pub fn has_pending_updates(&self) -> bool { + self.subtrees.iter().any(Interface::has_pending_updates) + } + + /// Applies pending writes, then shares every subtree equal to `base`'s. + /// + /// Subtree `k` of both lists has the same shape, so they rebase + /// pairwise; subtrees past `base`'s last have nothing to share. + pub fn rebase_on(&mut self, base: &Self) { + self.apply_updates(); + for (own, base) in self.subtrees.iter_mut().zip(&base.subtrees) { + own.rebase_on(base); + } + } + + /// Whether every committed subtree is the same allocation as `other`'s. + pub fn ptr_eq(&self, other: &Self) -> bool { + self.subtrees.len() == other.subtrees.len() + && self + .subtrees + .iter() + .zip(&other.subtrees) + .all(|(a, b)| a.ptr_eq(b)) + } + + /// The chain root, before the length mix-in: `hash(sub_0, hash(sub_1, + /// ... hash(sub_last, 0)))`. + fn chain_root(&self) -> Hash256 { + self.subtrees.iter().rev().fold([0u8; 32], |rest, subtree| { + hash_nodes(&Sha2Hasher, &subtree.root(), &rest) + }) + } +} + +impl> Default for ProgressiveList { + fn default() -> Self { + Self::empty() + } +} + +impl> From> for ProgressiveList { + fn from(values: Vec) -> Self { + Self::from_values(values) + } +} + +impl> Index for ProgressiveList { + type Output = T; + fn index(&self, index: usize) -> &T { + // `len` sums over every subtree, unlike `List`'s O(1) field read, so + // it is computed only on the panic path, not on every access. + self.get(index).unwrap_or_else(|| { + panic!( + "index {index} out of bounds for a list of length {}", + self.len() + ) + }) + } +} + +impl> IndexMut for ProgressiveList { + fn index_mut(&mut self, index: usize) -> &mut T { + // Locate once and index the subtree directly, rather than through + // `get`/`get_mut` (each of which re-locates on its own): `len`, + // needed only for the panic message, still costs nothing on the + // common, in-bounds path. + let Some((k, offset)) = self.locate(index) else { + panic!( + "index {index} out of bounds for a list of length {}", + self.len() + ); + }; + self.subtrees[k] + .get_mut(offset) + .expect("locate returned an in-bounds offset") + } +} + +impl> PartialEq for ProgressiveList { + fn eq(&self, other: &Self) -> bool { + if self.len() != other.len() { + return false; + } + let nothing_pending = !self.has_pending_updates() && !other.has_pending_updates(); + (nothing_pending && self.ptr_eq(other)) || self.iter().eq(other.iter()) + } +} + +impl> Eq for ProgressiveList {} + +impl> fmt::Debug for ProgressiveList { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.iter()).finish() + } +} + +impl<'a, T: Value, U: UpdateMap> IntoIterator for &'a ProgressiveList { + type Item = &'a T; + type IntoIter = ProgressiveIter<'a, T, U>; + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl> SszEncode for ProgressiveList { + fn is_fixed_size() -> bool { + false + } + + fn fixed_size() -> usize { + 0 + } + + fn encoded_len(&self) -> usize { + self.subtrees.iter().map(Interface::encoded_len).sum() + } + + fn ssz_append(&self, buf: &mut Vec) { + if ::is_fixed_size() { + for subtree in &self.subtrees { + subtree.ssz_append(buf); + } + return; + } + // Offsets are relative to the start of the whole list, so the + // offset table cannot be written subtree by subtree. + let len = self.len(); + let start = buf.len(); + buf.resize(start + len * BYTES_PER_LENGTH_OFFSET, 0); + for (index, value) in self.iter().enumerate() { + let offset = (buf.len() - start) as u32; + let position = start + index * BYTES_PER_LENGTH_OFFSET; + buf[position..position + BYTES_PER_LENGTH_OFFSET] + .copy_from_slice(&offset.to_le_bytes()); + value.ssz_append(buf); + } + } +} + +impl> SszDecode for ProgressiveList { + fn is_fixed_size() -> bool { + false + } + + fn fixed_size() -> usize { + 0 + } + + /// Matches `libssz_types::ProgressiveList`, which decodes as a plain + /// `Vec` with no length limit. Fixed-size elements stream straight + /// into the subtrees, as `Interface::from_ssz_bytes` does. + fn from_ssz_bytes(bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Ok(Self::empty()); + } + if !::is_fixed_size() { + let values = libssz::decode_list_with_max::(bytes, usize::MAX)?; + return Ok(Self::from_values(values)); + } + let size = ::fixed_size(); + if size == 0 || !bytes.len().is_multiple_of(size) { + return Err(DecodeError::InvalidByteLength { + expected: size, + got: bytes.len(), + }); + } + let mut error = None; + // `.fuse()` matters here: `map_while` alone is not a fused iterator, + // so once its closure returns `None` for a bad chunk, the + // `Peekable` in `from_values` would call it again on the *next* + // chunk rather than stopping, silently skipping the bad one and + // letting `error` end up holding the last failure instead of the + // first. + let values = bytes + .chunks_exact(size) + .map_while(|chunk| { + T::from_ssz_bytes(chunk) + .map_err(|err| error = Some(err)) + .ok() + }) + .fuse(); + let list = Self::from_values(values); + match error { + Some(err) => Err(err), + None => Ok(list), + } + } +} + +impl> HashTreeRoot for ProgressiveList { + /// Ignores `hasher` and uses SHA-256: see the crate docs. + fn hash_tree_root(&self, _hasher: &impl Sha256Hasher) -> Hash256 { + mix_in_length(&Sha2Hasher, &self.chain_root(), self.len()) + } +} + +/// An iterator over a [`ProgressiveList`], pending writes included. +pub struct ProgressiveIter<'a, T, U> { + subtrees: std::slice::Iter<'a, Interface>, + current: Option>, + remaining: usize, +} + +impl<'a, T: Value, U: UpdateMap> Iterator for ProgressiveIter<'a, T, U> { + type Item = &'a T; + + fn next(&mut self) -> Option<&'a T> { + loop { + if let Some(value) = self.current.as_mut().and_then(Iterator::next) { + self.remaining -= 1; + return Some(value); + } + self.current = Some(self.subtrees.next()?.iter_from(0)); + } + } + + fn size_hint(&self) -> (usize, Option) { + (self.remaining, Some(self.remaining)) + } +} + +impl> ExactSizeIterator for ProgressiveIter<'_, T, U> {} + +impl> FusedIterator for ProgressiveIter<'_, T, U> {} diff --git a/crates/common/ssz-tree/tests/model.rs b/crates/common/ssz-tree/tests/model.rs index e09a5f1b6..b3367404a 100644 --- a/crates/common/ssz-tree/tests/model.rs +++ b/crates/common/ssz-tree/tests/model.rs @@ -10,11 +10,11 @@ use std::collections::BTreeMap; use std::fmt::Debug; -use ethlambda_ssz_tree::{List, UpdateMap, Value, VecMap, Vector}; -use libssz::{SszDecode as _, SszEncode as _}; +use ethlambda_ssz_tree::{List, ProgressiveList, UpdateMap, Value, VecMap, Vector}; +use libssz::{DecodeError, SszDecode as _, SszEncode as _}; use libssz_derive::{HashTreeRoot, SszDecode, SszEncode}; use libssz_merkle::{HashTreeRoot, Sha2Hasher}; -use libssz_types::{SszList, SszVector}; +use libssz_types::{ProgressiveList as RefProgressiveList, SszList, SszVector}; use proptest::collection::vec; use proptest::prelude::*; @@ -521,3 +521,369 @@ proptest! { rebase_shrunk::>(orig_values, extra, hash_orig_first, hash_base_first)?; } } + +// ── ProgressiveList: parity with libssz_types::ProgressiveList (EIP-7916) ── + +/// Everything observable about `list` agrees with libssz's +/// `ProgressiveList` over `model`. +fn check_progressive_matches(list: &ProgressiveList, model: &[T]) +where + T: Value + Debug, + U: UpdateMap, +{ + let reference = RefProgressiveList::from(model.to_vec()); + assert_eq!(list.len(), model.len()); + for (index, value) in model.iter().enumerate() { + assert_eq!(list.get(index), Some(value), "index {index}"); + } + assert_eq!(list.get(model.len()), None); + assert_eq!(list.to_vec(), model); + assert_eq!(list.iter().len(), model.len()); + assert_eq!(root(list), root(&reference)); + let bytes = list.to_ssz(); + assert_eq!(bytes, reference.to_ssz()); + let decoded = ProgressiveList::::from_ssz_bytes(&bytes).expect("round trip"); + assert_eq!(root(&decoded), root(&reference)); +} + +fn run_progressive(initial: Vec, ops: Vec>) +where + T: Value + Debug, + U: UpdateMap, +{ + let mut list = ProgressiveList::::from(initial.clone()); + let mut model = initial; + check_progressive_matches(&list, &model); + for op in ops { + match op { + Op::Push(value) => { + list.push(value.clone()) + .expect("a progressive list has no limit"); + model.push(value); + } + Op::Set(index, value) if !model.is_empty() => { + let index = index % model.len(); + list[index] = value.clone(); + model[index] = value; + } + Op::Set(..) => {} + Op::Apply => list.apply_updates(), + } + check_progressive_matches(&list, &model); + } +} + +proptest! { + // Packed: 4 per chunk, so subtrees start at elements 0, 4, 20, 84, 340. + #[test] + fn progressive_u64(initial in vec(any::(), 0..400), ops in ops(any::())) { + run_progressive::>(initial, ops); + } + // Composite: one per chunk, subtrees start at 0, 1, 5, 21, 85. + #[test] + fn progressive_item(initial in vec(item(), 0..100), ops in ops(item())) { + run_progressive::>(initial, ops); + } + // Variable-size elements: the offset table spans every subtree. + #[test] + fn progressive_blob(initial in vec(blob(), 0..30), ops in ops(blob())) { + run_progressive::>(initial, ops); + } +} + +#[test] +fn progressive_boundaries_hash_like_libssz() { + // 0, 1, 5, 21, 85, 341 are the composite subtree boundaries (packed u64 + // boundaries are 4, 20, 84, 340); 6, 22, 86, 342 are one past each of + // those, so a subtree that has just opened with a single element is + // exercised too, not only subtrees that are exactly full or exactly one + // short. + for len in [0usize, 1, 2, 4, 5, 6, 20, 21, 22, 84, 85, 86, 340, 341, 342] { + let values: Vec = (0..len as u64).collect(); + check_progressive_matches(&ProgressiveList::::from(values.clone()), &values); + let items: Vec<[u8; 32]> = (0..len).map(|i| [i as u8; 32]).collect(); + check_progressive_matches(&ProgressiveList::<[u8; 32]>::from(items.clone()), &items); + } +} + +// ── ProgressiveList: iter_from ── + +#[test] +fn progressive_iter_from_starts_at_the_given_index() { + // u64 packed boundaries: subtrees start at elements 0, 4, 20, 84, 340. + let values: Vec = (0..50).collect(); + let list = ProgressiveList::::from(values.clone()); + + // Mid-subtree: inside subtree 2, which spans elements 20..84. + assert_eq!( + list.iter_from(30).copied().collect::>(), + values[30..] + ); + // Exactly at a subtree boundary: the first element of the next subtree. + assert_eq!(list.iter_from(4).copied().collect::>(), values[4..]); + assert_eq!( + list.iter_from(20).copied().collect::>(), + values[20..] + ); + // At and past the end. + assert_eq!(list.iter_from(50).count(), 0); + assert_eq!(list.iter_from(100).count(), 0); + assert_eq!(list.iter().len(), 50); +} + +// ── ProgressiveList: decode errors ── + +#[test] +fn progressive_decode_rejects_a_length_not_a_multiple_of_the_element_size() { + let bytes = vec![0u8; 7]; // u64 is 8 bytes wide; 7 does not divide evenly. + let err = ProgressiveList::::from_ssz_bytes(&bytes).unwrap_err(); + assert_eq!( + err, + DecodeError::InvalidByteLength { + expected: 8, + got: 7 + } + ); +} + +#[test] +fn progressive_decode_rejects_a_bad_offset_table_for_variable_size_elements() { + // First offset is 5, not a multiple of 4: the same bad input list.rs's + // own `decoding_a_first_offset_that_is_not_a_multiple_of_four_is_rejected_like_ssz_list` + // uses. + let bytes = vec![5u8, 0, 0, 0, 0]; + let ours = ProgressiveList::::from_ssz_bytes(&bytes).unwrap_err(); + let reference = RefProgressiveList::::from_ssz_bytes(&bytes).unwrap_err(); + assert_eq!(ours, reference); +} + +#[test] +fn progressive_decode_reports_the_first_bad_fixed_size_element_not_the_last() { + // Two invalid `bool` bytes (neither 0 nor 1). The non-fused `map_while` + // bug this guards against (see the `.fuse()` comment in + // progressive_list.rs) would keep decoding past the first bad byte and + // surface the second failure instead of the first. + let bytes = vec![1u8, 2, 1, 3, 1]; + let ours = ProgressiveList::::from_ssz_bytes(&bytes).unwrap_err(); + let reference = RefProgressiveList::::from_ssz_bytes(&bytes).unwrap_err(); + assert_eq!(ours, reference); + assert_eq!(ours, DecodeError::InvalidBooleanByte(2)); +} + +// ── ProgressiveList: rebase ── + +/// Applies `ops` to `list`, keeping `model` in sync. Mirrors `apply_ops`, +/// unlike `run_progressive`, makes no assertions: only the final state +/// matters to the rebase property. +fn apply_progressive_ops( + list: &mut ProgressiveList, + model: &mut Vec, + ops: Vec>, +) where + T: Value, + U: UpdateMap, +{ + for op in ops { + match op { + Op::Push(value) => { + list.push(value.clone()) + .expect("a progressive list has no limit"); + model.push(value); + } + Op::Set(index, value) => { + if !model.is_empty() { + let index = index % model.len(); + *list.get_mut(index).unwrap() = value.clone(); + model[index] = value; + } + } + Op::Apply => list.apply_updates(), + } + } +} + +/// After `orig.rebase_on(base)`: `orig`'s elements are unchanged and its root +/// matches a fresh `libssz_types::ProgressiveList` model of them; `base` is +/// untouched (elements and root); and if the two hold equal elements (so the +/// same number of subtrees too), `orig` shares every one of `base`'s +/// subtrees. `ProgressiveList::ptr_eq` only compares the whole list (per- +/// subtree sharing is not part of the public API), so unequal subtree counts +/// are not otherwise observable here; the deterministic test below checks +/// that case a different way. +fn check_progressive_rebase( + orig: &mut ProgressiveList, + orig_model: &[T], + base: &ProgressiveList, + base_model: &[T], +) -> Result<(), TestCaseError> +where + T: Value + Debug, + U: UpdateMap, +{ + orig.rebase_on(base); + + prop_assert_eq!(orig.to_vec(), orig_model.to_vec()); + let orig_reference = RefProgressiveList::from(orig_model.to_vec()); + prop_assert_eq!(root(orig), root(&orig_reference)); + + prop_assert_eq!(base.to_vec(), base_model.to_vec()); + let base_reference = RefProgressiveList::from(base_model.to_vec()); + prop_assert_eq!(root(base), root(&base_reference)); + + if orig_model == base_model { + prop_assert!(orig.ptr_eq(base)); + } + Ok(()) +} + +/// `orig` derived from `base` by a random edit sequence, so `base` ends up +/// with fewer (or equal) subtrees than `orig`: `Op::Push` only grows, +/// `Op::Set` never removes a subtree. +fn progressive_rebase_grown( + base_values: Vec, + ops: Vec>, + hash_base_first: bool, + hash_orig_first: bool, +) -> Result<(), TestCaseError> +where + T: Value + Debug, + U: UpdateMap, +{ + let base = ProgressiveList::::from(base_values.clone()); + if hash_base_first { + root(&base); + } + + let mut orig = base.clone(); + let mut orig_model = base_values.clone(); + apply_progressive_ops(&mut orig, &mut orig_model, ops); + orig.apply_updates(); + + // Round-trip through SSZ bytes: a freshly decoded list shares nothing + // with `base`'s allocation, so any sharing the rebase produces comes + // from its own content-equality walk. + let bytes = orig.to_ssz(); + let mut orig = ProgressiveList::::from_ssz_bytes(&bytes).unwrap(); + if hash_orig_first { + root(&orig); + } + + check_progressive_rebase(&mut orig, &orig_model, &base, &base_values) +} + +/// `base` derived from `orig` by pushing more elements onto a copy, so `base` +/// ends up with more (or equal) subtrees than `orig`. +fn progressive_rebase_shrunk( + orig_values: Vec, + extra: Vec, + hash_orig_first: bool, + hash_base_first: bool, +) -> Result<(), TestCaseError> +where + T: Value + Debug, + U: UpdateMap, +{ + let built = ProgressiveList::::from(orig_values.clone()); + let bytes = built.to_ssz(); + let mut orig = ProgressiveList::::from_ssz_bytes(&bytes).unwrap(); + if hash_orig_first { + root(&orig); + } + + let mut base = ProgressiveList::::from(orig_values.clone()); + let mut base_model = orig_values.clone(); + for value in extra { + base.push(value.clone()) + .expect("a progressive list has no limit"); + base_model.push(value); + } + base.apply_updates(); + if hash_base_first { + root(&base); + } + + check_progressive_rebase(&mut orig, &orig_values, &base, &base_model) +} + +proptest! { + #[test] + fn progressive_u64_rebase_grown( + base_values in vec(zero_prone_u64(), 0..400), + ops in ops(zero_prone_u64()), + hash_base_first in any::(), + hash_orig_first in any::(), + ) { + progressive_rebase_grown::>(base_values, ops, hash_base_first, hash_orig_first)?; + } + + #[test] + fn progressive_u64_rebase_shrunk( + orig_values in vec(zero_prone_u64(), 0..400), + extra in vec(zero_prone_u64(), 0..400), + hash_orig_first in any::(), + hash_base_first in any::(), + ) { + progressive_rebase_shrunk::>(orig_values, extra, hash_orig_first, hash_base_first)?; + } + + #[test] + fn progressive_item_rebase_grown( + base_values in vec(item(), 0..100), + ops in ops(item()), + hash_base_first in any::(), + hash_orig_first in any::(), + ) { + progressive_rebase_grown::>(base_values, ops, hash_base_first, hash_orig_first)?; + } + + #[test] + fn progressive_item_rebase_shrunk( + orig_values in vec(item(), 0..100), + extra in vec(item(), 0..100), + hash_orig_first in any::(), + hash_base_first in any::(), + ) { + progressive_rebase_shrunk::>(orig_values, extra, hash_orig_first, hash_base_first)?; + } +} + +/// `rebase_on` reads a subtree's *committed* content only. A pending push +/// that has just opened a new subtree (buffered, not yet folded into that +/// subtree's own tree) contributes nothing to share until `apply_updates` +/// runs. `orig` has the same subtree count as `base` (341 elements too), so +/// `rebase_on`'s positional zip actually reaches the pending subtree rather +/// than stopping short at `orig`'s length; its own value there differs from +/// `base`'s pending one, so adopting it (a bug) would be observable. `base` +/// is `&Self`, so rebasing another list onto it cannot change it either. A +/// second rebase, after `base.apply_updates()` makes the two lists equal +/// again, shows sharing resumes once there is something committed to share. +#[test] +fn progressive_rebase_ignores_a_pending_push_into_a_new_subtree() { + // u64 packs 4 per chunk; 340 elements exactly fill subtrees 0..3 + // (capacities 4, 16, 64, 256), so a 341st element opens subtree 4. + let full: Vec = (0..340).collect(); + let mut base = ProgressiveList::::from(full.clone()); + base.push(9999).expect("a progressive list has no limit"); // buffered + + let mut orig_values = full.clone(); + orig_values.push(1111); // differs from base's pending 9999 + let mut orig = ProgressiveList::::from(orig_values.clone()); + orig.rebase_on(&base); + + assert_eq!(orig.to_vec(), orig_values); + assert_eq!( + root(&orig), + root(&RefProgressiveList::from(orig_values.clone())) + ); + let mut base_expected = full.clone(); + base_expected.push(9999); + assert_eq!(base.to_vec(), base_expected); + + // Once the pending write is folded in, the two lists have the same + // shape and content again, so a fresh rebase reclaims every subtree, + // including the one that was pending above. + base.apply_updates(); + let mut orig = ProgressiveList::::from(base_expected); + orig.rebase_on(&base); + assert!(orig.ptr_eq(&base)); +} diff --git a/crates/common/types/tests/tree_bench.rs b/crates/common/types/tests/tree_bench.rs index e560b3d39..8f1421325 100644 --- a/crates/common/types/tests/tree_bench.rs +++ b/crates/common/types/tests/tree_bench.rs @@ -12,10 +12,12 @@ //! grepped out of the test harness's output. use std::alloc::{GlobalAlloc, Layout, System}; +use std::collections::BTreeMap; use std::hint::black_box; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Instant; +use ethlambda_ssz_tree::ProgressiveList; use ethlambda_types::beacon::containers::{Balances, Validator, Validators}; use ethlambda_types::beacon::preset; use ethlambda_types::beacon::primitives::BlsPubkey; @@ -313,3 +315,83 @@ fn tree_bench() { }) }); } + +/// Gloas turns the validator registry into a `ProgressiveList` (EIP-7916). +/// Same scale as [`tree_bench`], but a validators-only +/// [`Validators`](ethlambda_types::beacon::containers::Validators) baseline +/// is built in *this* test rather than reusing [`tree_bench`]'s +/// `memory_one_tree_state`, which holds `Validators` **and** `Balances` +/// together and so is not a like-for-like figure. `#[ignore]`d tests in this +/// file share one process-wide byte counter (see the module doc), so this +/// file must run with `--test-threads=1` for either figure to be trustworthy. +type ProgressiveValidators = ProgressiveList>; + +/// 1000 scattered registry indices, spread across the whole registry the way +/// [`block_validator_indices`] spreads its handful. At this sample size the +/// writes mostly land in the handful of large subtrees near the end of the +/// chain: a mainnet-sized registry's first six subtrees together hold only +/// 1365 of its elements (EIP-7916 boundaries at 0, 1, 5, 21, 85, 341, 1365, +/// ...), under one expected hit out of 1000 uniform-ish samples, so the small +/// subtrees near the start are rarely touched by this write set. +fn scattered_indices() -> impl Iterator { + (0..1000).map(|i| (i * 2_654_435_761) % VALIDATOR_COUNT) +} + +#[test] +#[ignore = "mainnet-scale benchmark: several GB and seconds to run"] +fn progressive_list_bench() { + let validators: Vec = (0..VALIDATOR_COUNT).map(validator).collect(); + + // The `List` baseline this compares against, same elements, same + // process, measured before the progressive list exists so the two + // memory deltas do not overlap. + let before = live_mib(); + let list_validators: Validators = time("list_build", || validators.clone().try_into().unwrap()); + println!( + "tree_bench memory_one_list_validators_state {:.0} MiB", + live_mib() - before + ); + time("list_cold_hash", || root(&list_validators)); + drop(list_validators); + + let before = live_mib(); + let progressive: ProgressiveValidators = + time("progressive_build", || validators.clone().into()); + println!( + "tree_bench memory_one_progressive_state {:.0} MiB", + live_mib() - before + ); + + // First root: nothing cached yet. + let first_root = time("progressive_cold_hash", || root(&progressive)); + + // 1000 scattered writes, applied, then rehashed. + let mut written = progressive.clone(); + let written_root = time("progressive_scattered_writes_apply_rehash", || { + for index in scattered_indices() { + written[index].exit_epoch = 1; + } + written.apply_updates(); + root(&written) + }); + assert_ne!( + first_root, written_root, + "the scattered writes must change the root" + ); + drop(validators); + + // An independent copy of the source data, built from scratch (sharing no + // `Arc` with `written`), then rebased onto it to reclaim every subtree + // the scattered writes did not touch. `rebase_on` only shares + // allocations; it does not adopt `written`'s content, so `independent` + // must still hash like the untouched registry. + let source: Vec = (0..VALIDATOR_COUNT).map(validator).collect(); + let mut independent: ProgressiveValidators = + time("progressive_independent_build", || source.into()); + time("progressive_rebase", || independent.rebase_on(&written)); + assert_eq!( + root(&independent), + first_root, + "a rebase must not change the contents" + ); +} From 75995bdc019a91ecf1710be0c85b39d9b937e701 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:49:51 -0300 Subject: [PATCH 004/132] test(beacon): run the v1.7.0-beta.2 spec fixtures Move the Beacon Chain spec-test fixtures from consensus-specs v1.6.1 to v1.7.0-beta.2, and take the BLS/KZG vectors from ethereum/cryptography-specs v0.1.0 instead of consensus-spec-tests' `general` config: consensus-specs stopped shipping them there as of v1.7.0-alpha.13 (ethereum/consensus-specs#5398), moving what was left of `general` (BLS) and KZG's cell/proof vectors into their own release. The v0.1.0 vectors are byte-identical to v1.6.1's, so every BLS and KZG case still passes unchanged. At this commit the suite is knowingly red on four groups the version bump surfaced, all fixed by the commits that follow: - `networking/gossip_*`: a new gossip-validation fixture format the release now ships for every fork phase0-fulu (893 minimal / 904 mainnet failures). Not modeled anywhere in this crate; fixed by merging beacon-chain-integration's own gossip validation and running these vectors against it. - Five new `ssz_static` containers this crate has no Rust type for yet: `NewPayloadRequest` (bellatrix through fulu) and `PartialDataColumnSidecar`, `PartialDataColumnPartsMetadata`, `PartialDataColumnHeader`, `PartialDataColumnGroupID` (fulu only) (1107 minimal / 45 mainnet failures). - A real phase0 fork-choice rule change (diffed against v1.6.1): a block's proposer-score boost now also requires its shuffling-dependent root to match the pre-block head's, and `is_head_weak`/`get_proposer_head` gained an equivocation-aware branch (19 minimal / 1 mainnet failures). - `transition/fulu/core/pyspec_tests/transition_with_deposit_request_right_after_fork`: a state-root mismatch, not yet root-caused (1 failure on both presets). Counts at this commit (passed/failed/ignored): - test-beacon-minimal --lib: 246/0/1 (unchanged from v1.6.1) - test-beacon-minimal beacon_spec_tests: 38688/2020/3692 - test-beacon-mainnet --lib: 245/0/1 (unchanged from v1.6.1) - test-beacon-mainnet beacon_spec_tests: 5823/951/152 --- .github/workflows/ci.yml | 12 +- .gitignore | 3 + CLAUDE.md | 22 +++- Makefile | 50 ++++++--- .../state_transition/src/beacon/bls.rs | 42 ++++--- .../state_transition/src/beacon/kzg.rs | 85 ++++++++------- .../state_transition/tests/beacon_spec/bls.rs | 20 ++-- .../tests/beacon_spec/harness.rs | 11 +- .../state_transition/tests/beacon_spec/kzg.rs | 41 ++++++- .../state_transition/tests/beacon_spec/mod.rs | 103 +++++++++++++++--- docs/beacon_stf.md | 49 ++++++--- 11 files changed, 295 insertions(+), 143 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b99cbf78e..52a5bab8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -257,10 +257,11 @@ jobs: preset: [mainnet, minimal] env: # Read by the Makefile, which declares this `?=` and so defers to the - # environment. A run reads its own preset's tree plus `general` and nothing - # else, so fetching the other preset's would be the largest single download - # in the workflow spent on files no case opens. - CONSENSUS_SPEC_TESTS_CONFIGS: general ${{ matrix.preset }} + # environment. A run reads its own preset's tree and nothing else, so + # fetching the other preset's would be the largest single download in the + # workflow spent on files no case opens. The preset-independent BLS/KZG + # vectors are a separate download (`make cryptography-specs`) below. + CONSENSUS_SPEC_TESTS_CONFIGS: ${{ matrix.preset }} steps: - uses: actions/checkout@v6 @@ -291,6 +292,9 @@ jobs: - name: Download the consensus spec test fixtures run: make consensus-spec-tests + - name: Download the BLS/KZG test vectors + run: make cryptography-specs + - name: Download the gossip validation test fixtures run: make consensus-spec-gossip-tests diff --git a/.gitignore b/.gitignore index 987e71652..8b917df2a 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,9 @@ target # Beacon Chain spec test fixtures, downloaded by `make consensus-spec-tests` /consensus-spec-tests +# BLS and KZG test vectors, downloaded by `make cryptography-specs` +/cryptography-specs + # The gossip validation vectors, from a newer release, downloaded by # `make consensus-spec-gossip-tests` /consensus-spec-tests-gossip diff --git a/CLAUDE.md b/CLAUDE.md index 6130d323f..522f37e24 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -586,8 +586,8 @@ existing once per chain. of a `[patch]` table, which `shadow/cargo-patch.toml` would collide with. - The state transition consuming these containers is **not** in this repo yet; it lives on `feat/beacon-chain-stf`, where these types are verified against - consensus-specs v1.6.1 (5705 mainnet / 40009 minimal cases). What runs here - is the containers' own round-trip and shape tests. + consensus-specs v1.7.0-beta.2. What runs here is the containers' own + round-trip and shape tests. - **`Validators` and `Balances` are `ethlambda_ssz_tree::List`s**, persistent Merkle trees that cache node hashes and share unchanged subtrees between states through `Arc`; they have no slices and no `iter_mut`. A leaf holds a @@ -652,7 +652,12 @@ transitions are in `ethlambda-types`, per the section above. Nothing above either wipes and re-downloads rather than leaving the old cases in place and silently green, or marking a partial tree complete. `CONSENSUS_SPEC_TESTS_CONFIGS` narrows the download: a run reads its own - preset's tree plus `general` and nothing else, which is what each CI job sets. + preset's tree and nothing else, which is what each CI job sets. The BLS and + KZG vectors are a separate, preset-independent download, + `make cryptography-specs`, pinned to an `ethereum/cryptography-specs` + release: consensus-specs shipped them itself, under a `general` config, + through v1.7.0-alpha.12, but v1.7.0-alpha.13 (consensus-specs #5398) moved + them out. - Preset is a **compile-time** choice (`preset-minimal` feature) because SSZ container bounds are const-generic arguments; fork scheduling is runtime because the `transition` suite moves fork epochs per case. @@ -689,8 +694,13 @@ transitions are in `ethlambda-types`, per the section above. Nothing above `IndexMut` for `ProgressiveList` (#37, an unmerged PR branch head for now). This module needs the `SszList` commit (#33) for the same reason. - **Status:** all seven forks (phase0 through fulu) have containers, fork - upgrades, state transitions, and epoch processing. Every fixture case passes - on both presets: mainnet is 5705 cases and minimal 40009. The crate's lib + upgrades, state transitions, and epoch processing. As of the v1.6.1 fixture + pin, every case passed on both presets: mainnet is 5705 cases and minimal + 40009. The v1.7.0-beta.2 bump adds failures the fixture release itself + brought: the new `networking/gossip_*` format, `ssz_static` cases for + `NewPayloadRequest` and the four new `PartialDataColumn*` containers, a + fork-choice rule change, and one `transition` case, each fixed by the + commits that follow the bump. The crate's lib target holds 200 tests with `beacon-spec-tests` on, 185 plus 15 ignored without; both figures cover lean's own unit tests as well, since the two chains now share one lib target. Fork choice is fixture-verified too: 150 mainnet @@ -699,7 +709,7 @@ transitions are in `ethlambda-types`, per the section above. Nothing above availability, and fulu's column data availability. - Nothing is ignored for being unimplemented. Ignored cases are the `LightClient*` containers (a different layer, out of scope) and the `gloas` - and `eip7805` fixture trees. Those two do not parse as a `ForkName`, so + and `heze` fixture trees. Those two do not parse as a `ForkName`, so `collect` would skip them silently; `UNMODELED_FORKS` names them and `fixture_forks/every_directory_is_accounted_for` fails on any fork directory that is neither parseable nor listed, so a new fork forces a decision. diff --git a/Makefile b/Makefile index dcce4680a..0e9a4ddc0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help fmt lint bench update cooldown-check docker-build shadow-build shadow-docker-build run-devnet test test-consensus test-node test-beacon test-beacon-mainnet test-beacon-minimal consensus-spec-tests consensus-spec-gossip-tests docs docs-deps docs-serve +.PHONY: help fmt lint bench update cooldown-check docker-build shadow-build shadow-docker-build run-devnet test test-consensus test-node test-beacon test-beacon-mainnet test-beacon-minimal consensus-spec-tests consensus-spec-gossip-tests cryptography-specs docs docs-deps docs-serve help: ## 📚 Show help for each of the Makefile recipes @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -46,16 +46,17 @@ test-node: leanSpec/fixtures ## 🧪 Run the node half of the workspace suite BEACON_TEST=cargo test -p ethlambda-state-transition --lib --test beacon_spec_tests --profile release-fast # The preset fixes SSZ container bounds at compile time, so each preset needs its -# own build, and a run walks its own fixture tree plus `general` and nothing else. -# Hence a target per preset rather than one recipe running both: CI gives each its -# own job, so the two build and run concurrently and each downloads only the trees -# its preset reads. +# own build, and a run walks its own preset's fixture tree and nothing else; the +# BLS and KZG vectors are a separate, preset-independent download +# (`cryptography-specs`). Hence a target per preset rather than one recipe +# running both: CI gives each its own job, so the two build and run +# concurrently and each downloads only the trees its preset reads. test-beacon: test-beacon-mainnet test-beacon-minimal ## 🧪 Run the Beacon Chain spec tests, both presets -test-beacon-mainnet: consensus-spec-tests consensus-spec-gossip-tests ## 🧪 Run the Beacon Chain spec tests, mainnet preset +test-beacon-mainnet: consensus-spec-tests consensus-spec-gossip-tests cryptography-specs ## 🧪 Run the Beacon Chain spec tests, mainnet preset $(BEACON_TEST) --features beacon-spec-tests -test-beacon-minimal: consensus-spec-tests consensus-spec-gossip-tests ## 🧪 Run the Beacon Chain spec tests, minimal preset +test-beacon-minimal: consensus-spec-tests consensus-spec-gossip-tests cryptography-specs ## 🧪 Run the Beacon Chain spec tests, minimal preset $(BEACON_TEST) --features beacon-spec-tests,preset-minimal # Used ONLY to resolve dependency updates: min-publish-age (.cargo/config.toml) @@ -143,15 +144,16 @@ leanSpec/fixtures: # definition of correctness for that module, so it should move only when we choose # to move it. The release publishes no checksums for these assets, so unlike the # leanSpec bundle below there is nothing to verify against. -CONSENSUS_SPEC_TESTS_VERSION ?= v1.6.1 +CONSENSUS_SPEC_TESTS_VERSION ?= v1.7.0-beta.2 CONSENSUS_SPEC_TESTS_BASE_URL ?= https://github.com/ethereum/consensus-specs/releases/download/$(CONSENSUS_SPEC_TESTS_VERSION) -# Which fixture trees to fetch. A run reads its own preset's tree plus `general`, -# the preset-independent BLS and KZG vectors, and nothing else, so a CI job pinned -# to one preset narrows this and skips the other preset's tree. That is worth -# doing: the three together are ~1.25 GiB compressed and several times that on -# disk, against a runner that has neither the space nor the time to spare. -CONSENSUS_SPEC_TESTS_CONFIGS ?= general minimal mainnet +# Which fixture trees to fetch. A run reads its own preset's tree and nothing +# else, so a CI job pinned to one preset narrows this and skips the other +# preset's tree. `general` is gone from the list: since v1.7.0-alpha.13 +# (consensus-specs #5398) its KZG vectors live only in +# ethereum/cryptography-specs, and what is left of it (the BLS suites) ships +# there too (see `cryptography-specs` below). +CONSENSUS_SPEC_TESTS_CONFIGS ?= minimal mainnet # The stamp is named after the version AND the configs, so changing either names # a file that does not exist and forces a fresh download. Depending on the @@ -228,6 +230,26 @@ $(CONSENSUS_SPEC_GOSSIP_TESTS_STAMP): done @touch $@ +# BLS and KZG test vectors, split out of consensus-spec-tests' `general` config +# since v1.7.0-alpha.13 (consensus-specs #5398). Preset-independent, so both +# beacon test targets share this one download. Flat layout, one zip: +# `tests/{bls,kzg}///`. +CRYPTOGRAPHY_SPECS_VERSION ?= v0.1.0 +CRYPTOGRAPHY_SPECS_URL ?= https://github.com/ethereum/cryptography-specs/releases/download/$(CRYPTOGRAPHY_SPECS_VERSION)/tests.zip +CRYPTOGRAPHY_SPECS_STAMP=cryptography-specs/.version-$(CRYPTOGRAPHY_SPECS_VERSION) + +cryptography-specs: $(CRYPTOGRAPHY_SPECS_STAMP) ## ⬇️ Download the BLS and KZG test vectors + +$(CRYPTOGRAPHY_SPECS_STAMP): + @command -v unzip >/dev/null || { echo "unzip is required to extract the BLS/KZG vectors"; exit 1; } + @rm -rf cryptography-specs + @mkdir -p cryptography-specs + @tmpdir=$$(mktemp -d); trap 'rm -rf "$$tmpdir"' EXIT; \ + echo "Downloading BLS/KZG test vectors ($(CRYPTOGRAPHY_SPECS_VERSION))"; \ + curl -L -f -o "$$tmpdir/tests.zip" "$(CRYPTOGRAPHY_SPECS_URL)" || exit 1; \ + unzip -q "$$tmpdir/tests.zip" -d cryptography-specs || exit 1 + @touch $@ + # lambdaclass fork of lean-quickstart: genesis keys come from `ethlambda keygen`, and the # partner clients run their devnet-5 images. An existing lean-quickstart/ is never # re-cloned, so delete it to pick up a new pin. diff --git a/crates/blockchain/state_transition/src/beacon/bls.rs b/crates/blockchain/state_transition/src/beacon/bls.rs index bf10923b9..f597885f9 100644 --- a/crates/blockchain/state_transition/src/beacon/bls.rs +++ b/crates/blockchain/state_transition/src/beacon/bls.rs @@ -451,39 +451,36 @@ mod tests { /// The root the two BLS handlers this module tests live under. /// /// BLS test vectors are configuration-independent (they do not touch any - /// preset constant), so they live under `general` rather than under a - /// preset name; see `crates/blockchain/state_transition/tests/beacon_spec/mod.rs` for the layout the - /// rest of the crate's spec tests share. This module keeps its own tiny, - /// local copy of just enough of that layout to run these two suites, - /// rather than depending on that harness. + /// preset constant). Since v1.7.0-alpha.13 (consensus-specs #5398) they ship + /// from `ethereum/cryptography-specs` rather than consensus-spec-tests, in a + /// flat `tests/bls///` layout with no suite level; see + /// `crates/blockchain/state_transition/tests/beacon_spec/mod.rs` for the + /// layout the rest of the crate's spec tests share. This module keeps its + /// own tiny, local copy of just enough of that layout to run these two + /// suites, rather than depending on that harness. fn handler_root(handler: &str) -> PathBuf { let root = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../../../consensus-spec-tests/tests/general/altair/bls") + .join("../../../cryptography-specs/tests/bls") .join(handler); assert!( root.is_dir(), - "BLS spec fixtures are missing from {}; run `make consensus-spec-tests`", + "BLS spec fixtures are missing from {}; run `make cryptography-specs`", root.display() ); root } - /// Every case's `data.yaml` under a handler: one directory level for the - /// handler's suite (named `bls` in every release seen so far), one for the - /// case itself. + /// Every case's `data.yaml` under a handler. fn fixture_cases(handler: &str) -> Vec { let mut cases = Vec::new(); - for suite in fs::read_dir(handler_root(handler)).unwrap() { - let suite_path = suite.unwrap().path(); - if !suite_path.is_dir() { + for case in fs::read_dir(handler_root(handler)).unwrap() { + let case_path = case.unwrap().path(); + if !case_path.is_dir() { continue; } - for case in fs::read_dir(&suite_path).unwrap() { - let case_path = case.unwrap().path(); - let data = case_path.join("data.yaml"); - if data.is_file() { - cases.push(data); - } + let data = case_path.join("data.yaml"); + if data.is_file() { + cases.push(data); } } cases @@ -515,7 +512,7 @@ mod tests { #[test] #[cfg_attr( not(feature = "beacon-spec-tests"), - ignore = "needs the consensus-spec fixture tree; run `make consensus-spec-tests`" + ignore = "needs the BLS test vectors; run `make cryptography-specs`" )] fn eth_aggregate_pubkeys_matches_spec_fixtures() { let mut executed = 0; @@ -588,7 +585,7 @@ mod tests { #[test] #[cfg_attr( not(feature = "beacon-spec-tests"), - ignore = "needs the consensus-spec fixture tree; run `make consensus-spec-tests`" + ignore = "needs the BLS test vectors; run `make cryptography-specs`" )] fn eth_fast_aggregate_verify_matches_spec_fixtures() { let mut executed = 0; @@ -608,7 +605,7 @@ mod tests { #[test] #[cfg_attr( not(feature = "beacon-spec-tests"), - ignore = "needs the consensus-spec fixture tree; run `make consensus-spec-tests`" + ignore = "needs the BLS test vectors; run `make cryptography-specs`" )] fn verify_accepts_a_known_good_vector_from_the_fixtures() { // `eth_fast_aggregate_verify_valid_0` has exactly one signer. A @@ -617,7 +614,6 @@ mod tests { // known-good input for `verify` without this module needing its own // signing function to produce one. let path = handler_root("eth_fast_aggregate_verify") - .join("bls") .join("eth_fast_aggregate_verify_valid_0") .join("data.yaml"); let (pubkeys, message, signature, expected) = parse_fast_aggregate_verify_case(&path); diff --git a/crates/blockchain/state_transition/src/beacon/kzg.rs b/crates/blockchain/state_transition/src/beacon/kzg.rs index f6c892a81..dbda0a4cf 100644 --- a/crates/blockchain/state_transition/src/beacon/kzg.rs +++ b/crates/blockchain/state_transition/src/beacon/kzg.rs @@ -444,41 +444,46 @@ mod tests { use super::*; - /// Root of the downloaded consensus spec test fixtures. + /// Root of the downloaded BLS/KZG test vectors. /// /// The fixture harness other stages of this module share does not exist /// yet (it lands in a later stage), so this test module loads its own /// small subset of it: KZG fixtures are plain YAML with hex strings, no - /// SSZ and no snappy compression, so no shared machinery is needed. + /// SSZ and no snappy compression, so no shared machinery is needed. Since + /// v1.7.0-alpha.13 (consensus-specs #5398) they ship from + /// `ethereum/cryptography-specs` rather than consensus-spec-tests. fn fixtures_root() -> PathBuf { - let root = - Path::new(env!("CARGO_MANIFEST_DIR")).join("../../../consensus-spec-tests/tests"); + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../../cryptography-specs/tests"); if !root.exists() { panic!( - "consensus spec test fixtures not found at {}; run `make consensus-spec-tests`", + "BLS/KZG test vectors not found at {}; run `make cryptography-specs`", root.display() ); } root } - /// Loads every `data.yaml` under `general//kzg//kzg-mainnet/*/`. - fn cases(fork: &str, handler: &str) -> Vec { - let dir = fixtures_root() - .join("general") - .join(fork) - .join("kzg") - .join(handler) - .join("kzg-mainnet"); + /// Loads every `data.yaml` under `kzg//*/`. + /// + /// The layout is flat, with no fork and no suite level: nothing about a + /// blob, a cell, or a KZG point is preset- or fork-dependent, so the + /// release ships one `kzg` tree, not one per fork. + fn cases(handler: &str) -> Vec { + let dir = fixtures_root().join("kzg").join(handler); let entries = fs::read_dir(&dir) .unwrap_or_else(|err| panic!("reading fixture directory {}: {err}", dir.display())); entries - .map(|entry| { + .filter_map(|entry| { let case_dir = entry.expect("readable directory entry").path(); + if !case_dir.is_dir() { + return None; + } let data = fs::read(case_dir.join("data.yaml")) .unwrap_or_else(|err| panic!("reading {}: {err}", case_dir.display())); - serde_yaml_ng::from_slice(&data) - .unwrap_or_else(|err| panic!("parsing {}: {err}", case_dir.display())) + Some( + serde_yaml_ng::from_slice(&data) + .unwrap_or_else(|err| panic!("parsing {}: {err}", case_dir.display())), + ) }) .collect() } @@ -532,10 +537,10 @@ mod tests { #[test] #[cfg_attr( not(feature = "beacon-spec-tests"), - ignore = "needs the consensus-spec fixture tree; run `make consensus-spec-tests`" + ignore = "needs the KZG test vectors; run `make cryptography-specs`" )] fn blob_to_kzg_commitment_matches_fixtures() { - let cases = cases("deneb", "blob_to_kzg_commitment"); + let cases = cases("blob_to_kzg_commitment"); assert!( !cases.is_empty(), "no blob_to_kzg_commitment fixtures found" @@ -556,10 +561,10 @@ mod tests { #[test] #[cfg_attr( not(feature = "beacon-spec-tests"), - ignore = "needs the consensus-spec fixture tree; run `make consensus-spec-tests`" + ignore = "needs the KZG test vectors; run `make cryptography-specs`" )] fn compute_kzg_proof_matches_fixtures() { - let cases = cases("deneb", "compute_kzg_proof"); + let cases = cases("compute_kzg_proof"); assert!(!cases.is_empty(), "no compute_kzg_proof fixtures found"); for case in &cases { let blob = hex_bytes(&case["input"]["blob"]); @@ -583,10 +588,10 @@ mod tests { #[test] #[cfg_attr( not(feature = "beacon-spec-tests"), - ignore = "needs the consensus-spec fixture tree; run `make consensus-spec-tests`" + ignore = "needs the KZG test vectors; run `make cryptography-specs`" )] fn compute_blob_kzg_proof_matches_fixtures() { - let cases = cases("deneb", "compute_blob_kzg_proof"); + let cases = cases("compute_blob_kzg_proof"); assert!( !cases.is_empty(), "no compute_blob_kzg_proof fixtures found" @@ -609,10 +614,10 @@ mod tests { #[test] #[cfg_attr( not(feature = "beacon-spec-tests"), - ignore = "needs the consensus-spec fixture tree; run `make consensus-spec-tests`" + ignore = "needs the KZG test vectors; run `make cryptography-specs`" )] fn verify_kzg_proof_matches_fixtures() { - let cases = cases("deneb", "verify_kzg_proof"); + let cases = cases("verify_kzg_proof"); assert!(!cases.is_empty(), "no verify_kzg_proof fixtures found"); for case in &cases { let expected = &case["output"]; @@ -637,10 +642,10 @@ mod tests { #[test] #[cfg_attr( not(feature = "beacon-spec-tests"), - ignore = "needs the consensus-spec fixture tree; run `make consensus-spec-tests`" + ignore = "needs the KZG test vectors; run `make cryptography-specs`" )] fn verify_blob_kzg_proof_matches_fixtures() { - let cases = cases("deneb", "verify_blob_kzg_proof"); + let cases = cases("verify_blob_kzg_proof"); assert!(!cases.is_empty(), "no verify_blob_kzg_proof fixtures found"); for case in &cases { let expected = &case["output"]; @@ -664,10 +669,10 @@ mod tests { #[test] #[cfg_attr( not(feature = "beacon-spec-tests"), - ignore = "needs the consensus-spec fixture tree; run `make consensus-spec-tests`" + ignore = "needs the KZG test vectors; run `make cryptography-specs`" )] fn verify_blob_kzg_proof_batch_matches_fixtures() { - let cases = cases("deneb", "verify_blob_kzg_proof_batch"); + let cases = cases("verify_blob_kzg_proof_batch"); assert!( !cases.is_empty(), "no verify_blob_kzg_proof_batch fixtures found" @@ -711,10 +716,10 @@ mod tests { #[test] #[cfg_attr( not(feature = "beacon-spec-tests"), - ignore = "needs the consensus-spec fixture tree; run `make consensus-spec-tests`" + ignore = "needs the KZG test vectors; run `make cryptography-specs`" )] fn compute_cells_matches_fixtures() { - let cases = cases("fulu", "compute_cells"); + let cases = cases("compute_cells"); assert!(!cases.is_empty(), "no compute_cells fixtures found"); for case in &cases { let blob = hex_bytes(&case["input"]["blob"]); @@ -736,10 +741,10 @@ mod tests { #[test] #[cfg_attr( not(feature = "beacon-spec-tests"), - ignore = "needs the consensus-spec fixture tree; run `make consensus-spec-tests`" + ignore = "needs the KZG test vectors; run `make cryptography-specs`" )] fn compute_cells_and_kzg_proofs_matches_fixtures() { - let cases = cases("fulu", "compute_cells_and_kzg_proofs"); + let cases = cases("compute_cells_and_kzg_proofs"); assert!( !cases.is_empty(), "no compute_cells_and_kzg_proofs fixtures found" @@ -768,10 +773,10 @@ mod tests { #[test] #[cfg_attr( not(feature = "beacon-spec-tests"), - ignore = "needs the consensus-spec fixture tree; run `make consensus-spec-tests`" + ignore = "needs the KZG test vectors; run `make cryptography-specs`" )] fn recover_cells_and_kzg_proofs_matches_fixtures() { - let cases = cases("fulu", "recover_cells_and_kzg_proofs"); + let cases = cases("recover_cells_and_kzg_proofs"); assert!( !cases.is_empty(), "no recover_cells_and_kzg_proofs fixtures found" @@ -817,10 +822,10 @@ mod tests { #[test] #[cfg_attr( not(feature = "beacon-spec-tests"), - ignore = "needs the consensus-spec fixture tree; run `make consensus-spec-tests`" + ignore = "needs the KZG test vectors; run `make cryptography-specs`" )] fn verify_cell_kzg_proof_batch_matches_fixtures() { - let cases = cases("fulu", "verify_cell_kzg_proof_batch"); + let cases = cases("verify_cell_kzg_proof_batch"); assert!( !cases.is_empty(), "no verify_cell_kzg_proof_batch fixtures found" @@ -870,10 +875,10 @@ mod tests { #[test] #[cfg_attr( not(feature = "beacon-spec-tests"), - ignore = "needs the consensus-spec fixture tree; run `make consensus-spec-tests`" + ignore = "needs the KZG test vectors; run `make cryptography-specs`" )] fn compute_challenge_matches_fixtures() { - let cases = cases("deneb", "compute_challenge"); + let cases = cases("compute_challenge"); assert!(!cases.is_empty(), "no compute_challenge fixtures found"); for case in &cases { let blob = hex_bytes(&case["input"]["blob"]); @@ -893,10 +898,10 @@ mod tests { #[test] #[cfg_attr( not(feature = "beacon-spec-tests"), - ignore = "needs the consensus-spec fixture tree; run `make consensus-spec-tests`" + ignore = "needs the KZG test vectors; run `make cryptography-specs`" )] fn compute_verify_cell_kzg_proof_batch_challenge_matches_fixtures() { - let cases = cases("fulu", "compute_verify_cell_kzg_proof_batch_challenge"); + let cases = cases("compute_verify_cell_kzg_proof_batch_challenge"); assert!( !cases.is_empty(), "no compute_verify_cell_kzg_proof_batch_challenge fixtures found" diff --git a/crates/blockchain/state_transition/tests/beacon_spec/bls.rs b/crates/blockchain/state_transition/tests/beacon_spec/bls.rs index df6371240..c2e22a989 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/bls.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/bls.rs @@ -7,17 +7,18 @@ //! that module's own doc for why they return the shapes they do //! (`crate::Result` for aggregation, plain `bool` for verification). //! -//! # Why `general`, not a preset +//! # Why `cryptography-specs`, not a preset //! //! Every other runner in this harness picks its fixture tree by [`super::PRESET`], //! because the functions it exercises read a preset constant somewhere in their //! call graph (committee sizes, epoch lengths, and so on). Neither BLS handler //! does: aggregating public keys and checking a pairing equation are BLS12-381 -//! operations with no notion of a validator count or a slots-per-epoch. The -//! release ships exactly one copy of these fixtures, under `general`, rather -//! than one per preset, which is what `general` signals across this whole -//! fixture tree (see `crates/blockchain/state_transition/tests/beacon_spec/mod.rs`'s module doc): a suite -//! that does not vary with the compiled-in preset. +//! operations with no notion of a validator count or a slots-per-epoch. Since +//! v1.7.0-alpha.13 (consensus-specs #5398) these vectors, along with KZG's, ship +//! from `ethereum/cryptography-specs` instead, downloaded by `make +//! cryptography-specs` into a tree of its own (see [`super::crypto_root`]): one +//! copy, not one per preset, which is what preset-independence means across +//! this whole harness. //! //! # What `output: null` means //! @@ -34,6 +35,7 @@ //! mode distinct from "the check did not pass", so every fixture under that //! handler ships a `true` or `false`, never a `null`. +use ethlambda_state_transition::beacon::ForkName; use ethlambda_state_transition::beacon::bls; use ethlambda_state_transition::beacon::primitives::{ BLS_PUBKEY_SIZE, BlsPubkey, BlsSignature, Root, @@ -152,10 +154,10 @@ fn eth_fast_aggregate_verify_case(case: &super::Case) -> Result<(), String> { /// This suite is not gated here: [`super::case_trial`] applies /// [`super::Case::in_scope`]'s gate itself, and every case this handler -/// collects lands under altair, which is always in scope, so nothing here -/// needs to check the fork. +/// collects is tagged [`ForkName::Altair`], which is always in scope, so +/// nothing here needs to check the fork. pub fn trials() -> Vec { - let cases = super::collect_all_handlers("general", "bls"); + let cases = super::collect_crypto("bls", |_| ForkName::Altair); let mut trials = vec![super::discovery_trial("bls", cases.len())]; for (handler, case) in cases { diff --git a/crates/blockchain/state_transition/tests/beacon_spec/harness.rs b/crates/blockchain/state_transition/tests/beacon_spec/harness.rs index de0b885a7..33386e176 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/harness.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/harness.rs @@ -16,7 +16,7 @@ use libssz::{SszDecode as _, SszEncode as _}; use libssz_derive::{HashTreeRoot, SszDecode, SszEncode}; use libtest_mimic::Trial; -use super::{Case, PRESET, collect, fixture_root}; +use super::{Case, PRESET, collect, crypto_root, fixture_root}; #[derive(serde::Deserialize)] struct RootFile { @@ -45,11 +45,10 @@ pub fn trials() -> Vec { "no {PRESET} fixtures under {}", root.display() ); - assert!( - root.join("general").is_dir(), - "no general fixtures under {}", - root.display() - ); + // `crypto_root` panics on its own if the BLS/KZG vectors are + // missing, which is the assertion this trial wants; calling it is + // enough. + crypto_root(); Ok(()) }), Trial::test("harness/discovery_finds_a_known_suite", || { diff --git a/crates/blockchain/state_transition/tests/beacon_spec/kzg.rs b/crates/blockchain/state_transition/tests/beacon_spec/kzg.rs index 6bdbe0bd7..b3aa9b43a 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/kzg.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/kzg.rs @@ -12,16 +12,17 @@ //! which predates this crate's shared fixture harness and is not wired into //! `spec_tests` or its per-case reporting. //! -//! # Why `general`, not a preset +//! # Why `cryptography-specs`, not a preset //! //! [`super::PRESET`] selects between `minimal` and `mainnet` because container //! bounds like `SLOTS_PER_EPOCH` are compiled in. Nothing about a blob, a cell, //! or a KZG point is preset-dependent in that sense: `BYTES_PER_BLOB`, //! `CELLS_PER_EXT_BLOB`, and the rest come from the KZG trusted setup and the //! EIP-4844/EIP-7594 scheme itself, which is one fixed size regardless of which -//! preset the beacon state around it uses. So the release ships exactly one -//! `kzg` tree per fork, under `general`, the same reason [`super::collect_all_handlers`]'s -//! own doc gives for BLS. +//! preset the beacon state around it uses. Since v1.7.0-alpha.13 +//! (consensus-specs #5398) these vectors ship from `ethereum/cryptography-specs` +//! instead, one flat `kzg` tree with no fork level, the same reason +//! [`super::collect_crypto`]'s own doc gives for BLS. //! //! # What `output: null` means //! @@ -57,12 +58,13 @@ //! ever need it. use c_kzg::Cell; +use ethlambda_state_transition::beacon::ForkName; use ethlambda_state_transition::beacon::kzg; use ethlambda_state_transition::beacon::primitives::{Bytes32, H256, KzgCommitment, KzgProof}; use libtest_mimic::Trial; use serde_yaml_ng::Value; -use super::{Case, collect_all_handlers}; +use super::Case; /// Decodes a `0x`-prefixed hex string into bytes of whatever length it has. /// @@ -459,8 +461,35 @@ fn run(handler: &str, case: &Case) -> Result<(), String> { } } +/// The handlers EIP-7594 (fulu) added; the rest are EIP-4844 (deneb). Neither +/// the fixture tree nor [`run`]'s own dispatch (see the module doc for why it +/// matches on the handler alone) carries a fork, so this list is what +/// [`trials`] hands [`super::collect_crypto`] to tag each case for +/// [`super::Case::in_scope`]. +/// +/// A handler missing from this list falls to the `else` branch below and is +/// tagged [`ForkName::Deneb`], not skipped: this list only ever needs a name +/// *added* to move a handler from deneb to fulu, never removed to stop one +/// being deneb. A release adding a thirteenth handler still reaches [`run`]'s +/// own `other` arm and fails loudly there, the same as any other unhandled +/// name; this list only decides which fork a *known* handler's cases are +/// tagged with, not whether they run at all. +const FULU_HANDLERS: &[&str] = &[ + "compute_cells", + "compute_cells_and_kzg_proofs", + "compute_verify_cell_kzg_proof_batch_challenge", + "recover_cells_and_kzg_proofs", + "verify_cell_kzg_proof_batch", +]; + pub fn trials() -> Vec { - let cases = collect_all_handlers("general", "kzg"); + let cases = super::collect_crypto("kzg", |handler| { + if FULU_HANDLERS.contains(&handler) { + ForkName::Fulu + } else { + ForkName::Deneb + } + }); let mut trials = vec![super::discovery_trial("kzg", cases.len())]; for (handler, case) in cases { diff --git a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs index 7b2024105..a92511c24 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs @@ -7,16 +7,25 @@ //! //! # Layout //! -//! The three release tarballs all unpack to `tests//...`, so they extract -//! into one directory: +//! The two release tarballs (mainnet, minimal) both unpack to +//! `tests//...`, so they extract into one directory: //! //! ```text //! consensus-spec-tests/tests/////// //! ``` //! -//! `` is `general` for the configuration-independent suites (BLS, KZG), -//! and the preset name otherwise. Since the preset is compiled in, a test run -//! walks only the tree matching its own build. +//! `` is the preset name. Since the preset is compiled in, a test run +//! walks only the tree matching its own build. The BLS and KZG vectors are a +//! separate, preset-independent download (`make cryptography-specs`; see +//! [`crypto_root`]), in a flatter layout with no fork or suite level: +//! +//! ```text +//! cryptography-specs/tests//// +//! ``` +//! +//! consensus-specs shipped these itself, under a `general` config, through +//! v1.7.0-alpha.12; v1.7.0-alpha.13 (consensus-specs #5398) moved them to +//! `ethereum/cryptography-specs` instead, and `general` is gone from this tree. //! //! # File formats //! @@ -91,6 +100,63 @@ pub fn fixture_root() -> PathBuf { root } +/// The root of the BLS and KZG vectors, a download of their own +/// (`make cryptography-specs`) since consensus-specs moved them out in +/// v1.7.0-alpha.13. +pub fn crypto_root() -> PathBuf { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../../cryptography-specs") + .join("tests"); + assert!( + root.is_dir(), + "BLS/KZG vectors are missing from {}; run `make cryptography-specs`", + root.display() + ); + root +} + +/// Every case under `cryptography-specs/tests/`, paired with its handler. +/// +/// The layout is flat, `///data.yaml`, with no fork or +/// suite level. `fork_of` supplies the fork each handler's function belongs +/// to, so a case still has one for [`Case::in_scope`]. The handler goes in +/// [`Case::suite`], which makes [`Case::id`] read `///`. +/// Files beside the case directories (`format.md` on cryptography-specs +/// master) are skipped. +pub fn collect_crypto(kind: &str, fork_of: impl Fn(&str) -> ForkName) -> Vec<(String, Case)> { + let mut out = Vec::new(); + for handler_entry in read_dir_sorted(&crypto_root().join(kind)) { + if !handler_entry.path().is_dir() { + continue; + } + let handler = handler_entry.file_name().to_string_lossy().into_owned(); + let fork = fork_of(&handler); + for case_entry in read_dir_sorted(&handler_entry.path()) { + if !case_entry.path().is_dir() { + continue; + } + out.push(( + handler.clone(), + // `suite: handler.clone()` is what makes `Case::id()` read + // `///` for one of these cases: + // `id()` climbs two directories above `self.path` (the case + // directory) and calls what it finds there `handler`, which + // for a crypto case is actually `kind` (`crypto_root().join(kind)` + // is two levels up from a case). Putting the real handler name + // in `suite` instead is what lines the two up; changing either + // side without the other breaks that reading. + Case { + path: case_entry.path(), + fork, + suite: handler.clone(), + name: case_entry.file_name().to_string_lossy().into_owned(), + }, + )); + } + } + out +} + /// The root of the gossip vector tree. /// /// The gossip vectors ship in a newer release than [`fixture_root`]'s, so they @@ -201,10 +267,10 @@ impl Case { /// Collects every case for one runner and handler across all supported forks. /// -/// `config` selects the fixture tree: [`PRESET`] for the preset-dependent -/// suites, or `"general"` for the configuration-independent ones. Forks this -/// crate does not implement are skipped, so an upstream release that adds a fork -/// does not break the build. +/// `config` selects the fixture tree; every caller here passes [`PRESET`], since +/// consensus-spec-tests no longer ships a preset-independent `general` tree (see +/// the module doc). Forks this crate does not implement are skipped, so an +/// upstream release that adds a fork does not break the build. pub fn collect(config: &str, runner: &str, handler: &str) -> Vec { collect_in(&fixture_root(), config, runner, handler) } @@ -298,10 +364,11 @@ fn read_dir_sorted(path: &Path) -> Vec { /// Fixture fork directories this crate deliberately does not model. /// -/// `gloas` is the fork after fulu, and this crate stops at fulu. `eip7805` is not -/// a fork in the sequence at all: the release ships a directory per in-flight EIP -/// whose cases are generated against a variant of some fork's rules, so there is -/// no [`ForkName`] for it to parse as. +/// `gloas` and `heze` are the two forks after fulu (in that order), and this +/// crate stops at fulu. At `v1.7.0-beta.2` the release ships no other directory +/// under either preset tree that [`ForkName::parse`] does not know: no +/// in-flight-EIP directory (`eipNNNN`) remains, unlike at `v1.6.1`, where +/// `eip7805` was one. /// /// Naming them is not bookkeeping for its own sake. A directory [`ForkName::parse`] /// does not recognize is how [`collect`] skips a fork, and that skip is *silent* @@ -311,7 +378,7 @@ fn read_dir_sorted(path: &Path) -> Vec { /// matching fails rather than reporting green, and an unparsed fork slips past /// [`HIGHEST_IMPLEMENTED_FORK`] entirely because the gate never sees the case. /// So [`fixture_fork_trials`] checks this list against the tree instead. -pub const UNMODELED_FORKS: &[&str] = &["gloas", "eip7805"]; +pub const UNMODELED_FORKS: &[&str] = &["gloas", "heze"]; /// Panics: a fixture case cannot be a lean case. /// @@ -347,9 +414,11 @@ pub fn fixture_fork_trials() -> Vec { let mut unknown: Vec = Vec::new(); let mut unmodeled: BTreeSet = BTreeSet::new(); - // Both trees, since `collect` is called with `general` for the - // configuration-independent suites as well as with the preset's own name. - for config in [PRESET, "general"] { + // Just the preset's own tree: `collect` is never called with any other + // config now that the BLS/KZG vectors live in `cryptography-specs`, whose + // flat `//` layout has no fork level for this check + // to walk in the first place. + for config in [PRESET] { for entry in read_dir_sorted(&fixture_root().join(config)) { if !entry.path().is_dir() { continue; diff --git a/docs/beacon_stf.md b/docs/beacon_stf.md index 17ebe686d..0665301ef 100644 --- a/docs/beacon_stf.md +++ b/docs/beacon_stf.md @@ -51,7 +51,8 @@ Correctness is defined by the released spec test fixtures, pinned in the ## Running the tests ```bash -make consensus-spec-tests # download the fixture tarballs (about 2.2 GB) +make consensus-spec-tests # download the fixture tarballs (mainnet and minimal) +make cryptography-specs # download the BLS/KZG test vectors make test-beacon # build and test once per preset ``` @@ -60,11 +61,14 @@ make test-beacon # build and test once per preset which is what CI does: the presets get a job each, so the two build and run concurrently instead of end to end. -A run reads its own preset's tree plus `general`, the preset-independent BLS and -KZG vectors, and opens nothing else. `CONSENSUS_SPEC_TESTS_CONFIGS` narrows the -download to that, and CI sets it, which takes roughly 1.25 GB of tarballs down to -0.8 GB for mainnet and 0.6 GB for minimal. Locally the default fetches all three, -so both presets can be run without re-downloading. +A run reads its own preset's tree and opens nothing else. +`CONSENSUS_SPEC_TESTS_CONFIGS` narrows the download to that, and CI sets it. +Locally the default fetches both, so both presets can be run without +re-downloading. The BLS and KZG vectors are a separate, preset-independent +download: consensus-specs shipped them itself, under a `general` config, +through v1.7.0-alpha.12, but v1.7.0-alpha.13 (consensus-specs #5398) moved them +to `ethereum/cryptography-specs`, which `make cryptography-specs` fetches into +its own `cryptography-specs/` tree. The download is stamped with the release it came from and the configs it holds (`consensus-spec-tests/.version--`), so changing either wipes @@ -319,9 +323,11 @@ Two details worth knowing: ## Fixture suites `consensus-spec-tests/tests///////`, -where `` is `general` for the configuration-independent suites and the -preset name otherwise. Container files are `.ssz_snappy`: SSZ compressed with -*raw* snappy, not the framed format. +where `` is the preset name. Container files are `.ssz_snappy`: SSZ +compressed with *raw* snappy, not the framed format. The BLS and KZG vectors +live in a separate tree, `cryptography-specs/tests////`, +flatter still: no fork and no suite level, since neither is preset- or +fork-dependent. Runners discover their cases from disk rather than listing them, so a fixture release that adds cases needs no code change. Two properties are deliberate: a @@ -391,13 +397,21 @@ Two things that arrangement has to be careful about: ## Status All seven forks, phase0 through fulu, have containers, fork upgrades, state -transitions, and epoch processing. Every fixture case passes on both presets: +transitions, and epoch processing. As of the v1.6.1 fixture pin, every case +passed on both presets: | Preset | Fixture cases | Ignored | Lib tests | |--------|---------------|---------|-----------| | mainnet | 5705, all green | 152 | 195 | | minimal | 40009, all green | 3692 | 196 | +The v1.7.0-beta.2 bump (see "Fixture suites" below) adds failures the fixture +release itself brought rather than any change here: the whole new `networking/gossip_*` +format, `ssz_static` cases for `NewPayloadRequest` and the four new +`PartialDataColumn*` containers, a fork-choice rule change, and one +`transition` case. The commits after the bump fix each in turn; this table is +stale until they land. + The lib counts were 244 and 245 while the containers, presets, configuration and primitives were defined here. Their 48 unit tests moved with them and run in `ethlambda-types`; the fixture counts, which are what defines correctness here, @@ -414,7 +428,7 @@ deliberate exclusions: - `LightClient*` containers under `ssz_static`, 5 container types across altair through fulu. The light-client sync protocol is a different layer from the state transition and fork choice, and is not in this module's scope. -- The `gloas` and `eip7805` fixture trees, one ignored entry each. See +- The `gloas` and `heze` fixture trees, one ignored entry each. See "Accounting for every fork directory" below. ## Accounting for every fork directory @@ -424,11 +438,10 @@ a name that does not parse is skipped. That skip is silent in a way the `HIGHEST_IMPLEMENTED_FORK` gate is not: the cases never become tests, so they are not counted as ignored either, and nothing in the output says they exist. -The release does ship two such trees. `gloas` is the fork after fulu, and -`eip7805` is not a fork in the sequence at all, being one of the per-EIP trees -generated against a variant of some fork's rules. Between them they hold 1898 -mainnet and 17539 minimal cases, all of which were previously dropped without a -trace, which is the opposite of what this harness promises. +The release does ship two such trees. `gloas` and `heze` are the two forks +after fulu, in that order. Between them they hold 3068 mainnet and 21137 +minimal cases, all of which were previously dropped without a trace, which is +the opposite of what this harness promises. So `UNMODELED_FORKS` names them, each reports as one ignored test, and `fixture_forks/every_directory_is_accounted_for` fails if the tree holds a fork @@ -437,8 +450,8 @@ forces a decision instead of quietly widening the gap. | Suite | Covers | |-------|--------| -| `general/bls` | Cryptography, fork- and preset-independent | -| `general/kzg` | Cryptography, fork- and preset-independent | +| `bls` (`cryptography-specs`) | Cryptography, fork- and preset-independent | +| `kzg` (`cryptography-specs`) | Cryptography, fork- and preset-independent | | `ssz_static` | Every fork's containers | | `shuffling` | Committee helpers | | `operations`, `epoch_processing` | Every fork's operation and epoch sub-function | From dd6ad0d729ddda20915d3347e2fc64c45459972c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 24 Sep 2026 16:40:31 -0300 Subject: [PATCH 005/132] refactor(beacon): read and write the registry by element From gloas on, the registry (validators, balances) moves to a tree-backed ssz_tree::ProgressiveList in place of ssz_tree::List, so no single accessor type can return "the list" for every fork. Element-level reads and writes (validator, balance, iter_validators, iter_balances, push_validator, ...) answer identically whichever list type backs a given fork's state, so this removes validators()/validators_mut()/balances()/balances_mut() from BeaconState in favor of them, ahead of gloas's own containers landing. bci's own callers (gossip validation, the validator client, p2p, rpc) already reached the registry through the single-element accessors this keeps, so merging bci needed no further migration beyond what this cherry-pick already carries. --- CLAUDE.md | 6 +- bin/ethlambda/src/checkpoint_sync.rs | 12 +- bin/ethlambda/src/main.rs | 2 +- .../state_transition/src/beacon/genesis.rs | 4 +- .../src/beacon/helpers/accessors.rs | 5 +- .../src/beacon/helpers/altair.rs | 4 +- .../src/beacon/helpers/electra.rs | 19 +- .../src/beacon/helpers/finality.rs | 3 +- .../src/beacon/helpers/mutators.rs | 16 +- .../state_transition/src/beacon/stf/altair.rs | 2 +- .../src/beacon/stf/capella.rs | 6 +- .../state_transition/src/beacon/stf/deneb.rs | 2 +- .../src/beacon/stf/electra.rs | 37 +-- .../src/beacon/stf/epoch/altair.rs | 12 +- .../src/beacon/stf/epoch/electra.rs | 70 ++--- .../src/beacon/stf/epoch/mod.rs | 26 +- .../src/beacon/stf/epoch/registry.rs | 4 +- .../src/beacon/stf/epoch/rewards.rs | 22 +- .../state_transition/src/beacon/stf/mod.rs | 11 +- .../src/beacon/stf/operations.rs | 11 +- .../state_transition/src/beacon/upgrade.rs | 8 +- .../common/types/src/beacon/containers/mod.rs | 294 ++++++++++++++++-- crates/net/rpc/src/beacon/states.rs | 12 +- crates/storage/src/store.rs | 7 +- docs/beacon_stf.md | 6 +- 25 files changed, 412 insertions(+), 189 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 522f37e24..2ef6677eb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -597,9 +597,9 @@ existing once per chain. buffered until `BeaconState::apply_pending_mutations`, which the state transition calls before every state-root computation. A state decoded from storage is rebased onto a cached one (`Store::get_state`). - - **`state.validator(i)` and `balances()[i]` are tree descents, not array - indexing.** A loop over the registry should walk `validators().iter()` - (zipped with `balances().iter()` where it needs both), not index per + - **`state.validator(i)` and `state.balance(i)` are tree descents, not array + indexing.** A loop over the registry should walk `iter_validators()` + (zipped with `iter_balances()` where it needs both), not index per validator: helpers that build the active-index `Vec` and then read each index back were the largest cost left in the import profile (`docs/beacon_stf.md`, "Registry and balances"). diff --git a/bin/ethlambda/src/checkpoint_sync.rs b/bin/ethlambda/src/checkpoint_sync.rs index 9030e33f7..5b01820f9 100644 --- a/bin/ethlambda/src/checkpoint_sync.rs +++ b/bin/ethlambda/src/checkpoint_sync.rs @@ -597,7 +597,7 @@ fn verify_beacon_checkpoint_state(state: &BeaconState) -> Result<(), CheckpointS return Err(CheckpointSyncError::SlotIsZero); } - if state.validators().is_empty() { + if state.validator_count() == 0 { return Err(CheckpointSyncError::NoValidators); } @@ -1051,9 +1051,13 @@ mod tests { #[test] fn a_beacon_anchor_with_no_validators_is_rejected() { let mut state = beacon_anchor_state(); - // `SszList`'s `DerefMut` target is a slice, which cannot shrink, so - // emptying the list means replacing it rather than mutating in place. - *state.validators_mut() = Default::default(); + // No element accessor can empty the registry, so this reaches into + // the fork this fixture actually decodes as (see + // `beacon_anchor_state`) directly. + let BeaconState::Phase0(inner) = &mut state else { + unreachable!("beacon_anchor_state always builds a Phase0 state"); + }; + inner.validators = Default::default(); assert!(matches!( verify_beacon_checkpoint_state(&state), diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index 5d6f07db6..bdbda91e2 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -1854,7 +1854,7 @@ async fn fetch_initial_beacon_state( info!( slot = state.slot(), fork = %state.fork_name(), - validators = state.validators().len(), + validators = state.validator_count(), finalized_epoch = state.finalized_checkpoint().epoch, anchor_block_slot = block.slot(), "Beacon checkpoint sync complete" diff --git a/crates/blockchain/state_transition/src/beacon/genesis.rs b/crates/blockchain/state_transition/src/beacon/genesis.rs index c48f444ae..9dc92fb3c 100644 --- a/crates/blockchain/state_transition/src/beacon/genesis.rs +++ b/crates/blockchain/state_transition/src/beacon/genesis.rs @@ -137,7 +137,7 @@ pub fn initialize_beacon_state_from_eth1( // one moment effective balance is set directly from the raw balance // rather than eased toward it over time: there is no previous effective // balance yet for hysteresis to protect. - for index in 0..state.validators().len() as ValidatorIndex { + for index in 0..state.validator_count() as ValidatorIndex { let balance = state.balance(index)?; let effective_balance = (balance - balance % preset::EFFECTIVE_BALANCE_INCREMENT) .min(preset::MAX_EFFECTIVE_BALANCE); @@ -154,7 +154,7 @@ pub fn initialize_beacon_state_from_eth1( // exist and what their keys are, and this root is what permanently // separates this chain, and every signature made on it, from any other // network that happens to run the same fork schedule. - let genesis_validators_root = state.validators().hash_tree_root(); + let genesis_validators_root = state.validators_root(); *state.genesis_validators_root_mut() = genesis_validators_root; Ok(state) diff --git a/crates/blockchain/state_transition/src/beacon/helpers/accessors.rs b/crates/blockchain/state_transition/src/beacon/helpers/accessors.rs index 6624360bf..9b6c83c57 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/accessors.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/accessors.rs @@ -83,8 +83,7 @@ pub fn get_randao_mix(state: &BeaconState, epoch: Epoch) -> Bytes32 { /// The validators active at `epoch`. pub fn get_active_validator_indices(state: &BeaconState, epoch: Epoch) -> Vec { state - .validators() - .iter() + .iter_validators() .enumerate() .filter(|(_, validator)| is_active_validator(validator, epoch)) .map(|(index, _)| index as ValidatorIndex) @@ -172,7 +171,7 @@ pub fn get_committee_count_per_slot(state: &BeaconState, epoch: Epoch) -> u64 { /// # Why the active set is not memoized on `epoch` or `seed` alone /// /// [`get_active_validator_indices`] reads `activation_epoch` and `exit_epoch` -/// off every validator in `state.validators()`, so it is a function of the +/// off every validator in `state.iter_validators()`, so it is a function of the /// state's registry, not of `epoch` or `seed` alone. Two different states can /// share an epoch number, or even a seed (it comes from a RANDAO mix fixed /// before either state's fork point, so two sibling branches diverging diff --git a/crates/blockchain/state_transition/src/beacon/helpers/altair.rs b/crates/blockchain/state_transition/src/beacon/helpers/altair.rs index 5c315582e..284af647a 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/altair.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/altair.rs @@ -326,7 +326,7 @@ pub fn get_flag_index_deltas( state: &BeaconState, flag_index: usize, ) -> Result<(Vec, Vec)> { - let validator_count = state.validators().len(); + let validator_count = state.validator_count(); let mut rewards = vec![0; validator_count]; let mut penalties = vec![0; validator_count]; @@ -416,7 +416,7 @@ pub fn get_inactivity_penalty_deltas( state: &BeaconState, config: &Config, ) -> Result<(Vec, Vec)> { - let validator_count = state.validators().len(); + let validator_count = state.validator_count(); let rewards = vec![0; validator_count]; let mut penalties = vec![0; validator_count]; diff --git a/crates/blockchain/state_transition/src/beacon/helpers/electra.rs b/crates/blockchain/state_transition/src/beacon/helpers/electra.rs index b2afcf7c0..6a25ed7d5 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/electra.rs @@ -730,10 +730,9 @@ pub fn queue_excess_active_balance(state: &mut BeaconState, index: ValidatorInde } let excess_balance = balance - preset::MIN_ACTIVATION_BALANCE; - // `state.balance(index)?` above already proved `index` is in range, and - // `balances` is always exactly as long as `validators`, so this indexing - // cannot panic. - state.balances_mut()[index as usize] = preset::MIN_ACTIVATION_BALANCE; + // `state.balance(index)?` above already proved `index` is in range, so + // this cannot fail. + *state.balance_mut(index)? = preset::MIN_ACTIVATION_BALANCE; let deposit = placeholder_pending_deposit(state.validator(index)?, excess_balance); electra_state(state, "queue_excess_active_balance")? @@ -760,8 +759,8 @@ pub fn queue_entire_balance_and_reset_validator( index: ValidatorIndex, ) -> Result<()> { let balance = state.balance(index)?; - // See `queue_excess_active_balance` for why this indexing cannot panic. - state.balances_mut()[index as usize] = 0; + // See `queue_excess_active_balance` for why this cannot fail. + *state.balance_mut(index)? = 0; let validator = state.validator_mut(index)?; validator.effective_balance = 0; @@ -1326,7 +1325,7 @@ mod tests { #[test] fn switch_to_compounding_validator_sets_the_prefix_and_queues_excess() { let mut state = electra_state_with_validators(4); - state.balances_mut()[0] = preset::MIN_ACTIVATION_BALANCE + 1_000_000_000; + *state.balance_mut(0).unwrap() = preset::MIN_ACTIVATION_BALANCE + 1_000_000_000; switch_to_compounding_validator(&mut state, 0).unwrap(); @@ -1340,7 +1339,7 @@ mod tests { fn queue_excess_active_balance_caps_the_balance_and_queues_the_rest() { let mut state = electra_state_with_validators(4); let pubkey = state.validator(0).unwrap().pubkey; - state.balances_mut()[0] = preset::MIN_ACTIVATION_BALANCE + 5_000_000_000; + *state.balance_mut(0).unwrap() = preset::MIN_ACTIVATION_BALANCE + 5_000_000_000; queue_excess_active_balance(&mut state, 0).unwrap(); @@ -1356,7 +1355,7 @@ mod tests { #[test] fn queue_excess_active_balance_does_nothing_at_or_below_the_minimum() { let mut state = electra_state_with_validators(4); - state.balances_mut()[0] = preset::MIN_ACTIVATION_BALANCE; + *state.balance_mut(0).unwrap() = preset::MIN_ACTIVATION_BALANCE; queue_excess_active_balance(&mut state, 0).unwrap(); let BeaconState::Electra(inner) = &state else { unreachable!("just built as Electra"); @@ -1367,7 +1366,7 @@ mod tests { #[test] fn queue_entire_balance_and_reset_validator_zeroes_the_validator() { let mut state = electra_state_with_validators(4); - state.balances_mut()[0] = 12_000_000_000; + *state.balance_mut(0).unwrap() = 12_000_000_000; queue_entire_balance_and_reset_validator(&mut state, 0).unwrap(); diff --git a/crates/blockchain/state_transition/src/beacon/helpers/finality.rs b/crates/blockchain/state_transition/src/beacon/helpers/finality.rs index 9c945c9bd..f310681e6 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/finality.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/finality.rs @@ -44,8 +44,7 @@ pub fn is_in_inactivity_leak(state: &BeaconState) -> bool { pub fn get_eligible_validator_indices(state: &BeaconState) -> Vec { let previous_epoch = get_previous_epoch(state); state - .validators() - .iter() + .iter_validators() .enumerate() .filter(|(_, validator)| { is_active_validator(validator, previous_epoch) diff --git a/crates/blockchain/state_transition/src/beacon/helpers/mutators.rs b/crates/blockchain/state_transition/src/beacon/helpers/mutators.rs index 21dffed57..e9d0879a6 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/mutators.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/mutators.rs @@ -17,10 +17,7 @@ use super::misc::compute_activation_exit_epoch; /// Adds `delta` to a validator's balance. pub fn increase_balance(state: &mut BeaconState, index: ValidatorIndex, delta: Gwei) -> Result<()> { - let balance = state - .balances_mut() - .get_mut(index as usize) - .ok_or(crate::beacon::Error::UnknownValidator(index))?; + let balance = state.balance_mut(index)?; *balance = balance.saturating_add(delta); Ok(()) } @@ -31,10 +28,7 @@ pub fn increase_balance(state: &mut BeaconState, index: ValidatorIndex, delta: G /// unsigned and explicitly floors this at zero, since a penalty larger than the /// remaining balance is normal rather than an error. pub fn decrease_balance(state: &mut BeaconState, index: ValidatorIndex, delta: Gwei) -> Result<()> { - let balance = state - .balances_mut() - .get_mut(index as usize) - .ok_or(crate::beacon::Error::UnknownValidator(index))?; + let balance = state.balance_mut(index)?; *balance = balance.saturating_sub(delta); Ok(()) } @@ -60,8 +54,7 @@ pub fn initiate_validator_exit( let earliest = compute_activation_exit_epoch(get_current_epoch(state)); let mut exit_queue_epoch = state - .validators() - .iter() + .iter_validators() .map(|validator| validator.exit_epoch) .filter(|epoch| *epoch != FAR_FUTURE_EPOCH) .chain(core::iter::once(earliest)) @@ -69,8 +62,7 @@ pub fn initiate_validator_exit( .unwrap_or(earliest); let churn_at_that_epoch = state - .validators() - .iter() + .iter_validators() .filter(|validator| validator.exit_epoch == exit_queue_epoch) .count() as u64; if churn_at_that_epoch >= get_validator_churn_limit(state, config) { diff --git a/crates/blockchain/state_transition/src/beacon/stf/altair.rs b/crates/blockchain/state_transition/src/beacon/stf/altair.rs index 22b320f52..14017a83b 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/altair.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/altair.rs @@ -342,7 +342,7 @@ pub fn process_sync_aggregate( seats_by_pubkey.entry(pubkey).or_default().push(seat); } let mut resolved: Vec> = vec![None; committee_pubkeys.len()]; - for (index, validator) in state.validators().iter().enumerate() { + for (index, validator) in state.iter_validators().enumerate() { if let Some(seats) = seats_by_pubkey.get(&validator.pubkey) { for &seat in seats { resolved[seat] = Some(index as ValidatorIndex); diff --git a/crates/blockchain/state_transition/src/beacon/stf/capella.rs b/crates/blockchain/state_transition/src/beacon/stf/capella.rs index b56639a98..6c42d739a 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/capella.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/capella.rs @@ -144,7 +144,7 @@ pub fn get_expected_withdrawals(state: &BeaconState) -> Result capella::ExecutionPayload { diff --git a/crates/blockchain/state_transition/src/beacon/stf/deneb.rs b/crates/blockchain/state_transition/src/beacon/stf/deneb.rs index 528892b21..e90ca6a6a 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/deneb.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/deneb.rs @@ -222,7 +222,7 @@ pub fn process_withdrawals( .ok_or(Error::ArithmeticOverflow("latest_withdrawal.index + 1"))?; } - let validator_count = state.validators().len() as ValidatorIndex; + let validator_count = state.validator_count() as ValidatorIndex; let next_validator_index = if expected_withdrawals.len() == preset::MAX_WITHDRAWALS_PER_PAYLOAD { let latest_withdrawal = expected_withdrawals diff --git a/crates/blockchain/state_transition/src/beacon/stf/electra.rs b/crates/blockchain/state_transition/src/beacon/stf/electra.rs index b7152b369..0be46b258 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/electra.rs @@ -331,12 +331,10 @@ pub fn add_validator_to_registry( withdrawal_credentials: Bytes32, amount: Gwei, ) -> Result<()> { - state.validators_mut().push(get_validator_from_deposit( - pubkey, - withdrawal_credentials, + state.push_validator( + get_validator_from_deposit(pubkey, withdrawal_credentials, amount), amount, - ))?; - state.balances_mut().push(amount)?; + )?; let mut fields = block_mut(state, "add_validator_to_registry")?; fields.epoch_participation_mut(true).push(0)?; @@ -405,7 +403,7 @@ pub fn apply_deposit( signature: &BlsSignature, config: &Config, ) -> Result<()> { - let already_registered = state.validators().iter().any(|v| v.pubkey == pubkey); + let already_registered = state.iter_validators().any(|v| v.pubkey == pubkey); if !already_registered { if is_valid_deposit_signature(pubkey, withdrawal_credentials, amount, signature, config) { // The registry entry starts at a zero balance; see this @@ -983,7 +981,7 @@ pub fn get_expected_withdrawals(state: &BeaconState) -> Result<(Vec Result<()> { - let validator_count = state.validators().len(); + let validator_count = state.validator_count(); let (previous_epoch_participation, current_epoch_participation, _) = state.altair_validator_lists_mut()?; @@ -256,7 +256,7 @@ mod tests { use super::*; use crate::beacon::fork::ForkName; use crate::beacon::helpers::altair::add_flag; - use crate::beacon::primitives::BlsPubkey; + use crate::beacon::primitives::{BlsPubkey, Gwei}; /// A deterministic but genuinely valid BLS public key for validator /// `index`. @@ -414,13 +414,13 @@ mod tests { let config = Config::mainnet(); let mut state = altair_state_with_validators(4); *state.slot_mut() = 0; - let balances_before = state.balances().clone(); + let balances_before: Vec = state.iter_balances().collect(); process_rewards_and_penalties(&mut state, &config).unwrap(); assert_eq!( - state.balances(), - &balances_before, + state.iter_balances().collect::>(), + balances_before, "the genesis epoch has no previous epoch to reward" ); } diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs index 2f6bf9fcf..42ff09bdd 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs @@ -133,8 +133,7 @@ pub fn process_registry_updates(state: &mut BeaconState, config: &Config) -> Res let finalized_epoch = state.finalized_checkpoint().epoch; let actions: Vec> = state - .validators() - .iter() + .iter_validators() .map(|validator| { if is_eligible_for_activation_queue(validator) { Some(RegistryAction::QueueForActivation) @@ -246,7 +245,7 @@ pub fn process_slashings(state: &mut BeaconState, _config: &Config) -> Result<() // reading a stable registry, the same reason // `super::registry::process_slashings` does. let mut penalties = Vec::new(); - for (index, validator) in state.validators().iter().enumerate() { + for (index, validator) in state.iter_validators().enumerate() { if validator.slashed && epoch + withdrawable_offset == validator.withdrawable_epoch { let effective_balance_increments = validator.effective_balance / increment; let penalty = penalty_per_effective_balance_increment @@ -315,7 +314,7 @@ pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Res // mutable borrows of `state` begin. Taking `pending_deposits` by value // here, rather than iterating it in place, is what frees `state` for // those: once the queue is a plain `Vec` of its own, reading - // `state.validators()` and crediting balances through `state` cannot + // `state.iter_validators()` and crediting balances through `state` cannot // conflict with walking the deposits that drive those reads and writes. let (deposit_requests_start_index, available_for_processing, deposits) = { let mut fields = pending_queue_fields(state, "process_pending_deposits")?; @@ -362,20 +361,16 @@ pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Res break; } - let (is_validator_exited, is_validator_withdrawn) = match state - .validators() - .iter() - .position(|validator| validator.pubkey == deposit.pubkey) - { - Some(index) => { - let validator = &state.validators()[index]; + let (is_validator_exited, is_validator_withdrawn) = state + .iter_validators() + .find(|validator| validator.pubkey == deposit.pubkey) + .map(|validator| { ( validator.exit_epoch < FAR_FUTURE_EPOCH, validator.withdrawable_epoch < next_epoch, ) - } - None => (false, false), - }; + }) + .unwrap_or((false, false)); if is_validator_withdrawn { apply_pending_deposit(state, deposit, config)?; @@ -444,8 +439,7 @@ fn apply_pending_deposit( config: &Config, ) -> Result<()> { let existing_index = state - .validators() - .iter() + .iter_validators() .position(|validator| validator.pubkey == deposit.pubkey); match existing_index { @@ -543,8 +537,7 @@ fn add_validator_from_pending_deposit( validator.effective_balance = (amount - amount % preset::EFFECTIVE_BALANCE_INCREMENT).min(max_effective_balance); - state.validators_mut().push(validator)?; - state.balances_mut().push(amount)?; + state.push_validator(validator, amount)?; pending_queue_fields(state, "add_validator_from_pending_deposit")? .push_empty_participation_and_inactivity() } @@ -642,26 +635,31 @@ pub fn process_effective_balance_updates(state: &mut BeaconState) -> Result<()> const UPWARD_THRESHOLD: Gwei = HYSTERESIS_INCREMENT * preset::HYSTERESIS_UPWARD_MULTIPLIER; // Two passes for the same reason `super::process_effective_balance_updates` - // needs them: `state` is an enum over per-fork structs, so there is no - // way to hold `validators` mutably while also reading `balances`, or - // (here) while calling `get_max_effective_balance` on the validator - // being decided on. + // needs them: `validator_mut` clones the element into the update buffer + // on every call, whether or not it is then written (see its own doc), so + // deciding and writing in one combined pass would buffer and rehash the + // whole registry instead of only the validators that actually move; that + // also keeps `get_max_effective_balance`, called here on the validator + // being decided on, reading rather than fighting a live mutable borrow. + debug_assert_eq!(state.iter_validators().len(), state.iter_balances().len()); let mut updates = Vec::new(); - for (index, validator) in state.validators().iter().enumerate() { - let balance = state.balances()[index]; + for (index, (validator, balance)) in state + .iter_validators() + .zip(state.iter_balances()) + .enumerate() + { if balance + DOWNWARD_THRESHOLD < validator.effective_balance || validator.effective_balance + UPWARD_THRESHOLD < balance { let max_effective_balance = get_max_effective_balance(validator); let effective = (balance - balance % preset::EFFECTIVE_BALANCE_INCREMENT) .min(max_effective_balance); - updates.push((index, effective)); + updates.push((index as ValidatorIndex, effective)); } } - let validators = state.validators_mut(); for (index, effective) in updates { - validators[index].effective_balance = effective; + state.validator_mut(index)?.effective_balance = effective; } Ok(()) } @@ -969,10 +967,10 @@ mod tests { .push(deposit) .unwrap(); - let validators_before = state.validators().len(); + let validators_before = state.validator_count(); process_pending_deposits(&mut state, &config).unwrap(); - assert_eq!(state.validators().len(), validators_before + 1); + assert_eq!(state.validator_count(), validators_before + 1); let new_index = validators_before as ValidatorIndex; let new_validator = state.validator(new_index).unwrap(); assert_eq!(new_validator.pubkey, pubkey); @@ -990,13 +988,13 @@ mod tests { BeaconState::Electra(inner) => { assert_eq!( inner.previous_epoch_participation.len(), - state.validators().len() + state.validator_count() ); assert_eq!( inner.current_epoch_participation.len(), - state.validators().len() + state.validator_count() ); - assert_eq!(inner.inactivity_scores.len(), state.validators().len()); + assert_eq!(inner.inactivity_scores.len(), state.validator_count()); } _ => unreachable!("electra_state_with_validators always builds an Electra state"), } @@ -1020,11 +1018,11 @@ mod tests { .push(deposit) .unwrap(); - let validators_before = state.validators().len(); + let validators_before = state.validator_count(); process_pending_deposits(&mut state, &config).unwrap(); assert_eq!( - state.validators().len(), + state.validator_count(), validators_before, "an invalid signature must not create a validator" ); @@ -1163,8 +1161,8 @@ mod tests { // Push both balances far above either ceiling, so the update fires // and both validators are actually capped, not merely nudged. let huge = preset::MAX_EFFECTIVE_BALANCE_ELECTRA + preset::EFFECTIVE_BALANCE_INCREMENT; - state.balances_mut()[0] = huge; - state.balances_mut()[1] = huge; + *state.balance_mut(0).unwrap() = huge; + *state.balance_mut(1).unwrap() = huge; process_effective_balance_updates(&mut state).unwrap(); diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs index f0fb05521..00659d1c4 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs @@ -271,27 +271,31 @@ pub fn process_effective_balance_updates(state: &mut BeaconState) -> Result<()> const DOWNWARD_THRESHOLD: Gwei = HYSTERESIS_INCREMENT * preset::HYSTERESIS_DOWNWARD_MULTIPLIER; const UPWARD_THRESHOLD: Gwei = HYSTERESIS_INCREMENT * preset::HYSTERESIS_UPWARD_MULTIPLIER; - // Decided in one pass and applied in another. The state is an enum over - // per-fork structs, so the accessors hand out a borrow of the whole state - // rather than of one field, and there is no way to hold `validators` mutably - // while reading `balances`. Collecting the decisions first keeps this - // fork-independent, which matters because every fork runs this step - // unchanged. + // Decided in one pass and applied in another: `validator_mut` clones the + // element into the update buffer on every call, whether or not it is + // then written (see its own doc), so deciding and writing in one combined + // pass would buffer and rehash the whole registry instead of only the + // validators that actually move. Collecting the decisions first also + // keeps this fork-independent, which matters because every fork runs + // this step unchanged. + debug_assert_eq!(state.iter_validators().len(), state.iter_balances().len()); let mut updates = Vec::new(); - for (index, validator) in state.validators().iter().enumerate() { - let balance = state.balances()[index]; + for (index, (validator, balance)) in state + .iter_validators() + .zip(state.iter_balances()) + .enumerate() + { if balance + DOWNWARD_THRESHOLD < validator.effective_balance || validator.effective_balance + UPWARD_THRESHOLD < balance { let effective = (balance - balance % preset::EFFECTIVE_BALANCE_INCREMENT) .min(preset::MAX_EFFECTIVE_BALANCE); - updates.push((index, effective)); + updates.push((index as ValidatorIndex, effective)); } } - let validators = state.validators_mut(); for (index, effective) in updates { - validators[index].effective_balance = effective; + state.validator_mut(index)?.effective_balance = effective; } Ok(()) } diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/registry.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/registry.rs index 5c8f30dc2..53df43d99 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/registry.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/registry.rs @@ -74,7 +74,7 @@ fn activation_churn_limit(state: &BeaconState, config: &Config) -> Result { /// fulu state silently activate validators under deneb's superseded rule. pub fn process_registry_updates(state: &mut BeaconState, config: &Config) -> Result<()> { let current_epoch = get_current_epoch(state); - let validator_count = state.validators().len() as ValidatorIndex; + let validator_count = state.validator_count() as ValidatorIndex; // `initiate_validator_exit` itself scans every validator's exit epoch to // find the queue's current tail, which needs `state` uncommitted to any @@ -191,7 +191,7 @@ pub fn process_slashings(state: &mut BeaconState, _config: &Config) -> Result<() // vector, not the validator being read here, but every other mutator in // this module needs the same shape, so this one follows suit. let mut penalties = Vec::new(); - for (index, validator) in state.validators().iter().enumerate() { + for (index, validator) in state.iter_validators().enumerate() { if validator.slashed && epoch + withdrawable_offset == validator.withdrawable_epoch { // Factored out from the penalty numerator to avoid a `uint64` // overflow, exactly as the specification does; multiplying before diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/rewards.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/rewards.rs index a6bfab32b..0c19b3c69 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/rewards.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/rewards.rs @@ -101,8 +101,8 @@ pub fn get_attestation_component_deltas( state: &BeaconState, attestations: &[PendingAttestation], ) -> Result<(Vec, Vec)> { - let mut rewards = vec![0; state.validators().len()]; - let mut penalties = vec![0; state.validators().len()]; + let mut rewards = vec![0; state.validator_count()]; + let mut penalties = vec![0; state.validator_count()]; let total_balance = get_total_active_balance(state)?; let unslashed_attesting_indices = get_unslashed_attesting_indices(state, attestations)?; @@ -170,7 +170,7 @@ pub fn get_inclusion_delay_deltas( state: &BeaconState, _config: &Config, ) -> Result<(Vec, Vec)> { - let mut rewards = vec![0; state.validators().len()]; + let mut rewards = vec![0; state.validator_count()]; let matching_source_attestations = get_matching_source_attestations(state, get_previous_epoch(state))?; @@ -214,7 +214,7 @@ pub fn get_inclusion_delay_deltas( } // No penalties are associated with inclusion delay. - let penalties = vec![0; state.validators().len()]; + let penalties = vec![0; state.validator_count()]; Ok((rewards, penalties)) } @@ -234,7 +234,7 @@ pub fn get_inactivity_penalty_deltas( state: &BeaconState, _config: &Config, ) -> Result<(Vec, Vec)> { - let mut penalties = vec![0; state.validators().len()]; + let mut penalties = vec![0; state.validator_count()]; if is_in_inactivity_leak(state) { let matching_target_attestations = @@ -269,7 +269,7 @@ pub fn get_inactivity_penalty_deltas( } // No rewards are associated with inactivity penalties. - let rewards = vec![0; state.validators().len()]; + let rewards = vec![0; state.validator_count()]; Ok((rewards, penalties)) } @@ -290,7 +290,7 @@ pub fn get_attestation_deltas( let (inclusion_delay_rewards, _) = get_inclusion_delay_deltas(state, config)?; let (_, inactivity_penalties) = get_inactivity_penalty_deltas(state, config)?; - let validator_count = state.validators().len(); + let validator_count = state.validator_count(); let mut rewards = vec![0; validator_count]; let mut penalties = vec![0; validator_count]; for i in 0..validator_count { @@ -318,7 +318,7 @@ pub fn process_rewards_and_penalties(state: &mut BeaconState, config: &Config) - } let (rewards, penalties) = get_attestation_deltas(state, config)?; - for index in 0..state.validators().len() as ValidatorIndex { + for index in 0..state.validator_count() as ValidatorIndex { increase_balance(state, index, rewards[index as usize])?; decrease_balance(state, index, penalties[index as usize])?; } @@ -369,13 +369,13 @@ mod tests { let config = Config::mainnet(); let mut state = crate::beacon::helpers::test_state::with_validators(4); *state.slot_mut() = 0; - let balances_before = state.balances().clone(); + let balances_before: Vec = state.iter_balances().collect(); process_rewards_and_penalties(&mut state, &config).unwrap(); assert_eq!( - state.balances(), - &balances_before, + state.iter_balances().collect::>(), + balances_before, "the genesis epoch has no previous epoch to reward" ); } diff --git a/crates/blockchain/state_transition/src/beacon/stf/mod.rs b/crates/blockchain/state_transition/src/beacon/stf/mod.rs index 743672863..fabe1aeac 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/mod.rs @@ -322,15 +322,13 @@ mod tests { #[test] fn process_slot_leaves_no_buffered_registry_writes() { let mut state = test_state::with_validators(4); - state.balances_mut()[0] += 1; + *state.balance_mut(0).unwrap() += 1; state.validator_mut(1).unwrap().effective_balance -= 1; - assert!(state.balances().has_pending_updates()); - assert!(state.validators().has_pending_updates()); + assert!(state.has_pending_mutations()); process_slot(&mut state).unwrap(); - assert!(!state.balances().has_pending_updates()); - assert!(!state.validators().has_pending_updates()); + assert!(!state.has_pending_mutations()); } /// Epoch processing (`process_rewards_and_penalties` here) writes every @@ -347,7 +345,6 @@ mod tests { process_slots(&mut state, target_slot, &config).unwrap(); - assert!(!state.balances().has_pending_updates()); - assert!(!state.validators().has_pending_updates()); + assert!(!state.has_pending_mutations()); } } diff --git a/crates/blockchain/state_transition/src/beacon/stf/operations.rs b/crates/blockchain/state_transition/src/beacon/stf/operations.rs index efe0d154a..9b7cd8bfd 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/operations.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/operations.rs @@ -447,12 +447,10 @@ pub fn add_validator_to_registry( ForkName::Lean => lean_state_unreachable("add_validator_to_registry"), }; - state.validators_mut().push(get_validator_from_deposit( - pubkey, - withdrawal_credentials, + state.push_validator( + get_validator_from_deposit(pubkey, withdrawal_credentials, amount), amount, - ))?; - state.balances_mut().push(amount)?; + )?; if grows_altair_lists { let (previous, current, scores) = state.altair_validator_lists_mut()?; @@ -487,8 +485,7 @@ pub fn apply_deposit( config: &Config, ) -> Result<()> { let existing_index = state - .validators() - .iter() + .iter_validators() .position(|validator| validator.pubkey == pubkey); match existing_index { diff --git a/crates/blockchain/state_transition/src/beacon/upgrade.rs b/crates/blockchain/state_transition/src/beacon/upgrade.rs index 8541ae8ff..d31eff1c9 100644 --- a/crates/blockchain/state_transition/src/beacon/upgrade.rs +++ b/crates/blockchain/state_transition/src/beacon/upgrade.rs @@ -696,8 +696,7 @@ pub fn upgrade_to_electra(pre: &BeaconState, config: &Config) -> Result = post - .validators() - .iter() + .iter_validators() .enumerate() .filter(|(_, validator)| validator.activation_epoch == constants::FAR_FUTURE_EPOCH) .map(|(index, _)| index as ValidatorIndex) @@ -705,7 +704,7 @@ pub fn upgrade_to_electra(pre: &BeaconState, config: &Config) -> Result Result = post - .validators() - .iter() + .iter_validators() .enumerate() .filter(|(_, validator)| { crate::beacon::helpers::electra::has_compounding_withdrawal_credential(validator) diff --git a/crates/common/types/src/beacon/containers/mod.rs b/crates/common/types/src/beacon/containers/mod.rs index 40a240d99..617f2b18d 100644 --- a/crates/common/types/src/beacon/containers/mod.rs +++ b/crates/common/types/src/beacon/containers/mod.rs @@ -461,8 +461,6 @@ shared_state_accessors!( (historical_roots, historical_roots_mut, HistoricalRoots), (eth1_data, eth1_data_mut, Eth1Data), (eth1_data_votes, eth1_data_votes_mut, Eth1DataVotes), - (validators, validators_mut, Validators), - (balances, balances_mut, Balances), (randao_mixes, randao_mixes_mut, RandaoMixes), (slashings, slashings_mut, Slashings), (justification_bits, justification_bits_mut, JustificationBits), @@ -520,21 +518,119 @@ impl BeaconState { } impl BeaconState { + /// The number of validators in the registry. + pub fn validator_count(&self) -> usize { + dispatch_state!(self, "validator_count", |state| state.validators.len()) + } + + /// Every validator, in index order, pending writes included. + /// + /// An iterator rather than a slice: from gloas on the registry is a + /// [`ethlambda_ssz_tree::ProgressiveList`] rather than a [`Validators`], + /// and the two list types agree on `Item` and `ExactSizeIterator` but not + /// on the concrete iterator type, so an iterator is what an accessor here + /// can promise across every fork. Prefer this over [`Self::validator`] in + /// a loop: each `validator()` call redoes the fork dispatch and a tree + /// descent, where this walks the registry once, sequentially. + pub fn iter_validators(&self) -> impl ExactSizeIterator { + dispatch_state!(self, "iter_validators", |state| state.validators.iter()) + } + + /// Every balance, in index order, pending writes included. See + /// [`Self::iter_validators`] for why this is an iterator. + pub fn iter_balances(&self) -> impl ExactSizeIterator { + dispatch_state!(self, "iter_balances", |state| state + .balances + .iter() + .copied()) + } + /// The validator at `index`. /// /// A named error rather than an `Option`, since the specification indexes the /// registry in many places and an out-of-range index is always a fault. pub fn validator(&self, index: ValidatorIndex) -> Result<&Validator> { - self.validators() - .get(index as usize) - .ok_or(Error::UnknownValidator(index)) + dispatch_state!(self, "validator", |state| state + .validators + .get(index as usize)) + .ok_or(Error::UnknownValidator(index)) } - /// The validator at `index`, mutably. + /// The validator at `index`, mutably. Call only when actually writing: + /// the underlying list clones the element into its update buffer on this + /// call alone, whether or not the caller goes on to change it. pub fn validator_mut(&mut self, index: ValidatorIndex) -> Result<&mut Validator> { - self.validators_mut() - .get_mut(index as usize) - .ok_or(Error::UnknownValidator(index)) + dispatch_state!(self, "validator_mut", |state| state + .validators + .get_mut(index as usize)) + .ok_or(Error::UnknownValidator(index)) + } + + /// The balance of the validator at `index`. + pub fn balance(&self, index: ValidatorIndex) -> Result { + dispatch_state!(self, "balance", |state| state + .balances + .get(index as usize) + .copied()) + .ok_or(Error::UnknownValidator(index)) + } + + /// The balance of the validator at `index`, mutably. Call only when + /// actually writing; see [`Self::validator_mut`] for why. + pub fn balance_mut(&mut self, index: ValidatorIndex) -> Result<&mut Gwei> { + dispatch_state!(self, "balance_mut", |state| state + .balances + .get_mut(index as usize)) + .ok_or(Error::UnknownValidator(index)) + } + + /// Appends a validator and its balance together, keeping the two + /// positionally parallel lists in step. From altair on there are three + /// more such lists (`previous_epoch_participation`, + /// `current_epoch_participation`, `inactivity_scores`); callers on those + /// forks still push to them separately, through + /// [`Self::altair_validator_lists_mut`], since this function only owns + /// the two lists every fork has. + pub fn push_validator(&mut self, validator: Validator, balance: Gwei) -> Result<()> { + dispatch_state!(self, "push_validator", |state| { + state.validators.push(validator)?; + state.balances.push(balance)?; + Ok(()) + }) + } + + /// The registry's own root: the value genesis records as + /// `genesis_validators_root`. + pub fn validators_root(&self) -> Root { + dispatch_state!(self, "validators_root", |state| state + .validators + .hash_tree_root()) + } + + /// Whether both validator lists are backed by the same committed tree: a + /// cheap check of sharing, not of equality. After `rebase_on` it holds + /// exactly when the two registries are equal in content. The storage + /// tests use it to prove `get_state` rebased a decoded state onto the + /// resident parent. Balances are not compared: a state whose balances + /// changed shares only its untouched subtrees, which a root-pointer + /// check cannot see. + pub fn validators_ptr_eq(&self, other: &BeaconState) -> bool { + match (self.registry(), other.registry()) { + (Some((v, _)), Some((ov, _))) => v.ptr_eq(ov), + _ => false, + } + } + + /// Both registry lists, or `None` for lean. Private: callers use the + /// element accessors above. + fn registry(&self) -> Option<(&Validators, &Balances)> { + match self { + BeaconState::Lean(_) => None, + _ => Some(dispatch_state!(self, "registry", |state| ( + &state.validators, + &state.balances + ))), + } } /// Folds every buffered write into the tree-backed fields (`validators`, @@ -549,8 +645,10 @@ impl BeaconState { if matches!(self, BeaconState::Lean(_)) { return; } - self.validators_mut().apply_updates(); - self.balances_mut().apply_updates(); + dispatch_state!(self, "apply_pending_mutations", |state| { + state.validators.apply_updates(); + state.balances.apply_updates(); + }) } /// Whether `validators` or `balances` has a write [`apply_pending_mutations`] @@ -563,10 +661,8 @@ impl BeaconState { /// /// [`apply_pending_mutations`]: BeaconState::apply_pending_mutations pub fn has_pending_mutations(&self) -> bool { - if matches!(self, BeaconState::Lean(_)) { - return false; - } - self.validators().has_pending_updates() || self.balances().has_pending_updates() + self.registry() + .is_some_and(|(v, b)| v.has_pending_updates() || b.has_pending_updates()) } /// Makes this state's tree-backed fields share every unchanged subtree @@ -577,19 +673,16 @@ impl BeaconState { /// Works across forks, since `validators` and `balances` have one type in /// every fork. A no-op if either state is lean. pub fn rebase_on(&mut self, base: &BeaconState) { - if matches!(self, BeaconState::Lean(_)) || matches!(base, BeaconState::Lean(_)) { + let Some((base_validators, base_balances)) = base.registry() else { + return; + }; + if matches!(self, BeaconState::Lean(_)) { return; } - self.validators_mut().rebase_on(base.validators()); - self.balances_mut().rebase_on(base.balances()); - } - - /// The balance of the validator at `index`. - pub fn balance(&self, index: ValidatorIndex) -> Result { - self.balances() - .get(index as usize) - .copied() - .ok_or(Error::UnknownValidator(index)) + dispatch_state!(self, "rebase_on", |state| { + state.validators.rebase_on(base_validators); + state.balances.rebase_on(base_balances); + }) } /// The randao mix for `epoch`, which the specification indexes modulo the @@ -1171,9 +1264,158 @@ signed_beacon_block_accessors!( #[cfg(test)] mod tests { use super::*; + use crate::beacon::constants; use crate::beacon::preset; use crate::beacon::primitives::{ExecutionAddress, Uint256}; + /// A Fulu state with `count` validators, each with a full effective + /// balance and otherwise eligible, and every other field at an all-zero + /// placeholder sized to the preset. Good enough for the registry accessor + /// tests below, which touch `validators`/`balances` only: nothing here + /// verifies a signature or aggregates a sync committee. + /// + /// `ethlambda-types` cannot reuse `ethlambda-state-transition`'s + /// `helpers::test_state` builder (dependency runs the other way), so this + /// is a small copy of its Fulu literal. + fn small_fulu_state(count: usize) -> BeaconState { + let validators: Vec = (0..count) + .map(|_| Validator { + effective_balance: preset::MAX_EFFECTIVE_BALANCE, + activation_eligibility_epoch: 0, + activation_epoch: 0, + exit_epoch: constants::FAR_FUTURE_EPOCH, + withdrawable_epoch: constants::FAR_FUTURE_EPOCH, + ..Default::default() + }) + .collect(); + let zero_root_vector = || -> BlockRoots { + vec![Root::ZERO; preset::SLOTS_PER_HISTORICAL_ROOT] + .try_into() + .expect("the vector is built at its exact length") + }; + let empty_sync_committee = || altair::SyncCommittee { + pubkeys: vec![Default::default(); preset::SYNC_COMMITTEE_SIZE] + .try_into() + .expect("built at exactly SYNC_COMMITTEE_SIZE"), + aggregate_pubkey: Default::default(), + }; + + BeaconState::Fulu(fulu::BeaconState { + genesis_time: 0, + genesis_validators_root: Root::ZERO, + slot: preset::SLOTS_PER_EPOCH, + fork: Default::default(), + latest_block_header: Default::default(), + block_roots: zero_root_vector(), + state_roots: zero_root_vector(), + historical_roots: Default::default(), + eth1_data: Default::default(), + eth1_data_votes: Default::default(), + eth1_deposit_index: 0, + validators: validators + .try_into() + .expect("count is far below VALIDATOR_REGISTRY_LIMIT"), + balances: vec![preset::MAX_EFFECTIVE_BALANCE; count] + .try_into() + .expect("count is far below VALIDATOR_REGISTRY_LIMIT"), + randao_mixes: vec![Bytes32::ZERO; preset::EPOCHS_PER_HISTORICAL_VECTOR] + .try_into() + .expect("the vector is built at its exact length"), + slashings: vec![0; preset::EPOCHS_PER_SLASHINGS_VECTOR] + .try_into() + .expect("the vector is built at its exact length"), + previous_epoch_participation: vec![0; count] + .try_into() + .expect("count is far below VALIDATOR_REGISTRY_LIMIT"), + current_epoch_participation: vec![0; count] + .try_into() + .expect("count is far below VALIDATOR_REGISTRY_LIMIT"), + justification_bits: Default::default(), + previous_justified_checkpoint: Default::default(), + current_justified_checkpoint: Default::default(), + finalized_checkpoint: Default::default(), + inactivity_scores: vec![0; count] + .try_into() + .expect("count is far below VALIDATOR_REGISTRY_LIMIT"), + current_sync_committee: empty_sync_committee(), + next_sync_committee: empty_sync_committee(), + latest_execution_payload_header: deneb::ExecutionPayloadHeader { + parent_hash: ExecutionBlockHash::ZERO, + fee_recipient: ExecutionAddress::ZERO, + state_root: Bytes32::ZERO, + receipts_root: Bytes32::ZERO, + logs_bloom: vec![0u8; preset::BYTES_PER_LOGS_BLOOM] + .try_into() + .expect("built at exactly BYTES_PER_LOGS_BLOOM"), + prev_randao: Bytes32::ZERO, + block_number: 0, + gas_limit: 0, + gas_used: 0, + timestamp: 0, + extra_data: Default::default(), + base_fee_per_gas: Uint256::ZERO, + block_hash: ExecutionBlockHash::ZERO, + transactions_root: Root::ZERO, + withdrawals_root: Root::ZERO, + blob_gas_used: 0, + excess_blob_gas: 0, + }, + next_withdrawal_index: 0, + next_withdrawal_validator_index: 0, + historical_summaries: Default::default(), + deposit_requests_start_index: constants::UNSET_DEPOSIT_REQUESTS_START_INDEX, + deposit_balance_to_consume: 0, + exit_balance_to_consume: 0, + earliest_exit_epoch: 0, + consolidation_balance_to_consume: 0, + earliest_consolidation_epoch: 0, + pending_deposits: Default::default(), + pending_partial_withdrawals: Default::default(), + pending_consolidations: Default::default(), + proposer_lookahead: vec![0; preset::PROPOSER_LOOKAHEAD_LENGTH] + .try_into() + .expect("the vector is built at its exact length"), + }) + } + + #[test] + fn registry_accessors_read_and_write_elements() { + let mut state = small_fulu_state(3); // 3 validators, balances 32 ETH + assert_eq!(state.validator_count(), 3); + assert_eq!(state.iter_validators().len(), 3); + assert_eq!( + state.iter_balances().sum::(), + 3 * preset::MAX_EFFECTIVE_BALANCE + ); + + *state.balance_mut(1).unwrap() += 5; + assert_eq!(state.balance(1).unwrap(), preset::MAX_EFFECTIVE_BALANCE + 5); + + let validator = state.validator(0).unwrap().clone(); + state.push_validator(validator, 7).unwrap(); + assert_eq!(state.validator_count(), 4); + assert_eq!(state.balance(3).unwrap(), 7); + assert!(state.balance_mut(4).is_err()); + assert!(state.validator(4).is_err()); + + // Order, the pending balance write, and the pending push all land + // where expected. + assert_eq!( + state.iter_balances().collect::>(), + vec![ + preset::MAX_EFFECTIVE_BALANCE, + preset::MAX_EFFECTIVE_BALANCE + 5, + preset::MAX_EFFECTIVE_BALANCE, + 7, + ] + ); + + // A `validator_mut` write is visible through `iter_validators`, not + // just through `validator`. + state.validator_mut(2).unwrap().effective_balance = 1; + assert_eq!(state.iter_validators().nth(2).unwrap().effective_balance, 1); + } + /// Single-validator lean state. The pubkeys are placeholders; nothing here /// verifies a signature. fn lean_state(genesis_time: u64, attestation_pubkey: u8) -> crate::state::State { diff --git a/crates/net/rpc/src/beacon/states.rs b/crates/net/rpc/src/beacon/states.rs index 47b0e4493..9128132d9 100644 --- a/crates/net/rpc/src/beacon/states.rs +++ b/crates/net/rpc/src/beacon/states.rs @@ -303,12 +303,11 @@ fn validators_response(store: &Store, state_id: &str, request: ValidatorsRequest }) }; let entries: Vec = state - .validators() - .iter() - .zip(state.balances().iter()) + .iter_validators() + .zip(state.iter_balances()) .enumerate() .filter(|(index, (validator, _))| selected(*index as ValidatorIndex, validator)) - .map(|(index, (validator, &balance))| { + .map(|(index, (validator, balance))| { let status = ValidatorStatus::of(validator, balance, epoch); (index as ValidatorIndex, balance, status, validator) }) @@ -463,7 +462,10 @@ mod tests { } fn pubkey_hex(state: &BeaconState, index: usize) -> String { - format!("0x{}", hex::encode(state.validators()[index].pubkey.0)) + format!( + "0x{}", + hex::encode(state.validator(index as u64).unwrap().pubkey.0) + ) } /// What `ethlambda validator` sends: its keys, to learn their indices. diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 28b147de7..0b7b02ff1 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -5798,8 +5798,7 @@ mod tests { effective_balance: i, ..Default::default() }; - parent.validators_mut().push(validator).unwrap(); - parent.balances_mut().push(i).unwrap(); + parent.push_validator(validator, i).unwrap(); } parent.apply_pending_mutations(); store @@ -5812,7 +5811,7 @@ mod tests { let mut child = parent; *child.slot_mut() = 11; child.latest_block_header_mut().parent_root = parent_root; - child.balances_mut()[0] += 1; + *child.balance_mut(0).unwrap() += 1; child.apply_pending_mutations(); store .insert_signed_block(child_root, beacon_test_block(11, parent_root)) @@ -5829,7 +5828,7 @@ mod tests { let resident_parent = cold.get_state(&parent_root).expect("get").expect("present"); let decoded = cold.get_state(&child_root).expect("get").expect("present"); - assert!(decoded.validators().ptr_eq(resident_parent.validators())); + assert!(decoded.validators_ptr_eq(&resident_parent)); assert_eq!(decoded.to_ssz(), child.to_ssz()); } diff --git a/docs/beacon_stf.md b/docs/beacon_stf.md index 0665301ef..a4b7a3907 100644 --- a/docs/beacon_stf.md +++ b/docs/beacon_stf.md @@ -284,7 +284,7 @@ lists lighthouse keeps its state in. store flushes a state before caching it, since a shared `Arc` cannot be flushed later. -The access pattern matters. `state.validator(i)` and `balances()[i]` are tree +The access pattern matters. `state.validator(i)` and `state.balance(i)` are tree descents, cheap next to a hash but far from an array index, and they add up when a helper calls them once per validator: `get_total_active_balance` builds the active-index `Vec` and then reads every @@ -292,8 +292,8 @@ index back, and runs several times per block (once per attestation through `get_base_reward_per_increment`, once per execution request through the churn limits). In the 2026-09-28 import profile, those per-index reads and the repeated whole-registry scans were the largest cost left after hashing. A loop -over the registry should walk `validators().iter()`, zipped with -`balances().iter()` where it needs both. The total active balance is the obvious +over the registry should walk `iter_validators()`, zipped with +`iter_balances()` where it needs both. The total active balance is the obvious candidate for computing once per epoch rather than per call, once it is shown that no block operation changes it mid-epoch. From 8443fd9f06d218e2fefd7ab191a2e3c38f796898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 24 Sep 2026 17:13:36 -0300 Subject: [PATCH 006/132] refactor(beacon): name every fork at the gates bci added A catch-all arm compiles unchanged when a fork is added and quietly applies an older fork's rule; with every variant named, the next fork fails to build at each site that has to decide. A sweep of every Rust file bci changed (gossip validation, precheck, the validator client, p2p, rpc) turned up exactly one such gate: HttpBeaconNode::aggregate_attestation accepted any fork not older than electra, so a fork added after fulu would silently be parsed as an electra-shaped aggregate. Named explicitly, matching the reject-pattern ProducedBlock::from_ssz already uses in the same crate. Every other catch-all in bci's code turned out not to be this kind of gate: the `_ =>`/`other =>` arms found either reject on a type other than ForkName/BeaconState/SignedBeaconBlock, are single- or dual-fork projections that already reject every other variant by construction, or dispatch on Lean-vs-every-beacon-fork where the non-Lean behavior does not vary by fork. Those are left as they are. --- crates/validator/src/beacon_node/http.rs | 29 ++++++++++++++++-------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/crates/validator/src/beacon_node/http.rs b/crates/validator/src/beacon_node/http.rs index caa709b6f..0eb0fa08a 100644 --- a/crates/validator/src/beacon_node/http.rs +++ b/crates/validator/src/beacon_node/http.rs @@ -591,15 +591,26 @@ impl BeaconNodeApi for HttpBeaconNode { )) })?; // Refused by name rather than left to fail as an opaque decode error, - // the same way a pre-electra block is. Electra widened `Attestation` - // with `committee_bits`, so an earlier fork's bytes are a different - // shape and this client has no container for them. - if fork < ForkName::Electra { - return Err(Error::InconsistentResponse(format!( - "node produced a {} aggregate, which this client does not publish; electra is \ - the earliest supported", - fork.as_str() - ))); + // the same way a pre-electra block is (see `ProducedBlock::from_ssz`). + // Electra widened `Attestation` with `committee_bits`, so an earlier + // fork's bytes are a different shape and this client has no container + // for them. Named explicitly rather than `fork < ForkName::Electra`, + // so a fork added after fulu is not silently waved through as + // electra-shaped. + match fork { + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Lean => { + return Err(Error::InconsistentResponse(format!( + "node produced a {} aggregate, which this client does not publish; electra \ + is the earliest supported", + fork.as_str() + ))); + } + ForkName::Electra | ForkName::Fulu => {} } let attestation = Attestation::try_from(&response.data)?; // The same contract the other two fetches carry, enforced here so a From eaefff02dda6ecd812af7ffcc300a1ab64923bfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 24 Sep 2026 16:12:13 -0300 Subject: [PATCH 007/132] feat(types): add NewPayloadRequest and the partial data column containers consensus-specs v1.7.0-beta.2's ssz_static suite covers five containers this crate did not model: NewPayloadRequest (bellatrix through fulu, one type per fork whose own ExecutionPayload shape differs, since the field's type is that fork's payload) and fulu's PartialDataColumnSidecar, PartialDataColumnPartsMetadata, PartialDataColumnHeader, and PartialDataColumnGroupID (gossipsub's Partial Message Extension wire types, from fulu/partial-columns/p2p-interface.md). Types only: nothing in the node uses them yet, and the state transition does not call the execution engine or speak partial columns. ssz_static failures for these handlers go from 1107 (minimal) / 45 (mainnet) to zero on both presets. --- .../tests/beacon_spec/ssz_static.rs | 36 ++++++ .../types/src/beacon/containers/bellatrix.rs | 17 +++ .../types/src/beacon/containers/capella.rs | 19 ++++ .../types/src/beacon/containers/deneb.rs | 28 ++++- .../types/src/beacon/containers/electra.rs | 22 +++- .../types/src/beacon/containers/fulu.rs | 107 ++++++++++++++++-- crates/common/types/src/beacon/primitives.rs | 3 + 7 files changed, 222 insertions(+), 10 deletions(-) diff --git a/crates/blockchain/state_transition/tests/beacon_spec/ssz_static.rs b/crates/blockchain/state_transition/tests/beacon_spec/ssz_static.rs index 2e6a092ad..198ea8058 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/ssz_static.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/ssz_static.rs @@ -278,6 +278,25 @@ pub fn trials() -> Vec { check::(case) } + // What `process_execution_payload` hands the execution engine + // to validate a proposed payload. Each fork whose own + // ExecutionPayload differs in shape needs its own type here, + // same as the ExecutionPayload arms above; fulu's payload is + // unchanged from electra's, so electra's request type covers + // both forks' cases. + "NewPayloadRequest" if fork == ForkName::Bellatrix => { + check::(case) + } + "NewPayloadRequest" if fork == ForkName::Capella => { + check::(case) + } + "NewPayloadRequest" if fork == ForkName::Deneb => { + check::(case) + } + "NewPayloadRequest" if fork >= ForkName::Electra => { + check::(case) + } + // Transcribed from fork-choice.md rather than beacon-chain.md, and // unchanged since the merge introduced it, so one type covers // every fork that carries it. @@ -334,6 +353,23 @@ pub fn trials() -> Vec { check::(case) } + // Partial columns (gossipsub's Partial Message Extension) are + // fulu-only too, and transcribed from the same + // `partial-columns/p2p-interface.md` rather than + // `beacon-chain.md`. + "PartialDataColumnSidecar" if fork == ForkName::Fulu => { + check::(case) + } + "PartialDataColumnPartsMetadata" if fork == ForkName::Fulu => { + check::(case) + } + "PartialDataColumnHeader" if fork == ForkName::Fulu => { + check::(case) + } + "PartialDataColumnGroupID" if fork == ForkName::Fulu => { + check::(case) + } + // No arm matched, so this crate has no container for this // handler/fork pair, and this is not `LightClient*` (that // prefix is filtered out above before this trial is even diff --git a/crates/common/types/src/beacon/containers/bellatrix.rs b/crates/common/types/src/beacon/containers/bellatrix.rs index 6e802e8ea..80993a8b7 100644 --- a/crates/common/types/src/beacon/containers/bellatrix.rs +++ b/crates/common/types/src/beacon/containers/bellatrix.rs @@ -14,6 +14,10 @@ //! it replaces `transactions` with `transactions_root`, the same substitution //! [`super::shared::BeaconBlockHeader`] makes for a beacon block's body. //! +//! [`NewPayloadRequest`] is what `process_execution_payload` hands the +//! execution engine to validate a proposed payload; bellatrix's carries only +//! `execution_payload`, and later forks append to it. +//! //! [`PowBlock`] is unrelated to either: it is transcribed from //! `fork-choice.md` rather than `beacon-chain.md`, and exists only to let fork //! choice check a candidate terminal proof-of-work block's total difficulty @@ -145,6 +149,19 @@ pub struct ExecutionPayloadHeader { pub transactions_root: Root, } +/// What `process_execution_payload` hands `execution_engine.verify_and_notify_new_payload` +/// to validate a proposed payload, bellatrix `beacon-chain.md`. +/// +/// One field: bellatrix has nothing else to check yet. Later forks append to +/// this (deneb's blob versioned hashes and beacon root, electra's execution +/// requests); each fork whose own [`ExecutionPayload`] differs from its +/// predecessor's needs its own `NewPayloadRequest` too, since the field's type +/// is that fork's own payload shape. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +pub struct NewPayloadRequest { + pub execution_payload: ExecutionPayload, +} + // --------------------------------------------------------------------------- // Blocks // --------------------------------------------------------------------------- diff --git a/crates/common/types/src/beacon/containers/capella.rs b/crates/common/types/src/beacon/containers/capella.rs index 8df81fb10..0ec2e397d 100644 --- a/crates/common/types/src/beacon/containers/capella.rs +++ b/crates/common/types/src/beacon/containers/capella.rs @@ -32,6 +32,12 @@ //! imported, since every field of both is repeated here with `withdrawals` //! (respectively `withdrawals_root`) appended, and a derive needs the whole //! field list in one struct. +//! +//! [`NewPayloadRequest`] does not gain a field here the way the containers +//! above do; the specification does not even redefine it for this fork. It +//! still gets its own type, because its one field's type is this fork's own +//! [`ExecutionPayload`], which carries `withdrawals` and so is not +//! `hash_tree_root`-compatible with bellatrix's. use libssz_derive::{HashTreeRoot, SszDecode, SszEncode}; use libssz_types::SszList; @@ -180,6 +186,19 @@ pub struct ExecutionPayloadHeader { pub withdrawals_root: Root, } +/// What `process_execution_payload` hands `execution_engine.verify_and_notify_new_payload` +/// to validate a proposed payload, capella `beacon-chain.md`. +/// +/// Capella does not redefine this container's field list (still just +/// `execution_payload`), but it still needs its own type here: the field's +/// type is this fork's own [`ExecutionPayload`], which differs in shape from +/// bellatrix's by the appended `withdrawals`, so the two forks' requests are +/// not `hash_tree_root`-compatible. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +pub struct NewPayloadRequest { + pub execution_payload: ExecutionPayload, +} + // --------------------------------------------------------------------------- // Blocks // --------------------------------------------------------------------------- diff --git a/crates/common/types/src/beacon/containers/deneb.rs b/crates/common/types/src/beacon/containers/deneb.rs index 9425b4223..3092844f8 100644 --- a/crates/common/types/src/beacon/containers/deneb.rs +++ b/crates/common/types/src/beacon/containers/deneb.rs @@ -25,6 +25,12 @@ //! the state transition document. Neither is ever stored in the state or a //! block body; a block only ever holds the commitments the sidecars are //! checked against. +//! +//! [`VersionedHash`] and [`VersionedHashes`] are new too: the versioned +//! hashes `kzg_commitment_to_versioned_hash` derives from +//! `blob_kzg_commitments`, which [`NewPayloadRequest`] passes to the +//! execution engine alongside `parent_beacon_block_root` so it can check a +//! payload's blob commitments without the blobs themselves. use libssz_derive::{HashTreeRoot, SszDecode, SszEncode}; use libssz_types::{SszList, SszVector}; @@ -42,7 +48,7 @@ use super::shared::{ use crate::beacon::preset; use crate::beacon::primitives::{ BlobIndex, BlsSignature, Bytes32, ExecutionAddress, ExecutionBlockHash, KzgCommitment, - KzgProof, Root, Slot, Uint256, ValidatorIndex, WithdrawalIndex, + KzgProof, Root, Slot, Uint256, ValidatorIndex, VersionedHash, WithdrawalIndex, }; // --------------------------------------------------------------------------- @@ -69,6 +75,11 @@ pub type KzgCommitments = SszList; +/// The versioned hashes [`NewPayloadRequest`] passes to the execution engine, +/// one per blob committed to by [`BeaconBlockBody::blob_kzg_commitments`], +/// computed from those commitments by `kzg_commitment_to_versioned_hash`. +pub type VersionedHashes = SszList; + // --------------------------------------------------------------------------- // Execution payload // --------------------------------------------------------------------------- @@ -142,6 +153,21 @@ pub struct ExecutionPayloadHeader { pub excess_blob_gas: u64, } +/// What `process_execution_payload` hands `execution_engine.verify_and_notify_new_payload` +/// to validate a proposed payload, deneb `beacon-chain.md`. +/// +/// Bellatrix's version carries only `execution_payload`; deneb appends +/// `versioned_hashes` (EIP-4844, so the engine can check blob commitments +/// against the payload without the blobs themselves) and +/// `parent_beacon_block_root` (EIP-4788, exposing the beacon root to the EVM). +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +pub struct NewPayloadRequest { + pub execution_payload: ExecutionPayload, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub versioned_hashes: VersionedHashes, + pub parent_beacon_block_root: Root, +} + // --------------------------------------------------------------------------- // Blocks // --------------------------------------------------------------------------- diff --git a/crates/common/types/src/beacon/containers/electra.rs b/crates/common/types/src/beacon/containers/electra.rs index af6a3eb90..4610957ca 100644 --- a/crates/common/types/src/beacon/containers/electra.rs +++ b/crates/common/types/src/beacon/containers/electra.rs @@ -33,7 +33,9 @@ //! withdrawal directly, instead of consensus replaying the deposit contract's //! event log or a validator signing a voluntary exit; EIP-7251 extends the //! same mechanism to consolidations. [`ExecutionRequests`] is the per-block -//! envelope the execution payload carries all three request kinds in. +//! envelope the execution payload carries all three request kinds in, and +//! [`NewPayloadRequest`] appends it to deneb's fields so the execution engine +//! validates those requests alongside the payload they arrived in. //! //! [`BeaconState`]'s field count crosses a power of two at electra, so its //! merkle tree gains a level relative to deneb's; see `docs/beacon_stf.md` @@ -45,7 +47,7 @@ use libssz_types::{SszBitlist, SszBitvector, SszList}; use super::altair::{SyncAggregate, SyncCommittee}; use super::bellatrix::LogsBloom; use super::capella::SignedBLSToExecutionChange; -use super::deneb::{ExecutionPayload, ExecutionPayloadHeader, KzgCommitments}; +use super::deneb::{ExecutionPayload, ExecutionPayloadHeader, KzgCommitments, VersionedHashes}; use super::shared::{ AttestationData, Balances, BeaconBlockHeader, BlockRoots, Checkpoint, Deposit, EpochParticipation, Eth1Data, Eth1DataVotes, Fork, HistoricalRoots, HistoricalSummaries, @@ -282,6 +284,22 @@ pub struct ExecutionRequests { SszList, } +/// What `process_execution_payload` hands `execution_engine.verify_and_notify_new_payload` +/// to validate a proposed payload, electra `beacon-chain.md`. +/// +/// Deneb's fields, with `execution_requests` (EIP-6110/7002/7251) appended so +/// the engine validates a block's execution-layer-triggered requests +/// alongside its payload. Electra does not redefine [`ExecutionPayload`] +/// itself, so this reuses deneb's, imported at the top of this module. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +pub struct NewPayloadRequest { + pub execution_payload: ExecutionPayload, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub versioned_hashes: VersionedHashes, + pub parent_beacon_block_root: Root, + pub execution_requests: ExecutionRequests, +} + // --------------------------------------------------------------------------- // Pending balance queues // --------------------------------------------------------------------------- diff --git a/crates/common/types/src/beacon/containers/fulu.rs b/crates/common/types/src/beacon/containers/fulu.rs index 73b5b8b10..83167aca2 100644 --- a/crates/common/types/src/beacon/containers/fulu.rs +++ b/crates/common/types/src/beacon/containers/fulu.rs @@ -13,12 +13,13 @@ //! the per-column grouping a sidecar presents; [`DataColumnsByRootIdentifier`] //! is how a request-response peer asks for specific columns of a specific //! block. `BeaconBlockBody`, `BeaconBlock`, `SignedBeaconBlock`, -//! `ExecutionPayload`, and `ExecutionPayloadHeader` are unchanged from electra: -//! a block still commits to the same `blob_kzg_commitments` it always has, -//! since sampling changes how the data behind those commitments travels over -//! the network, not what the block itself contains. This module defines none -//! of those five; state transition code should import them from -//! [`super::electra`] and [`super::deneb`] instead. +//! `ExecutionPayload`, `ExecutionPayloadHeader`, and `NewPayloadRequest` are +//! unchanged from electra: a block still commits to the same +//! `blob_kzg_commitments` it always has, since sampling changes how the data +//! behind those commitments travels over the network, not what the block +//! itself, or the request the execution engine validates it with, contains. +//! This module defines none of those six; state transition code should import +//! them from [`super::electra`] and [`super::deneb`] instead. //! //! The other change is [`BeaconState::proposer_lookahead`]. Every fork through //! electra computes each slot's proposer on demand from the active set and the @@ -27,9 +28,16 @@ //! window of upcoming proposers at each epoch boundary, so //! `get_beacon_proposer_index` becomes a lookup into that window rather than a //! shuffle. +//! +//! [`PartialDataColumnSidecar`], [`PartialDataColumnPartsMetadata`], +//! [`PartialDataColumnHeader`], and [`PartialDataColumnGroupID`] are +//! transcribed from `fulu/partial-columns/p2p-interface.md`: wire types for +//! gossipsub's Partial Message Extension, which exchanges the individual +//! cells and proofs behind a [`DataColumnSidecar`] instead of the whole +//! thing. use libssz_derive::{HashTreeRoot, SszDecode, SszEncode}; -use libssz_types::{SszList, SszVector}; +use libssz_types::{SszBitlist, SszList, SszVector}; use super::altair::SyncCommittee; use super::deneb::{ExecutionPayloadHeader, KzgCommitments}; @@ -165,6 +173,91 @@ pub struct DataColumnsByRootIdentifier { pub columns: ColumnIndices, } +// --------------------------------------------------------------------------- +// Partial columns +// --------------------------------------------------------------------------- +// +// Transcribed from `fulu/partial-columns/p2p-interface.md` rather than +// `beacon-chain.md`: gossipsub's Partial Message Extension lets a peer +// exchange individual cells and proofs of a `DataColumnSidecar` instead of +// the whole thing, and these are that mechanism's wire types. None of the +// four is ever stored in the state or a block body, the same way +// `DataColumnSidecar` itself is not: a node reconstructs the full sidecar +// from these parts before forwarding it to peers that do not support partial +// messages. + +/// A bitfield over the cells of one data column, one bit per blob in the +/// block, `CellsBitList`. +pub type CellsBitList = SszBitlist<{ preset::MAX_BLOB_COMMITMENTS_PER_BLOCK }>; + +/// A [`PartialDataColumnHeader`] that may or may not be present, encoded as a +/// list of length zero or one, `OptionalPartialDataColumnHeader`. +/// +/// A partial message can carry cells before its header has ever been sent, so +/// the header field has to be optional; SSZ has no `Option`, so the +/// specification spells "optional" as a list capped at one element instead. +pub type OptionalPartialDataColumnHeader = SszList; + +/// One column's worth of cells and proofs sent as a gossipsub partial +/// message, carrying only the cells `cells_present_bitmap` names rather than +/// a full [`DataColumn`]. +/// +/// `header` is only ever set on an eager push: a peer can only request cells +/// after it already has the header, so a request-driven response never needs +/// to repeat it. The column index itself is not a field here; it is inferred +/// from the gossipsub topic's subnet. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +pub struct PartialDataColumnSidecar { + #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + pub cells_present_bitmap: CellsBitList, + #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex_seq::serialize")] + pub partial_column: DataColumn, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub kzg_proofs: KzgProofs, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub header: OptionalPartialDataColumnHeader, +} + +/// The two bitmaps peers exchange to negotiate which cells to send: `available` +/// says which cells a peer holds, `requests` says which ones it wants. +/// +/// Having a cell but declining to provide it is functionally the same as not +/// having it, so no third state is needed beyond what the pair of bitmaps +/// already encodes. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +pub struct PartialDataColumnPartsMetadata { + #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + pub available: CellsBitList, + #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + pub requests: CellsBitList, +} + +/// The header common to every column of one block: which blobs it commits to, +/// proof that those commitments belong to the block, and enough of the block +/// itself to check both. +/// +/// Sent only on eager pushes, since a peer can only make a request once it +/// already holds this. `verify_partial_data_column_header_inclusion_proof` +/// checks `kzg_commitments_inclusion_proof` against +/// `signed_block_header.message.body_root`, the same shallow proof +/// [`DataColumnSidecar::kzg_commitments_inclusion_proof`] uses. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +pub struct PartialDataColumnHeader { + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub kzg_commitments: KzgCommitments, + pub signed_block_header: SignedBeaconBlockHeader, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub kzg_commitments_inclusion_proof: KzgCommitmentsInclusionProof, +} + +/// The gossipsub Partial Message group ID for a block's columns: the SSZ +/// encoding of this container, prefixed by a version byte, names the group +/// every partial message about that block's cells belongs to. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +pub struct PartialDataColumnGroupID { + pub beacon_block_root: Root, +} + // --------------------------------------------------------------------------- // State // --------------------------------------------------------------------------- diff --git a/crates/common/types/src/beacon/primitives.rs b/crates/common/types/src/beacon/primitives.rs index daf44ec88..43c2589b6 100644 --- a/crates/common/types/src/beacon/primitives.rs +++ b/crates/common/types/src/beacon/primitives.rs @@ -55,6 +55,9 @@ pub type Uint256 = U256; pub type ExecutionAddress = H160; /// An execution layer block hash. pub type ExecutionBlockHash = H256; +/// A versioned hash of a blob's KZG commitment (deneb `beacon-chain.md`, +/// `New VersionedHash`), derived by `kzg_commitment_to_versioned_hash`. +pub type VersionedHash = H256; /// A fork version. pub type Version = [u8; 4]; From 417e750d2d749c33e26efe594e7741976d38c7d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 24 Sep 2026 17:05:11 -0300 Subject: [PATCH 008/132] test(beacon): run the gossip vectors from the main fixture tree v1.7.0-beta.2's main consensus-spec-tests release now ships the networking/gossip_* vectors that used to live only in a v1.7.0-beta.1 pre-release, so the gossip runner that beacon-chain-integration gained with gossip validation no longer needs a tree of its own: drop the separate Makefile target, CI step, and .gitignore entry, and collect gossip_beacon_block/gossip_data_column_sidecar cases from the same tree every other runner reads. Every gossip_* handler and fork the tree ships that beacon::gossip does not validate yet (everything pre-fulu, plus attestations, slashings, sync committee messages, blob/partial-column sidecars, voluntary exits, BLS-to-execution changes) is now named and ignored rather than left for the tree walk to silently never mention, with a discovery test that fails if a future release adds a gossip_* handler this file does not list. --- .github/workflows/ci.yml | 3 - .gitignore | 4 - CLAUDE.md | 10 ++- Makefile | 46 +---------- .../tests/beacon_spec/gossip.rs | 81 +++++++++++++++++-- .../state_transition/tests/beacon_spec/mod.rs | 22 ----- .../tests/beacon_spec/networking.rs | 24 ++++-- docs/beacon_stf.md | 13 +-- 8 files changed, 110 insertions(+), 93 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52a5bab8c..f9a4bb233 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -295,9 +295,6 @@ jobs: - name: Download the BLS/KZG test vectors run: make cryptography-specs - - name: Download the gossip validation test fixtures - run: make consensus-spec-gossip-tests - - name: Run the Beacon Chain spec tests run: make test-beacon-${{ matrix.preset }} diff --git a/.gitignore b/.gitignore index 8b917df2a..b2e487fa2 100644 --- a/.gitignore +++ b/.gitignore @@ -37,10 +37,6 @@ target # BLS and KZG test vectors, downloaded by `make cryptography-specs` /cryptography-specs -# The gossip validation vectors, from a newer release, downloaded by -# `make consensus-spec-gossip-tests` -/consensus-spec-tests-gossip - # Personal Claude Code overrides (not shared with team) CLAUDE.local.md diff --git a/CLAUDE.md b/CLAUDE.md index 2ef6677eb..fcfcdce57 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -697,10 +697,12 @@ transitions are in `ethlambda-types`, per the section above. Nothing above upgrades, state transitions, and epoch processing. As of the v1.6.1 fixture pin, every case passed on both presets: mainnet is 5705 cases and minimal 40009. The v1.7.0-beta.2 bump adds failures the fixture release itself - brought: the new `networking/gossip_*` format, `ssz_static` cases for - `NewPayloadRequest` and the four new `PartialDataColumn*` containers, a - fork-choice rule change, and one `transition` case, each fixed by the - commits that follow the bump. The crate's lib + brought: `ssz_static` cases for `NewPayloadRequest` and the four new + `PartialDataColumn*` containers, a fork-choice rule change, and one + `transition` case, each fixed by the commits that follow the bump. (The + bump's new `networking/gossip_*` format is already handled: the gossip + validation vectors come from this same tree now, not the separate + v1.7.0-beta.1 download #38 used to fetch.) The crate's lib target holds 200 tests with `beacon-spec-tests` on, 185 plus 15 ignored without; both figures cover lean's own unit tests as well, since the two chains now share one lib target. Fork choice is fixture-verified too: 150 mainnet diff --git a/Makefile b/Makefile index 0e9a4ddc0..f8fe3c95d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help fmt lint bench update cooldown-check docker-build shadow-build shadow-docker-build run-devnet test test-consensus test-node test-beacon test-beacon-mainnet test-beacon-minimal consensus-spec-tests consensus-spec-gossip-tests cryptography-specs docs docs-deps docs-serve +.PHONY: help fmt lint bench update cooldown-check docker-build shadow-build shadow-docker-build run-devnet test test-consensus test-node test-beacon test-beacon-mainnet test-beacon-minimal consensus-spec-tests cryptography-specs docs docs-deps docs-serve help: ## 📚 Show help for each of the Makefile recipes @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -53,10 +53,10 @@ BEACON_TEST=cargo test -p ethlambda-state-transition --lib --test beacon_spec_te # concurrently and each downloads only the trees its preset reads. test-beacon: test-beacon-mainnet test-beacon-minimal ## 🧪 Run the Beacon Chain spec tests, both presets -test-beacon-mainnet: consensus-spec-tests consensus-spec-gossip-tests cryptography-specs ## 🧪 Run the Beacon Chain spec tests, mainnet preset +test-beacon-mainnet: consensus-spec-tests cryptography-specs ## 🧪 Run the Beacon Chain spec tests, mainnet preset $(BEACON_TEST) --features beacon-spec-tests -test-beacon-minimal: consensus-spec-tests consensus-spec-gossip-tests cryptography-specs ## 🧪 Run the Beacon Chain spec tests, minimal preset +test-beacon-minimal: consensus-spec-tests cryptography-specs ## 🧪 Run the Beacon Chain spec tests, minimal preset $(BEACON_TEST) --features beacon-spec-tests,preset-minimal # Used ONLY to resolve dependency updates: min-publish-age (.cargo/config.toml) @@ -190,46 +190,6 @@ $(CONSENSUS_SPEC_TESTS_STAMP): done @touch $@ -# The gossip validation vectors (`networking/gossip_*`) first ship in a -# pre-release, so they come from a tree of their own rather than moving the pin -# above. Only fulu's block, column, aggregate and attestation handlers are -# extracted, since those are the topics this node validates; the rest of each -# tarball is never unpacked. Fold this back into the main tree once that -# release is final. -CONSENSUS_SPEC_GOSSIP_TESTS_VERSION ?= v1.7.0-beta.1 -CONSENSUS_SPEC_GOSSIP_TESTS_BASE_URL ?= https://github.com/ethereum/consensus-specs/releases/download/$(CONSENSUS_SPEC_GOSSIP_TESTS_VERSION) -# Follows CONSENSUS_SPEC_TESTS_CONFIGS, so narrowing that one narrows this too. -# `general` is dropped because it has no `networking` runner: naming its -# members would fail the extraction. -CONSENSUS_SPEC_GOSSIP_TESTS_CONFIGS = $(filter-out general,$(CONSENSUS_SPEC_TESTS_CONFIGS)) -CONSENSUS_SPEC_GOSSIP_TESTS_HANDLERS = gossip_beacon_block gossip_data_column_sidecar gossip_beacon_aggregate_and_proof gossip_beacon_attestation -# Includes the handler list, not just the version and configs: growing the -# list must re-extract an existing tree, which a stamp keyed on version and -# configs alone would not notice, since neither of those changed. -CONSENSUS_SPEC_GOSSIP_TESTS_STAMP=consensus-spec-tests-gossip/.version-$(CONSENSUS_SPEC_GOSSIP_TESTS_VERSION)-$(subst $(space),-,$(sort $(CONSENSUS_SPEC_GOSSIP_TESTS_CONFIGS)))-$(subst $(space),-,$(sort $(CONSENSUS_SPEC_GOSSIP_TESTS_HANDLERS))) - -consensus-spec-gossip-tests: $(CONSENSUS_SPEC_GOSSIP_TESTS_STAMP) ## ⬇️ Download the gossip validation spec test fixtures - -# Member directories are named exactly rather than globbed: GNU tar needs -# `--wildcards` for a pattern and BSD tar rejects that flag, while a plain -# directory name extracts its whole subtree on both. -$(CONSENSUS_SPEC_GOSSIP_TESTS_STAMP): - @rm -rf consensus-spec-tests-gossip - @mkdir -p consensus-spec-tests-gossip - @for config in $(CONSENSUS_SPEC_GOSSIP_TESTS_CONFIGS); do \ - echo "Downloading $$config gossip test fixtures ($(CONSENSUS_SPEC_GOSSIP_TESTS_VERSION))"; \ - tmpdir=$$(mktemp -d); \ - trap 'rm -rf "$$tmpdir"' EXIT; \ - curl -L -f -o "$$tmpdir/$$config.tar.gz" "$(CONSENSUS_SPEC_GOSSIP_TESTS_BASE_URL)/$$config.tar.gz" || exit 1; \ - members=""; \ - for handler in $(CONSENSUS_SPEC_GOSSIP_TESTS_HANDLERS); do \ - members="$$members tests/$$config/fulu/networking/$$handler"; \ - done; \ - tar -xzf "$$tmpdir/$$config.tar.gz" -C consensus-spec-tests-gossip $$members || exit 1; \ - rm -rf "$$tmpdir"; \ - done - @touch $@ - # BLS and KZG test vectors, split out of consensus-spec-tests' `general` config # since v1.7.0-alpha.13 (consensus-specs #5398). Preset-independent, so both # beacon test targets share this one download. Flat layout, one zip: diff --git a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs index 854d5b2bd..0590fb5d5 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs @@ -2,8 +2,14 @@ //! vectors (`tests/formats/networking/gossip_validation.md` in //! consensus-specs), run against `beacon::gossip`. //! -//! These come from their own fixture tree; see [`super::gossip_fixture_root`]. +//! These used to come from a separate, newer-release fixture tree of their +//! own; v1.7.0-beta.2's main tree now ships the same vectors under +//! `networking/gossip_*` alongside every other runner, so they are collected +//! from there like any other handler (see [`super::collect`]). `networking.rs` +//! filters these same directories out of its own collection, so the two +//! runners partition the tree rather than both claiming it. +use std::collections::BTreeSet; use std::num::NonZeroUsize; use std::sync::Arc; @@ -22,11 +28,11 @@ use ethlambda_state_transition::beacon::helpers::accessors::CommitteeCache; use ethlambda_state_transition::beacon::primitives::Root; use ethlambda_storage::ForkCheckpoints; use libssz::SszDecode; -use libtest_mimic::Trial; +use libtest_mimic::{Failed, Trial}; -use super::{Case, PRESET, collect_gossip}; +use super::{Case, PRESET, collect, collect_all_handlers}; -/// The handlers this runner covers. +/// The handlers this runner covers, and runs. const HANDLERS: &[&str] = &[ "gossip_beacon_block", "gossip_data_column_sidecar", @@ -34,6 +40,25 @@ const HANDLERS: &[&str] = &[ "gossip_beacon_attestation", ]; +/// Every other `gossip_*` handler the fixture tree ships, none of which this +/// node validates yet: the topics it neither subscribes to nor has a +/// validator for. Reported as ignored, named individually, rather than left +/// for `HANDLERS` to silently not mention, so the gap is visible in the test +/// list instead of inferred from an absence. [`trials`]'s accounting test +/// fails if a fixture release adds a `gossip_*` handler that is in neither +/// list, the same way [`super::UNMODELED_FORKS`] forces a decision on a new +/// fork directory. +const IGNORED_HANDLERS: &[&str] = &[ + "gossip_attester_slashing", + "gossip_blob_sidecar", + "gossip_bls_to_execution_change", + "gossip_partial_data_column_sidecar", + "gossip_proposer_slashing", + "gossip_sync_committee_contribution_and_proof", + "gossip_sync_committee_message", + "gossip_voluntary_exit", +]; + /// Vectors that disagree with a deliberate deviation, by case name. const SKIPPED: &[(&str, &str)] = &[ ( @@ -336,15 +361,59 @@ fn run_case(case: &Case) -> Result<(), String> { pub fn trials() -> Vec { let mut trials = Vec::new(); for handler in HANDLERS { - let cases = collect_gossip(PRESET, handler); + let cases = collect(PRESET, "networking", handler); trials.push(super::discovery_trial( &format!("gossip/{handler}"), cases.len(), )); for case in cases { - let ignored = !case.in_scope() || SKIPPED.iter().any(|(name, _)| *name == case.name); + // The rules `beacon::gossip::block` and `beacon::gossip::column` + // implement are fulu's own `validate_beacon_block_gossip` and + // `validate_data_column_sidecar_gossip`; a case from any other + // fork is ignored rather than run, the same "known gap, not a + // silent one" treatment `case.in_scope()` already gives a fork + // past `HIGHEST_IMPLEMENTED_FORK`. `case.in_scope()` alone would + // pass every fork up to fulu, since the state transition handles + // them all; the gossip rules do not. + let ignored = !case.in_scope() + || case.fork != ForkName::Fulu + || SKIPPED.iter().any(|(name, _)| *name == case.name); trials.push(super::case_trial("gossip", case, run_case).with_ignored_flag(ignored)); } } + + // Every other `gossip_*` handler and fork this fixture tree ships: named + // and ignored rather than left for the loop above to never mention, and + // `unknown` is what makes a fixture release adding a handler neither run + // nor listed here fail loudly instead of vanishing. Mirrors + // `fixture_fork_trials`'s treatment of `UNMODELED_FORKS`. + let mut unknown: BTreeSet = BTreeSet::new(); + for (handler, case) in collect_all_handlers(PRESET, "networking") { + if !handler.starts_with("gossip_") || HANDLERS.contains(&handler.as_str()) { + continue; + } + if !IGNORED_HANDLERS.contains(&handler.as_str()) { + unknown.insert(handler); + continue; + } + let name = format!("gossip/{}", case.id()); + trials.push(Trial::test(name, || Ok(())).with_ignored_flag(true)); + } + + trials.push(Trial::test( + "gossip/every_handler_is_accounted_for", + move || { + if unknown.is_empty() { + return Ok(()); + } + Err(Failed::from(format!( + "the fixture release ships gossip_* handlers this runner neither runs nor lists \ + in IGNORED_HANDLERS, so every case under them is skipped without appearing \ + anywhere in the output: {}", + unknown.into_iter().collect::>().join(", ") + ))) + }, + )); + trials } diff --git a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs index a92511c24..54aec9af6 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs @@ -157,23 +157,6 @@ pub fn collect_crypto(kind: &str, fork_of: impl Fn(&str) -> ForkName) -> Vec<(St out } -/// The root of the gossip vector tree. -/// -/// The gossip vectors ship in a newer release than [`fixture_root`]'s, so they -/// live in a tree of their own; see `CONSENSUS_SPEC_GOSSIP_TESTS_VERSION` in -/// the Makefile. Panics when absent, for the same reason [`fixture_root`] does. -pub fn gossip_fixture_root() -> PathBuf { - let root = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../../../consensus-spec-tests-gossip") - .join("tests"); - assert!( - root.is_dir(), - "gossip test fixtures are missing from {}; run `make consensus-spec-gossip-tests`", - root.display() - ); - root -} - /// One fixture case: a directory of input and expected-output files. #[derive(Debug, Clone)] pub struct Case { @@ -275,11 +258,6 @@ pub fn collect(config: &str, runner: &str, handler: &str) -> Vec { collect_in(&fixture_root(), config, runner, handler) } -/// Cases for one handler of the `networking` runner, from the gossip tree. -pub fn collect_gossip(config: &str, handler: &str) -> Vec { - collect_in(&gossip_fixture_root(), config, "networking", handler) -} - /// [`collect`] over any fixture tree laid out like the release tarballs. fn collect_in(root: &Path, config: &str, runner: &str, handler: &str) -> Vec { let root = root.join(config); diff --git a/crates/blockchain/state_transition/tests/beacon_spec/networking.rs b/crates/blockchain/state_transition/tests/beacon_spec/networking.rs index ffccca2d1..5a28fd08f 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/networking.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/networking.rs @@ -8,6 +8,10 @@ //! `node_id` arrives as a decimal integer of up to 78 digits, wider than any //! Rust primitive, so it is read as a string and parsed into the 32 big-endian //! bytes the helper takes. +//! +//! The same runner directory also holds every `gossip_*` handler (the +//! specification's gossip validation vectors); those belong to [`super::gossip`] +//! instead, so [`trials`] filters them out here rather than failing on them. use ethlambda_state_transition::beacon::das::{ compute_columns_for_custody_group, get_custody_groups, @@ -91,11 +95,13 @@ fn check_compute_columns_for_custody_group(case: &Case) -> Result<(), String> { /// Dispatches one case to the check function its handler names. /// -/// The `other` arm is what makes a fixture release adding a third handler -/// under `networking/` visible: without it, an unrecognized handler would -/// simply contribute no trials, and neither `discovery_trial` counts by -/// handler name, so a silently-skipped handler would report nothing wrong. -/// Mirrors `kzg.rs`'s own dispatch for the same reason. +/// The `other` arm is what makes a fixture release adding a third +/// non-`gossip_*` handler under `networking/` visible: without it, an +/// unrecognized handler would simply contribute no trials, and neither +/// `discovery_trial` counts by handler name, so a silently-skipped handler +/// would report nothing wrong. `gossip_*` handlers never reach here at all; +/// `trials` filters them out before calling this. Mirrors `kzg.rs`'s own +/// dispatch for the same reason. fn run(handler: &str, case: &Case) -> Result<(), String> { match handler { "get_custody_groups" => check_get_custody_groups(case), @@ -105,7 +111,13 @@ fn run(handler: &str, case: &Case) -> Result<(), String> { } pub fn trials() -> Vec { - let cases = collect_all_handlers(PRESET, "networking"); + // `gossip_*` handlers live under this same runner directory but are + // `super::gossip`'s to collect and run; without this filter every one of + // their cases would also reach `run`'s `other` arm below and fail. + let cases: Vec<_> = collect_all_handlers(PRESET, "networking") + .into_iter() + .filter(|(handler, _)| !handler.starts_with("gossip_")) + .collect(); let groups_count = cases .iter() diff --git a/docs/beacon_stf.md b/docs/beacon_stf.md index a4b7a3907..6b922014c 100644 --- a/docs/beacon_stf.md +++ b/docs/beacon_stf.md @@ -406,11 +406,14 @@ passed on both presets: | minimal | 40009, all green | 3692 | 196 | The v1.7.0-beta.2 bump (see "Fixture suites" below) adds failures the fixture -release itself brought rather than any change here: the whole new `networking/gossip_*` -format, `ssz_static` cases for `NewPayloadRequest` and the four new -`PartialDataColumn*` containers, a fork-choice rule change, and one -`transition` case. The commits after the bump fix each in turn; this table is -stale until they land. +release itself brought rather than any change here: `ssz_static` cases for +`NewPayloadRequest` and the four new `PartialDataColumn*` containers, a +fork-choice rule change, and one `transition` case. (A fourth item, the whole +new `networking/gossip_*` format, is fixed: the gossip validation vectors this +crate's `gossip.rs` runner covers now come from this same tree instead of a +separate v1.7.0-beta.1 download, and every other `gossip_*` handler or fork is +ignored by name rather than left unmatched.) The remaining commits fix each of +the rest in turn; this table is stale until they land. The lib counts were 244 and 245 while the containers, presets, configuration and primitives were defined here. Their 48 unit tests moved with them and run in From e3f39cd8869f58d7a03363301143494949765182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:22:19 -0300 Subject: [PATCH 009/132] feat(types): add the gloas preset, constants and containers Gloas bundles EIP-7732 (ePBS: the execution payload moves out of the block into a builder-signed envelope, revealed and attested to a slot after the block that bid for it), EIP-7688 (progressive SSZ: most lists and every "big" container become unbounded and EIP-7495 progressive containers instead of a fixed shape), EIP-8045/8061/8282 (a gas limit schedule and builder deposit/exit requests). This lands the preset constants, spec constants, and every gloas SSZ container the types crate needs; nothing reads them yet (ForkName::Gloas wiring is a follow-up commit). The validator registry and balances stay tree-backed, on a new ethlambda_ssz_tree::ProgressiveList (mirroring List's element-level accessors) rather than switching to libssz_types' plain, Vec-backed one: a derived state still needs to share unchanged registry subtrees with its parent, and a block still needs to rehash only the records it touched. Every other progressive list is libssz_types::ProgressiveList / ProgressiveBitlist, now that the pinned libssz rev carries DerefMut / IndexMut on both. Every container was cross-checked against consensus-spec-tests' ssz_static/*/ssz_random/case_0 fixtures (decode, hash_tree_root, and re-encode-matches-input) on both mainnet and minimal, through a throwaway test removed before this commit: all 30 gloas-defined containers, plus all 38 containers gloas reuses unmodified from earlier forks, for the 68 non-LightClient handlers the fixture tree lists. Field order and SSZ attributes were also checked against libssz's own alpha.13 reference model (~/libssz/spec-tests/src/types/gloas.rs), with field sets taken from the beta.2 spec text where the two disagree. Two gaps found against that alpha.13 reference, both confirmed from the beta.2 spec text and covered by the fixture cross-check: NewPayloadRequest has no gloas variant there (added here, since ExecutionPayload's shape does change), and PartialDataColumnSidecar / PartialDataColumnPartsMetadata / PartialDataColumnGroupID are reused from fulu there, but gloas's own partial-columns/p2p-interface.md modifies all three (CellsBitList becomes progressive, PartialDataColumnSidecar drops its header field entirely, PartialDataColumnGroupID gains slot), so this commit gives gloas its own versions instead. --- crates/common/types/src/beacon/constants.rs | 50 + .../types/src/beacon/containers/gloas.rs | 1111 +++++++++++++++++ .../common/types/src/beacon/containers/mod.rs | 1 + .../types/src/beacon/containers/shared.rs | 14 +- crates/common/types/src/beacon/preset.rs | 132 +- .../common/types/src/beacon/serde_helpers.rs | 53 + 6 files changed, 1358 insertions(+), 3 deletions(-) create mode 100644 crates/common/types/src/beacon/containers/gloas.rs diff --git a/crates/common/types/src/beacon/constants.rs b/crates/common/types/src/beacon/constants.rs index cee085c01..2606be382 100644 --- a/crates/common/types/src/beacon/constants.rs +++ b/crates/common/types/src/beacon/constants.rs @@ -141,6 +141,14 @@ pub const DOMAIN_CONTRIBUTION_AND_PROOF: DomainType = [0x09, 0x00, 0x00, 0x00]; /// [`BLS_WITHDRAWAL_PREFIX`] credentials over to /// [`ETH1_ADDRESS_WITHDRAWAL_PREFIX`]. pub const DOMAIN_BLS_TO_EXECUTION_CHANGE: DomainType = [0x0a, 0x00, 0x00, 0x00]; +/// Domain for a `SignedExecutionPayloadBid` (gloas). +pub const DOMAIN_BEACON_BUILDER: DomainType = [0x0b, 0x00, 0x00, 0x00]; +/// Domain for a `PayloadAttestationMessage` (gloas). +pub const DOMAIN_PTC_ATTESTER: DomainType = [0x0c, 0x00, 0x00, 0x00]; +/// Domain for a `SignedProposerPreferences` (gloas). +pub const DOMAIN_PROPOSER_PREFERENCES: DomainType = [0x0d, 0x00, 0x00, 0x00]; +/// Domain for a `BuilderDepositRequest`'s `signature` (gloas). +pub const DOMAIN_BUILDER_DEPOSIT: DomainType = [0x0e, 0x00, 0x00, 0x00]; // --------------------------------------------------------------------------- // Participation flags and incentivization weights (altair) @@ -354,6 +362,44 @@ pub const MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: u64 = 4096; /// neighbours have no use for. pub const MAXIMUM_GOSSIP_CLOCK_DISPARITY: u64 = 500; +// --------------------------------------------------------------------------- +// Gloas (EIP-7732, EIP-8282) +// --------------------------------------------------------------------------- + +/// Set on a `ValidatorIndex` to mean a [`crate::beacon::containers::gloas::BuilderIndex`] +/// instead, so one `Withdrawal` container can pay out to either a validator +/// or a builder without a second, near-identical container. +pub const BUILDER_INDEX_FLAG: u64 = 1 << 40; + +/// The builder index a proposer's own `ExecutionPayloadBid` carries when it +/// is building its own payload rather than buying one from a registered +/// builder. +pub const BUILDER_INDEX_SELF_BUILD: u64 = u64::MAX; + +/// Numerator of the fraction of a bid's value a builder's payment must clear +/// before `settle_builder_payment` pays it out in full rather than partially. +pub const BUILDER_PAYMENT_THRESHOLD_NUMERATOR: u64 = 6; +/// Denominator paired with [`BUILDER_PAYMENT_THRESHOLD_NUMERATOR`]. +pub const BUILDER_PAYMENT_THRESHOLD_DENOMINATOR: u64 = 10; + +/// The first byte of a builder's withdrawal credentials, `Validator`'s +/// withdrawal-prefix family extended to builders. +pub const BUILDER_WITHDRAWAL_PREFIX: u8 = 0xb0; + +/// The `Builder.version` an onboarded or newly deposited builder is given. +/// Only one version exists so far; the field exists so a future format +/// change has somewhere to record which shape a builder's record is in. +pub const PAYLOAD_BUILDER_VERSION: u8 = 0; + +/// The first byte of an execution-layer request list entry (EIP-8282) that +/// identifies it as a builder deposit request, continuing electra's +/// `DEPOSIT_REQUEST_TYPE`/`WITHDRAWAL_REQUEST_TYPE`/`CONSOLIDATION_REQUEST_TYPE` +/// sequence. +pub const BUILDER_DEPOSIT_REQUEST_TYPE: u8 = 0x03; +/// The first byte identifying an execution layer request as a builder exit +/// request. +pub const BUILDER_EXIT_REQUEST_TYPE: u8 = 0x04; + #[cfg(test)] mod tests { use super::*; @@ -385,6 +431,10 @@ mod tests { DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF, DOMAIN_CONTRIBUTION_AND_PROOF, DOMAIN_BLS_TO_EXECUTION_CHANGE, + DOMAIN_BEACON_BUILDER, + DOMAIN_PTC_ATTESTER, + DOMAIN_PROPOSER_PREFERENCES, + DOMAIN_BUILDER_DEPOSIT, ] { let masked = [ domain[0] & DOMAIN_APPLICATION_MASK[0], diff --git a/crates/common/types/src/beacon/containers/gloas.rs b/crates/common/types/src/beacon/containers/gloas.rs new file mode 100644 index 000000000..8c6731e03 --- /dev/null +++ b/crates/common/types/src/beacon/containers/gloas.rs @@ -0,0 +1,1111 @@ +//! Containers whose shape is specific to gloas. +//! +//! Gloas bundles several EIPs. EIP-7732 (ePBS: enshrined proposer-builder +//! separation) moves the execution payload out of the beacon block and into +//! a separate, builder-signed [`ExecutionPayloadEnvelope`]: a proposer now +//! only commits to a builder's [`ExecutionPayloadBid`] at block-production +//! time, and the payload itself is revealed and attested to afterward, by a +//! payload timeliness committee ([`PayloadAttestation`]) rather than by the +//! attesters who voted on the block. [`Builder`] and its pending-payment and +//! pending-withdrawal queues are the registry this introduces alongside the +//! validator one, since a builder now escrows a balance and receives +//! payments the same way a validator earns rewards. +//! +//! EIP-7688 makes most of the specification's lists, and every "big" +//! container (`Attestation`, `BeaconBlockBody`, `BeaconState`, +//! `ExecutionPayload`, `ExecutionRequests`, `NewPayloadRequest`, and their +//! nested attestation containers), progressive: an EIP-7916 `ProgressiveList` +//! or an EIP-7495 progressive container in place of a bounded `SszList` or a +//! fixed field layout. `BeaconState.validators` and `.balances` are the one +//! exception that stays tree-backed rather than becoming a plain +//! `libssz_types::ProgressiveList`: see [`super::shared::ProgressiveValidators`] +//! and [`super::shared::ProgressiveBalances`]. +//! +//! EIP-8282 adds builder deposit and exit requests to [`ExecutionRequests`], +//! the execution-layer-triggered path [`Builder`]s are onboarded and retired +//! through (mirroring EIP-6110/7002/7251's validator deposit, withdrawal, and +//! consolidation requests). EIP-8045 and EIP-8061 do not reshape any +//! container this module defines. +//! +//! `BeaconBlockBody`, `Attestation`, `IndexedAttestation`, and +//! `AttesterSlashing` are the phase0/electra shapes that gained the fields +//! ePBS moves out or in (`signed_execution_payload_bid`, +//! `payload_attestations`, `parent_execution_requests` on the body) or that +//! EIP-7688 alone reshapes into a progressive container without changing a +//! single field. `AggregateAndProof` and `SignedAggregateAndProof` are the +//! electra shape, unmodified except that `aggregate` is now this module's own +//! wider [`Attestation`]. `DataColumnSidecar` drops the header and inclusion +//! proof fulu's carried, since a Gloas sidecar no longer needs to prove its +//! commitments against a block body directly: they are read from +//! `signed_execution_payload_bid.message.blob_kzg_commitments` instead. +//! `PartialDataColumnSidecar`, `PartialDataColumnPartsMetadata`, and +//! `PartialDataColumnGroupID` are gloas's own versions of fulu's Partial +//! Message Extension wire types (`gloas/partial-columns/p2p-interface.md`), +//! not reused from [`super::fulu`]: `CellsBitList` becomes progressive here, +//! which changes the merkleization of every container holding one even where +//! its field list looks unchanged, `PartialDataColumnSidecar` drops `header` +//! entirely, and `PartialDataColumnGroupID` gains `slot`. + +use libssz_derive::{HashTreeRoot, SszDecode, SszEncode}; +use libssz_types::{ProgressiveBitlist, ProgressiveList, SszBitvector, SszList, SszVector}; + +use super::altair::{SyncAggregate, SyncCommittee}; +use super::bellatrix::{ExtraData, LogsBloom}; +use super::capella::{SignedBLSToExecutionChange, Withdrawal}; +use super::deneb::VersionedHashes; +use super::electra::{ + CommitteeBits, ConsolidationRequest, DepositRequest, PendingConsolidation, PendingDeposit, + PendingPartialWithdrawal, WithdrawalRequest, +}; +use super::fulu::{Cell, ProposerLookahead}; +use super::shared::{ + AttestationData, BeaconBlockHeader, BlockRoots, Checkpoint, Deposit, Eth1Data, Eth1DataVotes, + Fork, HistoricalRoots, HistoricalSummaries, JustificationBits, ProgressiveBalances, + ProgressiveValidators, ProposerSlashing, RandaoMixes, SignedVoluntaryExit, Slashings, + StateRoots, +}; +use crate::beacon::preset; +use crate::beacon::primitives::{ + BlsPubkey, BlsSignature, Bytes32, ColumnIndex, Epoch, ExecutionAddress, ExecutionBlockHash, + Gwei, KzgCommitment, KzgProof, ParticipationFlags, Root, Slot, Uint256, ValidatorIndex, + WithdrawalIndex, +}; + +// --------------------------------------------------------------------------- +// Collection aliases (EIP-7688: progressive where the spec says so) +// --------------------------------------------------------------------------- + +/// The index of a [`Builder`] in the builder registry. A plain `u64` alias, +/// the same convention [`crate::beacon::primitives`] uses for every other +/// spec index type, kept here rather than there because nothing outside +/// gloas reads it. +pub type BuilderIndex = u64; + +/// The attesters covered by an [`Attestation`]. Unbounded from gloas on, +/// unlike every earlier fork's `SszBitlist`. +pub type AggregationBits = ProgressiveBitlist; +/// The attesters covered by an [`IndexedAttestation`], named explicitly. +pub type AttestingIndices = ProgressiveList; +/// Attestations included in a block. +pub type Attestations = ProgressiveList; +/// Evidence of conflicting attestations included in a block. +pub type AttesterSlashings = ProgressiveList; +/// Proposer slashings included in a block. +pub type ProposerSlashings = ProgressiveList; +/// Deposits included in a block. +pub type Deposits = ProgressiveList; +/// Voluntary exits included in a block. +pub type VoluntaryExits = ProgressiveList; +/// BLS-to-execution changes included in a block. +pub type BlsToExecutionChanges = ProgressiveList; +/// The KZG commitments an [`ExecutionPayloadBid`] carries for the blobs its +/// payload will reveal. +pub type BlobKzgCommitments = ProgressiveList; +/// An opaque execution-layer transaction. Unbounded from gloas on, unlike +/// [`super::bellatrix::Transaction`]'s `SszList`. +pub type Transaction = ProgressiveList; +/// The transactions in an [`ExecutionPayload`]. +pub type Transactions = ProgressiveList; +/// The withdrawals in an [`ExecutionPayload`], or the sweep's expected set +/// cached on the state ([`BeaconState::payload_expected_withdrawals`]). +pub type Withdrawals = ProgressiveList; +/// The serialized block access list of an [`ExecutionPayload`] (EIP-7928). +pub type BlockAccessList = ProgressiveList; +/// Execution-layer-triggered deposit requests. +pub type DepositRequests = ProgressiveList; +/// Execution-layer-triggered withdrawal requests. +pub type WithdrawalRequests = ProgressiveList; +/// Execution-layer-triggered consolidation requests. +pub type ConsolidationRequests = ProgressiveList; +/// Execution-layer-triggered builder deposit requests (EIP-8282). +pub type BuilderDepositRequests = ProgressiveList; +/// Execution-layer-triggered builder exit requests (EIP-8282). +pub type BuilderExitRequests = ProgressiveList; +/// Per-validator participation flags for one epoch, positionally parallel to +/// the registry. +pub type EpochParticipation = ProgressiveList; +/// Per-validator inactivity scores, positionally parallel to the registry. +pub type InactivityScores = ProgressiveList; +/// Deposits known but not yet credited to the validator registry. +pub type PendingDeposits = ProgressiveList; +/// Partial withdrawals known but not yet paid out. +pub type PendingPartialWithdrawals = ProgressiveList; +/// Validator consolidations known but not yet applied. +pub type PendingConsolidations = ProgressiveList; +/// The builder registry (EIP-7732), the builder-side counterpart of the +/// validator registry. +pub type Builders = ProgressiveList; +/// Builder withdrawals known but not yet paid out, the builder-side +/// counterpart of [`PendingPartialWithdrawals`]. +pub type BuilderPendingWithdrawals = ProgressiveList; +/// Builder payments owed for the previous and current epoch. A fixed-length +/// vector rather than a progressive list: the specification bounds it to +/// exactly two epochs' worth, since a payment is settled well before a third +/// epoch could accumulate. +pub type BuilderPendingPayments = + SszVector; +/// Bits tracking payload availability for recent slots, indexed by slot +/// modulo `SLOTS_PER_HISTORICAL_ROOT`, the same indexing [`BlockRoots`] uses. +pub type ExecutionPayloadAvailability = SszBitvector<{ preset::SLOTS_PER_HISTORICAL_ROOT }>; +/// Payload attestations included in a block. +pub type PayloadAttestations = ProgressiveList; +/// The payload timeliness committee of a slot, named explicitly. +pub type PayloadTimelinessCommittee = SszVector; +/// The payload timeliness committee members an [`IndexedPayloadAttestation`] +/// names, as a bounded list rather than the fixed-size +/// [`PayloadTimelinessCommittee`]: an attester subset, not the whole +/// committee. +pub type PayloadTimelinessCommitteeIndices = SszList; +/// One bit per payload timeliness committee member, in committee order. +pub type PayloadTimelinessCommitteeBits = SszBitvector<{ preset::PTC_SIZE }>; +/// The cached window of payload timeliness committees +/// [`BeaconState::ptc_window`] holds: the previous, current, and lookahead +/// epochs. +pub type PayloadTimelinessCommitteeWindow = + SszVector; +/// One data column, with at most one cell per blob. Unbounded from gloas on, +/// unlike fulu's `SszList`. +pub type DataColumn = ProgressiveList; +/// The KZG cell proofs behind a [`DataColumn`]. +pub type KzgProofs = ProgressiveList; +/// A bitfield over the cells of a column, one bit per blob (`fulu/partial-columns/p2p-interface.md`, +/// modified for gloas). Unbounded from gloas on, unlike fulu's +/// `SszBitlist`. +pub type CellsBitList = ProgressiveBitlist; + +// --------------------------------------------------------------------------- +// Builders +// --------------------------------------------------------------------------- + +/// A registered builder's record in the builder registry (EIP-7732), the +/// builder-side counterpart of [`super::shared::Validator`]. +#[derive( + Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, +)] +pub struct Builder { + pub pubkey: BlsPubkey, + /// Which shape this record is in. Only [`crate::beacon::constants::PAYLOAD_BUILDER_VERSION`] + /// exists today; the field exists so a future format change has + /// somewhere to record it. + pub version: u8, + pub execution_address: ExecutionAddress, + pub balance: Gwei, + /// The epoch this builder's deposit was placed, which + /// `is_active_builder` compares against the finalized checkpoint before + /// treating the builder as eligible. + pub deposit_epoch: Epoch, + /// `FAR_FUTURE_EPOCH` until this builder initiates an exit, the same + /// sentinel convention [`super::shared::Validator::withdrawable_epoch`] + /// uses. + pub withdrawable_epoch: Epoch, +} + +/// A builder payment owed to a proposer, not yet settled. +/// +/// `weight` accumulates the attesting stake behind the block the payment is +/// for, so `settle_builder_payment` can compare it against the quorum +/// threshold before paying `withdrawal` out. +#[derive( + Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, +)] +pub struct BuilderPendingPayment { + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub weight: Gwei, + pub withdrawal: BuilderPendingWithdrawal, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub proposer_index: ValidatorIndex, +} + +/// A builder withdrawal queued but not yet paid out, the builder-side +/// counterpart of [`super::capella::Withdrawal`]. +#[derive( + Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, +)] +pub struct BuilderPendingWithdrawal { + pub fee_recipient: ExecutionAddress, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub amount: Gwei, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub builder_index: BuilderIndex, +} + +/// An execution-layer-triggered builder deposit (EIP-8282), the builder-side +/// counterpart of [`super::electra::DepositRequest`]. +#[derive( + Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, +)] +pub struct BuilderDepositRequest { + pub pubkey: BlsPubkey, + pub withdrawal_credentials: Bytes32, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub amount: Gwei, + pub signature: BlsSignature, +} + +/// An execution-layer-triggered builder exit (EIP-8282), the builder-side +/// counterpart of [`super::electra::WithdrawalRequest`]'s full-exit case. +#[derive( + Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, +)] +pub struct BuilderExitRequest { + pub source_address: ExecutionAddress, + pub pubkey: BlsPubkey, +} + +// --------------------------------------------------------------------------- +// Payload attestations +// --------------------------------------------------------------------------- + +/// What the payload timeliness committee attests to: whether the slot's +/// payload was revealed on time and whether its blob data is available. +#[derive( + Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, +)] +pub struct PayloadAttestationData { + pub beacon_block_root: Root, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub slot: Slot, + pub payload_present: bool, + pub blob_data_available: bool, +} + +/// An aggregate payload timeliness attestation, as included in a block. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[ssz(progressive_container)] +pub struct PayloadAttestation { + #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + pub aggregation_bits: PayloadTimelinessCommitteeBits, + pub data: PayloadAttestationData, + pub signature: BlsSignature, +} + +/// One payload timeliness committee member's unaggregated vote, gossiped +/// before an aggregator folds it into a [`PayloadAttestation`]. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +pub struct PayloadAttestationMessage { + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub validator_index: ValidatorIndex, + pub data: PayloadAttestationData, + pub signature: BlsSignature, +} + +/// A [`PayloadAttestation`] with its attesters named rather than bit-encoded, +/// the payload-attestation counterpart of [`IndexedAttestation`]. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[ssz(progressive_container)] +pub struct IndexedPayloadAttestation { + #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + pub attesting_indices: PayloadTimelinessCommitteeIndices, + pub data: PayloadAttestationData, + pub signature: BlsSignature, +} + +// --------------------------------------------------------------------------- +// Execution +// --------------------------------------------------------------------------- + +/// A builder's commitment to reveal a payload for a slot (EIP-7732): what a +/// proposer chooses among, and signs over, in place of embedding a payload in +/// the block directly. +#[derive( + Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, +)] +#[ssz(progressive_container)] +pub struct ExecutionPayloadBid { + pub parent_block_hash: ExecutionBlockHash, + pub parent_block_root: Root, + pub block_hash: ExecutionBlockHash, + pub prev_randao: Bytes32, + pub fee_recipient: ExecutionAddress, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub gas_limit: u64, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub builder_index: BuilderIndex, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub slot: Slot, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub value: Gwei, + /// What the builder is actually paid, which may fall short of `value` if + /// the payload is never revealed or attested to as available. + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub execution_payment: Gwei, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub blob_kzg_commitments: BlobKzgCommitments, + /// The root of the [`ExecutionRequests`] this payload's envelope will + /// carry, committed to here so the bid is binding on the requests too, + /// not only the payload itself. + pub execution_requests_root: Root, +} + +#[derive( + Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, +)] +pub struct SignedExecutionPayloadBid { + pub message: ExecutionPayloadBid, + pub signature: BlsSignature, +} + +/// The execution-layer-triggered deposit, withdrawal, consolidation, builder +/// deposit, and builder exit requests carried by one execution payload. +/// +/// Electra's three request kinds (EIP-6110/7002/7251), plus `builder_deposits` +/// and `builder_exits` (EIP-8282), the execution-layer path [`Builder`]s are +/// onboarded and retired through from the fork onward. +#[derive( + Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, +)] +#[ssz(progressive_container)] +pub struct ExecutionRequests { + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub deposits: DepositRequests, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub withdrawals: WithdrawalRequests, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub consolidations: ConsolidationRequests, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub builder_deposits: BuilderDepositRequests, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub builder_exits: BuilderExitRequests, +} + +/// The execution layer's block contents, no longer carried inside +/// [`BeaconBlockBody`] but revealed afterward in an [`ExecutionPayloadEnvelope`] +/// (EIP-7732). +/// +/// Bellatrix's fields through `block_hash`, unchanged; `transactions` and +/// `withdrawals` become progressive (EIP-7688); `block_access_list` +/// (EIP-7928) and `slot_number` (EIP-8061) are appended. Not +/// `#[derive(Default)]`: `logs_bloom` is an [`SszVector`], which, like every +/// earlier fork's payload, has no meaningful empty value. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[ssz(progressive_container)] +pub struct ExecutionPayload { + pub parent_hash: ExecutionBlockHash, + pub fee_recipient: ExecutionAddress, + pub state_root: Bytes32, + pub receipts_root: Bytes32, + #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + pub logs_bloom: LogsBloom, + pub prev_randao: Bytes32, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub block_number: u64, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub gas_limit: u64, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub gas_used: u64, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub timestamp: u64, + #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + pub extra_data: ExtraData, + pub base_fee_per_gas: Uint256, + pub block_hash: ExecutionBlockHash, + #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex_seq::serialize")] + pub transactions: Transactions, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub withdrawals: Withdrawals, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub blob_gas_used: u64, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub excess_blob_gas: u64, + /// The serialized block access list this payload's execution produced + /// (EIP-7928). + #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + pub block_access_list: BlockAccessList, + /// The slot this payload was built for (EIP-8061's gas limit schedule + /// reads it against the schedule rather than the enclosing block's own + /// slot, since a payload can be revealed a slot late). + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub slot_number: u64, +} + +/// A builder-signed reveal of the payload it bid for: the payload itself, +/// the execution-layer-triggered requests it carries, and enough to bind it +/// to the block whose [`ExecutionPayloadBid`] it fulfills. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[ssz(progressive_container)] +pub struct ExecutionPayloadEnvelope { + pub payload: ExecutionPayload, + pub execution_requests: ExecutionRequests, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub builder_index: BuilderIndex, + pub beacon_block_root: Root, + pub parent_beacon_block_root: Root, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +pub struct SignedExecutionPayloadEnvelope { + pub message: ExecutionPayloadEnvelope, + pub signature: BlsSignature, +} + +/// What `process_execution_payload` hands `execution_engine.verify_and_notify_new_payload` +/// to validate a revealed payload, gloas `beacon-chain.md`. +/// +/// Electra's fields (deneb's blob versioned hashes and beacon root, electra's +/// execution requests), with `execution_payload` and `execution_requests` now +/// this module's own progressive shapes. Not `#[derive(Default)]`, for the +/// reason [`ExecutionPayload`]'s own doc gives. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[ssz(progressive_container)] +pub struct NewPayloadRequest { + pub execution_payload: ExecutionPayload, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub versioned_hashes: VersionedHashes, + pub parent_beacon_block_root: Root, + pub execution_requests: ExecutionRequests, +} + +// --------------------------------------------------------------------------- +// Attestations +// --------------------------------------------------------------------------- + +/// An aggregate attestation, as gossiped and as included in a block. +/// +/// Electra's shape (EIP-7549's wide `committee_bits`/`aggregation_bits`), +/// with `aggregation_bits` now the unbounded [`AggregationBits`] rather than +/// electra's bounded `SszBitlist`. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[ssz(progressive_container)] +pub struct Attestation { + #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + pub aggregation_bits: AggregationBits, + pub data: AttestationData, + pub signature: BlsSignature, + #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + pub committee_bits: CommitteeBits, +} + +/// An attestation with its attesters named rather than bit-encoded, the same +/// role electra's plays, with `attesting_indices` now the unbounded +/// [`AttestingIndices`]. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[ssz(progressive_container)] +pub struct IndexedAttestation { + #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + pub attesting_indices: AttestingIndices, + pub data: AttestationData, + pub signature: BlsSignature, +} + +/// Evidence that a set of validators made two conflicting attestations. +/// Unchanged in shape from electra; what changed is [`IndexedAttestation`] +/// itself. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +pub struct AttesterSlashing { + pub attestation_1: IndexedAttestation, + pub attestation_2: IndexedAttestation, +} + +/// An aggregate together with proof that its aggregator was selected to +/// produce it. Unchanged in shape from electra: `aggregate` simply carries +/// gloas's own, wider [`Attestation`] now. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +pub struct AggregateAndProof { + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub aggregator_index: ValidatorIndex, + pub aggregate: Attestation, + pub selection_proof: BlsSignature, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +pub struct SignedAggregateAndProof { + pub message: AggregateAndProof, + pub signature: BlsSignature, +} + +// --------------------------------------------------------------------------- +// Blocks +// --------------------------------------------------------------------------- + +/// The contents of a block. +/// +/// *Removed* relative to electra: `execution_payload`, `blob_kzg_commitments`, +/// and `execution_requests`, which now live in [`ExecutionPayloadEnvelope`] +/// instead, revealed by the builder after the block itself is. *Added*: +/// `signed_execution_payload_bid` (the proposer's binding commitment to a +/// builder's bid), `payload_attestations` (the payload timeliness +/// committee's votes on the *previous* slot's payload), and +/// `parent_execution_requests` (the previous slot's execution requests, +/// carried forward so a payload attestation can be checked without a +/// separate fetch). Every other field is electra's, now progressive +/// (EIP-7688). +#[derive( + Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, +)] +#[ssz(progressive_container)] +pub struct BeaconBlockBody { + /// The proposer's contribution to the chain's randomness, which is a + /// signature over the current epoch and so cannot be chosen freely. + pub randao_reveal: BlsSignature, + /// The proposer's vote on the execution chain's deposit state. + pub eth1_data: Eth1Data, + /// Arbitrary proposer-chosen bytes, which consensus never reads. + pub graffiti: Bytes32, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub proposer_slashings: ProposerSlashings, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub attester_slashings: AttesterSlashings, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub attestations: Attestations, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub deposits: Deposits, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub voluntary_exits: VoluntaryExits, + /// The aggregated sync committee signature over the previous slot's + /// block root, plus which members contributed. + pub sync_aggregate: SyncAggregate, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub bls_to_execution_changes: BlsToExecutionChanges, + /// The proposer's binding commitment to the builder's bid for this + /// slot's payload (EIP-7732); the payload itself is revealed afterward + /// in a [`SignedExecutionPayloadEnvelope`]. + pub signed_execution_payload_bid: SignedExecutionPayloadBid, + /// The payload timeliness committee's votes on the *previous* slot's + /// payload. + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub payload_attestations: PayloadAttestations, + /// The previous slot's execution-layer-triggered requests, carried + /// forward so `process_payload_attestation` and friends do not need a + /// separate fetch of the previous envelope. + pub parent_execution_requests: ExecutionRequests, +} + +impl BeaconBlockBody { + /// An empty body: no operations of any kind, and a default (all-zero) + /// execution payload bid. Unlike every earlier fork's `empty()`, this can + /// delegate to `Default::default()`: `execution_payload` is no longer a + /// field of this container, so nothing here is an [`SszVector`] without + /// a meaningful empty value the way `execution_payload.logs_bloom` was. + /// Kept as its own method, rather than leaving callers to write + /// `BeaconBlockBody::default()`, for parity with every earlier fork's + /// `BeaconBlockBody::empty()`, which callers building a block for a test + /// already reach for by that name. + pub fn empty() -> Self { + Self::default() + } +} + +/// A block. +#[derive( + Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, +)] +pub struct BeaconBlock { + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub slot: Slot, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub proposer_index: ValidatorIndex, + pub parent_root: Root, + /// The root of the state after this block is applied, which the state + /// transition recomputes and compares. + pub state_root: Root, + pub body: BeaconBlockBody, +} + +#[derive( + Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, +)] +pub struct SignedBeaconBlock { + pub message: BeaconBlock, + pub signature: BlsSignature, +} + +// --------------------------------------------------------------------------- +// State +// --------------------------------------------------------------------------- + +/// The gloas beacon state: 46 fields, in the specification's order. +/// +/// Field order is load-bearing, as in every fork: SSZ encoding and this +/// progressive container's merkleization both follow declaration order. +/// +/// Fields through `pending_consolidations` are fulu's, field for field, with +/// `validators`, `balances`, `previous_epoch_participation`, +/// `current_epoch_participation`, `inactivity_scores`, `pending_deposits`, +/// `pending_partial_withdrawals`, and `pending_consolidations` now +/// progressive (EIP-7688). `proposer_lookahead` is unchanged from fulu. +/// `latest_execution_payload_header` is *removed*: a gloas state no longer +/// holds a payload header at all, only `latest_block_hash` (the previous +/// payload's own hash) and `latest_execution_payload_bid` (the bid the +/// *current* payload, if any, was bought under). Every field from `builders` +/// onward is new, EIP-7732's builder registry, payment queues, availability +/// tracking, and cached payload timeliness committee window. Not +/// `#[derive(Default)]`: `block_roots`, `state_roots`, `randao_mixes`, +/// `slashings`, `builder_pending_payments`, and `ptc_window` are all +/// [`SszVector`]s, none of which has a meaningful empty value. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[ssz(progressive_container)] +pub struct BeaconState { + // -- Versioning -- + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub genesis_time: u64, + /// The root of the genesis validator registry, which separates this + /// chain from any other running the same fork schedule. + pub genesis_validators_root: Root, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub slot: Slot, + pub fork: Fork, + + // -- History -- + /// The most recent block's header, with `state_root` left zero until the + /// slot advances, since a block cannot commit to the root of the state + /// containing it. + pub latest_block_header: BeaconBlockHeader, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub block_roots: BlockRoots, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub state_roots: StateRoots, + /// Frozen since capella: further history is committed to by + /// `historical_summaries` instead. + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub historical_roots: HistoricalRoots, + + // -- Eth1 -- + pub eth1_data: Eth1Data, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub eth1_data_votes: Eth1DataVotes, + /// How many deposits from the contract have been processed, which is + /// where the next one will be read from. + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub eth1_deposit_index: u64, + + // -- Registry (EIP-7688) -- + /// The validator registry, tree-backed and progressively merkleized. See + /// [`ProgressiveValidators`]. + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub validators: ProgressiveValidators, + #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + pub balances: ProgressiveBalances, + + // -- Randomness -- + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub randao_mixes: RandaoMixes, + + // -- Slashings -- + #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + pub slashings: Slashings, + + // -- Participation (EIP-7688) -- + #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + pub previous_epoch_participation: EpochParticipation, + #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + pub current_epoch_participation: EpochParticipation, + + // -- Finality -- + #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + pub justification_bits: JustificationBits, + pub previous_justified_checkpoint: Checkpoint, + pub current_justified_checkpoint: Checkpoint, + pub finalized_checkpoint: Checkpoint, + + // -- Inactivity (EIP-7688) -- + #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + pub inactivity_scores: InactivityScores, + + // -- Sync committees -- + pub current_sync_committee: SyncCommittee, + pub next_sync_committee: SyncCommittee, + + // -- Execution (EIP-7732) -- + /// The most recently revealed payload's own hash. Replaces + /// `latest_execution_payload_header`: a gloas state no longer holds a + /// payload header, only this hash and `latest_execution_payload_bid`. + pub latest_block_hash: ExecutionBlockHash, + + // -- Withdrawals -- + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub next_withdrawal_index: WithdrawalIndex, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub next_withdrawal_validator_index: ValidatorIndex, + + // -- History (capella) -- + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub historical_summaries: HistoricalSummaries, + + // -- Deposits, exits, and consolidations (electra) -- + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub deposit_requests_start_index: u64, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub deposit_balance_to_consume: Gwei, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub exit_balance_to_consume: Gwei, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub earliest_exit_epoch: Epoch, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub consolidation_balance_to_consume: Gwei, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub earliest_consolidation_epoch: Epoch, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub pending_deposits: PendingDeposits, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub pending_partial_withdrawals: PendingPartialWithdrawals, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub pending_consolidations: PendingConsolidations, + + // -- Proposer lookahead (fulu) -- + #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + pub proposer_lookahead: ProposerLookahead, + + // -- Builders (EIP-7732) -- + /// The builder registry. + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub builders: Builders, + /// Where the builder withdrawal sweep last stopped, the builder-side + /// counterpart of `next_withdrawal_validator_index`. + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub next_withdrawal_builder_index: BuilderIndex, + /// Bits tracking, for each of the last `SLOTS_PER_HISTORICAL_ROOT` slots, + /// whether that slot's payload was made available. + #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + pub execution_payload_availability: ExecutionPayloadAvailability, + /// Builder payments owed for the previous and current epoch, settled by + /// `process_builder_pending_payments`. + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub builder_pending_payments: BuilderPendingPayments, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub builder_pending_withdrawals: BuilderPendingWithdrawals, + /// The bid the payload currently expected for this slot, if any, was + /// bought under. + pub latest_execution_payload_bid: ExecutionPayloadBid, + /// The withdrawals `get_expected_withdrawals` computed for the payload + /// this slot expects, cached here so a late-revealed payload's + /// `process_execution_payload` does not have to recompute the sweep. + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub payload_expected_withdrawals: Withdrawals, + /// The cached window of payload timeliness committees for the previous, + /// current, and lookahead epochs, refreshed each epoch by + /// `process_ptc_window`. + #[serde(serialize_with = "crate::beacon::serde_helpers::nested_quoted_u64_seq::serialize")] + pub ptc_window: PayloadTimelinessCommitteeWindow, +} + +// --------------------------------------------------------------------------- +// Data columns +// --------------------------------------------------------------------------- + +/// One column's worth of one block's blob data, as gossiped and served over +/// request-response. +/// +/// Drops the `signed_block_header`, `kzg_commitments`, and +/// `kzg_commitments_inclusion_proof` fields fulu's carried: those existed to +/// let a sidecar prove its commitments against a block body on its own, and +/// gloas no longer needs that proof, since the commitments a sidecar's +/// column must match are read from +/// `block.body.signed_execution_payload_bid.message.blob_kzg_commitments` +/// for the block named by `beacon_block_root` instead. `slot` and +/// `beacon_block_root` are new, naming that block directly. +#[derive( + Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, +)] +pub struct DataColumnSidecar { + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub index: ColumnIndex, + #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex_seq::serialize")] + pub column: DataColumn, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub kzg_proofs: KzgProofs, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub slot: Slot, + pub beacon_block_root: Root, +} + +// --------------------------------------------------------------------------- +// Partial columns +// --------------------------------------------------------------------------- +// +// Transcribed from `gloas/partial-columns/p2p-interface.md`, which modifies +// fulu's own partial-column wire types (see [`super::fulu`]'s module doc): +// `CellsBitList` becomes progressive, so every container holding one gains a +// different merkleization even where its field list is textually unchanged, +// and `PartialDataColumnSidecar` drops `header` entirely (no header/inclusion +// proof survives in gloas; see [`DataColumnSidecar`]'s own doc for why). +// `PartialDataColumnGroupID` gains `slot`, since a partial message can no +// longer be reasoned about by block root alone once payloads are revealed a +// slot after the block that bid for them. + +/// One column's worth of cells and proofs sent as a gossipsub partial +/// message, carrying only the cells `cells_present_bitmap` names rather than +/// a full [`DataColumn`]. +/// +/// Unlike fulu's, this carries no `header`: a partial message's cells are +/// checked against `signed_execution_payload_bid.message.blob_kzg_commitments` +/// on the block named by the enclosing [`PartialDataColumnGroupID`], the same +/// way [`DataColumnSidecar`] no longer carries its own header either. +#[derive( + Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, +)] +pub struct PartialDataColumnSidecar { + #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + pub cells_present_bitmap: CellsBitList, + #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex_seq::serialize")] + pub partial_column: DataColumn, + #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + pub kzg_proofs: KzgProofs, +} + +/// The two bitmaps peers exchange to negotiate which cells to send. Same +/// field list as fulu's, but a different shape underneath: both fields are +/// now the progressive [`CellsBitList`]. +#[derive( + Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, +)] +pub struct PartialDataColumnPartsMetadata { + #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + pub available: CellsBitList, + #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + pub requests: CellsBitList, +} + +/// The gossipsub Partial Message group ID for a block's columns. +/// +/// `slot` is new relative to fulu's: a payload can be revealed a slot after +/// the block whose bid it fulfills, so naming the block root alone no longer +/// pins down which slot's partial messages a group covers. +#[derive( + Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, +)] +pub struct PartialDataColumnGroupID { + pub beacon_block_root: Root, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub slot: Slot, +} + +// --------------------------------------------------------------------------- +// Proposer preferences +// --------------------------------------------------------------------------- + +/// A proposer's broadcast, ahead of its slot, of the fee recipient and gas +/// limit it wants a builder's bid to target (EIP-7732 p2p-interface.md). +#[derive( + Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, +)] +pub struct ProposerPreferences { + /// The root of the beacon state the proposer duty this message announces + /// was computed from, so a receiver can check the message against its + /// own view of the same duty. + pub dependent_root: Root, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub proposal_slot: Slot, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub validator_index: ValidatorIndex, + pub fee_recipient: ExecutionAddress, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub target_gas_limit: u64, +} + +#[derive( + Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, +)] +pub struct SignedProposerPreferences { + pub message: ProposerPreferences, + pub signature: BlsSignature, +} + +#[cfg(test)] +mod tests { + use libssz::{SszDecode as _, SszEncode as _}; + + use super::*; + use crate::beacon::primitives::HashTreeRoot as _; + + /// A zeroed [`LogsBloom`], built explicitly rather than through + /// `Default`: [`SszVector`] has none, the same reason every earlier + /// fork's `BeaconBlockBody::empty()` builds one by hand. + fn zero_logs_bloom() -> LogsBloom { + LogsBloom::try_from(vec![0u8; preset::BYTES_PER_LOGS_BLOOM]) + .expect("BYTES_PER_LOGS_BLOOM zeros fit LogsBloom's exact length") + } + + /// A default-valued [`ExecutionPayload`], filling in only the one field + /// `#[derive(Default)]` cannot reach: `logs_bloom`. + fn zero_execution_payload() -> ExecutionPayload { + ExecutionPayload { + parent_hash: Default::default(), + fee_recipient: Default::default(), + state_root: Default::default(), + receipts_root: Default::default(), + logs_bloom: zero_logs_bloom(), + prev_randao: Default::default(), + block_number: 0, + gas_limit: 0, + gas_used: 0, + timestamp: 0, + extra_data: Default::default(), + base_fee_per_gas: Default::default(), + block_hash: Default::default(), + transactions: Default::default(), + withdrawals: Default::default(), + blob_gas_used: 0, + excess_blob_gas: 0, + block_access_list: Default::default(), + slot_number: 0, + } + } + + /// A minimal but complete gloas state, every field present at an + /// all-zero or empty placeholder. Built field by field rather than + /// through `Default`, which [`BeaconState`] cannot derive: `block_roots`, + /// `state_roots`, `randao_mixes`, `slashings`, + /// `builder_pending_payments`, and `ptc_window` are all [`SszVector`]s, + /// none of which has a meaningful empty value. + fn zero_state() -> BeaconState { + let zero_root_vector = || -> BlockRoots { + vec![Root::ZERO; preset::SLOTS_PER_HISTORICAL_ROOT] + .try_into() + .expect("the vector is built at its exact length") + }; + let empty_sync_committee = || SyncCommittee { + pubkeys: vec![Default::default(); preset::SYNC_COMMITTEE_SIZE] + .try_into() + .expect("built at exactly SYNC_COMMITTEE_SIZE"), + aggregate_pubkey: Default::default(), + }; + let empty_ptc: PayloadTimelinessCommittee = vec![0u64; preset::PTC_SIZE] + .try_into() + .expect("built at exactly PTC_SIZE"); + + BeaconState { + genesis_time: 0, + genesis_validators_root: Root::ZERO, + slot: 0, + fork: Default::default(), + latest_block_header: Default::default(), + block_roots: zero_root_vector(), + state_roots: zero_root_vector(), + historical_roots: Default::default(), + eth1_data: Default::default(), + eth1_data_votes: Default::default(), + eth1_deposit_index: 0, + validators: Default::default(), + balances: Default::default(), + randao_mixes: vec![Bytes32::ZERO; preset::EPOCHS_PER_HISTORICAL_VECTOR] + .try_into() + .expect("the vector is built at its exact length"), + slashings: vec![0; preset::EPOCHS_PER_SLASHINGS_VECTOR] + .try_into() + .expect("the vector is built at its exact length"), + previous_epoch_participation: Default::default(), + current_epoch_participation: Default::default(), + justification_bits: Default::default(), + previous_justified_checkpoint: Default::default(), + current_justified_checkpoint: Default::default(), + finalized_checkpoint: Default::default(), + inactivity_scores: Default::default(), + current_sync_committee: empty_sync_committee(), + next_sync_committee: empty_sync_committee(), + latest_block_hash: ExecutionBlockHash::ZERO, + next_withdrawal_index: 0, + next_withdrawal_validator_index: 0, + historical_summaries: Default::default(), + deposit_requests_start_index: + crate::beacon::constants::UNSET_DEPOSIT_REQUESTS_START_INDEX, + deposit_balance_to_consume: 0, + exit_balance_to_consume: 0, + earliest_exit_epoch: 0, + consolidation_balance_to_consume: 0, + earliest_consolidation_epoch: 0, + pending_deposits: Default::default(), + pending_partial_withdrawals: Default::default(), + pending_consolidations: Default::default(), + proposer_lookahead: vec![0; preset::PROPOSER_LOOKAHEAD_LENGTH] + .try_into() + .expect("the vector is built at its exact length"), + builders: Default::default(), + next_withdrawal_builder_index: 0, + execution_payload_availability: Default::default(), + builder_pending_payments: vec![ + BuilderPendingPayment::default(); + preset::BUILDER_PENDING_PAYMENTS_LENGTH + ] + .try_into() + .expect("the vector is built at its exact length"), + builder_pending_withdrawals: Default::default(), + latest_execution_payload_bid: Default::default(), + payload_expected_withdrawals: Default::default(), + ptc_window: vec![empty_ptc; preset::PTC_WINDOW_LENGTH] + .try_into() + .expect("the vector is built at its exact length"), + } + } + + #[test] + fn a_zeroed_state_round_trips_through_ssz() { + let state = zero_state(); + let bytes = state.to_ssz(); + let decoded = BeaconState::from_ssz_bytes(&bytes).expect("decodes"); + assert_eq!(decoded, state); + assert_eq!(decoded.hash_tree_root(), state.hash_tree_root()); + } + + #[test] + fn a_bid_and_an_envelope_round_trip_through_ssz() { + let bid = SignedExecutionPayloadBid::default(); + assert_eq!( + SignedExecutionPayloadBid::from_ssz_bytes(&bid.to_ssz()).unwrap(), + bid + ); + + // `SignedExecutionPayloadEnvelope` cannot derive `Default` (its + // `ExecutionPayload` holds a `logs_bloom` `SszVector`), so its + // payload is built explicitly instead. + let envelope = SignedExecutionPayloadEnvelope { + message: ExecutionPayloadEnvelope { + payload: zero_execution_payload(), + execution_requests: Default::default(), + builder_index: 0, + beacon_block_root: Root::ZERO, + parent_beacon_block_root: Root::ZERO, + }, + signature: Default::default(), + }; + assert_eq!( + SignedExecutionPayloadEnvelope::from_ssz_bytes(&envelope.to_ssz()).unwrap(), + envelope + ); + } + + #[test] + fn an_empty_block_body_round_trips_through_ssz() { + let body = BeaconBlockBody::empty(); + let bytes = body.to_ssz(); + assert_eq!(BeaconBlockBody::from_ssz_bytes(&bytes).unwrap(), body); + } + + #[test] + fn a_data_column_sidecar_round_trips_through_ssz() { + let sidecar = DataColumnSidecar { + index: 3, + column: vec![Cell::try_from(vec![7u8; preset::BYTES_PER_CELL]).unwrap(); 2].into(), + kzg_proofs: vec![KzgProof([1; crate::beacon::primitives::KZG_POINT_SIZE]); 2].into(), + slot: 9, + beacon_block_root: Root::repeat_byte(4), + }; + + let bytes = sidecar.to_ssz(); + assert_eq!(DataColumnSidecar::from_ssz_bytes(&bytes).unwrap(), sidecar); + } + + #[test] + fn variable_length_containers_carry_offsets() { + assert!(!::is_fixed_size()); + assert!(!::is_fixed_size()); + assert!(!::is_fixed_size()); + assert!(!::is_fixed_size()); + assert!(!::is_fixed_size()); + assert!(!::is_fixed_size()); + assert!(!::is_fixed_size()); + assert!(!::is_fixed_size()); + assert!(!::is_fixed_size()); + } + + #[test] + fn new_fixed_size_containers_have_no_offsets() { + assert!(::is_fixed_size()); + assert!(::is_fixed_size()); + assert!(::is_fixed_size()); + assert!(::is_fixed_size()); + assert!(::is_fixed_size()); + assert!(::is_fixed_size()); + assert!(::is_fixed_size()); + assert!(::is_fixed_size()); + } +} diff --git a/crates/common/types/src/beacon/containers/mod.rs b/crates/common/types/src/beacon/containers/mod.rs index 617f2b18d..0b0be4b69 100644 --- a/crates/common/types/src/beacon/containers/mod.rs +++ b/crates/common/types/src/beacon/containers/mod.rs @@ -45,6 +45,7 @@ pub mod capella; pub mod deneb; pub mod electra; pub mod fulu; +pub mod gloas; pub mod phase0; pub mod shared; diff --git a/crates/common/types/src/beacon/containers/shared.rs b/crates/common/types/src/beacon/containers/shared.rs index 610a98ffd..fb2e65b67 100644 --- a/crates/common/types/src/beacon/containers/shared.rs +++ b/crates/common/types/src/beacon/containers/shared.rs @@ -9,7 +9,7 @@ use std::collections::BTreeMap; -use ethlambda_ssz_tree::List; +use ethlambda_ssz_tree::{List, ProgressiveList}; use libssz_derive::{HashTreeRoot, SszDecode, SszEncode}; use libssz_types::{SszBitvector, SszList, SszVector}; @@ -97,6 +97,18 @@ pub type InactivityScores = SszList; /// as the commitment to history from capella onward. pub type HistoricalSummaries = SszList; +/// The gloas validator registry (EIP-7688): unbounded and progressively +/// merkleized, tree-backed like [`Validators`] for the same reason (a +/// derived state shares the unchanged part of the registry, and re-hashing +/// after a block only rehashes the records the block touched). Writes are +/// buffered sparsely in a `BTreeMap`, the same as [`Validators`]. +pub type ProgressiveValidators = ProgressiveList>; + +/// The gloas balances, positionally parallel to [`ProgressiveValidators`]. +/// Tree-backed like [`Balances`], with writes buffered densely (the default +/// `VecMap`), since epoch processing writes every balance. +pub type ProgressiveBalances = ProgressiveList; + // --------------------------------------------------------------------------- // Misc // --------------------------------------------------------------------------- diff --git a/crates/common/types/src/beacon/preset.rs b/crates/common/types/src/beacon/preset.rs index f820f2b1c..c7ab9bbf8 100644 --- a/crates/common/types/src/beacon/preset.rs +++ b/crates/common/types/src/beacon/preset.rs @@ -18,7 +18,7 @@ //! The specification defines exactly two presets, `mainnet` and `minimal` //! (`minimal` exists to keep spec test fixtures and local devnets fast). This //! crate mirrors that: [`mainnet`] and [`minimal`] each define every preset -//! constant the specification uses from phase0 through fulu, and the top-level +//! constant the specification uses from phase0 through gloas, and the top-level //! re-export picks one at compile time, gated on the `preset-minimal` feature. //! Downstream code should import from here (`preset::SLOTS_PER_EPOCH`, not //! `preset::mainnet::SLOTS_PER_EPOCH`), so that it automatically follows whichever @@ -27,7 +27,7 @@ //! # Why some constants are `usize` and others are `u64` //! //! A constant that bounds an SSZ collection (`List`, `Vector`, `Bitlist`, -//! `Bitvector`) in some container, in any fork from phase0 through fulu, is typed +//! `Bitvector`) in some container, in any fork from phase0 through gloas, is typed //! `usize` here, because it gets threaded through this crate as a const-generic //! parameter on that collection's Rust representation, and const generics require //! `usize`. Every other preset constant (reward/penalty divisors, Gwei amounts, @@ -501,6 +501,56 @@ pub mod mainnet { /// precomputed proposer indices fulu adds to the state. pub const PROPOSER_LOOKAHEAD_LENGTH: usize = (MIN_SEED_LOOKAHEAD as usize + 1) * SLOTS_PER_EPOCH as usize; + + // ================================================================ + // Gloas + // ================================================================ + + // --- Misc --- + + /// Members of a slot's payload timeliness committee. Bounds + /// `PayloadTimelinessCommittee` (`Vector`), + /// `PayloadAttestation.aggregation_bits`/`IndexedPayloadAttestation.attesting_indices`, + /// and each row of `BeaconState.ptc_window`. + pub const PTC_SIZE: usize = 512; + + // --- Max operations per block --- + + /// Bounds `BeaconBlockBody.payload_attestations` + /// (`List`). + pub const MAX_PAYLOAD_ATTESTATIONS: u64 = 4; + + // --- Execution --- + + /// Bounds `ExecutionRequests.builder_deposits` + /// (`List`). + pub const MAX_BUILDER_DEPOSIT_REQUESTS_PER_PAYLOAD: u64 = 64; + + /// Bounds `ExecutionRequests.builder_exits` + /// (`List`). + pub const MAX_BUILDER_EXIT_REQUESTS_PER_PAYLOAD: u64 = 16; + + // --- Withdrawals processing --- + + /// Cap on how many builders the withdrawal sweep scans per slot. A loop + /// bound, not a container length, so it stays `u64`, the same convention + /// as `MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP`. + pub const MAX_BUILDERS_PER_WITHDRAWALS_SWEEP: u64 = 16_384; + + // --- State list lengths --- + + /// `(MIN_SEED_LOOKAHEAD + 2) * SLOTS_PER_EPOCH`, the specification's + /// formula for the length of `BeaconState.ptc_window` + /// (`Vector`): the cached + /// committees for the previous, current, and lookahead epochs. + pub const PTC_WINDOW_LENGTH: usize = + (MIN_SEED_LOOKAHEAD as usize + 2) * SLOTS_PER_EPOCH as usize; + + /// `2 * SLOTS_PER_EPOCH`, the specification's formula for the length of + /// `BeaconState.builder_pending_payments` + /// (`Vector`): + /// builder payments owed for the previous and current epoch. + pub const BUILDER_PENDING_PAYMENTS_LENGTH: usize = 2 * SLOTS_PER_EPOCH as usize; } /// The minimal preset: the same shape as [`mainnet`], scaled down so spec test @@ -955,6 +1005,54 @@ pub mod minimal { /// precomputed proposer indices fulu adds to the state. pub const PROPOSER_LOOKAHEAD_LENGTH: usize = (MIN_SEED_LOOKAHEAD as usize + 1) * SLOTS_PER_EPOCH as usize; + + // ================================================================ + // Gloas + // ================================================================ + + // --- Misc --- + + /// Members of a slot's payload timeliness committee. `[customized]`: a + /// small network needs far fewer members to keep the committee + /// meaningful at minimal's validator counts. + pub const PTC_SIZE: usize = 16; + + // --- Max operations per block --- + + /// Bounds `BeaconBlockBody.payload_attestations` + /// (`List`). + pub const MAX_PAYLOAD_ATTESTATIONS: u64 = 4; + + // --- Execution --- + + /// Bounds `ExecutionRequests.builder_deposits` + /// (`List`). + pub const MAX_BUILDER_DEPOSIT_REQUESTS_PER_PAYLOAD: u64 = 64; + + /// Bounds `ExecutionRequests.builder_exits` + /// (`List`). + pub const MAX_BUILDER_EXIT_REQUESTS_PER_PAYLOAD: u64 = 16; + + // --- Withdrawals processing --- + + /// Cap on how many builders the withdrawal sweep scans per slot. + /// `[customized]`, the same way `MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP` is. + pub const MAX_BUILDERS_PER_WITHDRAWALS_SWEEP: u64 = 16; + + // --- State list lengths --- + + /// `(MIN_SEED_LOOKAHEAD + 2) * SLOTS_PER_EPOCH`, the specification's + /// formula for the length of `BeaconState.ptc_window` + /// (`Vector`): the cached + /// committees for the previous, current, and lookahead epochs. + pub const PTC_WINDOW_LENGTH: usize = + (MIN_SEED_LOOKAHEAD as usize + 2) * SLOTS_PER_EPOCH as usize; + + /// `2 * SLOTS_PER_EPOCH`, the specification's formula for the length of + /// `BeaconState.builder_pending_payments` + /// (`Vector`): + /// builder payments owed for the previous and current epoch. + pub const BUILDER_PENDING_PAYMENTS_LENGTH: usize = 2 * SLOTS_PER_EPOCH as usize; } #[cfg(not(feature = "preset-minimal"))] @@ -1322,6 +1420,12 @@ mod tests { assert_eq!(mainnet::MAX_VALIDATORS_PER_SLOT, 131_072); assert_eq!(minimal::MAX_VALIDATORS_PER_SLOT, 8192); + + assert_eq!(mainnet::PTC_SIZE, 512); + assert_eq!(minimal::PTC_SIZE, 16); + + assert_eq!(mainnet::MAX_BUILDERS_PER_WITHDRAWALS_SWEEP, 16_384); + assert_eq!(minimal::MAX_BUILDERS_PER_WITHDRAWALS_SWEEP, 16); } /// Every derived constant computed from more than one preset value, checked @@ -1382,6 +1486,24 @@ mod tests { minimal::MAX_VALIDATORS_PER_SLOT, minimal::MAX_COMMITTEES_PER_SLOT * minimal::MAX_VALIDATORS_PER_COMMITTEE ); + + assert_eq!( + mainnet::PTC_WINDOW_LENGTH, + (mainnet::MIN_SEED_LOOKAHEAD as usize + 2) * mainnet::SLOTS_PER_EPOCH as usize + ); + assert_eq!( + minimal::PTC_WINDOW_LENGTH, + (minimal::MIN_SEED_LOOKAHEAD as usize + 2) * minimal::SLOTS_PER_EPOCH as usize + ); + + assert_eq!( + mainnet::BUILDER_PENDING_PAYMENTS_LENGTH, + 2 * mainnet::SLOTS_PER_EPOCH as usize + ); + assert_eq!( + minimal::BUILDER_PENDING_PAYMENTS_LENGTH, + 2 * minimal::SLOTS_PER_EPOCH as usize + ); } /// Every name this module defines must resolve, at the same type, through @@ -1478,5 +1600,11 @@ mod tests { FIELD_ELEMENTS_PER_EXT_BLOB, CELLS_PER_EXT_BLOB, NUMBER_OF_COLUMNS, BYTES_PER_CELL, PROPOSER_LOOKAHEAD_LENGTH, ); + + in_both!(usize: PTC_SIZE, PTC_WINDOW_LENGTH, BUILDER_PENDING_PAYMENTS_LENGTH); + in_both!(u64: + MAX_PAYLOAD_ATTESTATIONS, MAX_BUILDER_DEPOSIT_REQUESTS_PER_PAYLOAD, + MAX_BUILDER_EXIT_REQUESTS_PER_PAYLOAD, MAX_BUILDERS_PER_WITHDRAWALS_SWEEP, + ); } } diff --git a/crates/common/types/src/beacon/serde_helpers.rs b/crates/common/types/src/beacon/serde_helpers.rs index 91599b286..976b28901 100644 --- a/crates/common/types/src/beacon/serde_helpers.rs +++ b/crates/common/types/src/beacon/serde_helpers.rs @@ -181,6 +181,59 @@ pub mod quoted_u64_seq { } } +/// A sequence of sequences of integers, each innermost value written quoted. +/// +/// [`quoted_u64_seq`] drives one flat sequence of `Display` scalars; this +/// drives it once per outer element, for a field like +/// `gloas::BeaconState.ptc_window` +/// (`SszVector, PTC_WINDOW_LENGTH>`) +/// whose elements are themselves a foreign `SszVector` with no `Serialize` +/// of their own. Neither [`seq`] (which needs the element itself to already +/// implement `Serialize`) nor [`quoted_u64_seq`] alone (which needs the +/// element to already be the scalar) applies to a field shaped like this. +pub mod nested_quoted_u64_seq { + use serde::ser::SerializeSeq as _; + + /// Wraps one inner sequence so [`serde::ser::SerializeSeq::serialize_element`] + /// drives it through [`super::quoted_u64_seq::serialize`] instead of + /// requiring the inner sequence itself to implement `Serialize`. + struct Inner(C); + + impl serde::Serialize for Inner + where + C: IntoIterator + Copy, + C::IntoIter: ExactSizeIterator, + T: std::fmt::Display, + { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + super::quoted_u64_seq::serialize(self.0, serializer) + } + } + + pub fn serialize(values: C, serializer: S) -> Result + where + S: serde::Serializer, + C: IntoIterator, + C::IntoIter: ExactSizeIterator, + // `D` is the outer iterator's item, an inner sequence reached by + // reference (e.g. `&SszVector`), so it is `Copy` the same + // way any shared reference is. + D: IntoIterator + Copy, + D::IntoIter: ExactSizeIterator, + T: std::fmt::Display, + { + let iter = values.into_iter(); + let mut seq = serializer.serialize_seq(Some(iter.len()))?; + for item in iter { + seq.serialize_element(&Inner(item))?; + } + seq.end() + } +} + /// Anything SSZ-encodable, written as `0x`-prefixed hex of that encoding. /// /// This is how the Beacon API carries bitfields: `SszBitlist` and From 5aab222f5a60ed082b71e8f90799bd5de8164b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:00:18 -0300 Subject: [PATCH 010/132] feat(beacon): add the gloas fork variant and config Gloas is a new ForkName, between fulu and lean, so every existing fork-keyed match (BeaconState, SignedBeaconBlock, ForkName itself) gains a Gloas arm. Per the spec-port convention, no match grows a `_` catch-all for it: naming every variant explicitly is what makes the next fork after gloas break the build instead of silently defaulting somewhere. Most of these arms are `Error::UnsupportedForFork` (or the local not-a-Result idiom: a panic, a `None`, or a decode refusal), since the gloas state transition, fork choice, p2p decode, and validator client integration are all follow-up work; a few (execution_payload_timestamp, genesis_anchor_block, the container-level registry accessors) already have a real gloas answer because the container itself is enough to give one. The validator registry needed its own abstraction rather than a Gloas arm on the existing `dispatch_state!` macro: gloas's registry is EIP-7688's progressive, unbounded list (ProgressiveValidators/ProgressiveBalances) rather than every earlier fork's bounded, tree-backed List. Accessors that hand back an *element* (validator_count, validator, balance, push_validator, validators_root, apply_pending_mutations) still typecheck against both list kinds unchanged, since ProgressiveList mirrors List's element-level API. The few that hand back the *list itself* cannot: `registry`/`registry_mut` now return a `Registry`/`RegistryMut` enum over the two kinds, and `iter_validators`/`iter_balances` return a private `RegistryIter` wrapping either list's own iterator type. `rebase_on` and `validators_ptr_eq` match same-kind pairs and no-op (or return false) across a fork boundary, since there is no shared tree to reuse between a bounded and a progressive registry anyway. Config gains the gloas fork version/epoch, its timing deadlines (attestation/aggregate/sync-message/contribution *_GLOAS, payload_due_bps, payload_attestation_due_bps), its validator churn quotients (EIP-8061; these are the validator registry's own churn, not builders' -- see `min_builder_withdrawability_delay` for the one field that is), and min_builder_withdrawability_delay. DB_VERSION becomes 5 because Config is SSZ-encoded under Metadata["config"], so widening it changes what a directory's stored bytes decode into: an existing data directory has to be re-synced, the same as any prior DB_VERSION bump. HEZE_* and EIP8321_* (each its own later fork schedule), GAS_LIMIT_SCHEDULE, and heze's own data keys (INCLUSION_LIST_DUE_BPS and its three siblings, plus CONFIRMATION_BYZANTINE_THRESHOLD, the Fast Confirmation Rule) are still unclaimed and still reported as ignored keys. This also hardens every place a gloas message or anchor can reach this build from the network, beyond the bare wiring above: - `fork_choice::get_forkchoice_store` refuses a successfully decoded gloas anchor state outright: checkpoint sync decodes one structurally today (the container exists, so nothing about the bytes themselves fails), and a loaded network scheduling GLOAS_FORK_EPOCH: 0 reaches the same decoded anchor through `genesis_anchor_block`; either way, without this check the node would boot a store it can never advance past. The two startup call sites keep that reason distinguishable from a genuinely mismatched anchor pair (`CheckpointSyncError::UnsupportedFork` vs `AnchorPairingMismatch`) rather than collapsing both into one message. - Gossip validation now `Ignore`s a gloas-shaped message instead of `Reject`ing it: a `Reject` down-scores the sending peer in gossipsub, and every honest peer sends exactly this once this node's own clock reaches gloas, so scoring it would punish the whole mesh for a gap in this build. Covers beacon blocks (`gossip::block::cheap_checks`), aggregates and attester slashings (`p2p::beacon::decode`'s new `DecodeError::UnsupportedFork`). Data column sidecars decode first, unlike the other two: gossip topic subscriptions are frozen at startup, so a node running across the boundary stays on its fulu-digest topic where a late but legitimate fulu sidecar can still arrive, and a clock check ahead of the decode would drop that one too. Only on a decode failure does `triage_data_column` consult the wall clock (`decode::current_fork`, an approximation of the topic's own fork, which is not threaded down to the handler today) to decide `Ignore` vs `Reject`. - `process_or_pend_block` refuses a gloas block once, before persisting it: without this, a gloas block whose parent was missing would be tracked as pending, and every later block naming it as an ancestor would walk back, fetch it out of storage, and requeue it -- paying the clone-parent-state round trip again on every single delivery, forever, since nothing ever marks it imported. This also makes `data_availability_for`'s and `fork_choice::block_operations`'s own gloas arms unreachable from network input; both are left as explicit, harmless refusals rather than `unreachable!()`, in case a future change reintroduces a path to them. Range sync itself does not loop on the same slot range: its own `beacon_fetched_through` bookkeeping advances on delivery, not on import. It will, however, keep fetching (and discarding) every new gloas-era block a peer produces for as long as this build cannot import any of them -- that is a real, unbounded-over-time cost this change does not address, since deciding whether range sync should stop advancing past a fork this build cannot process at all is a product decision, not a wiring one. - Startup now warns explicitly when `gloas_fork_epoch` is scheduled (`network::warn_if_gloas_scheduled`, shared by `BuiltInNetwork::resolve` and a loaded `NetworkDir`), since the ignored-keys warning no longer implies it now that GLOAS_* keys are claimed. Sepolia's own boundary log line (`beacon::wire_params`) is now a `warn!` and no longer suggests a restart crosses it. In practice this build's Sepolia follower stops making progress at the first gloas-era block: gossip already refuses it via `Ignore`, and `process_or_pend_block` refuses it again before persisting, so `upgrade_state`'s own epoch-boundary refusal is the fallback for an empty-slot epoch crossing, not the usual path. Notes on how the change fits the code it lands on: - There are more fork-keyed matches than a first survey finds (gossip validation, the validator crate, p2p column checks, committee-cache changes, block production); every one the compiler or a search found got a named Gloas arm, including the two wildcard arms in block production. - `max_request_payloads` was already a Config field before this change, and was never reported as ignored, so it is not added here. - `registry()` already returned `(&Validators, &Balances)` uniformly; this commit replaces it with the `Registry` enum, plus a `registry_mut`/ `RegistryMut` pair. The pair is needed because `rebase_on` matches self's registry kind against a distinct `base` value's kind and needs both mutably and immutably at once. - `bin/main.rs`'s `a_genesis_anchor_block_hashes_to_the_states_own_header_at_every_fork` test chains `upgrade_state` across every `ForkName::ALL` member; since `upgrade_to_gloas` does not exist yet, the loop now stops before Gloas (asserting `upgrade_state` still refuses it, so the assertion itself starts failing the day that changes), leaving `genesis_anchor_block`'s own gloas arm uncovered there until then. - `gossip/every_handler_is_accounted_for`'s IGNORED_HANDLERS list gained gloas's four new gossip topics (execution_payload_bid, execution_payload_envelope, payload_attestation_message, proposer_preferences), which only became visible once gloas's fixture directory started parsing; the list is alphabetized throughout. - `UNMODELED_FORKS` drops "gloas" (now parses); "heze" stays. Nothing on the node's own paths panics on a gloas boundary: fork digest, next-fork-boundary, and ENR computation are Ord/list-based rather than exhaustive matches and handle Gloas today; checkpoint sync decodes a gloas state through the new `BeaconState::from_ssz` arm. --- CLAUDE.md | 25 +- bin/ethlambda/src/beacon.rs | 14 + bin/ethlambda/src/checkpoint_sync.rs | 11 + bin/ethlambda/src/main.rs | 79 +++- bin/ethlambda/src/network/built_in.rs | 11 +- bin/ethlambda/src/network/config_file.rs | 24 +- bin/ethlambda/src/network/dir.rs | 1 + bin/ethlambda/src/network/mod.rs | 28 ++ .../fixtures/networks/devnet/config.yaml | 8 +- crates/blockchain/src/beacon_engine.rs | 8 + crates/blockchain/src/lib.rs | 38 ++ .../src/beacon/block_production.rs | 27 +- .../src/beacon/fork_choice.rs | 43 ++ .../src/beacon/gossip/block.rs | 14 + .../state_transition/src/beacon/gossip/mod.rs | 8 + .../src/beacon/helpers/accessors.rs | 15 +- .../src/beacon/helpers/altair.rs | 9 + .../src/beacon/helpers/mutators.rs | 9 + .../src/beacon/helpers/test_state.rs | 7 + .../state_transition/src/beacon/stf/block.rs | 8 + .../src/beacon/stf/epoch/mod.rs | 16 +- .../src/beacon/stf/epoch/registry.rs | 10 +- .../src/beacon/stf/operations.rs | 16 +- .../state_transition/src/beacon/upgrade.rs | 9 + .../tests/beacon_spec/epoch_processing.rs | 24 + .../tests/beacon_spec/fork_choice.rs | 3 + .../tests/beacon_spec/gossip.rs | 21 +- .../state_transition/tests/beacon_spec/mod.rs | 17 +- .../tests/beacon_spec/operations.rs | 42 ++ .../tests/beacon_spec/rewards.rs | 6 + crates/common/types/src/beacon/config.rs | 109 ++++- .../common/types/src/beacon/containers/mod.rs | 434 ++++++++++++++++-- crates/common/types/src/beacon/fork.rs | 46 +- crates/common/types/src/beacon/preset.rs | 19 +- crates/net/p2p/src/beacon/decode.rs | 44 ++ crates/net/p2p/src/gossipsub/handler.rs | 39 ++ crates/storage/src/store.rs | 6 +- .../src/beacon_node/block_contents.rs | 5 + crates/validator/src/beacon_node/http.rs | 4 + docs/beacon_stf.md | 22 +- docs/cli.md | 12 +- 41 files changed, 1162 insertions(+), 129 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index fcfcdce57..ace330ad8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -448,9 +448,15 @@ Hoodi's against fork digests published in their own bootnode ENRs), and at runtime by checkpoint sync and resume, which both check the anchor state against them. `beacon::tests::the_built_in_network_derives_what_it_always_did` pins the parsed mainnet config to `Config::mainnet()`, so the file and the -Rust constant cannot drift apart. Sepolia's config schedules gloas, which this -build cannot process, so a Sepolia follower stops tracking the chain at -`GLOAS_FORK_EPOCH`; the ignored-keys warning at startup names it. +Rust constant cannot drift apart. Sepolia's config schedules gloas: its +`GLOAS_*` keys are claimed (`ForkName::Gloas` exists, and `Config` carries the +fork's schedule, timing and churn fields), but its state transition and fork +choice do not exist yet, so a Sepolia follower stops tracking the chain at +`GLOAS_FORK_EPOCH` regardless. `network::warn_if_gloas_scheduled` (shared by +`BuiltInNetwork::resolve` and a loaded `NetworkDir`) warns about this +explicitly at startup (checked against `config.gloas_fork_epoch`), separately +from the ignored-keys warning, which now only ever names heze's keys and +`GAS_LIMIT_SCHEDULE`/`INCLUSION_LIST_DUE_BPS`. A loaded network decodes its own `genesis.ssz` instead, at whatever fork its own schedule names for epoch 0. Every `config.yaml`, built-in or loaded, goes @@ -809,12 +815,13 @@ behavior. which is what keeps a parked column from satisfying the availability gate. Its only index is the chain actor's in-memory `sidecars_awaiting_parent`, so `start_actor` clears the whole table at startup. -- `DB_VERSION` is 4: `Config` gained `PRESET_BASE` and `CONFIG_NAME` (as - `ConfigName`, a bounded string) at the front of its encoding, and it is - SSZ-encoded under `KEY_CONFIG`, so a data directory written by an earlier - version decodes into the wrong fields. (3 was the runtime keys a - `config.yaml` supplies.) `Store::from_db_state` refuses any other version - outright; there is no migration. +- `DB_VERSION` is 5: `Config` gained the gloas schedule, timing and churn + keys, and it is SSZ-encoded under `KEY_CONFIG`, so a data directory written + by an earlier version decodes into the wrong fields. (4 was `PRESET_BASE` and + `CONFIG_NAME`, as `ConfigName`, a bounded string, at the front of the + encoding; 3 was the runtime keys a `config.yaml` supplies.) + `Store::from_db_state` refuses any other version outright; there is no + migration. ### State Root Computation - Always computed via `hash_tree_root()` after full state transition diff --git a/bin/ethlambda/src/beacon.rs b/bin/ethlambda/src/beacon.rs index 26a670bb3..3b15d23a4 100644 --- a/bin/ethlambda/src/beacon.rs +++ b/bin/ethlambda/src/beacon.rs @@ -189,7 +189,21 @@ pub fn wire_params( // The digest is computed once. Crossing a boundary while running strands // this node on topic names nobody publishes to, so say when that is. + // + // The gloas boundary gets its own message rather than the generic one: a + // restart there recomputes the digest and resubscribes to the right + // topics, the same as any other boundary, but it does not make this + // build any more able to import a gloas block or cross a gloas epoch + // boundary (`state_transition` refuses both). Suggesting a restart + // "crosses" it would read as a stall a restart clears, which this is + // not. match next_fork_boundary(&chain, epoch) { + Some(boundary) if boundary == chain.gloas_fork_epoch => warn!( + boundary_epoch = boundary, + boundary_unix_time = time_at_epoch(&chain, genesis.genesis_time, boundary), + "The fork digest changes at this boundary, but this build cannot follow past it: \ + a restart resubscribes to the right topics, not past the gloas gap itself" + ), Some(boundary) => info!( boundary_epoch = boundary, boundary_unix_time = time_at_epoch(&chain, genesis.genesis_time, boundary), diff --git a/bin/ethlambda/src/checkpoint_sync.rs b/bin/ethlambda/src/checkpoint_sync.rs index 5b01820f9..6eb50c154 100644 --- a/bin/ethlambda/src/checkpoint_sync.rs +++ b/bin/ethlambda/src/checkpoint_sync.rs @@ -180,6 +180,17 @@ pub enum CheckpointSyncError { BlockHeaderJustifiedRootMismatch, #[error("anchor block does not match anchor state")] AnchorPairingMismatch, + /// `get_forkchoice_store` refused the anchor outright because this build + /// cannot process its fork yet (currently only gloas): a real anchor, not + /// a peer serving a mismatched pair, so it is named separately from + /// [`Self::AnchorPairingMismatch`] rather than folded into it. + #[error( + "this build cannot follow {fork} yet; the anchor is at {fork} and nothing past this \ + point will import until that support lands" + )] + UnsupportedFork { + fork: ethlambda_types::beacon::fork::ForkName, + }, #[error("no checkpoint urls configured")] NoCheckpointUrls, #[error("failed to insert anchor signed block into store")] diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index bdbda91e2..e0c9f782b 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -59,7 +59,7 @@ use ethlambda_types::{ aggregator::AggregatorController, beacon::config::Config, beacon::containers::{ - BeaconState, SignedBeaconBlock, altair, bellatrix, capella, deneb, electra, phase0, + BeaconState, SignedBeaconBlock, altair, bellatrix, capella, deneb, electra, gloas, phase0, }, beacon::fork::ForkName, genesis::{GenesisConfig, verify_state_genesis}, @@ -1530,6 +1530,8 @@ fn first_config_difference(persisted: &Config, supplied: &Config) -> Option Option Option Option Option Option checkpoint_sync::CheckpointSyncError { + match err { + ethlambda_state_transition::beacon::Error::UnsupportedForFork { fork, .. } => { + checkpoint_sync::CheckpointSyncError::UnsupportedFork { fork } + } + _ => checkpoint_sync::CheckpointSyncError::AnchorPairingMismatch, + } +} + /// Fetch the initial state for a beacon node. /// /// The beacon twin of [`fetch_initial_state`], with the same precedence: a @@ -1838,7 +1874,7 @@ async fn fetch_initial_beacon_state( ); return fork_choice::get_forkchoice_store(backend, state, block, &config) .inspect_err(|err| error!(%err, "Failed to initialize store from the genesis state")) - .map_err(|_| checkpoint_sync::CheckpointSyncError::AnchorPairingMismatch); + .map_err(anchor_store_error); } info!(?checkpoint_urls, "Starting beacon checkpoint sync"); @@ -1862,7 +1898,7 @@ async fn fetch_initial_beacon_state( fork_choice::get_forkchoice_store(backend, state, block, &config) .inspect_err(|err| error!(%err, "Failed to initialize store from anchor state and block")) - .map_err(|_| checkpoint_sync::CheckpointSyncError::AnchorPairingMismatch) + .map_err(anchor_store_error) } /// The block the specification pairs with a genesis anchor state. @@ -1963,6 +1999,16 @@ fn genesis_anchor_block(state: &BeaconState) -> SignedBeaconBlock { }, signature: Default::default(), }), + ForkName::Gloas => SignedBeaconBlock::Gloas(gloas::SignedBeaconBlock { + message: gloas::BeaconBlock { + slot, + proposer_index, + parent_root, + state_root, + body: gloas::BeaconBlockBody::empty(), + }, + signature: Default::default(), + }), // Never reached: this is only called on a network's own genesis // state, and every `NetworkSource` decodes a beacon fork there // (`NetworkDir::load` resolves the fork from `Config::fork_at_epoch`, @@ -2759,12 +2805,38 @@ validators: /// from genesis still names phase0's empty-body root; it is re-stamped /// to each new fork's own empty body below, exactly as a genesis /// generator targeting that fork directly would have to. + /// + /// Stops at fulu: `upgrade_state` has no `upgrade_to_gloas` yet (it + /// returns `Error::UnsupportedForFork` for `ForkName::Gloas` until a + /// later task adds one), so this loop cannot chain a gloas state the way + /// it does every earlier fork. `genesis_anchor_block`'s own gloas arm + /// therefore has no coverage here until that lands. #[test] fn a_genesis_anchor_block_hashes_to_the_states_own_header_at_every_fork() { let config = Config::mainnet(); let mut state = beacon::mainnet_genesis_state().unwrap(); for fork in ForkName::ALL { + if fork == ForkName::Gloas { + // Pinned so this test starts failing, loudly, the moment + // `upgrade_to_gloas` lands: that failure is the cue to + // delete this early break and extend the loop past it. + let err = ethlambda_state_transition::beacon::upgrade::upgrade_state( + &state, fork, &config, + ) + .expect_err("upgrade_to_gloas does not exist yet"); + assert!( + matches!( + err, + ethlambda_state_transition::beacon::Error::UnsupportedForFork { + fork: ForkName::Gloas, + .. + } + ), + "expected UnsupportedForFork, got {err:?}" + ); + break; + } if fork != ForkName::Phase0 { state = ethlambda_state_transition::beacon::upgrade::upgrade_state( &state, fork, &config, @@ -2781,6 +2853,7 @@ validators: ForkName::Electra | ForkName::Fulu => { electra::BeaconBlockBody::empty().hash_tree_root() } + ForkName::Gloas => gloas::BeaconBlockBody::empty().hash_tree_root(), ForkName::Lean => unreachable!("ForkName::ALL excludes Lean"), }; state.latest_block_header_mut().body_root = empty_body_root; diff --git a/bin/ethlambda/src/network/built_in.rs b/bin/ethlambda/src/network/built_in.rs index 1ff084d36..6a6ee6d64 100644 --- a/bin/ethlambda/src/network/built_in.rs +++ b/bin/ethlambda/src/network/built_in.rs @@ -62,14 +62,19 @@ impl BuiltInNetwork { .wrap_err_with(|| format!("the built-in {} config.yaml did not parse", self.name()))?; super::check_preset(parsed.config.preset_base.as_str())?; super::check_constants(&parsed.config)?; - // A built-in config can carry a fork this build cannot process (Sepolia - // schedules gloas), and saying so at every start is the point: the - // node follows the chain only up to that fork's epoch. + // A built-in config can carry a fork whose *keys* this build claims + // but whose state transition it does not implement yet (gloas), or + // one whose keys it does not even claim (heze). The ignored-keys + // warning below only ever said the second half of that; it used to + // say both, back when GLOAS_* were unclaimed too. `warn_if_gloas_scheduled` + // says the first half explicitly instead of leaving it to be + // discovered as a stall. parsed.warn_about_ignored_keys(); let genesis = chain.genesis(); let mut config = parsed.config; super::derive_genesis_fields(&mut config, genesis.genesis_time); + super::warn_if_gloas_scheduled(self.name(), &config); Ok(BuiltIn { config, diff --git a/bin/ethlambda/src/network/config_file.rs b/bin/ethlambda/src/network/config_file.rs index 93a1e914b..789afaf91 100644 --- a/bin/ethlambda/src/network/config_file.rs +++ b/bin/ethlambda/src/network/config_file.rs @@ -53,9 +53,9 @@ impl ConfigFile { /// Log what was ignored, as one line naming each key. /// /// One line rather than one per key: a current `config.yaml` carries the - /// gloas and heze schedule, so per-key lines would flood every valid - /// startup with one warning per ignored key and bury a real typo among - /// them. + /// heze schedule, which this build does not claim, so per-key lines + /// would flood every valid startup with one warning per ignored key and + /// bury a real typo among them. pub(crate) fn warn_about_ignored_keys(&self) { if self.ignored.is_empty() { return; @@ -142,12 +142,9 @@ mod tests { fn the_forks_this_build_cannot_process_are_reported_as_ignored() { let parsed = ConfigFile::parse(DEVNET).unwrap(); for key in [ - "GLOAS_FORK_VERSION", - "GLOAS_FORK_EPOCH", "HEZE_FORK_VERSION", "HEZE_FORK_EPOCH", "GAS_LIMIT_SCHEDULE", - "PAYLOAD_DUE_BPS", "INCLUSION_LIST_DUE_BPS", ] { assert!( @@ -157,6 +154,21 @@ mod tests { } } + #[test] + fn the_gloas_keys_are_claimed_rather_than_ignored() { + // GLOAS_* and PAYLOAD_DUE_BPS used to be reported as ignored + // alongside heze's; now that `Config` has fields for them, they must + // not be. + let parsed = ConfigFile::parse(DEVNET).unwrap(); + for key in ["GLOAS_FORK_VERSION", "GLOAS_FORK_EPOCH", "PAYLOAD_DUE_BPS"] { + assert!( + !parsed.ignored.contains(&key.to_string()), + "{key} reported as ignored" + ); + } + assert_eq!(parsed.config.gloas_fork_version, [0x07, 0, 0, 0]); + } + #[test] fn preset_base_and_config_name_are_not_reported_as_ignored() { // Both are `Config` fields now, so the derive claims them. diff --git a/bin/ethlambda/src/network/dir.rs b/bin/ethlambda/src/network/dir.rs index 17306c21f..8505cf062 100644 --- a/bin/ethlambda/src/network/dir.rs +++ b/bin/ethlambda/src/network/dir.rs @@ -129,6 +129,7 @@ impl NetworkDir { // once, before anything reads the config. let mut config = parsed.config; super::derive_genesis_fields(&mut config, genesis_state.genesis_time()); + super::warn_if_gloas_scheduled(config.config_name.as_str(), &config); Ok(Self { config, diff --git a/bin/ethlambda/src/network/mod.rs b/bin/ethlambda/src/network/mod.rs index 10875fea9..9f6495eae 100644 --- a/bin/ethlambda/src/network/mod.rs +++ b/bin/ethlambda/src/network/mod.rs @@ -232,6 +232,34 @@ fn derive_genesis_fields(config: &mut Config, genesis_time: u64) { config.slot_duration_ms = config.seconds_per_slot * 1_000; } +/// Warn at startup when `config` schedules gloas, whichever way this network +/// was resolved. +/// +/// Shared by a loaded directory and the built-in networks for the same +/// reason [`derive_genesis_fields`] is. The ignored-keys warning +/// (`ConfigFile::warn_about_ignored_keys`) no longer implies this the way it +/// used to, now that `GLOAS_*` keys are claimed rather than reported as +/// unknown, so this says it explicitly instead of leaving it to be +/// discovered as a stall: this build's state transition and fork choice +/// refuse every gloas block and epoch boundary, so a follower stops making +/// progress there regardless of how cleanly its config parsed. A loaded +/// network reaches this the same as a built-in one, and can even schedule +/// gloas at epoch 0, in which case its own genesis state already decodes as +/// gloas and `fork_choice::get_forkchoice_store` refuses it outright at +/// startup; this warning fires first either way. +fn warn_if_gloas_scheduled(network: &str, config: &Config) { + if config.gloas_fork_epoch != ethlambda_types::beacon::constants::FAR_FUTURE_EPOCH { + tracing::warn!( + network, + gloas_fork_epoch = config.gloas_fork_epoch, + "This build stops following this chain at its gloas fork epoch; \ + crossing it is not a stall a restart clears, since state_transition \ + refuses every gloas block and epoch boundary until a later release \ + implements it" + ); + } +} + /// A resolved network: everything startup needs before it can build a swarm. /// /// The built-in arm holds what the binary carries; the loaded arm holds what diff --git a/bin/ethlambda/tests/fixtures/networks/devnet/config.yaml b/bin/ethlambda/tests/fixtures/networks/devnet/config.yaml index b027ee509..e56bdf93c 100644 --- a/bin/ethlambda/tests/fixtures/networks/devnet/config.yaml +++ b/bin/ethlambda/tests/fixtures/networks/devnet/config.yaml @@ -203,9 +203,11 @@ BLOB_SCHEDULE: - EPOCH: 419072 # January 7, 2026, 01:01:11am UTC MAX_BLOBS_PER_BLOCK: 21 -# Keys a kurtosis-generated config carries that mainnet's does not. The gloas -# and heze entries are deliberately present: this build cannot process either -# fork, so they must land in the ignored-key warning rather than in Config. +# Keys a kurtosis-generated config carries that mainnet's does not. GLOAS_* and +# PAYLOAD_DUE_BPS are claimed by `Config` now; GAS_LIMIT_SCHEDULE, +# HEZE_FORK_VERSION, HEZE_FORK_EPOCH, and INCLUSION_LIST_DUE_BPS are still +# deliberately present: this build cannot process heze, so they must land in +# the ignored-key warning rather than in Config. MAX_REQUEST_PAYLOADS: 128 CONSOLIDATION_CHURN_LIMIT_QUOTIENT: 65536 GAS_LIMIT_SCHEDULE: [] diff --git a/crates/blockchain/src/beacon_engine.rs b/crates/blockchain/src/beacon_engine.rs index b33f6e9ad..c643914bc 100644 --- a/crates/blockchain/src/beacon_engine.rs +++ b/crates/blockchain/src/beacon_engine.rs @@ -41,6 +41,13 @@ pub struct NewPayloadRequest<'a> { /// and `engine_newPayloadV4` would reject their payloads as an unsupported fork /// anyway. Supporting them would mean the V1 through V3 methods too, for chains /// this follower cannot reach. +/// +/// Gloas is `None` too, genuinely rather than temporarily: ePBS (EIP-7732) +/// moves the payload out of the block into a separately gossiped envelope +/// (see `containers::gloas::SignedExecutionPayloadEnvelope`), so a gloas +/// block carries nothing this function could ask about at all. A later task +/// answers the engine question gloas actually asks, from that envelope +/// rather than from the block. pub fn new_payload_request(block: &SignedBeaconBlock) -> Option> { let inner = match block { SignedBeaconBlock::Electra(inner) | SignedBeaconBlock::Fulu(inner) => inner, @@ -49,6 +56,7 @@ pub fn new_payload_request(block: &SignedBeaconBlock) -> Option return None, }; diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 06f255128..0a809b496 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -840,6 +840,19 @@ fn data_availability_for( | SignedBeaconBlock::Altair(_) | SignedBeaconBlock::Bellatrix(_) | SignedBeaconBlock::Capella(_) => Some(fork_choice::DataAvailability::NotRequired), + // ePBS (EIP-7732) moves the availability question onto the payload + // envelope's data columns, not this block's own (nonexistent) + // `blob_kzg_commitments`, so this gate has nothing gloas-shaped to + // check yet. Unreachable from network input today: + // `process_or_pend_block` refuses every gloas block before this + // function is ever called, precisely so a gloas block never reaches + // this gate, `fork_choice::block_operations`, or `on_block`'s own + // `Error::UnsupportedForFork`, none of which it can pay for + // productively. `None` here regardless, defensively: were that gate + // ever bypassed, holding the block rather than answering + // `NotRequired` keeps it from being treated as available and handed + // to `on_block` under a false pretense. + SignedBeaconBlock::Gloas(_) => None, // `process_block` dispatches a lean block to `store::on_block` before // this function is ever reached, so this arm is never observed for // one; named on its own rather than folded into the group above so @@ -2579,6 +2592,31 @@ impl BlockChainServer { let proposer = signed_block.proposer_index(); timings.guards_start = Some(Instant::now()); + // Refused before anything else, including the columns/finalized-slot + // checks below: `state_transition` refuses every gloas block anyway + // (`Error::UnsupportedForFork`), so persisting one as pending and + // discovering that on the other end of a clone-parent-state-then-fail + // round trip is pure waste. Worse than waste were this not here + // first: a gloas block whose parent is missing would otherwise be + // `insert_pending_block`-ed and tracked in `pending_block_parents`, + // and every later block naming it as an ancestor would walk back to + // it, fetch it out of storage, and requeue it, paying that same + // round trip again on every single delivery, forever, since nothing + // ever marks it imported. `discard_pending_subtree` clears any + // children already queued under it before this landed; logged once, + // here, rather than once per retry, since after this there is no + // retry. + if matches!(signed_block, SignedBeaconBlock::Gloas(_)) { + warn!( + %slot, + block_root = %ShortRoot(&block_root.0), + "Refusing a gloas block: this build does not implement the gloas state \ + transition yet" + ); + self.discard_pending_subtree(block_root); + return None; + } + // Asked before the parent check, so that an absent `columns_wait` row // can be read two ways rather than one: the columns were never // missing, or they landed while this block was held for its parent. diff --git a/crates/blockchain/state_transition/src/beacon/block_production.rs b/crates/blockchain/state_transition/src/beacon/block_production.rs index 0c412cb42..77b974543 100644 --- a/crates/blockchain/state_transition/src/beacon/block_production.rs +++ b/crates/blockchain/state_transition/src/beacon/block_production.rs @@ -44,6 +44,7 @@ use super::helpers::accessors::{ use super::helpers::electra::{ get_attesting_indices, get_indexed_attestation, is_valid_indexed_attestation, }; +use super::lean_boundary::lean_state_unreachable; use super::stf::{self, ExecutionEngine}; /// `state` advanced through empty slots to `slot`, as a block for `slot` is @@ -75,7 +76,19 @@ pub fn payload_inputs(state: &BeaconState, config: &Config) -> Result state.latest_execution_payload_header.block_hash, BeaconState::Fulu(state) => state.latest_execution_payload_header.block_hash, - _ => return Err(Error::SpecAssert("block production is electra and later")), + BeaconState::Phase0(_) + | BeaconState::Altair(_) + | BeaconState::Bellatrix(_) + | BeaconState::Capella(_) + | BeaconState::Deneb(_) => { + return Err(Error::SpecAssert("block production is electra and later")); + } + // Gloas builds its block around a builder's bid rather than an + // execution payload the proposer requests, which nothing here models. + BeaconState::Gloas(_) => { + return Err(Error::SpecAssert("block production does not support gloas")); + } + BeaconState::Lean(_) => lean_state_unreachable("payload_inputs"), }; Ok(PayloadInputs { timestamp: stf::bellatrix::compute_timestamp_at_slot(state, state.slot(), config), @@ -334,7 +347,17 @@ pub fn assemble_block( let wrapped = match state { BeaconState::Electra(_) => SignedBeaconBlock::Electra(signed), BeaconState::Fulu(_) => SignedBeaconBlock::Fulu(signed), - _ => return Err(Error::SpecAssert("block production is electra and later")), + BeaconState::Phase0(_) + | BeaconState::Altair(_) + | BeaconState::Bellatrix(_) + | BeaconState::Capella(_) + | BeaconState::Deneb(_) => { + return Err(Error::SpecAssert("block production is electra and later")); + } + BeaconState::Gloas(_) => { + return Err(Error::SpecAssert("block production does not support gloas")); + } + BeaconState::Lean(_) => lean_state_unreachable("assemble_block"), }; let mut post = state.clone(); stf::block::process_block( diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 7332a2327..176f989ab 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -167,6 +167,7 @@ use crate::beacon::constants; use crate::beacon::containers::{AttestationData, BeaconState, Checkpoint, SignedBeaconBlock}; use crate::beacon::containers::{bellatrix, deneb, electra, fulu, phase0}; use crate::beacon::error::{Error, Result, verify}; +use crate::beacon::fork::ForkName; use crate::beacon::helpers::accessors::{ CommitteeCache, get_active_validator_indices, get_beacon_proposer_index, get_current_epoch, get_total_active_balance, @@ -459,6 +460,18 @@ pub fn block_operations(block: &SignedBeaconBlock) -> (Vec, Vec (Vec::new(), Vec::new()), SignedBeaconBlock::Lean(_) => lean_block_unreachable("fork_choice::block_operations"), } } @@ -892,6 +905,22 @@ pub fn get_forkchoice_store( anchor_block: SignedBeaconBlock, config: &Config, ) -> Result { + // Refused before any other check: a gloas anchor (checkpoint sync + // decodes one structurally today, and a loaded network scheduling + // `GLOAS_FORK_EPOCH: 0` reaches this from `genesis_anchor_block` too) + // would otherwise boot a store this build can never advance past: + // `state_transition` already refuses every gloas block and epoch + // boundary, so a node anchored here would sit at genesis forever, + // looking alive while importing nothing. Checked ahead of the + // fork-matching `verify` below so a gloas anchor reports its real reason + // rather than a spurious "fork mismatch". + if anchor_state.fork_name() == ForkName::Gloas { + return Err(Error::UnsupportedForFork { + function: "get_forkchoice_store", + fork: ForkName::Gloas, + }); + } + // The specification's `BeaconState` and `BeaconBlock` are already one // fork's own types, so a mismatch between them cannot even be expressed // there; here both are enums, so this module has to enforce the invariant @@ -2357,6 +2386,20 @@ pub fn on_block( | SignedBeaconBlock::Altair(_) | SignedBeaconBlock::Bellatrix(_) | SignedBeaconBlock::Capella(_) => {} + // ePBS (EIP-7732) moves the availability question from the block's + // blob/column commitments to the separately gossiped execution + // payload envelope; the check above the match on `signed_block` does + // not apply to a gloas block at all. `state_transition` below is the + // gloas gap this repository has not closed yet (Part B), so this + // returns the same `Error::UnsupportedForFork` any other + // fork-mismatched call in this crate does, rather than skipping the + // availability question with a silent `{}` arm. + SignedBeaconBlock::Gloas(_) => { + return Err(Error::UnsupportedForFork { + function: "fork_choice::on_block", + fork: ForkName::Gloas, + }); + } SignedBeaconBlock::Lean(_) => lean_block_unreachable("fork_choice::on_block"), } diff --git a/crates/blockchain/state_transition/src/beacon/gossip/block.rs b/crates/blockchain/state_transition/src/beacon/gossip/block.rs index 02360ef2b..177b2b637 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/block.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/block.rs @@ -8,6 +8,7 @@ use super::{ finalized_start_slot, is_future_slot, }; use crate::beacon::containers::SignedBeaconBlock; +use crate::beacon::fork::ForkName; use crate::beacon::fork_choice::Store; use crate::beacon::helpers::misc::compute_epoch_at_slot; use crate::beacon::precheck::{self, PrecheckError, Reference, precheck_block}; @@ -25,6 +26,19 @@ pub fn cheap_checks( ) -> Result<(), Outcome> { let config = store.config(); let slot = block.slot(); + // [IGNORE] This build does not validate a gloas-shaped block against + // these (fulu) rules: EIP-7732 changes what a block commits to enough + // that accepting one here would propagate something this node cannot + // itself import (`fork_choice::on_block` refuses every gloas block with + // `Error::UnsupportedForFork`). Checked first, so a gloas block never + // reaches a rule below whose fields it happens to still answer (its + // `blob_kzg_commitment_count` reads the builder's bid) but whose verdict + // would misdescribe it. `Ignore`, not `Reject`: the sender did nothing + // wrong, and every honest peer sends exactly this once this node's own + // clock reaches gloas. + if block.fork_name() == ForkName::Gloas { + return Err(Outcome::Ignore(IgnoreReason::UnsupportedFork)); + } // [IGNORE] The block is not from a future slot. if is_future_slot(&config, slot, now_ms) { return Err(Outcome::Ignore(IgnoreReason::FutureSlot)); diff --git a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs index 2acc7077c..bd6657079 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs @@ -98,6 +98,13 @@ pub enum IgnoreReason { Overloaded, /// Validation panicked. Internal, + /// The message's own fork is one this build does not validate or decode + /// (currently gloas): not the sender's fault, and not malformed, so this + /// is `Ignore` rather than `Reject`. A `Reject` down-scores the peer in + /// gossipsub, and every honest peer on the network sends these once this + /// node's own clock crosses that fork's activation, which would punish + /// the whole mesh for a gap in this build rather than in the message. + UnsupportedFork, /// An attestation's slot is in neither the current nor the previous epoch. OutsideEpochWindow, /// An aggregate adds no bit that one already accepted for the same data @@ -124,6 +131,7 @@ impl IgnoreReason { Self::NoConsumer => "no_consumer", Self::Overloaded => "overloaded", Self::Internal => "internal", + Self::UnsupportedFork => "unsupported_fork", Self::OutsideEpochWindow => "outside_epoch_window", Self::CoveredBits => "covered_bits", Self::UnknownBlock => "unknown_block", diff --git a/crates/blockchain/state_transition/src/beacon/helpers/accessors.rs b/crates/blockchain/state_transition/src/beacon/helpers/accessors.rs index 9b6c83c57..94e3c3a96 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/accessors.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/accessors.rs @@ -16,7 +16,7 @@ pub use ethlambda_storage::{CommitteeCache, Lookup, ShufflingKey}; use crate::beacon::config::Config; use crate::beacon::constants; use crate::beacon::containers::BeaconState; -use crate::beacon::error::Result; +use crate::beacon::error::{Error, Result}; use crate::beacon::fork::ForkName; use crate::beacon::hash::hash; use crate::beacon::lean_state_unreachable; @@ -366,6 +366,18 @@ pub fn get_beacon_proposer_index(state: &BeaconState) -> Result | ForkName::Capella | ForkName::Deneb | ForkName::Electra => {} + // Gloas's own `proposer_lookahead` is unchanged from fulu (see + // `containers::gloas`'s module doc), so this almost certainly + // reduces to `super::fulu::get_beacon_proposer_index` unmodified. + // Left as an explicit gap rather than assumed, since ePBS separates + // the block proposer from the payload's builder and this repository + // has not yet ported that distinction. + ForkName::Gloas => { + return Err(Error::UnsupportedForFork { + function: "get_beacon_proposer_index", + fork: ForkName::Gloas, + }); + } ForkName::Lean => lean_state_unreachable("get_beacon_proposer_index"), } @@ -393,6 +405,7 @@ pub fn get_beacon_proposer_index(state: &BeaconState) -> Result |index| Ok(state.validator(index)?.effective_balance), ), ForkName::Fulu => unreachable!("Fulu returns via the dispatch match above"), + ForkName::Gloas => unreachable!("Gloas returns via the dispatch match above"), ForkName::Lean => lean_state_unreachable("get_beacon_proposer_index"), } } diff --git a/crates/blockchain/state_transition/src/beacon/helpers/altair.rs b/crates/blockchain/state_transition/src/beacon/helpers/altair.rs index 284af647a..b48b77ed1 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/altair.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/altair.rs @@ -163,6 +163,15 @@ pub fn get_next_sync_committee(state: &BeaconState) -> Result get_next_sync_committee_indices(state)?, + // Almost certainly reuses electra's indices function, the way fulu + // does, since gloas keeps electra's effective-balance ceiling. Left + // as an explicit gap rather than assumed; a later task ports it. + ForkName::Gloas => { + return Err(Error::UnsupportedForFork { + function: "get_next_sync_committee", + fork: ForkName::Gloas, + }); + } ForkName::Lean => lean_state_unreachable("get_next_sync_committee"), }; let mut pubkeys = Vec::with_capacity(indices.len()); diff --git a/crates/blockchain/state_transition/src/beacon/helpers/mutators.rs b/crates/blockchain/state_transition/src/beacon/helpers/mutators.rs index e9d0879a6..131706411 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/mutators.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/mutators.rs @@ -140,6 +140,15 @@ fn initiate_validator_exit_for_fork( | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb => initiate_validator_exit(state, index, config), + // Gloas (EIP-8061) redefines the validator exit churn itself + // (`churn_limit_quotient_gloas`/`max_per_epoch_activation_churn_limit_gloas` + // in `Config`, read by `get_activation_churn_limit`/ + // `get_exit_churn_limit`), which is more than a retuned constant; + // left as an explicit gap for the STF port rather than assumed. + ForkName::Gloas => Err(Error::UnsupportedForFork { + function: "initiate_validator_exit_for_fork", + fork: ForkName::Gloas, + }), ForkName::Lean => lean_state_unreachable("initiate_validator_exit_for_fork"), } } diff --git a/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs b/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs index 4344108e5..8b31a8093 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs @@ -140,6 +140,13 @@ pub fn with_validators_at(fork: ForkName, count: usize) -> BeaconState { ForkName::Deneb => BeaconState::Deneb(deneb_state(count)), ForkName::Electra => BeaconState::Electra(electra_state(count)), ForkName::Fulu => BeaconState::Fulu(fulu_state(count)), + // No gloas literal builder exists yet: its registry is progressive + // rather than bounded (see `containers::gloas`'s module doc), so this + // cannot reuse `fulu_state`'s placeholder fields unchanged. A later + // task adds `gloas_state` alongside the others above. + ForkName::Gloas => { + unimplemented!("with_validators_at: gloas has no test-state builder yet") + } // A test asking for a lean state from a beacon builder, which no fixture // fork name can produce; the `fork:` form, since the argument is the // whole input. diff --git a/crates/blockchain/state_transition/src/beacon/stf/block.rs b/crates/blockchain/state_transition/src/beacon/stf/block.rs index 234d1cef3..d7500ca75 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/block.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/block.rs @@ -69,6 +69,14 @@ pub fn process_block( containers::SignedBeaconBlock::Fulu(signed) => { fulu::process_block(state, &signed.message, config, engine, committees) } + // ePBS (EIP-7732) restructures block processing around a builder's + // bid rather than an embedded payload; a later task ports it. + containers::SignedBeaconBlock::Gloas(_) => { + Err(crate::beacon::error::Error::UnsupportedForFork { + function: "process_block", + fork: crate::beacon::fork::ForkName::Gloas, + }) + } containers::SignedBeaconBlock::Lean(_) => { crate::beacon::lean_block_unreachable("process_block") } diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs index 00659d1c4..1ec25e41a 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs @@ -24,7 +24,7 @@ pub mod rewards; use crate::beacon::containers::phase0::PendingAttestation; use crate::beacon::containers::{BeaconState, HistoricalBatch}; -use crate::beacon::error::{Result, verify}; +use crate::beacon::error::{Error, Result, verify}; use crate::beacon::fork::ForkName; use crate::beacon::helpers::accessors::{ CommitteeCache, CommitteeCacheExt, get_block_root, get_block_root_at_slot, get_current_epoch, @@ -88,6 +88,13 @@ pub fn process_epoch(state: &mut BeaconState, config: &Config) -> Result<()> { ForkName::Deneb => capella::process_epoch(state, config), ForkName::Electra => electra::process_epoch(state, config), ForkName::Fulu => fulu::process_epoch(state, config), + // EIP-7732's builder registry, payment queues, and payload + // availability each need their own epoch-boundary steps; a later + // task writes gloas's own driver. + ForkName::Gloas => Err(Error::UnsupportedForFork { + function: "process_epoch", + fork: ForkName::Gloas, + }), ForkName::Lean => lean_state_unreachable("process_epoch"), } } @@ -116,6 +123,13 @@ pub fn process_justification_and_finalization( | ForkName::Deneb | ForkName::Electra | ForkName::Fulu => altair::process_justification_and_finalization(state), + // Almost certainly unchanged from altair (nothing in gloas's own + // `beacon-chain.md` section touches justification), but left as an + // explicit gap rather than assumed until confirmed against the spec. + ForkName::Gloas => Err(Error::UnsupportedForFork { + function: "process_justification_and_finalization", + fork: ForkName::Gloas, + }), ForkName::Lean => lean_state_unreachable("process_justification_and_finalization"), } } diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/registry.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/registry.rs index 53df43d99..f6985d73c 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/registry.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/registry.rs @@ -44,10 +44,12 @@ fn activation_churn_limit(state: &BeaconState, config: &Config) -> Result { Ok(churn_limit) } ForkName::Deneb => Ok(config.max_per_epoch_activation_churn_limit.min(churn_limit)), - fork @ (ForkName::Electra | ForkName::Fulu) => Err(Error::UnsupportedForFork { - function: "process_registry_updates", - fork, - }), + fork @ (ForkName::Electra | ForkName::Fulu | ForkName::Gloas) => { + Err(Error::UnsupportedForFork { + function: "process_registry_updates", + fork, + }) + } ForkName::Lean => lean_state_unreachable("activation_churn_limit"), } } diff --git a/crates/blockchain/state_transition/src/beacon/stf/operations.rs b/crates/blockchain/state_transition/src/beacon/stf/operations.rs index 9b7cd8bfd..6836a063f 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/operations.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/operations.rs @@ -82,12 +82,15 @@ pub fn process_operations( // Deneb never reaches here: it has its own `process_operations` // (`deneb.rs`), which calls its own `process_attestation` for this loop // instead of this shared one. Electra and fulu have their own - // `process_operations` too (`electra.rs`, which fulu reuses), so all - // three are refused rather than guessed at. + // `process_operations` too (`electra.rs`, which fulu reuses), and gloas + // will need a third (its bodies carry progressive attestations and + // slashings, plus the payload-attestation and builder-registry + // operations this signature has no parameters for), so all four are + // refused rather than guessed at. let altair_attestations = match state.fork_name() { ForkName::Phase0 => false, ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => true, - fork @ (ForkName::Deneb | ForkName::Electra | ForkName::Fulu) => { + fork @ (ForkName::Deneb | ForkName::Electra | ForkName::Fulu | ForkName::Gloas) => { return Err(Error::UnsupportedForFork { function: "process_operations", fork, @@ -438,7 +441,12 @@ pub fn add_validator_to_registry( let grows_altair_lists = match state.fork_name() { ForkName::Phase0 => false, ForkName::Altair | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb => true, - fork @ (ForkName::Electra | ForkName::Fulu) => { + // Gloas queues deposits the way electra does (see the pending-deposit + // queue in `containers::gloas`), and its three altair-onward lists + // are progressive rather than `SszList` (see `BeaconState::altair_validator_lists`'s + // own documentation), so it is refused for both reasons rather than + // folded into either branch above. + fork @ (ForkName::Electra | ForkName::Fulu | ForkName::Gloas) => { return Err(Error::UnsupportedForFork { function: "add_validator_to_registry", fork, diff --git a/crates/blockchain/state_transition/src/beacon/upgrade.rs b/crates/blockchain/state_transition/src/beacon/upgrade.rs index d31eff1c9..b9730601e 100644 --- a/crates/blockchain/state_transition/src/beacon/upgrade.rs +++ b/crates/blockchain/state_transition/src/beacon/upgrade.rs @@ -841,6 +841,15 @@ pub fn upgrade_state(state: &BeaconState, to: ForkName, config: &Config) -> Resu ForkName::Deneb => upgrade_to_deneb(state, config), ForkName::Electra => upgrade_to_electra(state, config), ForkName::Fulu => upgrade_to_fulu(state, config), + // `upgrade_to_gloas` does not exist yet: EIP-7688 replaces the + // registry's list kind and EIP-7732 removes + // `latest_execution_payload_header` outright, so this is not a + // field-by-field carry-over the way `upgrade_to_fulu` is. A later + // task writes it. + ForkName::Gloas => Err(Error::UnsupportedForFork { + function: "upgrade_state", + fork: to, + }), // The `fork:` form, not `state:`: this dispatches on the requested // target, so it is the argument that is wrong, not what `state` holds. ForkName::Lean => lean_fork_unreachable("upgrade_state"), diff --git a/crates/blockchain/state_transition/tests/beacon_spec/epoch_processing.rs b/crates/blockchain/state_transition/tests/beacon_spec/epoch_processing.rs index a3ab100bb..7283b0136 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/epoch_processing.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/epoch_processing.rs @@ -83,6 +83,12 @@ fn apply( | ForkName::Deneb | ForkName::Electra | ForkName::Fulu => epoch::altair::process_rewards_and_penalties(state, config), + ForkName::Gloas => Err( + ethlambda_state_transition::beacon::Error::UnsupportedForFork { + function: "rewards_and_penalties", + fork: ForkName::Gloas, + }, + ), ForkName::Lean => lean_is_not_a_fixture_fork("rewards_and_penalties"), }, // Deneb's own change (EIP-7514's activation-churn cap) is selected @@ -103,6 +109,12 @@ fn apply( | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb => epoch::registry::process_registry_updates(state, config), + ForkName::Gloas => Err( + ethlambda_state_transition::beacon::Error::UnsupportedForFork { + function: "registry_updates", + fork: ForkName::Gloas, + }, + ), ForkName::Lean => lean_is_not_a_fixture_fork("registry_updates"), }, // Altair's and bellatrix's own changes here are scoped to the @@ -121,6 +133,12 @@ fn apply( | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb => epoch::registry::process_slashings(state, config), + ForkName::Gloas => Err( + ethlambda_state_transition::beacon::Error::UnsupportedForFork { + function: "slashings", + fork: ForkName::Gloas, + }, + ), ForkName::Lean => lean_is_not_a_fixture_fork("slashings"), }, "eth1_data_reset" => epoch::process_eth1_data_reset(state), @@ -138,6 +156,12 @@ fn apply( | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb => epoch::process_effective_balance_updates(state), + ForkName::Gloas => Err( + ethlambda_state_transition::beacon::Error::UnsupportedForFork { + function: "effective_balance_updates", + fork: ForkName::Gloas, + }, + ), ForkName::Lean => lean_is_not_a_fixture_fork("effective_balance_updates"), }, "slashings_reset" => epoch::process_slashings_reset(state), diff --git a/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs b/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs index fbbe69533..6c92e24e7 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs @@ -287,6 +287,7 @@ pub(super) fn decode_anchor_block(case: &Case) -> Result Err("gloas not implemented".into()), ForkName::Lean => lean_is_not_a_fixture_fork("fork_choice"), } } @@ -314,6 +315,7 @@ fn decode_attestation(case: &Case, name: &str) -> Result Ok(fork_choice::Attestation::Phase0(decode(case, name)?)), + ForkName::Gloas => Err("gloas not implemented".into()), ForkName::Lean => lean_is_not_a_fixture_fork("fork_choice"), } } @@ -333,6 +335,7 @@ fn decode_attester_slashing( | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb => Ok(fork_choice::AttesterSlashing::Phase0(decode(case, name)?)), + ForkName::Gloas => Err("gloas not implemented".into()), ForkName::Lean => lean_is_not_a_fixture_fork("fork_choice"), } } diff --git a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs index 0590fb5d5..aae025f99 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs @@ -48,11 +48,24 @@ const HANDLERS: &[&str] = &[ /// fails if a fixture release adds a `gossip_*` handler that is in neither /// list, the same way [`super::UNMODELED_FORKS`] forces a decision on a new /// fork directory. +/// +/// `gossip_execution_payload_bid`, `gossip_execution_payload_envelope`, +/// `gossip_payload_attestation_message`, and `gossip_proposer_preferences` +/// are gloas's own topics (EIP-7732 ePBS): the builder's bid, the revealed +/// payload envelope, the payload timeliness committee's vote, and a +/// builder's advertised preferences, respectively. None of them existed +/// until gloas's fixture directory started parsing (`ForkName::Gloas`), so +/// they land here rather than silently in `unknown` the first time this +/// runner sees them. Alphabetized with the rest rather than kept together. const IGNORED_HANDLERS: &[&str] = &[ "gossip_attester_slashing", "gossip_blob_sidecar", "gossip_bls_to_execution_change", + "gossip_execution_payload_bid", + "gossip_execution_payload_envelope", "gossip_partial_data_column_sidecar", + "gossip_payload_attestation_message", + "gossip_proposer_preferences", "gossip_proposer_slashing", "gossip_sync_committee_contribution_and_proof", "gossip_sync_committee_message", @@ -134,7 +147,9 @@ fn decode_block(case: &Case, name: &str) -> Result { /// needed one yet): every fork through deneb shares phase0's shape, and /// electra and fulu share electra's, exactly the split /// [`SignedAggregateAndProof`]'s own doc describes for -/// [`SignedBeaconBlock::Fulu`]. +/// [`SignedBeaconBlock::Fulu`]. Gloas's own aggregate has no modeled variant, +/// and this runner does not run its cases (see [`trials`]), so it is refused +/// by name. fn decode_signed_aggregate(case: &Case, name: &str) -> Result { let bytes = case.ssz_bytes(name); match case.fork { @@ -150,7 +165,9 @@ fn decode_signed_aggregate(case: &Case, name: &str) -> Result Err(format!("no aggregate shape for fork {other:?}")), + ForkName::Gloas | ForkName::Lean => { + Err(format!("no aggregate shape for fork {:?}", case.fork)) + } } } diff --git a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs index 54aec9af6..9c03ed7e8 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs @@ -342,13 +342,16 @@ fn read_dir_sorted(path: &Path) -> Vec { /// Fixture fork directories this crate deliberately does not model. /// -/// `gloas` and `heze` are the two forks after fulu (in that order), and this -/// crate stops at fulu. At `v1.7.0-beta.2` the release ships no other directory -/// under either preset tree that [`ForkName::parse`] does not know: no -/// in-flight-EIP directory (`eipNNNN`) remains, unlike at `v1.6.1`, where -/// `eip7805` was one. +/// `heze` is the one fork after fulu that [`ForkName::parse`] still does not +/// know: `gloas` moved out of this list once [`ForkName`] gained that variant, +/// since its directory now parses and its cases fall under +/// [`HIGHEST_IMPLEMENTED_FORK`]'s ordinary out-of-scope handling +/// ([`Case::in_scope`]) instead of this one. At `v1.7.0-beta.2` the release +/// ships no other directory under either preset tree that `ForkName::parse` +/// does not know: no in-flight-EIP directory (`eipNNNN`) remains, unlike at +/// `v1.6.1`, where `eip7805` was one. /// -/// Naming them is not bookkeeping for its own sake. A directory [`ForkName::parse`] +/// Naming it is not bookkeeping for its own sake. A directory [`ForkName::parse`] /// does not recognize is how [`collect`] skips a fork, and that skip is *silent* /// in a way [`Case::in_scope`] is not: the cases never become tests at all, so /// they are not counted as ignored either, and nothing in the output says they @@ -356,7 +359,7 @@ fn read_dir_sorted(path: &Path) -> Vec { /// matching fails rather than reporting green, and an unparsed fork slips past /// [`HIGHEST_IMPLEMENTED_FORK`] entirely because the gate never sees the case. /// So [`fixture_fork_trials`] checks this list against the tree instead. -pub const UNMODELED_FORKS: &[&str] = &["gloas", "heze"]; +pub const UNMODELED_FORKS: &[&str] = &["heze"]; /// Panics: a fixture case cannot be a lean case. /// diff --git a/crates/blockchain/state_transition/tests/beacon_spec/operations.rs b/crates/blockchain/state_transition/tests/beacon_spec/operations.rs index dbddf6d79..b084f33ea 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/operations.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/operations.rs @@ -199,6 +199,12 @@ fn apply( block.body.hash_tree_root(), ) } + ForkName::Gloas => Err( + ethlambda_state_transition::beacon::Error::UnsupportedForFork { + function: "block_header", + fork: ForkName::Gloas, + }, + ), // No `other` arm: the patterns above already cover every `ForkName` // there is, so a catch-all here would be dead code rather than a // safety net. `Lean` is covered by name for the same reason, rather @@ -231,6 +237,12 @@ fn apply( let attestation: electra::Attestation = case.ssz("attestation"); electra_stf::process_attestation(state, &attestation, &committees) } + ForkName::Gloas => Err( + ethlambda_state_transition::beacon::Error::UnsupportedForFork { + function: "attestation", + fork: ForkName::Gloas, + }, + ), ForkName::Lean => lean_is_not_a_fixture_fork("attestation"), }, "attester_slashing" => match case.fork { @@ -251,6 +263,12 @@ fn apply( let slashing: electra::AttesterSlashing = case.ssz("attester_slashing"); electra_stf::process_attester_slashing(state, &slashing, config) } + ForkName::Gloas => Err( + ethlambda_state_transition::beacon::Error::UnsupportedForFork { + function: "attester_slashing", + fork: ForkName::Gloas, + }, + ), ForkName::Lean => lean_is_not_a_fixture_fork("attester_slashing"), }, // `ProposerSlashing` never changes shape, and no fork's specification @@ -279,6 +297,12 @@ fn apply( let deposit: shared::Deposit = case.ssz("deposit"); electra_stf::process_deposit(state, &deposit, config) } + ForkName::Gloas => Err( + ethlambda_state_transition::beacon::Error::UnsupportedForFork { + function: "deposit", + fork: ForkName::Gloas, + }, + ), ForkName::Lean => lean_is_not_a_fixture_fork("deposit"), }, "voluntary_exit" => match case.fork { @@ -301,6 +325,12 @@ fn apply( let exit: shared::SignedVoluntaryExit = case.ssz("voluntary_exit"); electra_stf::process_voluntary_exit(state, &exit, config) } + ForkName::Gloas => Err( + ethlambda_state_transition::beacon::Error::UnsupportedForFork { + function: "voluntary_exit", + fork: ForkName::Gloas, + }, + ), ForkName::Lean => lean_is_not_a_fixture_fork("voluntary_exit"), }, // New in altair, and never listed as modified again, so this needs no @@ -376,6 +406,12 @@ fn apply( "execution_payload has no handler for fork `{fork}`" )); } + ForkName::Gloas => Err( + ethlambda_state_transition::beacon::Error::UnsupportedForFork { + function: "execution_payload", + fork: ForkName::Gloas, + }, + ), ForkName::Lean => lean_is_not_a_fixture_fork("execution_payload"), } } @@ -416,6 +452,12 @@ fn apply( fork @ (ForkName::Phase0 | ForkName::Altair | ForkName::Bellatrix) => { return Err(format!("withdrawals has no handler for fork `{fork}`")); } + ForkName::Gloas => Err( + ethlambda_state_transition::beacon::Error::UnsupportedForFork { + function: "withdrawals", + fork: ForkName::Gloas, + }, + ), ForkName::Lean => lean_is_not_a_fixture_fork("withdrawals"), }, // All three are new in electra, alongside `withdrawals`; fulu's diff --git a/crates/blockchain/state_transition/tests/beacon_spec/rewards.rs b/crates/blockchain/state_transition/tests/beacon_spec/rewards.rs index d97853f54..fef71f3ab 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/rewards.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/rewards.rs @@ -153,6 +153,12 @@ fn components( altair_helpers::get_inactivity_penalty_deltas(state, config), ), ], + // Gloas's `previous_epoch_participation`/`current_epoch_participation` + // are progressive lists (EIP-7688), a different Rust type from the + // `SszList` `altair_helpers::get_flag_index_deltas` reads, so this is + // not altair's list unchanged even though the reward rule itself is; + // a later task writes gloas's own copy over the progressive type. + ForkName::Gloas => return Err("gloas not implemented".into()), ForkName::Lean => lean_is_not_a_fixture_fork("rewards"), }; diff --git a/crates/common/types/src/beacon/config.rs b/crates/common/types/src/beacon/config.rs index 9e0b03bcf..6d6908eaa 100644 --- a/crates/common/types/src/beacon/config.rs +++ b/crates/common/types/src/beacon/config.rs @@ -11,11 +11,19 @@ //! //! # What is left out //! -//! - **Forks this build cannot process.** `GLOAS_*`, `HEZE_*` and the timing -//! values that arrived with them, plus `GAS_LIMIT_SCHEDULE`, which is -//! gloas-era and is a list rather than a scalar. A fork version stored here -//! would give [`Config::fork_at_epoch`] a fork [`crate::beacon::fork::ForkName`] -//! has no variant for, so these are reported as unknown keys instead. +//! - **Forks this build cannot process.** `HEZE_FORK_VERSION`/`HEZE_FORK_EPOCH` +//! and `EIP8321_FORK_VERSION`/`EIP8321_FORK_EPOCH` (its own later fork +//! schedule) stay out for the same reason `GLOAS_*` used to: a fork version +//! stored here would give [`Config::fork_at_epoch`] a fork +//! [`crate::beacon::fork::ForkName`] has no variant for, so each is +//! reported as an unknown key instead. `GAS_LIMIT_SCHEDULE` (gloas-era) +//! stays out too, since it is a list rather than a scalar. The rest are +//! heze-era data keys, not gloas's, with no reader in this build at all: +//! `INCLUSION_LIST_DUE_BPS`, `MAX_REQUEST_INCLUSION_LIST`, +//! `MIN_SLOTS_FOR_INCLUSION_LISTS_REQUESTS`, and +//! `MAX_TRANSACTIONS_BYTES_PER_INCLUSION_LIST` (EIP-7805's fork-choice-enforced +//! inclusion lists), and `CONFIRMATION_BYZANTINE_THRESHOLD` (the Fast +//! Confirmation Rule). //! //! `PRESET_BASE` and `CONFIG_NAME` used to be left out as well, because they //! are strings and this struct is SSZ-encoded into the database. They are here @@ -309,6 +317,13 @@ pub struct Config { /// is not scheduled. #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub fulu_fork_epoch: Epoch, + /// The `Fork.current_version` a gloas block or attestation signs under. + #[serde(with = "crate::beacon::serde_helpers::hex_array")] + pub gloas_fork_version: Version, + /// The epoch gloas activates at, or [`constants::FAR_FUTURE_EPOCH`] if it + /// is not scheduled. + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub gloas_fork_epoch: Epoch, // -- Time parameters --------------------------------------------------- /// Wall-clock seconds per slot. Deprecated in favor of @@ -364,6 +379,36 @@ pub struct Config { /// contribution is due (altair); read by `get_contribution_due_ms`. #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub contribution_due_bps: u64, + /// Gloas: [`Self::attestation_due_bps`]'s replacement, now that ePBS + /// (EIP-7732) moves the attestation deadline earlier in the slot to make + /// room for the payload and payload-attestation windows below. + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub attestation_due_bps_gloas: u64, + /// Gloas: [`Self::aggregate_due_bps`]'s replacement, for the reason + /// [`Self::attestation_due_bps_gloas`] gives. + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub aggregate_due_bps_gloas: u64, + /// Gloas: [`Self::sync_message_due_bps`]'s replacement, for the reason + /// [`Self::attestation_due_bps_gloas`] gives. + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub sync_message_due_bps_gloas: u64, + /// Gloas: [`Self::contribution_due_bps`]'s replacement, for the reason + /// [`Self::attestation_due_bps_gloas`] gives. + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub contribution_due_bps_gloas: u64, + /// Gloas (EIP-7732): basis points of [`Self::slot_duration_ms`] by which + /// the builder's execution payload is due. + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub payload_due_bps: u64, + /// Gloas (EIP-7732): basis points of [`Self::slot_duration_ms`] by which + /// a payload attestation is due. + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub payload_attestation_due_bps: u64, + /// Gloas: epochs a builder must wait after its exit is processed before + /// its collateral becomes withdrawable, the builder-registry counterpart + /// to [`Self::min_validator_withdrawability_delay`]. + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub min_builder_withdrawability_delay: Epoch, // -- Validator cycle ----------------------------------------------------- /// Score points added to a validator's inactivity score for each epoch it @@ -406,6 +451,19 @@ pub struct Config { /// consolidations (`get_activation_exit_churn_limit`). #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub max_per_epoch_activation_exit_churn_limit: Gwei, + /// Gloas (EIP-8061): the *validator* registry's own churn quotient for + /// this fork, read by `get_activation_churn_limit`/`get_exit_churn_limit`. + /// Not a builder-registry value, despite the name pattern this crate uses + /// elsewhere for gloas-specific fields. + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub churn_limit_quotient_gloas: u64, + /// Gloas (EIP-8061): the ceiling on how much validator activation churn + /// one epoch may admit, this fork's counterpart to + /// [`Self::max_per_epoch_activation_churn_limit`] (deneb) and + /// [`Self::max_per_epoch_activation_exit_churn_limit`] (electra). Not a + /// builder-registry value; see [`Self::churn_limit_quotient_gloas`]. + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] + pub max_per_epoch_activation_churn_limit_gloas: Gwei, // -- Fork choice --------------------------------------------------------- /// Percentage boost, relative to a single committee's weight, given to a @@ -659,6 +717,8 @@ impl Config { electra_fork_epoch: 364_032, fulu_fork_version: [0x06, 0x00, 0x00, 0x00], fulu_fork_epoch: 411_392, + gloas_fork_version: [0x07, 0x00, 0x00, 0x00], + gloas_fork_epoch: constants::FAR_FUTURE_EPOCH, seconds_per_slot: 12, slot_duration_ms: 12_000, @@ -671,6 +731,13 @@ impl Config { proposer_reorg_cutoff_bps: 1_667, sync_message_due_bps: 3_333, contribution_due_bps: 6_667, + attestation_due_bps_gloas: 2_500, + aggregate_due_bps_gloas: 5_000, + sync_message_due_bps_gloas: 2_500, + contribution_due_bps_gloas: 5_000, + payload_due_bps: 5_000, + payload_attestation_due_bps: 7_500, + min_builder_withdrawability_delay: 64, inactivity_score_bias: 4, inactivity_score_recovery_rate: 16, @@ -680,6 +747,8 @@ impl Config { max_per_epoch_activation_churn_limit: 8, min_per_epoch_churn_limit_electra: 128_000_000_000, max_per_epoch_activation_exit_churn_limit: 256_000_000_000, + churn_limit_quotient_gloas: 32_768, + max_per_epoch_activation_churn_limit_gloas: 256_000_000_000, proposer_score_boost: 40, reorg_head_weight_threshold: 20, @@ -782,6 +851,8 @@ impl Config { electra_fork_epoch: constants::FAR_FUTURE_EPOCH, fulu_fork_version: [0x06, 0x00, 0x00, 0x01], fulu_fork_epoch: constants::FAR_FUTURE_EPOCH, + gloas_fork_version: [0x07, 0x00, 0x00, 0x01], + gloas_fork_epoch: constants::FAR_FUTURE_EPOCH, seconds_per_slot: 6, slot_duration_ms: 6_000, @@ -794,6 +865,13 @@ impl Config { proposer_reorg_cutoff_bps: 1_667, sync_message_due_bps: 3_333, contribution_due_bps: 6_667, + attestation_due_bps_gloas: 2_500, + aggregate_due_bps_gloas: 5_000, + sync_message_due_bps_gloas: 2_500, + contribution_due_bps_gloas: 5_000, + payload_due_bps: 5_000, + payload_attestation_due_bps: 7_500, + min_builder_withdrawability_delay: 2, inactivity_score_bias: 4, inactivity_score_recovery_rate: 16, @@ -803,6 +881,8 @@ impl Config { max_per_epoch_activation_churn_limit: 4, min_per_epoch_churn_limit_electra: 64_000_000_000, max_per_epoch_activation_exit_churn_limit: 128_000_000_000, + churn_limit_quotient_gloas: 16, + max_per_epoch_activation_churn_limit_gloas: 128_000_000_000, proposer_score_boost: 40, reorg_head_weight_threshold: 20, @@ -870,7 +950,7 @@ impl Config { /// are chosen so that a beacon-shaped gate reading this by mistake fails /// closed: every fork epoch is `FAR_FUTURE_EPOCH`, so no fork ever reads as /// activated, rather than epoch 0, which would read as "activated at - /// genesis" for all seven of them. + /// genesis" for all eight of them. pub fn lean(genesis_time: u64, slot_duration_ms: u64) -> Self { Self { // A lean `config.yaml` has no `PRESET_BASE` or `CONFIG_NAME` key, @@ -891,6 +971,7 @@ impl Config { deneb_fork_epoch: constants::FAR_FUTURE_EPOCH, electra_fork_epoch: constants::FAR_FUTURE_EPOCH, fulu_fork_epoch: constants::FAR_FUTURE_EPOCH, + gloas_fork_epoch: constants::FAR_FUTURE_EPOCH, ..Config::mainnet() } } @@ -973,6 +1054,7 @@ impl Config { ForkName::Deneb => self.deneb_fork_version, ForkName::Electra => self.electra_fork_version, ForkName::Fulu => self.fulu_fork_version, + ForkName::Gloas => self.gloas_fork_version, ForkName::Lean => lean_fork_unreachable("Config::fork_version"), } } @@ -990,6 +1072,7 @@ impl Config { ForkName::Deneb => self.deneb_fork_epoch, ForkName::Electra => self.electra_fork_epoch, ForkName::Fulu => self.fulu_fork_epoch, + ForkName::Gloas => self.gloas_fork_epoch, ForkName::Lean => lean_fork_unreachable("Config::fork_epoch"), } } @@ -1014,6 +1097,7 @@ impl Config { ForkName::Deneb => self.deneb_fork_epoch = epoch, ForkName::Electra => self.electra_fork_epoch = epoch, ForkName::Fulu => self.fulu_fork_epoch = epoch, + ForkName::Gloas => self.gloas_fork_epoch = epoch, ForkName::Lean => lean_fork_unreachable("Config::with_fork_epoch"), } self @@ -1030,6 +1114,7 @@ impl Config { ForkName::Deneb => self.deneb_fork_version = version, ForkName::Electra => self.electra_fork_version = version, ForkName::Fulu => self.fulu_fork_version = version, + ForkName::Gloas => self.gloas_fork_version = version, // Matches `fork_version`'s own arm: reaching this means a caller // dispatched on the wrong chain, which is a bug in the caller. ForkName::Lean => lean_fork_unreachable("Config::with_fork_version"), @@ -1147,6 +1232,18 @@ mod tests { assert_eq!(config.fork_at_epoch(1_000_000), ForkName::Altair); } + #[test] + fn gloas_keys_are_claimed_and_parse() { + let yaml = "GLOAS_FORK_VERSION: 0x07000000\nGLOAS_FORK_EPOCH: 12\n\ + CHURN_LIMIT_QUOTIENT_GLOAS: 32768\nMIN_BUILDER_WITHDRAWABILITY_DELAY: 64\n\ + PAYLOAD_DUE_BPS: 5000\nPAYLOAD_ATTESTATION_DUE_BPS: 7500\nMAX_REQUEST_PAYLOADS: 128\n"; + let config: Config = serde_yaml_ng::from_str(yaml).unwrap(); + assert_eq!(config.gloas_fork_version, [0x07, 0, 0, 0]); + assert_eq!(config.fork_epoch(ForkName::Gloas), 12); + assert_eq!(config.fork_at_epoch(12), ForkName::Gloas); + assert_eq!(config.max_request_payloads, 128); + } + #[test] fn max_blobs_per_block_selects_the_active_schedule_entry() { let config = Config::mainnet(); diff --git a/crates/common/types/src/beacon/containers/mod.rs b/crates/common/types/src/beacon/containers/mod.rs index 0b0be4b69..aa4cca202 100644 --- a/crates/common/types/src/beacon/containers/mod.rs +++ b/crates/common/types/src/beacon/containers/mod.rs @@ -77,6 +77,7 @@ macro_rules! dispatch_state { BeaconState::Deneb($state) => $body, BeaconState::Electra($state) => $body, BeaconState::Fulu($state) => $body, + BeaconState::Gloas($state) => $body, BeaconState::Lean(_) => lean_state_unreachable($function), } }; @@ -106,6 +107,7 @@ macro_rules! dispatch_state_including_lean { BeaconState::Deneb($state) => $body, BeaconState::Electra($state) => $body, BeaconState::Fulu($state) => $body, + BeaconState::Gloas($state) => $body, BeaconState::Lean($state) => $body, } }; @@ -150,6 +152,7 @@ macro_rules! dispatch_block { SignedBeaconBlock::Deneb($block) => $body, SignedBeaconBlock::Electra($block) => $body, SignedBeaconBlock::Fulu($block) => $body, + SignedBeaconBlock::Gloas($block) => $body, SignedBeaconBlock::Lean(_) => lean_block_unreachable($function), } }; @@ -174,6 +177,7 @@ macro_rules! dispatch_block_including_lean { SignedBeaconBlock::Deneb($block) => $body, SignedBeaconBlock::Electra($block) => $body, SignedBeaconBlock::Fulu($block) => $body, + SignedBeaconBlock::Gloas($block) => $body, SignedBeaconBlock::Lean($block) => $body, } }; @@ -199,6 +203,7 @@ pub enum BeaconState { Deneb(deneb::BeaconState), Electra(electra::BeaconState), Fulu(fulu::BeaconState), + Gloas(gloas::BeaconState), Lean(crate::state::State), } @@ -242,6 +247,7 @@ impl serde::Serialize for BeaconState { BeaconState::Deneb(state) => state.serialize(serializer), BeaconState::Electra(state) => state.serialize(serializer), BeaconState::Fulu(state) => state.serialize(serializer), + BeaconState::Gloas(state) => state.serialize(serializer), BeaconState::Lean(_) => Err(serde::ser::Error::custom( "a lean state has no JSON encoding: /lean/v0/states/finalized serves SSZ only", )), @@ -282,6 +288,7 @@ impl BeaconState { BeaconState::Deneb(_) => ForkName::Deneb, BeaconState::Electra(_) => ForkName::Electra, BeaconState::Fulu(_) => ForkName::Fulu, + BeaconState::Gloas(_) => ForkName::Gloas, BeaconState::Lean(_) => ForkName::Lean, } } @@ -344,6 +351,9 @@ impl BeaconState { bytes, )?)), ForkName::Fulu => Ok(BeaconState::Fulu(fulu::BeaconState::from_ssz_bytes(bytes)?)), + ForkName::Gloas => Ok(BeaconState::Gloas(gloas::BeaconState::from_ssz_bytes( + bytes, + )?)), ForkName::Lean => Ok(BeaconState::Lean(crate::state::State::from_ssz_bytes( bytes, )?)), @@ -518,6 +528,69 @@ impl BeaconState { } } +/// The registry of whichever list kind this state's fork uses. +/// +/// Every fork through fulu backs `validators`/`balances` with the bounded, +/// tree-backed [`List`](ethlambda_ssz_tree::List); gloas (EIP-7688) backs them +/// with the unbounded, progressively merkleized +/// [`ProgressiveList`](ethlambda_ssz_tree::ProgressiveList) instead. The two +/// types share every method this crate calls on them (`len`, `get`, `get_mut`, +/// `push`, `apply_updates`, `hash_tree_root`, `has_pending_updates`, +/// `rebase_on`, `ptr_eq`), which is what lets [`dispatch_state!`] keep one body +/// for most of the accessors below. This enum exists only for the handful that +/// return the list itself rather than an element: `dispatch_state!`'s +/// `$body` still has to typecheck as one Rust type across every arm, and +/// `&Validators` and `&ProgressiveValidators` are not that. +enum Registry<'a> { + Bounded(&'a Validators, &'a Balances), + Progressive(&'a ProgressiveValidators, &'a ProgressiveBalances), +} + +/// [`Registry`], mutably. A separate type rather than a lifetime trick on the +/// same one: an enum cannot hold either `&'a T` or `&'a mut T` depending on +/// the caller, so [`BeaconState::rebase_on`] needs its own mutable version to +/// match against. +enum RegistryMut<'a> { + Bounded(&'a mut Validators, &'a mut Balances), + Progressive(&'a mut ProgressiveValidators, &'a mut ProgressiveBalances), +} + +/// An iterator over either registry list kind, so [`BeaconState::iter_validators`] +/// and [`BeaconState::iter_balances`] can promise one return type across every +/// fork despite [`Registry`]'s two underlying list types having different +/// concrete iterators. Private like `Registry`/`RegistryMut`: both accessors +/// hand it back only as the opaque `impl ExactSizeIterator` those two already +/// promised, so nothing outside this module ever names the concrete type. +enum RegistryIter<'a, T, U> { + Bounded(ethlambda_ssz_tree::Iter<'a, T, U>), + Progressive(ethlambda_ssz_tree::ProgressiveIter<'a, T, U>), +} + +impl<'a, T: ethlambda_ssz_tree::Value, U: ethlambda_ssz_tree::UpdateMap> Iterator + for RegistryIter<'a, T, U> +{ + type Item = &'a T; + + fn next(&mut self) -> Option<&'a T> { + match self { + Self::Bounded(it) => it.next(), + Self::Progressive(it) => it.next(), + } + } + + fn size_hint(&self) -> (usize, Option) { + match self { + Self::Bounded(it) => it.size_hint(), + Self::Progressive(it) => it.size_hint(), + } + } +} + +impl> ExactSizeIterator + for RegistryIter<'_, T, U> +{ +} + impl BeaconState { /// The number of validators in the registry. pub fn validator_count(&self) -> usize { @@ -534,16 +607,22 @@ impl BeaconState { /// a loop: each `validator()` call redoes the fork dispatch and a tree /// descent, where this walks the registry once, sequentially. pub fn iter_validators(&self) -> impl ExactSizeIterator { - dispatch_state!(self, "iter_validators", |state| state.validators.iter()) + match self.registry() { + Some(Registry::Bounded(v, _)) => RegistryIter::Bounded(v.iter()), + Some(Registry::Progressive(v, _)) => RegistryIter::Progressive(v.iter()), + None => lean_state_unreachable("iter_validators"), + } } /// Every balance, in index order, pending writes included. See /// [`Self::iter_validators`] for why this is an iterator. pub fn iter_balances(&self) -> impl ExactSizeIterator { - dispatch_state!(self, "iter_balances", |state| state - .balances - .iter() - .copied()) + match self.registry() { + Some(Registry::Bounded(_, b)) => RegistryIter::Bounded(b.iter()), + Some(Registry::Progressive(_, b)) => RegistryIter::Progressive(b.iter()), + None => lean_state_unreachable("iter_balances"), + } + .copied() } /// The validator at `index`. @@ -617,20 +696,60 @@ impl BeaconState { /// check cannot see. pub fn validators_ptr_eq(&self, other: &BeaconState) -> bool { match (self.registry(), other.registry()) { - (Some((v, _)), Some((ov, _))) => v.ptr_eq(ov), + (Some(Registry::Bounded(v, _)), Some(Registry::Bounded(ov, _))) => v.ptr_eq(ov), + (Some(Registry::Progressive(v, _)), Some(Registry::Progressive(ov, _))) => v.ptr_eq(ov), _ => false, } } - /// Both registry lists, or `None` for lean. Private: callers use the - /// element accessors above. - fn registry(&self) -> Option<(&Validators, &Balances)> { + /// Both registry lists, whichever kind this fork uses, or `None` for + /// lean. Private: callers use the element accessors above. + /// + /// Written out as its own match rather than through `dispatch_state!`: + /// that macro inserts one `$body` verbatim into every arm, so it needs + /// every arm's result to be the same Rust type. `Registry` exists + /// precisely because [`Validators`] and [`ProgressiveValidators`] are not + /// that type from gloas on. + fn registry(&self) -> Option> { + match self { + BeaconState::Phase0(s) => Some(Registry::Bounded(&s.validators, &s.balances)), + BeaconState::Altair(s) => Some(Registry::Bounded(&s.validators, &s.balances)), + BeaconState::Bellatrix(s) => Some(Registry::Bounded(&s.validators, &s.balances)), + BeaconState::Capella(s) => Some(Registry::Bounded(&s.validators, &s.balances)), + BeaconState::Deneb(s) => Some(Registry::Bounded(&s.validators, &s.balances)), + BeaconState::Electra(s) => Some(Registry::Bounded(&s.validators, &s.balances)), + BeaconState::Fulu(s) => Some(Registry::Bounded(&s.validators, &s.balances)), + BeaconState::Gloas(s) => Some(Registry::Progressive(&s.validators, &s.balances)), + BeaconState::Lean(_) => None, + } + } + + /// [`Self::registry`], mutably. Kept separate rather than folded into one + /// generic helper: the borrow checker needs the `&mut self` match here to + /// be visibly distinct from the `&self` one above. + fn registry_mut(&mut self) -> Option> { match self { + BeaconState::Phase0(s) => { + Some(RegistryMut::Bounded(&mut s.validators, &mut s.balances)) + } + BeaconState::Altair(s) => { + Some(RegistryMut::Bounded(&mut s.validators, &mut s.balances)) + } + BeaconState::Bellatrix(s) => { + Some(RegistryMut::Bounded(&mut s.validators, &mut s.balances)) + } + BeaconState::Capella(s) => { + Some(RegistryMut::Bounded(&mut s.validators, &mut s.balances)) + } + BeaconState::Deneb(s) => Some(RegistryMut::Bounded(&mut s.validators, &mut s.balances)), + BeaconState::Electra(s) => { + Some(RegistryMut::Bounded(&mut s.validators, &mut s.balances)) + } + BeaconState::Fulu(s) => Some(RegistryMut::Bounded(&mut s.validators, &mut s.balances)), + BeaconState::Gloas(s) => { + Some(RegistryMut::Progressive(&mut s.validators, &mut s.balances)) + } BeaconState::Lean(_) => None, - _ => Some(dispatch_state!(self, "registry", |state| ( - &state.validators, - &state.balances - ))), } } @@ -662,8 +781,11 @@ impl BeaconState { /// /// [`apply_pending_mutations`]: BeaconState::apply_pending_mutations pub fn has_pending_mutations(&self) -> bool { - self.registry() - .is_some_and(|(v, b)| v.has_pending_updates() || b.has_pending_updates()) + match self.registry() { + Some(Registry::Bounded(v, b)) => v.has_pending_updates() || b.has_pending_updates(), + Some(Registry::Progressive(v, b)) => v.has_pending_updates() || b.has_pending_updates(), + None => false, + } } /// Makes this state's tree-backed fields share every unchanged subtree @@ -671,19 +793,32 @@ impl BeaconState { /// of the registry. The state's contents do not change, only which /// allocations back them. /// - /// Works across forks, since `validators` and `balances` have one type in - /// every fork. A no-op if either state is lean. + /// Works across every fork that shares a registry list kind with `base`, + /// since `List::rebase_on` and `ProgressiveList::rebase_on` each take a + /// `&Self`, not some common trait object. A no-op if either state is + /// lean, or if the two disagree on list kind (a gloas state rebased onto + /// a pre-gloas one, or vice versa): that pairing only happens across the + /// fork boundary itself, where there is no shared tree to reuse anyway. pub fn rebase_on(&mut self, base: &BeaconState) { - let Some((base_validators, base_balances)) = base.registry() else { + let Some(base_registry) = base.registry() else { return; }; - if matches!(self, BeaconState::Lean(_)) { - return; + match (self.registry_mut(), base_registry) { + (Some(RegistryMut::Bounded(v, b)), Registry::Bounded(bv, bb)) => { + v.rebase_on(bv); + b.rebase_on(bb); + } + (Some(RegistryMut::Progressive(v, b)), Registry::Progressive(bv, bb)) => { + v.rebase_on(bv); + b.rebase_on(bb); + } + // Self is lean (no registry at all), or the two states disagree + // on list kind: exactly the fork-boundary case this function's + // own doc names as a no-op. + (None, _) + | (Some(RegistryMut::Bounded(..)), Registry::Progressive(..)) + | (Some(RegistryMut::Progressive(..)), Registry::Bounded(..)) => {} } - dispatch_state!(self, "rebase_on", |state| { - state.validators.rebase_on(base_validators); - state.balances.rebase_on(base_balances); - }) } /// The randao mix for `epoch`, which the specification indexes modulo the @@ -696,14 +831,17 @@ impl BeaconState { /// The withdrawal sweep's cursor: how many withdrawals the chain has ever /// made, and which validator the next sweep resumes from. /// - /// Both exist from capella on, so they cannot join + /// Both exist from capella on, gloas included, so they cannot join /// `shared_state_accessors`' fork-invariant lists, and they are read /// through here rather than through a per-fork projection to a concrete /// state struct because the sweep that reads them is genuinely shared: deneb /// reuses capella's `get_expected_withdrawals` unchanged, and a projection /// returning `&capella::BeaconState` cannot serve a deneb state at all. That /// mistake was made once here and cost a runtime `UnsupportedForFork` on - /// every deneb block carrying a withdrawal. + /// every deneb block carrying a withdrawal. Both fields keep their exact + /// pre-gloas types (`WithdrawalIndex`, `ValidatorIndex`), unlike the three + /// lists [`Self::altair_validator_lists`] reads, which is why gloas joins + /// this accessor's `carried_by` rather than its `absent_from`. pub fn withdrawal_cursor(&self) -> Result<(WithdrawalIndex, ValidatorIndex)> { dispatch_state_from!( self, @@ -712,7 +850,7 @@ impl BeaconState { state.next_withdrawal_index, state.next_withdrawal_validator_index, ), - carried_by: [Capella, Deneb, Electra, Fulu], + carried_by: [Capella, Deneb, Electra, Fulu, Gloas], absent_from: [Phase0, Altair, Bellatrix], ) } @@ -726,7 +864,7 @@ impl BeaconState { &mut state.next_withdrawal_index, &mut state.next_withdrawal_validator_index, ), - carried_by: [Capella, Deneb, Electra, Fulu], + carried_by: [Capella, Deneb, Electra, Fulu, Gloas], absent_from: [Phase0, Altair, Bellatrix], ) } @@ -742,6 +880,15 @@ impl BeaconState { /// Rust type, so a projection typed to return `&altair::BeaconState` can /// only ever answer for an altair state. /// + /// Gloas is in `absent_from` here, not `carried_by`: EIP-7688 makes all + /// three progressive lists, so `gloas::BeaconState`'s own + /// `previous_epoch_participation`, `current_epoch_participation` and + /// `inactivity_scores` are a different Rust type from every other fork's + /// (`ProgressiveList` rather than `SszList`), and this accessor's return + /// type is fixed to the pre-gloas one. A gloas caller reaches them through + /// a per-fork projection instead, the way `withdrawal_cursor`'s doc above + /// explains a projection cannot serve *this* accessor. + /// /// Handed back together rather than one accessor per field for the same /// reason [`Self::altair_validator_lists_mut`] does: the fork condition /// that gates all three is identical, so one match serves every caller, @@ -759,7 +906,7 @@ impl BeaconState { &state.inactivity_scores, ), carried_by: [Altair, Bellatrix, Capella, Deneb, Electra, Fulu], - absent_from: [Phase0], + absent_from: [Phase0, Gloas], ) } @@ -796,27 +943,27 @@ impl BeaconState { &mut state.inactivity_scores, ), carried_by: [Altair, Bellatrix, Capella, Deneb, Electra, Fulu], - absent_from: [Phase0], + absent_from: [Phase0, Gloas], ) } /// The current and next sync committee, by reference. /// /// Both exist from altair on, byte-for-byte the same field in every later - /// fork (see, for instance, bellatrix's own state doc), so they cannot - /// join `shared_state_accessors`' lists, since phase0 predates sync - /// committees entirely. A per-fork projection cannot serve here either: + /// fork including gloas (see, for instance, bellatrix's own state doc), so + /// they cannot join `shared_state_accessors`' lists, since phase0 predates + /// sync committees entirely. A per-fork projection cannot serve here either: /// the beacon STF's `stf::altair::process_sync_aggregate` is called for every - /// fork from altair through fulu (see that function's own documentation), + /// fork from altair on (see that function's own documentation), /// and a projection typed to return `&altair::BeaconState` can only ever answer - /// for an altair state, not for the bellatrix, capella, deneb, electra, or - /// fulu ones the same call site also has to serve. + /// for an altair state, not for the bellatrix, capella, deneb, electra, + /// fulu, or gloas ones the same call site also has to serve. pub fn sync_committees(&self) -> Result<(&altair::SyncCommittee, &altair::SyncCommittee)> { dispatch_state_from!( self, "BeaconState::sync_committees", |state| (&state.current_sync_committee, &state.next_sync_committee), - carried_by: [Altair, Bellatrix, Capella, Deneb, Electra, Fulu], + carried_by: [Altair, Bellatrix, Capella, Deneb, Electra, Fulu, Gloas], absent_from: [Phase0], ) } @@ -839,7 +986,7 @@ impl BeaconState { &mut state.current_sync_committee, &mut state.next_sync_committee, ), - carried_by: [Altair, Bellatrix, Capella, Deneb, Electra, Fulu], + carried_by: [Altair, Bellatrix, Capella, Deneb, Electra, Fulu, Gloas], absent_from: [Phase0], ) } @@ -1027,6 +1174,7 @@ pub enum SignedBeaconBlock { /// Fulu's block. See the enum doc for why this wraps /// [`electra::SignedBeaconBlock`] instead of a `fulu` type. Fulu(electra::SignedBeaconBlock), + Gloas(gloas::SignedBeaconBlock), /// The Lean consensus protocol's block. /// @@ -1067,9 +1215,18 @@ impl SignedBeaconBlock { /// /// Named arms rather than a catch-all `_`, so a fork added to the enum /// breaks this match instead of silently defaulting to "no payload". + /// + /// `None` for [`Self::Gloas`] too, for a different reason than the + /// pre-merge forks: ePBS (EIP-7732) removes the payload from the block + /// entirely, so a gloas block carries only a builder's *bid* on a payload + /// (`signed_execution_payload_bid`), not the payload itself. The block's + /// own EL hash depends on whether that payload is later revealed and + /// attested available, which this accessor, reading only the block, + /// cannot answer. Part B replaces this once the payload envelope and + /// availability tracking exist. pub fn execution_block_hash(&self) -> Option { match self { - Self::Phase0(_) | Self::Altair(_) | Self::Lean(_) => None, + Self::Phase0(_) | Self::Altair(_) | Self::Gloas(_) | Self::Lean(_) => None, Self::Bellatrix(block) => Some(block.message.body.execution_payload.block_hash), Self::Capella(block) => Some(block.message.body.execution_payload.block_hash), Self::Deneb(block) => Some(block.message.body.execution_payload.block_hash), @@ -1084,6 +1241,14 @@ impl SignedBeaconBlock { /// /// The one body field `beacon_block` gossip validation bounds before it /// consults any state. + /// + /// ePBS (EIP-7732) moves the list out of the body directly: a gloas + /// block's commitments live on the builder's bid + /// (`signed_execution_payload_bid.message.blob_kzg_commitments`), not on + /// a `blob_kzg_commitments` field of the body itself. Unlike + /// [`Self::execution_block_hash`], the block still carries this count on + /// its own, so it is read from the bid rather than answered with a + /// placeholder. pub fn blob_kzg_commitment_count(&self) -> usize { match self { Self::Phase0(_) @@ -1095,16 +1260,27 @@ impl SignedBeaconBlock { Self::Electra(block) | Self::Fulu(block) => { block.message.body.blob_kzg_commitments.len() } + Self::Gloas(block) => block + .message + .body + .signed_execution_payload_bid + .message + .blob_kzg_commitments + .len(), } } /// This block's execution payload timestamp, if it carries a payload. /// /// `None` before bellatrix, for the same reason as - /// [`Self::execution_block_hash`]. + /// [`Self::execution_block_hash`], and `None` for [`Self::Gloas`] for the + /// reason given there too: the builder's bid + /// (`signed_execution_payload_bid.message`) carries no `timestamp` field, + /// since ePBS's payload envelope, not the block, is what a timestamp + /// would describe. pub fn execution_payload_timestamp(&self) -> Option { match self { - Self::Phase0(_) | Self::Altair(_) | Self::Lean(_) => None, + Self::Phase0(_) | Self::Altair(_) | Self::Gloas(_) | Self::Lean(_) => None, Self::Bellatrix(block) => Some(block.message.body.execution_payload.timestamp), Self::Capella(block) => Some(block.message.body.execution_payload.timestamp), Self::Deneb(block) => Some(block.message.body.execution_payload.timestamp), @@ -1129,6 +1305,7 @@ impl SignedBeaconBlock { SignedBeaconBlock::Deneb(_) => ForkName::Deneb, SignedBeaconBlock::Electra(_) => ForkName::Electra, SignedBeaconBlock::Fulu(_) => ForkName::Fulu, + SignedBeaconBlock::Gloas(_) => ForkName::Gloas, SignedBeaconBlock::Lean(_) => ForkName::Lean, } } @@ -1162,6 +1339,9 @@ impl SignedBeaconBlock { ForkName::Fulu => Ok(SignedBeaconBlock::Fulu( electra::SignedBeaconBlock::from_ssz_bytes(bytes)?, )), + ForkName::Gloas => Ok(SignedBeaconBlock::Gloas( + gloas::SignedBeaconBlock::from_ssz_bytes(bytes)?, + )), ForkName::Lean => Ok(SignedBeaconBlock::Lean( crate::block::SignedBlock::from_ssz_bytes(bytes)?, )), @@ -1379,6 +1559,101 @@ mod tests { }) } + /// A Gloas state with `count` validators, mirroring [`small_fulu_state`] + /// but for gloas's progressive registry + /// ([`ProgressiveValidators`]/[`ProgressiveBalances`]) and its own + /// builder/payload fields, each at an all-zero or empty placeholder. + /// Good enough for the registry accessor tests below, which touch + /// `validators`/`balances` only. + fn small_gloas_state(count: usize) -> BeaconState { + let validators: Vec = (0..count) + .map(|_| Validator { + effective_balance: preset::MAX_EFFECTIVE_BALANCE, + activation_eligibility_epoch: 0, + activation_epoch: 0, + exit_epoch: constants::FAR_FUTURE_EPOCH, + withdrawable_epoch: constants::FAR_FUTURE_EPOCH, + ..Default::default() + }) + .collect(); + let zero_root_vector = || -> BlockRoots { + vec![Root::ZERO; preset::SLOTS_PER_HISTORICAL_ROOT] + .try_into() + .expect("the vector is built at its exact length") + }; + let empty_sync_committee = || altair::SyncCommittee { + pubkeys: vec![Default::default(); preset::SYNC_COMMITTEE_SIZE] + .try_into() + .expect("built at exactly SYNC_COMMITTEE_SIZE"), + aggregate_pubkey: Default::default(), + }; + let empty_ptc: gloas::PayloadTimelinessCommittee = vec![0u64; preset::PTC_SIZE] + .try_into() + .expect("built at exactly PTC_SIZE"); + + BeaconState::Gloas(gloas::BeaconState { + genesis_time: 0, + genesis_validators_root: Root::ZERO, + slot: preset::SLOTS_PER_EPOCH, + fork: Default::default(), + latest_block_header: Default::default(), + block_roots: zero_root_vector(), + state_roots: zero_root_vector(), + historical_roots: Default::default(), + eth1_data: Default::default(), + eth1_data_votes: Default::default(), + eth1_deposit_index: 0, + validators: validators.into(), + balances: vec![preset::MAX_EFFECTIVE_BALANCE; count].into(), + randao_mixes: vec![Bytes32::ZERO; preset::EPOCHS_PER_HISTORICAL_VECTOR] + .try_into() + .expect("the vector is built at its exact length"), + slashings: vec![0; preset::EPOCHS_PER_SLASHINGS_VECTOR] + .try_into() + .expect("the vector is built at its exact length"), + previous_epoch_participation: vec![0; count].into(), + current_epoch_participation: vec![0; count].into(), + justification_bits: Default::default(), + previous_justified_checkpoint: Default::default(), + current_justified_checkpoint: Default::default(), + finalized_checkpoint: Default::default(), + inactivity_scores: vec![0; count].into(), + current_sync_committee: empty_sync_committee(), + next_sync_committee: empty_sync_committee(), + latest_block_hash: ExecutionBlockHash::ZERO, + next_withdrawal_index: 0, + next_withdrawal_validator_index: 0, + historical_summaries: Default::default(), + deposit_requests_start_index: constants::UNSET_DEPOSIT_REQUESTS_START_INDEX, + deposit_balance_to_consume: 0, + exit_balance_to_consume: 0, + earliest_exit_epoch: 0, + consolidation_balance_to_consume: 0, + earliest_consolidation_epoch: 0, + pending_deposits: Default::default(), + pending_partial_withdrawals: Default::default(), + pending_consolidations: Default::default(), + proposer_lookahead: vec![0; preset::PROPOSER_LOOKAHEAD_LENGTH] + .try_into() + .expect("the vector is built at its exact length"), + builders: Default::default(), + next_withdrawal_builder_index: 0, + execution_payload_availability: Default::default(), + builder_pending_payments: vec![ + gloas::BuilderPendingPayment::default(); + preset::BUILDER_PENDING_PAYMENTS_LENGTH + ] + .try_into() + .expect("the vector is built at its exact length"), + builder_pending_withdrawals: Default::default(), + latest_execution_payload_bid: Default::default(), + payload_expected_withdrawals: Default::default(), + ptc_window: vec![empty_ptc; preset::PTC_WINDOW_LENGTH] + .try_into() + .expect("the vector is built at its exact length"), + }) + } + #[test] fn registry_accessors_read_and_write_elements() { let mut state = small_fulu_state(3); // 3 validators, balances 32 ETH @@ -1417,6 +1692,83 @@ mod tests { assert_eq!(state.iter_validators().nth(2).unwrap().effective_balance, 1); } + /// Same coverage as `registry_accessors_read_and_write_elements`, on a + /// gloas state instead of a fulu one: the registry there is a + /// [`ethlambda_ssz_tree::ProgressiveList`] (`Registry::Progressive`), not + /// a bounded [`List`], and this is what proves the element accessors + /// take that arm too rather than only ever exercising + /// `Registry::Bounded`. + #[test] + fn gloas_registry_accessors_read_and_write_elements() { + let mut state = small_gloas_state(3); // 3 validators, balances 32 ETH + assert_eq!(state.validator_count(), 3); + assert_eq!(state.iter_validators().len(), 3); + assert_eq!( + state.iter_balances().sum::(), + 3 * preset::MAX_EFFECTIVE_BALANCE + ); + + *state.balance_mut(1).unwrap() += 5; + assert_eq!(state.balance(1).unwrap(), preset::MAX_EFFECTIVE_BALANCE + 5); + + let validator = state.validator(0).unwrap().clone(); + state.push_validator(validator, 7).unwrap(); + assert_eq!(state.validator_count(), 4); + assert_eq!(state.balance(3).unwrap(), 7); + assert!(state.balance_mut(4).is_err()); + assert!(state.validator(4).is_err()); + + assert_eq!( + state.iter_balances().collect::>(), + vec![ + preset::MAX_EFFECTIVE_BALANCE, + preset::MAX_EFFECTIVE_BALANCE + 5, + preset::MAX_EFFECTIVE_BALANCE, + 7, + ] + ); + + state.validator_mut(2).unwrap().effective_balance = 1; + assert_eq!(state.iter_validators().nth(2).unwrap().effective_balance, 1); + } + + /// [`BeaconState::rebase_on`] and [`BeaconState::validators_ptr_eq`] + /// across two gloas states: the `Registry::Progressive` arm of both, + /// which nothing above exercises. Two independently built states with + /// equal content start out backed by different allocations + /// (`validators_ptr_eq` false); `rebase_on` shares every subtree the + /// content agrees on, so `validators_ptr_eq` becomes true afterward, the + /// same invariant the storage crate's tests rely on for a decoded state + /// rebased onto its resident parent. + #[test] + fn gloas_registry_rebases_onto_an_equal_gloas_parent() { + let base = small_gloas_state(3); + let mut derived = small_gloas_state(3); + assert!(!derived.validators_ptr_eq(&base)); + + derived.rebase_on(&base); + assert!(derived.validators_ptr_eq(&base)); + } + + /// A fulu state and a gloas state disagree on registry list kind + /// (`Registry::Bounded` vs `Registry::Progressive`), which is exactly + /// the fork boundary [`BeaconState::rebase_on`]'s own documentation + /// describes as a no-op: there is no shared tree to reuse between a + /// bounded and a progressive registry. Pins that the mismatch is + /// silently inert rather than a panic, and changes nothing about either + /// state. + #[test] + fn a_fulu_and_a_gloas_registry_do_not_rebase_across_the_fork_boundary() { + let fulu = small_fulu_state(3); + let mut gloas = small_gloas_state(3); + let before = gloas.clone(); + + gloas.rebase_on(&fulu); + + assert_eq!(gloas, before, "a mixed-kind rebase must be a no-op"); + assert!(!gloas.validators_ptr_eq(&fulu)); + } + /// Single-validator lean state. The pubkeys are placeholders; nothing here /// verifies a signature. fn lean_state(genesis_time: u64, attestation_pubkey: u8) -> crate::state::State { diff --git a/crates/common/types/src/beacon/fork.rs b/crates/common/types/src/beacon/fork.rs index 9e6192b4e..be895bd6d 100644 --- a/crates/common/types/src/beacon/fork.rs +++ b/crates/common/types/src/beacon/fork.rs @@ -40,7 +40,7 @@ const BEACON_SNAPSHOT_INTERVAL: u64 = preset::SLOTS_PER_EPOCH; /// a target of the beacon STF's `upgrade`-style traversal. See /// [`ForkName::ALL`]. /// -/// Forks after fulu exist upstream but are out of scope for this crate. +/// Forks after gloas exist upstream but are out of scope for this crate. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum ForkName { Phase0, @@ -50,6 +50,7 @@ pub enum ForkName { Deneb, Electra, Fulu, + Gloas, /// The Lean consensus protocol, which this repository implements alongside /// the Beacon Chain. Not a Beacon Chain fork, and not in [`ForkName::ALL`]. Lean, @@ -61,9 +62,9 @@ impl ForkName { /// [`ForkName::Lean`] is not here: it is not a Beacon Chain fork. Because /// `parse`, `previous`, `next`, and the spec-fixture harness all search this /// array, its absence is what makes `parse("lean")` return `None`, keeps - /// `Fulu.next()` at `None`, and stops any fixture directory from resolving + /// `Gloas.next()` at `None`, and stops any fixture directory from resolving /// to a lean case. - pub const ALL: [ForkName; 7] = [ + pub const ALL: [ForkName; 8] = [ ForkName::Phase0, ForkName::Altair, ForkName::Bellatrix, @@ -71,6 +72,7 @@ impl ForkName { ForkName::Deneb, ForkName::Electra, ForkName::Fulu, + ForkName::Gloas, ]; /// The lowercase name the specification and its fixture paths use. @@ -83,6 +85,7 @@ impl ForkName { ForkName::Deneb => "deneb", ForkName::Electra => "electra", ForkName::Fulu => "fulu", + ForkName::Gloas => "gloas", ForkName::Lean => "lean", } } @@ -114,8 +117,8 @@ impl ForkName { /// deriving the on-disk tag from it too would mean a reorder made for the /// ordering's sake silently reinterpreted every state already written. /// - /// [`ForkName::Lean`] takes 255 rather than 7 so that the beacon forks after - /// fulu can keep taking the next free value as they land. + /// [`ForkName::Lean`] takes 255 rather than 8 so that the beacon forks after + /// gloas can keep taking the next free value as they land. pub const fn selector(self) -> u8 { match self { ForkName::Phase0 => 0, @@ -125,6 +128,7 @@ impl ForkName { ForkName::Deneb => 4, ForkName::Electra => 5, ForkName::Fulu => 6, + ForkName::Gloas => 7, ForkName::Lean => 255, } } @@ -148,7 +152,8 @@ impl ForkName { | ForkName::Capella | ForkName::Deneb | ForkName::Electra - | ForkName::Fulu => BEACON_SNAPSHOT_INTERVAL, + | ForkName::Fulu + | ForkName::Gloas => BEACON_SNAPSHOT_INTERVAL, } } @@ -165,6 +170,7 @@ impl ForkName { 4 => Some(ForkName::Deneb), 5 => Some(ForkName::Electra), 6 => Some(ForkName::Fulu), + 7 => Some(ForkName::Gloas), 255 => Some(ForkName::Lean), _ => None, } @@ -193,13 +199,23 @@ mod tests { for fork in ForkName::ALL { assert_eq!(ForkName::parse(fork.as_str()), Some(fork)); } - assert_eq!(ForkName::parse("gloas"), None); + assert_eq!(ForkName::parse("gloas"), Some(ForkName::Gloas)); + assert_eq!(ForkName::parse("heze"), None); + } + + #[test] + fn gloas_is_the_last_beacon_fork() { + assert_eq!(ForkName::Fulu.next(), Some(ForkName::Gloas)); + // Also guards the reason Lean is kept out of ALL: adding it there + // would make this None into Some(Lean) and let `upgrade` walk off + // the end. + assert_eq!(ForkName::Gloas.next(), None); } #[test] fn neighbours_terminate_at_the_ends() { assert_eq!(ForkName::Phase0.previous(), None); - assert_eq!(ForkName::Fulu.next(), None); + assert_eq!(ForkName::Gloas.next(), None); assert_eq!(ForkName::Altair.previous(), Some(ForkName::Phase0)); assert_eq!(ForkName::Altair.next(), Some(ForkName::Bellatrix)); } @@ -223,13 +239,6 @@ mod tests { assert_eq!(ForkName::Lean.previous(), None); } - #[test] - fn fulu_is_still_the_last_beacon_fork() { - // Guards the reason Lean is kept out of ALL: adding it there would make - // this None into Some(Lean) and let `upgrade` walk off the end. - assert_eq!(ForkName::Fulu.next(), None); - } - #[test] fn lean_has_a_name() { assert_eq!(ForkName::Lean.as_str(), "lean"); @@ -270,9 +279,10 @@ mod tests { // derived from the variant order, which the derived Ord already owns. assert_eq!(ForkName::Phase0.selector(), 0); assert_eq!(ForkName::Fulu.selector(), 6); - // Lean sits at the top of the byte range so gloas and heze can keep - // taking the next free value after fulu. + assert_eq!(ForkName::Gloas.selector(), 7); + // Lean sits at the top of the byte range so heze can keep taking the + // next free value after gloas. assert_eq!(ForkName::Lean.selector(), 255); - assert_eq!(ForkName::from_selector(7), None); + assert_eq!(ForkName::from_selector(8), None); } } diff --git a/crates/common/types/src/beacon/preset.rs b/crates/common/types/src/beacon/preset.rs index c7ab9bbf8..7f8600125 100644 --- a/crates/common/types/src/beacon/preset.rs +++ b/crates/common/types/src/beacon/preset.rs @@ -1178,7 +1178,8 @@ pub mod retuned { | ForkName::Capella | ForkName::Deneb | ForkName::Electra - | ForkName::Fulu => super::PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX, + | ForkName::Fulu + | ForkName::Gloas => super::PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX, ForkName::Lean => lean_fork_unreachable("proportional_slashing_multiplier"), } } @@ -1200,7 +1201,9 @@ pub mod retuned { ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb => { super::MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX } - ForkName::Electra | ForkName::Fulu => super::MIN_SLASHING_PENALTY_QUOTIENT_ELECTRA, + ForkName::Electra | ForkName::Fulu | ForkName::Gloas => { + super::MIN_SLASHING_PENALTY_QUOTIENT_ELECTRA + } ForkName::Lean => lean_fork_unreachable("min_slashing_penalty_quotient"), } } @@ -1214,7 +1217,9 @@ pub mod retuned { /// small one. pub fn whistleblower_reward_quotient(fork: ForkName) -> u64 { match fork { - ForkName::Electra | ForkName::Fulu => super::WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA, + ForkName::Electra | ForkName::Fulu | ForkName::Gloas => { + super::WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA + } ForkName::Phase0 | ForkName::Altair | ForkName::Bellatrix @@ -1239,7 +1244,8 @@ pub mod retuned { | ForkName::Capella | ForkName::Deneb | ForkName::Electra - | ForkName::Fulu => super::INACTIVITY_PENALTY_QUOTIENT_BELLATRIX, + | ForkName::Fulu + | ForkName::Gloas => super::INACTIVITY_PENALTY_QUOTIENT_BELLATRIX, ForkName::Lean => lean_fork_unreachable("inactivity_penalty_quotient"), } } @@ -1275,6 +1281,7 @@ pub mod retuned { ForkName::Deneb, ForkName::Electra, ForkName::Fulu, + ForkName::Gloas, ]; assert_eq!( @@ -1324,7 +1331,7 @@ pub mod retuned { "{fork} must use bellatrix's penalty divisor", ); } - for fork in [ForkName::Electra, ForkName::Fulu] { + for fork in [ForkName::Electra, ForkName::Fulu, ForkName::Gloas] { assert_eq!( min_slashing_penalty_quotient(fork), MIN_SLASHING_PENALTY_QUOTIENT_ELECTRA, @@ -1345,7 +1352,7 @@ pub mod retuned { "{fork} predates electra's whistleblower reward change", ); } - for fork in [ForkName::Electra, ForkName::Fulu] { + for fork in [ForkName::Electra, ForkName::Fulu, ForkName::Gloas] { assert_eq!( whistleblower_reward_quotient(fork), WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA, diff --git a/crates/net/p2p/src/beacon/decode.rs b/crates/net/p2p/src/beacon/decode.rs index 0be81c439..54a223630 100644 --- a/crates/net/p2p/src/beacon/decode.rs +++ b/crates/net/p2p/src/beacon/decode.rs @@ -31,6 +31,7 @@ use ethlambda_types::beacon::containers::{ use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::preset; use ethlambda_types::beacon::primitives::Slot; +use ethlambda_types::time::unix_now_ms; use libssz::SszDecode as _; use super::topics; @@ -93,6 +94,12 @@ pub enum DecodeError { Truncated, /// SSZ rejected the payload for the fork the slot selected. Ssz, + /// The message's own fork (gloas, today) has no container modeled here + /// yet. Distinct from [`Self::Ssz`] on purpose: an honest peer on a fork + /// this build cannot decode is not sending malformed bytes, so a caller + /// mapping this to a gossipsub verdict must not score it as `Reject` the + /// way [`Self::Ssz`] is; see `gossipsub::handler`'s callers. + UnsupportedFork, } impl std::fmt::Display for DecodeError { @@ -101,6 +108,7 @@ impl std::fmt::Display for DecodeError { Self::UnknownTopic => write!(f, "unsubscribed topic"), Self::Truncated => write!(f, "payload truncated before the slot"), Self::Ssz => write!(f, "ssz decode failed"), + Self::UnsupportedFork => write!(f, "message's fork has no container modeled yet"), } } } @@ -175,6 +183,25 @@ pub fn fork_at_slot(config: &Config, slot: Slot) -> ForkName { config.fork_at_epoch(slot / preset::SLOTS_PER_EPOCH) } +/// The fork this node's own wall clock says is active right now. +/// +/// [`decode_data_column_sidecar`] has no slot to look a fork up by the way +/// [`fork_at_slot`] does: only fulu's container is modeled, so nothing reads +/// a slot out of the bytes first. A caller on the gossip path uses this to +/// approximate, once that decode has already failed, whether the failure is +/// this build's own gap rather than the sender's fault. An approximation +/// only, since gossip topic subscriptions are frozen at startup and this +/// reads the live clock instead of whatever fork the topic was actually +/// built for; see [`decode_data_column_sidecar`]'s own doc and its caller in +/// `gossipsub::handler::triage_data_column`. +pub fn current_fork(config: &Config) -> ForkName { + let slot = unix_now_ms() + .saturating_sub(config.genesis_time_ms()) + .checked_div(config.slot_duration_ms) + .unwrap_or(0); + fork_at_slot(config, slot) +} + /// Decode a `beacon_block` payload, at the fork its own slot names. /// /// Separate from [`decode_gossip`] because the two topics worth logging in @@ -190,6 +217,15 @@ pub fn decode_block(config: &Config, bytes: &[u8]) -> Result Result { fulu::DataColumnSidecar::from_ssz_bytes(bytes).map_err(|_| DecodeError::Ssz) } @@ -211,6 +247,11 @@ pub fn decode_aggregate_and_proof( | ForkName::Capella | ForkName::Deneb => phase0::SignedAggregateAndProof::from_ssz_bytes(bytes) .map(SignedAggregateAndProof::Phase0), + // Gloas's own `AggregateAndProof` has no modeled variant here yet + // (its attestation is progressive-list-shaped, EIP-7549 continued), + // so this is refused rather than mis-decoded as electra's. + // `UnsupportedFork`, not `Ssz`: the sender did nothing wrong. + ForkName::Gloas => return Err(DecodeError::UnsupportedFork), ForkName::Lean => { unreachable!("fork_at_slot never returns Lean: it is absent from ForkName::ALL") } @@ -290,6 +331,9 @@ pub fn decode_gossip( | ForkName::Deneb => { phase0::AttesterSlashing::from_ssz_bytes(bytes).map(AttesterSlashing::Phase0) } + // Gloas's own `AttesterSlashing` has no modeled variant here + // yet; see `decode_aggregate_and_proof`'s matching arm. + ForkName::Gloas => return Err(DecodeError::UnsupportedFork), ForkName::Lean => { unreachable!("fork_at_slot never returns Lean: it is absent from ForkName::ALL") } diff --git a/crates/net/p2p/src/gossipsub/handler.rs b/crates/net/p2p/src/gossipsub/handler.rs index a1797a813..4e6f28988 100644 --- a/crates/net/p2p/src/gossipsub/handler.rs +++ b/crates/net/p2p/src/gossipsub/handler.rs @@ -13,6 +13,7 @@ use ethlambda_types::{ ShortRoot, attestation::{SignedAggregatedAttestation, SignedAttestation}, beacon::containers::{SignedBeaconBlock, electra::SingleAttestation}, + beacon::fork::ForkName, block::SignedBlock, primitives::HashTreeRoot as _, time::unix_now_ms, @@ -290,11 +291,36 @@ fn triage_block(server: &P2PServer, wire: &BeaconWire, payload: &[u8]) -> Dispat /// Decode a data column sidecar and run its cheap gossip checks. Same shape as /// [`triage_block`]. +/// +/// Decodes first, unlike the clock-based gate this used to be: gossipsub +/// topic subscriptions are frozen at startup (`build_swarm` does not +/// resubscribe as a fork boundary is crossed), so a node running across the +/// gloas boundary stays on its fulu-digest topic the whole time, where a +/// late but perfectly legitimate fulu sidecar can still legally arrive. A +/// clock check ahead of the decode would drop that one too, mistaking it for +/// gloas-shaped just because the clock has moved on. Only on a decode +/// failure does the clock matter, and only as an approximation: this node's +/// *topic* fork (whichever one gossip actually subscribed under at startup) +/// is not threaded down to this handler today, so [`beacon_decode::current_fork`] +/// (the wall clock) stands in for it. That is exactly backwards for a node +/// stuck on stale fulu topics past the boundary, the same case this doc +/// opens with: a genuinely malformed fulu sidecar arriving there reads as +/// `Ignore` instead of `Reject`, since the clock alone cannot tell "stale +/// topic, bad bytes" apart from "current topic, gloas-shaped bytes". Safe +/// either way, since `Ignore` never down-scores a peer; a future change that +/// carries the topic's own fork into `BeaconWire` (or wherever else carries +/// the fork digest to this handler) can make this exact instead of merely +/// safe. fn triage_data_column(server: &P2PServer, payload: &[u8], subnet_id: u64) -> Dispatch { const KIND: &str = beacon_topics::DATA_COLUMN_SIDECAR_KIND; let sidecar = match beacon_decode::decode_data_column_sidecar(payload) { Ok(sidecar) => sidecar, Err(err) => { + let config = server.store.config(); + if beacon_decode::current_fork(&config) == ForkName::Gloas { + metrics::inc_beacon_gossip(KIND, "unsupported_fork"); + return Dispatch::Report(Outcome::Ignore(IgnoreReason::UnsupportedFork)); + } metrics::inc_beacon_gossip(KIND, "decode_failed"); debug!(?err, "Dropping an undecodable data column sidecar"); return Dispatch::Report(Outcome::Reject(RejectReason::Decode)); @@ -326,6 +352,13 @@ fn triage_aggregate( const KIND: &str = beacon_topics::BEACON_AGGREGATE_AND_PROOF; let aggregate = match beacon_decode::decode_aggregate_and_proof(&wire.config, payload) { Ok(aggregate) => aggregate, + // `UnsupportedFork` is not the sender's fault (an honest gloas peer + // sends exactly this once this node's own clock reaches gloas), so it + // must not score like every other decode failure does. + Err(beacon_decode::DecodeError::UnsupportedFork) => { + metrics::inc_beacon_gossip(KIND, "unsupported_fork"); + return Dispatch::Report(Outcome::Ignore(IgnoreReason::UnsupportedFork)); + } Err(err) => { metrics::inc_beacon_gossip(KIND, "decode_failed"); debug!(kind = KIND, %err, bytes = payload.len(), "Beacon gossip decode failed"); @@ -447,6 +480,12 @@ fn triage_other(wire: &BeaconWire, kind: &str, payload: &[u8]) -> Dispatch { ); Outcome::Ignore(IgnoreReason::NoConsumer) } + // See `triage_aggregate`'s matching arm for why this scores as + // `Ignore` rather than `Reject`. + Err(beacon_decode::DecodeError::UnsupportedFork) => { + metrics::inc_beacon_gossip(kind, "unsupported_fork"); + Outcome::Ignore(IgnoreReason::UnsupportedFork) + } Err(err) => { metrics::inc_beacon_gossip(kind, "decode_failed"); debug!(kind, %err, bytes = payload.len(), "Beacon gossip decode failed"); diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 0b7b02ff1..0bfefb7e6 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -172,7 +172,11 @@ const KEY_ANCHOR_SLOT: &[u8] = b"anchor_slot"; /// /// 4 added `PRESET_BASE` and `CONFIG_NAME` to `Config`, at the front of its /// encoding, for the same reason and with the same consequence as 3. -pub const DB_VERSION: u64 = 4; +/// +/// 5 widened `Config` with the gloas schedule, timing and churn keys +/// (`max_request_payloads` was already claimed before this version bump), so a +/// directory written by version 4 decodes into the wrong fields. +pub const DB_VERSION: u64 = 5; /// The consensus protocol a data directory holds. /// diff --git a/crates/validator/src/beacon_node/block_contents.rs b/crates/validator/src/beacon_node/block_contents.rs index 4ea71218e..7dcb40197 100644 --- a/crates/validator/src/beacon_node/block_contents.rs +++ b/crates/validator/src/beacon_node/block_contents.rs @@ -230,11 +230,16 @@ impl ProducedBlock { }, }) } + // ePBS (EIP-7732) replaces the embedded payload this client + // decodes here with a builder's bid, a shape neither + // `Contents` variant models yet, so this is refused the same + // way a pre-electra fork is rather than mis-decoded. ForkName::Phase0 | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb + | ForkName::Gloas | ForkName::Lean => Err(Error::InconsistentResponse(format!( "beacon node produced a block for fork {}, which this client does not propose \ under; electra is the earliest supported", diff --git a/crates/validator/src/beacon_node/http.rs b/crates/validator/src/beacon_node/http.rs index 0eb0fa08a..dee38b01d 100644 --- a/crates/validator/src/beacon_node/http.rs +++ b/crates/validator/src/beacon_node/http.rs @@ -598,11 +598,15 @@ impl BeaconNodeApi for HttpBeaconNode { // so a fork added after fulu is not silently waved through as // electra-shaped. match fork { + // Gloas's own attestation shape is not this client's electra one + // (EIP-7549 continues to change under EIP-7688), so it is + // refused by name too rather than mis-decoded. ForkName::Phase0 | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb + | ForkName::Gloas | ForkName::Lean => { return Err(Error::InconsistentResponse(format!( "node produced a {} aggregate, which this client does not publish; electra \ diff --git a/docs/beacon_stf.md b/docs/beacon_stf.md index 6b922014c..ac4b74817 100644 --- a/docs/beacon_stf.md +++ b/docs/beacon_stf.md @@ -425,14 +425,18 @@ runs two runners mainnet does not: `genesis`'s `initialization` and `validity`. The release ships no mainnet `genesis` fixtures, so that whole runner is gated behind the `preset-minimal` feature (`tests/spec/genesis.rs`). -Nothing is ignored for being unimplemented. Every ignored case is one of two +Nothing is ignored for being unimplemented, with one exception now that +`ForkName::Gloas` exists: every gloas case is individually ignored, by the +same `HIGHEST_IMPLEMENTED_FORK`/`Case::in_scope` gate that would ignore any +other fork past the last one this crate implements (`HIGHEST_IMPLEMENTED_FORK` +stays `Fulu` through Part A). Every other ignored case is one of two deliberate exclusions: - `LightClient*` containers under `ssz_static`, 5 container types across altair through fulu. The light-client sync protocol is a different layer from the state transition and fork choice, and is not in this module's scope. -- The `gloas` and `heze` fixture trees, one ignored entry each. See - "Accounting for every fork directory" below. +- The `heze` fixture tree, one ignored entry. See "Accounting for every fork + directory" below. ## Accounting for every fork directory @@ -441,12 +445,14 @@ a name that does not parse is skipped. That skip is silent in a way the `HIGHEST_IMPLEMENTED_FORK` gate is not: the cases never become tests, so they are not counted as ignored either, and nothing in the output says they exist. -The release does ship two such trees. `gloas` and `heze` are the two forks -after fulu, in that order. Between them they hold 3068 mainnet and 21137 -minimal cases, all of which were previously dropped without a trace, which is -the opposite of what this harness promises. +`gloas` no longer takes this silent path: `ForkName::parse("gloas")` succeeds, +so its cases become ordinary tests, individually named and ignored through ordinary +`Case::in_scope` gating, the same as fulu's cases would be if +`HIGHEST_IMPLEMENTED_FORK` were dropped back a fork. `heze`, the one fork after +gloas, is still unparseable and still silently skipped without this section's +own accounting. -So `UNMODELED_FORKS` names them, each reports as one ignored test, and +So `UNMODELED_FORKS` names `heze` alone, it reports as one ignored test, and `fixture_forks/every_directory_is_accounted_for` fails if the tree holds a fork directory that is neither parseable nor listed. A release that adds a fork now forces a decision instead of quietly widening the gap. diff --git a/docs/cli.md b/docs/cli.md index 4b665e167..d9086f3b2 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -163,12 +163,16 @@ of its own the way `node` does: those are read off the resolved network instead of being separate operator input. A `config.yaml` (a directory's, or a built-in network's embedded one) is read permissively: absent keys fall back to mainnet's values, numbers are accepted quoted or bare, and unrecognised keys (on a current -config, the gloas and heze schedule this build cannot process) are dropped +config, the heze schedule this build does not claim) are dropped with one warning line naming each. Its `PRESET_BASE` is checked against the compiled preset; a mismatch is a hard startup error naming the cargo feature -that would fix it. The keys this build runs on compile-time constants for (the -custody and subnet counts, the `MAX_REQUEST_*` limits, `MAX_PAYLOAD_SIZE`, the -snappy message domains and `MAXIMUM_GOSSIP_CLOCK_DISPARITY`) must equal those +that would fix it. Sepolia's config schedules gloas, whose keys this build +does claim but whose state transition it does not implement yet: a separate, +explicit warning at startup names that boundary, since the chain follower +stops tracking the chain there regardless of what parsed cleanly. The keys this +build runs on compile-time constants for (the custody and subnet counts, the +`MAX_REQUEST_*` limits, `MAX_PAYLOAD_SIZE`, the snappy message domains and +`MAXIMUM_GOSSIP_CLOCK_DISPARITY`) must equal those constants, or startup fails naming each key that differs: the node cannot follow a network that sets them otherwise, and `/eth/v1/config/spec` would report values it does not use. For a directory, both checks run before its From a6923f31867e71e5021637c6821c2f42e8e63404 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 25 Sep 2026 10:41:35 -0300 Subject: [PATCH 011/132] fix(beacon): stop retiring the eth1-bridge deposit path a second time in fulu transition/fulu/core/pyspec_tests/transition_with_deposit_request_right_after_fork failed because process_deposit_request set deposit_requests_start_index for a fulu block, differing from the reference implementation's post-state root. v1.7.0-beta.2 gave fulu's beacon-chain.md three "Modified" sections our port never picked up (v1.6.1's fulu spec did not have them, per the tag diff): process_operations, process_deposit_request, and process_pending_deposits all drop the former eth1-bridge deposit mechanism's last bookkeeping. process_deposit_request no longer sets deposit_requests_start_index the first time a request is seen; process_pending_deposits no longer gates a deposit request behind "every eth1-bridge deposit ahead of it has drained" (deposit_requests_start_index compared against eth1_deposit_index). Our fulu block and epoch processing called electra's shared functions for both, so process_deposit_request kept writing the field. Fixing only that write without also dropping process_pending_deposits' gate would have regressed every later fulu deposit request: with the field permanently UNSET_DEPOSIT_REQUESTS_START_INDEX, electra's gate (eth1_deposit_index < deposit_requests_start_index) is true forever, wedging the pending-deposits queue. Both functions, plus process_operations for spec-text fidelity (it reaches the same verdict as electra's shared version on any state that finished the electra transition, but the spec explicitly redefines it), now have their own fulu copies. Two spec-test harness handlers (epoch_processing.rs's "pending_deposits" and operations.rs's "deposit_request") still called electra's function for every fork, including fulu; they passed only because every fixture case that exercises either handler starts with deposit_requests_start_index already set, so electra's gate and index-setting were no-ops for them. Routed both to the fulu functions on ForkName::Fulu, with an explicit arm per fork. Added unit tests exercising the one case the fixtures do not: an UNSET start index plus a deposit-request-sourced pending deposit, processed by fulu's function and stalled forever by electra's. Rebased onto the gloas fork variant (1a6829e4): the two spec-test harness matches this cherry-pick added on ForkName (epoch_processing.rs's "pending_deposits", operations.rs's "deposit_request") now each name a Gloas arm returning Error::UnsupportedForFork, matching every other handler's own gloas refusal in these files. --- .../src/beacon/stf/electra.rs | 27 ++ .../src/beacon/stf/epoch/electra.rs | 40 ++- .../src/beacon/stf/epoch/fulu.rs | 273 +++++++++++++++++- .../src/beacon/stf/epoch/mod.rs | 7 +- .../state_transition/src/beacon/stf/fulu.rs | 210 ++++++++++++-- .../src/beacon/stf/operations.rs | 12 +- .../tests/beacon_spec/epoch_processing.rs | 35 ++- .../tests/beacon_spec/operations.rs | 67 ++++- 8 files changed, 606 insertions(+), 65 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/stf/electra.rs b/crates/blockchain/state_transition/src/beacon/stf/electra.rs index 0be46b258..6a6a2dcfd 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/electra.rs @@ -1126,6 +1126,11 @@ pub fn process_withdrawals( /// [`electra::BeaconBlockBody::execution_requests`] rather than carrying them /// as three separate body fields the way every other operation list is /// carried. +/// +/// Electra's own version. Fulu redefines the deposit-request leg +/// ([`super::fulu::process_deposit_request`]) to drop the former deposit +/// mechanism's bookkeeping, so it keeps a copy of this whole loop +/// ([`super::fulu::process_execution_requests`]) rather than sharing this one. pub fn process_execution_requests( state: &mut BeaconState, requests: &electra::ExecutionRequests, @@ -1161,6 +1166,22 @@ pub fn process_execution_requests( /// [`process_withdrawal_request`] and [`process_consolidation_request`] /// trust the execution layer's own request rather than re-deriving a /// signature for it. +/// +/// Electra's own version, in effect only through this fork. This is the +/// function that *releases* electra's own gate in +/// [`super::epoch::electra::process_pending_deposits`]: setting +/// `deposit_requests_start_index` the first time a request is seen is what +/// eventually turns that gate's `eth1_deposit_index < deposit_requests_start_index` +/// false. By fulu the former deposit mechanism that field tracks is fully +/// retired (`beacon-chain.md`'s "Modified `process_deposit_request`" for that +/// fork), so [`super::fulu::process_deposit_request`] queues the same +/// [`electra::PendingDeposit`] without touching that field at all, and +/// [`super::epoch::fulu::process_pending_deposits`] drops the gate to match. +/// The two changes are a package: pairing fulu's deposit handler (which never +/// sets the field) with electra's still-gated pending-deposits step, rather +/// than fulu's own, would leave the field permanently unset and the gate +/// permanently true, wedging the queue for good; see that function's own +/// doc. pub fn process_deposit_request( state: &mut BeaconState, request: &electra::DepositRequest, @@ -1687,6 +1708,12 @@ pub fn get_execution_requests_list(requests: &electra::ExecutionRequests) -> Vec /// `eth1_deposit_index_limit - state.eth1_deposit_index` cannot underflow: /// the surrounding `if` already established `state.eth1_deposit_index < /// eth1_deposit_index_limit`. +/// +/// Electra's own version. Fulu removes the old deposit-contract path outright +/// (`beacon-chain.md`'s "Modified `process_operations`": `assert +/// len(body.deposits) == 0`, unconditionally, with no `process_deposit` call +/// at all), so it keeps its own copy, [`super::fulu::process_operations`], +/// rather than calling this one. pub fn process_operations( state: &mut BeaconState, body: &electra::BeaconBlockBody, diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs index 42ff09bdd..6608119c6 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs @@ -303,6 +303,13 @@ pub fn process_slashings(state: &mut BeaconState, _config: &Config) -> Result<() /// nor withdrawn) consumes churn and is credited, unless doing so would /// exceed the budget, in which case processing stops for this epoch as /// described above. +/// +/// Electra's own version, in effect only through this fork. Fulu retires the +/// eth1-bridge-ahead-of-requests gate outright (`beacon-chain.md`'s "Modified +/// `process_pending_deposits`"), so [`super::fulu::process_pending_deposits`] +/// keeps its own copy of this loop rather than sharing this one; see that +/// function's own doc for why running this version under fulu would be +/// wrong, not merely redundant. pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Result<()> { let next_epoch = get_current_epoch(state) + 1; let churn_limit = get_activation_exit_churn_limit(state, config)?; @@ -433,7 +440,12 @@ pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Res /// this, still builds a new validator phase0's way; see /// [`add_validator_from_pending_deposit`]'s doc for exactly how reusing it /// would go wrong here. -fn apply_pending_deposit( +/// +/// `pub(crate)`, not private: fulu's own `process_pending_deposits` +/// ([`super::fulu::process_pending_deposits`]) shares this step unchanged, +/// only dropping the eth1-bridge ordering gate ahead of it; see that +/// function's own doc. +pub(crate) fn apply_pending_deposit( state: &mut BeaconState, deposit: &electra::PendingDeposit, config: &Config, @@ -678,7 +690,14 @@ pub fn process_effective_balance_updates(state: &mut BeaconState) -> Result<()> /// though its shape, an electra-or-fulu match, is identical: fulu keeps every /// field this covers unchanged (see `crate::beacon::helpers::electra`'s own module /// doc for why that module's projection accepts fulu too). -enum PendingQueueFields<'a> { +/// +/// `pub(crate)`: fulu's own [`super::fulu::process_pending_deposits`] reads +/// [`Self::deposit_balance_to_consume`] and writes through +/// [`Self::pending_deposits_mut`] and [`Self::deposit_balance_to_consume_mut`] +/// the same way this module's own [`process_pending_deposits`] does; it never +/// reads [`Self::deposit_requests_start_index`], the one field that function +/// stops consulting. +pub(crate) enum PendingQueueFields<'a> { Electra(&'a mut electra::BeaconState), Fulu(&'a mut fulu::BeaconState), } @@ -688,7 +707,14 @@ impl<'a> PendingQueueFields<'a> { /// from crediting deposits off `Eth1Data` votes to crediting them off /// `DepositRequest`s directly, read by [`process_pending_deposits`] to /// know whether any eth1-bridge deposit is still outstanding. - fn deposit_requests_start_index(&self) -> u64 { + /// + /// `pub(crate)`: fulu's own tests (`super::fulu`) read this to assert a + /// test state starts with it at + /// [`crate::beacon::constants::UNSET_DEPOSIT_REQUESTS_START_INDEX`], the + /// precondition [`super::fulu::process_pending_deposits`]'s own doc + /// explains. [`super::fulu::process_pending_deposits`] itself never reads + /// this field. + pub(crate) fn deposit_requests_start_index(&self) -> u64 { match self { PendingQueueFields::Electra(state) => state.deposit_requests_start_index, PendingQueueFields::Fulu(state) => state.deposit_requests_start_index, @@ -696,14 +722,14 @@ impl<'a> PendingQueueFields<'a> { } /// How much of this epoch's deposit balance churn limit remains unused. - fn deposit_balance_to_consume(&self) -> Gwei { + pub(crate) fn deposit_balance_to_consume(&self) -> Gwei { match self { PendingQueueFields::Electra(state) => state.deposit_balance_to_consume, PendingQueueFields::Fulu(state) => state.deposit_balance_to_consume, } } - fn deposit_balance_to_consume_mut(&mut self) -> &mut Gwei { + pub(crate) fn deposit_balance_to_consume_mut(&mut self) -> &mut Gwei { match self { PendingQueueFields::Electra(state) => &mut state.deposit_balance_to_consume, PendingQueueFields::Fulu(state) => &mut state.deposit_balance_to_consume, @@ -711,7 +737,7 @@ impl<'a> PendingQueueFields<'a> { } /// Deposits known but not yet credited to the validator registry. - fn pending_deposits_mut(&mut self) -> &mut electra::PendingDeposits { + pub(crate) fn pending_deposits_mut(&mut self) -> &mut electra::PendingDeposits { match self { PendingQueueFields::Electra(state) => &mut state.pending_deposits, PendingQueueFields::Fulu(state) => &mut state.pending_deposits, @@ -750,7 +776,7 @@ impl<'a> PendingQueueFields<'a> { /// The electra-or-fulu state, mutably, through [`PendingQueueFields`]. See /// its own doc for why this is a second projection rather than a call into /// [`crate::beacon::helpers::electra::electra_state`]. -fn pending_queue_fields<'a>( +pub(crate) fn pending_queue_fields<'a>( state: &'a mut BeaconState, function: &'static str, ) -> Result> { diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/fulu.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/fulu.rs index 0975158e9..802a8d9d3 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/fulu.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/fulu.rs @@ -1,11 +1,14 @@ //! Fulu-specific epoch processing. //! -//! Electra's whole step list carries over unchanged: fulu's "Epoch processing" -//! section (`beacon-chain.md`) redefines `process_epoch` only to append one new -//! step, [`process_proposer_lookahead`] (EIP-7917), after electra's last one. -//! Nothing about how any existing step behaves changes; the state simply grows -//! one more piece of bookkeeping for [`process_proposer_lookahead`] to -//! maintain. +//! Fulu's "Epoch processing" section (`beacon-chain.md`) redefines +//! `process_epoch` in two ways: it appends one new step, +//! [`process_proposer_lookahead`] (EIP-7917), after electra's last one, and it +//! retires the eth1-bridge deposit mechanism's last trace in +//! [`process_pending_deposits`] (this module's own, not electra's shared one; +//! see that function's own doc for why sharing it would be wrong here, not +//! merely redundant). Every other step carries over unchanged: the state +//! simply grows one more piece of bookkeeping for +//! [`process_proposer_lookahead`] to maintain. //! //! [`process_proposer_lookahead`] keeps `BeaconState::proposer_lookahead` //! (`crate::beacon::containers::fulu::BeaconState`) a fixed-length rolling window: it @@ -43,21 +46,176 @@ //! only the registry and balance state matters for the ordering here. use crate::beacon::config::Config; +use crate::beacon::constants::FAR_FUTURE_EPOCH; use crate::beacon::containers::BeaconState; -use crate::beacon::error::Result; +use crate::beacon::containers::electra as electra_containers; +use crate::beacon::error::{Error, Result}; use crate::beacon::helpers::accessors::get_current_epoch; +use crate::beacon::helpers::electra::get_activation_exit_churn_limit; use crate::beacon::helpers::fulu::{fulu_state, get_beacon_proposer_indices}; +use crate::beacon::helpers::misc::compute_start_slot_at_epoch; use crate::beacon::preset; +use crate::beacon::primitives::Gwei; use super::electra; -/// Fulu's epoch-boundary driver: electra's, unchanged, with -/// [`process_proposer_lookahead`] appended at the end. +/// Fulu's epoch-boundary driver, in the specification's own order +/// (`beacon-chain.md`'s "Modified `process_epoch`"): electra's own steps, +/// with this module's own [`process_pending_deposits`] standing in for +/// [`electra::process_pending_deposits`], and [`process_proposer_lookahead`] +/// appended at the end. pub fn process_epoch(state: &mut BeaconState, config: &Config) -> Result<()> { - electra::process_epoch(state, config)?; + super::altair::process_justification_and_finalization(state)?; + super::altair::process_inactivity_updates(state, config)?; + super::altair::process_rewards_and_penalties(state, config)?; + electra::process_registry_updates(state, config)?; + electra::process_slashings(state, config)?; + super::process_eth1_data_reset(state)?; + process_pending_deposits(state, config)?; + electra::process_pending_consolidations(state, config)?; + electra::process_effective_balance_updates(state)?; + super::process_slashings_reset(state)?; + super::process_randao_mixes_reset(state)?; + super::capella::process_historical_summaries_update(state)?; + super::altair::process_participation_flag_updates(state)?; + super::altair::process_sync_committee_updates(state)?; + // [New in Fulu:EIP7917] process_proposer_lookahead(state) } +// --------------------------------------------------------------------------- +// Pending deposits +// --------------------------------------------------------------------------- + +/// Drains a balance-churn-limited amount of [`electra_containers::PendingDeposit`]s +/// into the validator registry. +/// +/// [`electra::process_pending_deposits`]'s own doc covers the shape this +/// shares with it in full: strict front-to-back draining, the three outcomes +/// a dequeued entry can have, and why a deposit that would blow this epoch's +/// budget stops the whole pass rather than being skipped over. +/// +/// The one thing this drops is that function's first gate, the one holding +/// every deposit *request* back until every eth1-bridge deposit ahead of it +/// (tracked by `deposit_requests_start_index`) has drained, which is exactly +/// what `beacon-chain.md`'s "Modified `process_pending_deposits`" for this +/// fork says to remove. That gate stalls the queue whenever +/// `eth1_deposit_index < deposit_requests_start_index`: either +/// `deposit_requests_start_index` is still +/// [`crate::beacon::constants::UNSET_DEPOSIT_REQUESTS_START_INDEX`] (no +/// deposit request has ever been seen), or it is set but `eth1_deposit_index` +/// has not yet drained up to it. Once `eth1_deposit_index` *has* caught up, +/// which every real network is expected to have well before it reaches fulu +/// (the eth1-bridge drain itself takes on the order of a day; electra ran +/// about seven months ahead of fulu on mainnet, sepolia, and hoodi), the gate +/// is already a no-op, which is why dropping it here changes nothing +/// observable on mainnet. The case this function actually exists for is a +/// chain that reaches a deposit request under fulu with the field either +/// still unset or set but not yet caught up: see +/// [`super::super::fulu::process_deposit_request`]'s own doc for why fulu +/// never writes it, and why pairing that with electra's gate here would stall +/// the queue for good rather than merely once. This function reads neither +/// field. +// [Modified in Fulu] +pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Result<()> { + let next_epoch = get_current_epoch(state) + 1; + let churn_limit = get_activation_exit_churn_limit(state, config)?; + let finalized_slot = compute_start_slot_at_epoch(state.finalized_checkpoint().epoch); + + // See `electra::process_pending_deposits`'s own doc for why the queue is + // taken by value here rather than iterated in place. + let (available_for_processing, deposits) = { + let mut fields = electra::pending_queue_fields(state, "process_pending_deposits")?; + let available_for_processing = fields + .deposit_balance_to_consume() + .checked_add(churn_limit) + .ok_or(Error::ArithmeticOverflow( + "deposit_balance_to_consume + get_activation_exit_churn_limit", + ))?; + let deposits: Vec = + core::mem::take(fields.pending_deposits_mut()).into_inner(); + (available_for_processing, deposits) + }; + + let mut processed_amount: Gwei = 0; + let mut next_deposit_index = 0usize; + let mut deposits_to_postpone: Vec = Vec::new(); + let mut is_churn_limit_reached = false; + + for deposit in &deposits { + // A deposit whose queue position could still be reorged out must + // wait: crediting it now and reverting later is not an option, since + // nothing else in the state transition undoes a balance change. + if deposit.slot > finalized_slot { + break; + } + + if next_deposit_index >= preset::MAX_PENDING_DEPOSITS_PER_EPOCH as usize { + break; + } + + let (is_validator_exited, is_validator_withdrawn) = state + .iter_validators() + .find(|validator| validator.pubkey == deposit.pubkey) + .map(|validator| { + ( + validator.exit_epoch < FAR_FUTURE_EPOCH, + validator.withdrawable_epoch < next_epoch, + ) + }) + .unwrap_or((false, false)); + + if is_validator_withdrawn { + electra::apply_pending_deposit(state, deposit, config)?; + } else if is_validator_exited { + deposits_to_postpone.push(deposit.clone()); + } else { + match processed_amount.checked_add(deposit.amount) { + Some(sum) if sum <= available_for_processing => { + processed_amount = sum; + electra::apply_pending_deposit(state, deposit, config)?; + } + // Either the sum overflowed (certainly too much) or it fit in + // a `u64` but still exceeded the budget: both mean this + // epoch's processing stops here. + _ => { + is_churn_limit_reached = true; + break; + } + } + } + + next_deposit_index += 1; + } + + let remaining: Vec = deposits + .into_iter() + .skip(next_deposit_index) + .chain(deposits_to_postpone) + .collect(); + + // Leftover churn is only worth remembering when it was actually the + // reason processing stopped: if the queue simply ran out, or the + // finality or per-epoch-cap gate stopped it first, next epoch's budget + // starts fresh rather than inheriting room this epoch never even tried + // to spend. + let deposit_balance_to_consume = if is_churn_limit_reached { + available_for_processing + .checked_sub(processed_amount) + .ok_or(Error::ArithmeticOverflow( + "available_for_processing - processed_amount", + ))? + } else { + 0 + }; + + let mut fields = electra::pending_queue_fields(state, "process_pending_deposits")?; + *fields.pending_deposits_mut() = electra_containers::PendingDeposits::try_from(remaining)?; + *fields.deposit_balance_to_consume_mut() = deposit_balance_to_consume; + + Ok(()) +} + /// Shifts `proposer_lookahead` forward by one epoch. /// /// Drops the window's first `SLOTS_PER_EPOCH` entries (the epoch that just @@ -95,8 +253,10 @@ pub fn process_proposer_lookahead(state: &mut BeaconState) -> Result<()> { #[cfg(test)] mod tests { use super::*; + use crate::beacon::constants; use crate::beacon::fork::ForkName; use crate::beacon::helpers::fulu::initialize_proposer_lookahead; + use crate::beacon::primitives::BlsSignature; /// A fulu state with `count` fully active, full-balance validators and a /// `proposer_lookahead` filled by @@ -177,4 +337,97 @@ mod tests { let mut phase0_state = crate::beacon::helpers::test_state::with_validators(4); assert!(process_proposer_lookahead(&mut phase0_state).is_err()); } + + // ----------------------------------------------------------------------- + // process_pending_deposits + // ----------------------------------------------------------------------- + + /// A pending deposit shaped like one a deposit *request* would queue + /// (`slot > GENESIS_SLOT`), topping up `index`'s own existing balance so + /// [`electra::apply_pending_deposit`] takes the + /// signature-free top-up branch rather than needing a real one. + fn request_sourced_pending_deposit( + state: &BeaconState, + index: usize, + ) -> electra_containers::PendingDeposit { + let validator = state.validator(index as u64).unwrap(); + electra_containers::PendingDeposit { + pubkey: validator.pubkey, + withdrawal_credentials: validator.withdrawal_credentials, + amount: preset::EFFECTIVE_BALANCE_INCREMENT, + signature: BlsSignature::default(), + slot: constants::GENESIS_SLOT + 1, + } + } + + #[test] + fn a_test_state_starts_with_deposit_requests_start_index_unset() { + let mut state = fulu_state_with_validators(2); + assert_eq!( + electra::pending_queue_fields(&mut state, "test assertion") + .unwrap() + .deposit_requests_start_index(), + constants::UNSET_DEPOSIT_REQUESTS_START_INDEX + ); + } + + #[test] + fn an_unset_start_index_does_not_stall_fulus_own_pending_deposits() { + let config = Config::mainnet(); + let mut state = fulu_state_with_validators(2); + // Past the deposit's own slot, so the unrelated finality gate + // (`deposit.slot > finalized_slot`) cannot be what lets, or blocks, + // this deposit through; only the eth1-bridge gate is under test. + state.finalized_checkpoint_mut().epoch = 1; + let deposit = request_sourced_pending_deposit(&state, 0); + electra::pending_queue_fields(&mut state, "test setup") + .unwrap() + .pending_deposits_mut() + .push(deposit) + .unwrap(); + + let balance_before = state.balance(0).unwrap(); + process_pending_deposits(&mut state, &config).unwrap(); + + // Applied: the queue's only entry was a top-up for an active, + // non-exited validator, so nothing here should postpone or block it. + assert_eq!( + state.balance(0).unwrap(), + balance_before + preset::EFFECTIVE_BALANCE_INCREMENT + ); + let mut fields = electra::pending_queue_fields(&mut state, "test assertion").unwrap(); + assert!(fields.pending_deposits_mut().is_empty()); + } + + #[test] + fn the_same_queue_stalls_forever_under_electras_own_gate() { + let config = Config::mainnet(); + let mut state = fulu_state_with_validators(2); + // See the previous test: past the deposit's own slot, so only the + // eth1-bridge gate, not the unrelated finality gate, is under test. + state.finalized_checkpoint_mut().epoch = 1; + let deposit = request_sourced_pending_deposit(&state, 0); + electra::pending_queue_fields(&mut state, "test setup") + .unwrap() + .pending_deposits_mut() + .push(deposit.clone()) + .unwrap(); + + let balance_before = state.balance(0).unwrap(); + // `electra::process_pending_deposits` accepts a fulu state too (its + // `PendingQueueFields` projection covers both), which is exactly what + // let the pre-fix code call it here unnoticed. + electra::process_pending_deposits(&mut state, &config).unwrap(); + + // Blocked: `deposit_requests_start_index` is still + // `UNSET_DEPOSIT_REQUESTS_START_INDEX`, so electra's + // `eth1_deposit_index < deposit_requests_start_index` gate holds for + // this request-sourced entry and the whole pass breaks before ever + // reaching it. + assert_eq!(state.balance(0).unwrap(), balance_before); + let mut fields = electra::pending_queue_fields(&mut state, "test assertion").unwrap(); + let remaining = fields.pending_deposits_mut(); + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0], deposit); + } } diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs index 1ec25e41a..b8e50f47c 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs @@ -77,8 +77,11 @@ pub fn process_epoch_phase0(state: &mut BeaconState, config: &Config) -> Result< /// across forks, and never redefines `process_epoch` itself, so this reuses /// capella's driver. /// - Electra's and fulu's sections each give a full, modified `process_epoch` -/// (electra adds the pending-deposit and pending-consolidation steps; -/// fulu appends the proposer-lookahead step), so each gets its own stub. +/// (electra adds the pending-deposit and pending-consolidation steps; fulu +/// appends the proposer-lookahead step and, less visibly in this listing, +/// swaps in its own `process_pending_deposits`, which drops the +/// eth1-bridge gate electra's version still has), so each gets its own +/// stub. pub fn process_epoch(state: &mut BeaconState, config: &Config) -> Result<()> { match state.fork_name() { ForkName::Phase0 => process_epoch_phase0(state, config), diff --git a/crates/blockchain/state_transition/src/beacon/stf/fulu.rs b/crates/blockchain/state_transition/src/beacon/stf/fulu.rs index 9cd8d3267..9b81dd6ef 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/fulu.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/fulu.rs @@ -1,19 +1,23 @@ //! Fulu-specific block processing. //! -//! Fulu changes no field of a block and redefines no step of `process_block` -//! itself: `beacon-chain.md`'s "Block processing" section for this fork -//! contains exactly one item, a modified `process_execution_payload`. So -//! [`process_block`] dispatches on [`electra::BeaconBlock`], the same type -//! fulu's own [`crate::beacon::containers::SignedBeaconBlock::Fulu`] variant wraps, -//! rather than a `fulu::BeaconBlock` that does not exist, and every step -//! but one is [`super::electra`]'s own function, called directly rather than +//! `beacon-chain.md`'s "Block processing" section for this fork redefines two +//! of `process_block`'s own steps: a modified `process_execution_payload` +//! (EIP-7892's blob schedule, below) and a modified `process_operations`, +//! which reaches one level deeper (through `process_execution_requests`) to +//! also redefine `process_deposit_request` (both retire the last of the +//! former eth1-bridge deposit mechanism EIP-6110 began superseding back in +//! electra; see [`process_operations`]'s own doc). So [`process_block`] +//! dispatches on [`electra::BeaconBlock`], the same type fulu's own +//! [`crate::beacon::containers::SignedBeaconBlock::Fulu`] variant wraps, rather +//! than a `fulu::BeaconBlock` that does not exist, and every step but those +//! two is [`super::electra`]'s own function, called directly rather than //! transcribed. //! -//! The one change is why a block's blob commitment count stops being checked -//! against a single network-wide constant. Through electra, a block could -//! carry at most [`Config::max_blobs_per_block_electra`] blobs, each -//! downloaded and verified whole by every node that wants to check it. Fulu -//! moves to a sampling model instead (`das-core.md`): a blob is +//! The blob-schedule change is why a block's blob commitment count stops +//! being checked against a single network-wide constant. Through electra, a +//! block could carry at most [`Config::max_blobs_per_block_electra`] blobs, +//! each downloaded and verified whole by every node that wants to check it. +//! Fulu moves to a sampling model instead (`das-core.md`): a blob is //! erasure-coded into a wide row of columns, and a node gains the same //! confidence that the data behind a commitment is available by sampling a //! handful of those columns rather than downloading the blob itself. That @@ -28,11 +32,9 @@ //! [`Config::max_blobs_per_block`], in place of electra's fixed field. //! //! [`super::block::process_block_header`], [`super::block::process_randao`], -//! and [`super::block::process_eth1_data`] are reused because they always -//! were fork-shared; [`super::electra::process_withdrawals`], -//! [`super::electra::process_operations`], and -//! [`super::altair::process_sync_aggregate`] are reused because fulu's -//! specification never mentions any of the three. [`process_execution_payload`] +//! [`super::block::process_eth1_data`], [`super::electra::process_withdrawals`], +//! and [`super::altair::process_sync_aggregate`] are reused because fulu's +//! specification never mentions any of the five. [`process_execution_payload`] //! itself cannot be [`super::electra::process_execution_payload`] called //! unchanged, for the same reason electra's own version could not be //! deneb's: it reads a different [`Config`] field for the one check that @@ -42,6 +44,7 @@ use crate::beacon::config::Config; use crate::beacon::containers::{BeaconState, deneb, electra}; use crate::beacon::error::{Result, verify}; use crate::beacon::helpers::accessors::{CommitteeCache, get_current_epoch, get_randao_mix}; +use crate::beacon::helpers::electra::electra_state; use crate::beacon::helpers::fulu::{fulu_state, fulu_state_ref}; use crate::beacon::primitives::{Bytes32, HashTreeRoot as _}; @@ -52,9 +55,10 @@ use super::ExecutionEngine; // --------------------------------------------------------------------------- /// Fulu's block processing: electra's own steps, in electra's own order, -/// with [`process_execution_payload`] standing in for -/// [`super::electra::process_execution_payload`]. See the module docs for -/// why that is the only step this fork's specification asks to change. +/// with [`process_execution_payload`] and [`process_operations`] standing in +/// for [`super::electra::process_execution_payload`] and +/// [`super::electra::process_operations`]. See the module docs for why these +/// are the only steps this fork's specification asks to change. pub fn process_block( state: &mut BeaconState, block: &electra::BeaconBlock, @@ -73,7 +77,7 @@ pub fn process_block( process_execution_payload(state, &block.body, config, engine)?; super::block::process_randao(state, &block.body.randao_reveal)?; super::block::process_eth1_data(state, &block.body.eth1_data)?; - super::electra::process_operations(state, &block.body, config, committees)?; + process_operations(state, &block.body, config, committees)?; super::altair::process_sync_aggregate(state, &block.body.sync_aggregate)?; Ok(()) } @@ -186,6 +190,120 @@ pub fn process_execution_payload( Ok(()) } +// --------------------------------------------------------------------------- +// Operations +// --------------------------------------------------------------------------- + +/// Runs every operation in a fulu block, in the specification's order. +/// +/// [`super::electra::process_operations`]'s own doc explains why this takes +/// the whole `electra::BeaconBlockBody` rather than one parameter per list; +/// the same reasoning applies here unchanged. +/// +/// The former eth1-bridge deposit mechanism EIP-6110 began superseding back +/// in electra (`deposit_requests_start_index`, `body.deposits`, +/// `process_deposit`) is fully retired as of this fork: `beacon-chain.md`'s +/// "Modified `process_operations`" replaces electra's conditional +/// deposit-count check with an unconditional `assert len(body.deposits) == +/// 0` and drops the `process_deposit` call outright, since every validator +/// deposit arrives as a [`process_deposit_request`] by now. Electra's own +/// [`super::electra::process_operations`] still reaches the same verdict on +/// every state that has genuinely finished that transition (the branch it +/// takes once `state.eth1_deposit_index` has caught up to +/// `deposit_requests_start_index` is this same empty-deposits assertion), so +/// the two do not diverge in practice; this copy exists to match the +/// specification's own text for this fork rather than to fix an observed +/// bug in the shared version. +pub fn process_operations( + state: &mut BeaconState, + body: &electra::BeaconBlockBody, + config: &Config, + committees: &CommitteeCache, +) -> Result<()> { + // [Modified in Fulu] + verify(body.deposits.is_empty(), "len(body.deposits) == 0")?; + + for proposer_slashing in body.proposer_slashings.iter() { + super::operations::process_proposer_slashing(state, proposer_slashing, config)?; + } + for attester_slashing in body.attester_slashings.iter() { + super::electra::process_attester_slashing(state, attester_slashing, config)?; + } + for attestation in body.attestations.iter() { + super::electra::process_attestation(state, attestation, committees)?; + } + // [Modified in Fulu] + // Removed `process_deposit` + for voluntary_exit in body.voluntary_exits.iter() { + super::electra::process_voluntary_exit(state, voluntary_exit, config)?; + } + for signed_change in body.bls_to_execution_changes.iter() { + super::electra::process_bls_to_execution_change(state, signed_change, config)?; + } + process_execution_requests(state, &body.execution_requests, config)?; + + Ok(()) +} + +/// Runs every execution-layer-triggered request in `requests`, in the +/// specification's order: deposits, then withdrawals, then consolidations. +/// +/// [`super::electra::process_execution_requests`]'s own copy, with +/// [`process_deposit_request`] standing in for +/// [`super::electra::process_deposit_request`]; withdrawal and consolidation +/// requests are unmentioned by this fork's specification, so those two legs +/// still call electra's own functions directly. +pub fn process_execution_requests( + state: &mut BeaconState, + requests: &electra::ExecutionRequests, + config: &Config, +) -> Result<()> { + for deposit in requests.deposits.iter() { + process_deposit_request(state, deposit)?; + } + for withdrawal in requests.withdrawals.iter() { + super::electra::process_withdrawal_request(state, withdrawal, config)?; + } + for consolidation in requests.consolidations.iter() { + super::electra::process_consolidation_request(state, consolidation, config)?; + } + Ok(()) +} + +/// Records an execution-layer-triggered deposit (EIP-6110) as a +/// [`electra::PendingDeposit`], the same queue-then-drain destination every +/// other deposit source feeds. +/// +/// [`super::electra::process_deposit_request`]'s own doc explains +/// `deposit_requests_start_index` and why no signature check happens here. +/// This fork's `beacon-chain.md` ("Modified `process_deposit_request`") drops +/// that function's other statement, the one setting +/// `deposit_requests_start_index` to this request's index, but only the +/// first time it is ever seen (electra's own `if` guards it on the field +/// still being [`crate::beacon::constants::UNSET_DEPOSIT_REQUESTS_START_INDEX`]). +/// A chain that reaches fulu having never processed a deposit request under +/// electra still has the field at that sentinel, and calling electra's +/// function unchanged for such a chain's first fulu request would set it, +/// which is exactly the case [`super::epoch::fulu::process_pending_deposits`]'s +/// own doc explains this fork's specification means to prevent: the field +/// stays unset for good, not merely until this fork's own first request. +pub fn process_deposit_request( + state: &mut BeaconState, + request: &electra::DepositRequest, +) -> Result<()> { + let deposit = electra::PendingDeposit { + pubkey: request.pubkey, + withdrawal_credentials: request.withdrawal_credentials, + amount: request.amount, + signature: request.signature, + slot: state.slot(), + }; + electra_state(state, "process_deposit_request")? + .pending_deposits_mut() + .push(deposit)?; + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -395,4 +513,54 @@ mod tests { process_execution_payload(&mut state.clone(), &over_limit, &config, &engine).is_err() ); } + + // ----------------------------------------------------------------------- + // process_deposit_request + // ----------------------------------------------------------------------- + + #[test] + fn leaves_deposit_requests_start_index_unset_and_queues_the_deposit() { + use crate::beacon::constants; + use crate::beacon::primitives::BlsPubkey; + + let mut state = fulu_state_with_validators(1, empty_execution_payload_header()); + let BeaconState::Fulu(before) = &state else { + unreachable!("built as fulu above") + }; + assert_eq!( + before.deposit_requests_start_index, + constants::UNSET_DEPOSIT_REQUESTS_START_INDEX + ); + assert!(before.pending_deposits.is_empty()); + let slot_before = state.slot(); + + let request = electra::DepositRequest { + pubkey: BlsPubkey::default(), + withdrawal_credentials: Bytes32::ZERO, + amount: preset::EFFECTIVE_BALANCE_INCREMENT, + signature: crate::beacon::primitives::BlsSignature::default(), + index: 64, + }; + process_deposit_request(&mut state, &request).unwrap(); + + let BeaconState::Fulu(after) = &state else { + unreachable!("still fulu") + }; + // Unlike electra's own version, this must leave the field alone: + // fulu's specification drops the statement that sets it entirely. + assert_eq!( + after.deposit_requests_start_index, + constants::UNSET_DEPOSIT_REQUESTS_START_INDEX + ); + assert_eq!(after.pending_deposits.len(), 1); + let queued = &after.pending_deposits[0]; + assert_eq!(queued.pubkey, request.pubkey); + assert_eq!( + queued.withdrawal_credentials, + request.withdrawal_credentials + ); + assert_eq!(queued.amount, request.amount); + assert_eq!(queued.signature, request.signature); + assert_eq!(queued.slot, slot_before); + } } diff --git a/crates/blockchain/state_transition/src/beacon/stf/operations.rs b/crates/blockchain/state_transition/src/beacon/stf/operations.rs index 6836a063f..43b0a2394 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/operations.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/operations.rs @@ -81,12 +81,12 @@ pub fn process_operations( // // Deneb never reaches here: it has its own `process_operations` // (`deneb.rs`), which calls its own `process_attestation` for this loop - // instead of this shared one. Electra and fulu have their own - // `process_operations` too (`electra.rs`, which fulu reuses), and gloas - // will need a third (its bodies carry progressive attestations and - // slashings, plus the payload-attestation and builder-registry - // operations this signature has no parameters for), so all four are - // refused rather than guessed at. + // instead of this shared one. Electra and fulu each have their own + // `process_operations` too (`electra.rs` and `fulu.rs` respectively), and + // gloas will need one of its own as well (its bodies carry progressive + // attestations and slashings, plus the payload-attestation and + // builder-registry operations this signature has no parameters for), so + // all four are refused rather than guessed at. let altair_attestations = match state.fork_name() { ForkName::Phase0 => false, ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => true, diff --git a/crates/blockchain/state_transition/tests/beacon_spec/epoch_processing.rs b/crates/blockchain/state_transition/tests/beacon_spec/epoch_processing.rs index 7283b0136..4be921334 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/epoch_processing.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/epoch_processing.rs @@ -36,8 +36,12 @@ //! different handler name rather than a different fork's version of this //! one, starting at capella. //! - Electra adds `pending_deposits` and `pending_consolidations`, with no -//! earlier-fork counterpart at all; fulu carries both forward unchanged and -//! adds `proposer_lookahead` of its own, likewise with no earlier-fork +//! earlier-fork counterpart at all. Fulu carries `pending_consolidations` +//! forward unchanged, but its own `beacon-chain.md` lists a modified +//! `pending_deposits` (it drops the eth1-bridge-ahead-of-requests gate, +//! which stalls if `deposit_requests_start_index` is still unset or not +//! yet drained when a deposit request reaches it), and adds +//! `proposer_lookahead` of its own, likewise with no earlier-fork //! counterpart. use std::sync::Arc; @@ -188,11 +192,28 @@ fn apply( "inactivity_updates" => epoch::altair::process_inactivity_updates(state, config), "participation_flag_updates" => epoch::altair::process_participation_flag_updates(state), "sync_committee_updates" => epoch::altair::process_sync_committee_updates(state), - // New in electra (EIP-7251); fulu carries both pending queues forward - // unchanged, calling the very same functions (see - // `electra::process_epoch`'s own doc for why fulu's driver needs no - // rewrite of either). - "pending_deposits" => epoch::electra::process_pending_deposits(state, config), + // New in electra (EIP-7251). Fulu's driver keeps calling the very + // same `process_pending_consolidations` (see `electra::process_epoch`'s + // own doc for why that one needs no rewrite), but `pending_deposits` + // has its own fulu-specific function: `epoch::fulu::process_pending_deposits`'s + // own doc explains why running electra's version under fulu would be + // wrong, not merely redundant. + "pending_deposits" => match fork { + ForkName::Fulu => epoch::fulu::process_pending_deposits(state, config), + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra => epoch::electra::process_pending_deposits(state, config), + ForkName::Gloas => Err( + ethlambda_state_transition::beacon::Error::UnsupportedForFork { + function: "pending_deposits", + fork: ForkName::Gloas, + }, + ), + ForkName::Lean => lean_is_not_a_fixture_fork("pending_deposits"), + }, "pending_consolidations" => epoch::electra::process_pending_consolidations(state, config), // New in fulu (EIP-7917); no earlier fork has this handler at all. "proposer_lookahead" => epoch::fulu::process_proposer_lookahead(state), diff --git a/crates/blockchain/state_transition/tests/beacon_spec/operations.rs b/crates/blockchain/state_transition/tests/beacon_spec/operations.rs index b084f33ea..31c7e42cb 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/operations.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/operations.rs @@ -81,9 +81,17 @@ //! to hold a differently-typed `payload` parameter, not different logic. //! - `consolidation_request`, `deposit_request`, and `withdrawal_request` are //! all new in electra, alongside `withdrawals`, `process_attestation`, -//! `process_deposit`, and `process_voluntary_exit`; fulu's specification -//! lists none of the seven as modified again, so electra's functions serve -//! fulu's cases too. +//! `process_deposit`, and `process_voluntary_exit`. Fulu's specification +//! lists `consolidation_request`, `withdrawal_request`, `withdrawals`, +//! `process_attestation`, and `process_voluntary_exit` as unmodified, so +//! electra's functions serve fulu's cases for those five too, and it lists +//! `process_deposit` as removed outright rather than modified (beta.2 ships +//! no fulu `operations/deposit` fixtures, so that arm is untested either +//! way). `deposit_request` is the one exception with a genuine fulu-only +//! function: it drops the statement that sets +//! `deposit_requests_start_index`, since fulu retires the eth1-bridge +//! deposit mechanism outright (`fulu_stf::process_deposit_request`'s own +//! doc). use std::sync::Arc; use ethlambda_state_transition::beacon::ForkName; @@ -291,8 +299,15 @@ fn apply( } // Electra queues a deposit's amount instead of crediting it // directly (EIP-7251), so the epoch boundary can rate-limit - // activation by balance rather than by validator count; fulu's - // specification makes no further change. + // activation by balance rather than by validator count. Fulu's + // specification removes `process_deposit` outright rather than + // modifying it (`beacon-chain.md`'s "Modified `process_operations`": + // `assert len(body.deposits) == 0`, no call at all), so this + // fulu arm is dead in practice: beta.2 ships no fulu + // `operations/deposit` fixtures. Kept rather than dropped so a + // future fixture release that does ship one fails loudly through + // whatever electra's function rejects it for, not silently + // through `unhandled operation`. ForkName::Electra | ForkName::Fulu => { let deposit: shared::Deposit = case.ssz("deposit"); electra_stf::process_deposit(state, &deposit, config) @@ -460,17 +475,45 @@ fn apply( ), ForkName::Lean => lean_is_not_a_fixture_fork("withdrawals"), }, - // All three are new in electra, alongside `withdrawals`; fulu's - // specification lists none of them as modified, so electra's own - // functions serve fulu's cases too. + // `consolidation_request` and `withdrawal_request` are new in + // electra, alongside `withdrawals`; fulu's specification lists + // neither as modified, so electra's own functions serve fulu's cases + // too. "consolidation_request" => { let request: electra::ConsolidationRequest = case.ssz("consolidation_request"); electra_stf::process_consolidation_request(state, &request, config) } - "deposit_request" => { - let request: electra::DepositRequest = case.ssz("deposit_request"); - electra_stf::process_deposit_request(state, &request) - } + // `deposit_request` is new in electra too, but fulu's specification + // does list a modified version (`beacon-chain.md`'s "Modified + // `process_deposit_request`"): it drops the statement that sets + // `deposit_requests_start_index` the first time a request is seen, + // since fulu retires the eth1-bridge deposit mechanism outright (see + // `fulu_stf::process_deposit_request`'s own doc). Routing a fulu + // case to electra's version regardless would still pass every fixture + // shipped so far, since pyspec's own fulu genesis already sets that + // field, but it is not the function fulu's specification names. + "deposit_request" => match case.fork { + ForkName::Fulu => { + let request: electra::DepositRequest = case.ssz("deposit_request"); + fulu_stf::process_deposit_request(state, &request) + } + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra => { + let request: electra::DepositRequest = case.ssz("deposit_request"); + electra_stf::process_deposit_request(state, &request) + } + ForkName::Gloas => Err( + ethlambda_state_transition::beacon::Error::UnsupportedForFork { + function: "deposit_request", + fork: ForkName::Gloas, + }, + ), + ForkName::Lean => lean_is_not_a_fixture_fork("deposit_request"), + }, "withdrawal_request" => { let request: electra::WithdrawalRequest = case.ssz("withdrawal_request"); electra_stf::process_withdrawal_request(state, &request, config) From baa549c87300f1f4c6173b6d1ab7282acdb88bbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 25 Sep 2026 12:09:26 -0300 Subject: [PATCH 012/132] fix(beacon): port the v1.7.0-beta.2 fork-choice changes The fixture bump to consensus-specs v1.7.0-beta.2 left 19 minimal and 1 mainnet `fork_choice` cases failing: `fork-choice.md` changed between v1.6.1 and beta.2 for phase0 (inherited by every later fork through fulu) and again for fulu itself, while this module still implemented the v1.6.1 algorithm. Ported, diffing v1.6.1 against beta.2 (gloas/heze/fast-confirmation.md out of scope): - `update_proposer_boost_root` (phase0 `fork-choice.md`): the boost is now gated on the new block sharing the pre-import head's proposer shuffling, via `get_shuffling_dependent_root`/`compute_shuffling_dependent_slot`/ `compute_shuffling_lookahead_start_slot` (`MIN_SEED_LOOKAHEAD`). `on_block` computes that pre-import head (`compute_head`, split out of `get_head`) only when the block is timely and unopposed, since those are cheap and already decide most blocks (a synced block is never timely), before the computation `compute_weights`' own documentation calls out as expensive at scale. - `get_attestation_score` (phase0): split out of `get_weight`, the vote-only half of the weight with no proposer boost, which `is_head_weak`/ `is_parent_strong` now call directly instead of going through `get_weight`. - `is_head_weak` (phase0): now also adds the effective balance of every equivocating validator in the head slot's committees (bci's `CommitteeCache`, not a fresh shuffle), so the weight it reads is monotonic under later equivocation evidence. - `is_parent_strong` (phase0): takes the head root rather than the parent root and derives the parent internally, matching the spec's own `store.blocks[root].parent_root`. - `is_proposer_equivocation` (phase0, new): whether more than one block is known for a root's own (slot, proposer); scans the in-memory block index for same-slot candidates before paying for `get_signed_block`, rather than the spec's whole-history scan. - `get_proposer_head` (phase0, modified again in fulu/EIP-7917): gains the `head_weak and current_time_ok and proposer_equivocation` re-org branch. Fulu drops the `shuffling_stable` requirement; kept as one function whose `shuffling_stable` folds to `true` from fulu on (read off the head block's own fork, not `Config::fork_at_epoch`, since the minimal-preset spec-test config never schedules a fork away from `FAR_FUTURE_EPOCH`) rather than two near-identical copies. `should_override_forkchoice_update` (bellatrix) is deleted outright, along with its harness check: it is not merely relocated at `v1.7.0-beta.2`, it is gone from the specification entirely (checked the whole tree, including `fast-confirmation.md`), and it never had a caller of its own. Also, from review: - `get_ancestor`-based walks used to fail past a non-genesis (checkpoint- synced) anchor once the current epoch was close enough to the anchor's own that a shuffling-dependent slot fell below it: the anchor's own `parent_root` is real history this store never held, so the walk stepped into a gap instead of stopping. `get_shuffling_dependent_root` now stops at the lowest indexed ancestor instead (regression test with a non-genesis anchor), which is what the import benchmark's `replay.rs` hit on its very first block, since it sets the clock to that block's own slot before importing it. - Every fallible step of the new proposer-boost logic (only `compute_head` itself) now runs before `on_block` mutates the store; `record_block_timeliness` and `update_proposer_boost_root` are infallible once the anchor-walk fix above lands, so nothing after the block joins the store can turn into an `Err`. - `on_block` now returns early on an already-known root (`Store::has_block`), the specification's own guard; the chain actor's own `has_state`-based dedup at the call site is unchanged and still the one that matters on that path. - `CLAUDE.md` and `docs/beacon_stf.md` both still credited the deleted `should_override_forkchoice_update` among what the mainnet `fork_choice` fixtures cover; updated both sentences to name the proposer-boost dependent-root gate and `get_proposer_head`'s equivocation branch instead. `get_proposer_head`/`is_head_weak`/`is_parent_strong` have no caller outside this module and its test harness today, but `on_block`'s proposer-boost gating and `get_head`'s head selection run on every live beacon import, so this changes head selection and proposer boost on the live node as well as in the spec suite. fork_choice suite: 19 minimal + 1 mainnet failures -> 0 (328/328 minimal, 152/152 mainnet). Lib unit tests (32, six new: `is_proposer_equivocation` true/false, the new `is_head_weak`/`get_proposer_head` equivocation branches, the anchor-walk regression), `make fmt`, clippy on both presets, and `cargo build -p ethlambda-blockchain`/`-p ethlambda` all pass. Rebased onto the gloas fork variant (1a6829e4) and F6 (the fulu deposit-request fix, ee7e9283/15002399): the cherry-pick applied without conflicts, since 1a6829e4's gloas refusals in this module (get_forkchoice_store's anchor refusal, block_operations' and on_block's own Gloas arms) sit in code this change never touches, and it adds no new exhaustive ForkName/BeaconState match of its own (only one non-exhaustive `>= ForkName::Fulu` comparison in get_proposer_head's shuffling-stable fold). Fixed during integration, found by a livelocked workspace test run (ethlambda-blockchain's `a_childs_fan_out_does_not_livelock_a_held_parent` spun at 100% CPU): on_block's new early-return guard checked `Store::has_block`, matching the specification's `block_root in store.blocks` literally, but this store's `blocks` table does not mean what the specification's does. The specification's `store.blocks` only gains a root at the very end of `on_block`, once a post-state exists; this store writes a held block's row (one row for a beacon block, through `Store::insert_pending_block`) before import, while it is parked waiting for missing data columns, without the `LiveChain` entry that marks a block admitted. Guarding on `has_block` therefore answered `Ok(())` for a held-but-never-imported root without importing it, so every child naming it as an ancestor found it "already known" and gave up without importing anything either, forever. The guard now checks `Store::has_state`, which is what `store.blocks` membership actually maps to here; `lib.rs`'s own comment on its parallel `has_state` dedup check is updated to match. Added `on_block_imports_a_block_already_persisted_with_no_post_state` (`fork_choice.rs`), which persists a block the way `hold_block_for_columns` does (`insert_pending_block`: readable by root, no `LiveChain` entry, no post-state), builds it a real BLS-signed, correctly-rooted successor to a genuine anchor pair, delivers it timely so `on_block`'s pre-import `compute_head` call also runs, and asserts `on_block` still imports it. --- CLAUDE.md | 3 +- crates/blockchain/src/lib.rs | 48 +- .../src/beacon/fork_choice.rs | 921 ++++++++++++++---- .../tests/beacon_spec/fork_choice.rs | 71 +- .../tests/beacon_spec/sync.rs | 4 +- docs/beacon_stf.md | 5 +- 6 files changed, 810 insertions(+), 242 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ace330ad8..d3e7a93fe 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -713,7 +713,8 @@ transitions are in `ethlambda-types`, per the section above. Nothing above without; both figures cover lean's own unit tests as well, since the two chains now share one lib target. Fork choice is fixture-verified too: 150 mainnet `fork_choice` cases pass, covering bellatrix's `on_merge_block`/terminal-PoW - validation, `should_override_forkchoice_update`, deneb's blob data + validation, the `v1.7.0` proposer-boost dependent-root gate and + `get_proposer_head`'s proposer-equivocation branch, deneb's blob data availability, and fulu's column data availability. - Nothing is ignored for being unimplemented. Ignored cases are the `LightClient*` containers (a different layer, out of scope) and the `gloas` diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 0a809b496..5a075f1ee 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -2222,16 +2222,18 @@ impl BlockChainServer { /// Re-run beacon fork choice and republish the head gauge. /// - /// `fork_choice::on_block` does not compute a head, and `Store::beacon_head` - /// only reads back whatever the last [`fork_choice::get_head`] recorded, so - /// without this an import moves no head at all: it just adds a block and a - /// post-state. Until the block path called this too, [`Self::on_tick`] was - /// the only caller, and it is one message per slot in the same mailbox as - /// every arriving block. A follower catching up imports back-to-back and - /// never drains that mailbox, so the tick did not run, the head stayed - /// pinned at the checkpoint-sync anchor, and `lean_head_slot` sat flat for - /// the entire catch-up even while imports were landing every few seconds. - /// That is what "the head is not advancing" looked like on the dashboard. + /// `fork_choice::on_block` may compute a head of its own now (to gate its + /// proposer-boost logic), but never records it: `Store::beacon_head` only + /// reads back whatever the last [`fork_choice::get_head`] recorded, so + /// without this an import moves no *visible* head at all: it just adds a + /// block and a post-state. Until the block path called this too, + /// [`Self::on_tick`] was the only caller, and it is one message per slot in + /// the same mailbox as every arriving block. A follower catching up + /// imports back-to-back and never drains that mailbox, so the tick did not + /// run, the head stayed pinned at the checkpoint-sync anchor, and + /// `lean_head_slot` sat flat for the entire catch-up even while imports + /// were landing every few seconds. That is what "the head is not + /// advancing" looked like on the dashboard. /// /// A failure here means fork choice could not find a head (for instance /// every known block is unjustifiable), which is a condition to log and @@ -2646,15 +2648,23 @@ impl BlockChainServer { } // Beacon: a block whose post-state is already here needs no work. - // `fork_choice::on_block` does not short-circuit on a known root: it - // goes straight from cloning the parent state to `state_transition`, - // so a re-delivery pays the entire import a second time. On mainnet - // 2026-09-08 that was 37 of 116 imports, a third of the actor's import - // budget, spent recomputing post-states the store already held. - // Children are still collected: this root did import, so anything - // pending on it is ready whether or not this delivery is the one that - // imported it. Beacon-only, because lean's `store::on_block` has its - // own already-imported early return. + // `fork_choice::on_block` also returns early on a root that already + // has a post-state now (`Store::has_state`, the specification's own + // `store.blocks` guard, mapped onto what that membership means here; + // see that call site's own doc for why `has_block` alone is the + // wrong check), so calling it again costs little more than that one + // lookup. This check still + // matters because `on_block` answers a bare `Ok(())` either way, with + // no signal that nothing happened, while this call site needs to + // collect the root's pending children and answer + // `ImportOutcome::Imported` regardless of which delivery actually did + // the importing. Before `on_block` had its own guard, that + // distinction was also a real cost: on mainnet 2026-09-08, a + // re-delivery recomputing the whole post-state was 37 of 116 imports, + // a third of the actor's import budget. Children are still collected: + // this root did import, so anything pending on it is ready whether or + // not this delivery is the one that imported it. Beacon-only, because + // lean's `store::on_block` has its own already-imported early return. if self.store.chain() == Chain::Beacon && self .store diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 176f989ab..96f0c6186 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -10,10 +10,12 @@ //! ([`on_tick`], [`on_block`], [`on_attestation`], [`on_attester_slashing`]) //! are the only ones the specification lists as sole ways to change it, //! matching its own framing: "Invalid calls to handlers must not modify -//! `store`." Every other function in this file takes `&Store`, with one -//! exception: [`get_head`] takes `&mut Store` too, since it records the head -//! it just computed; see its own documentation for why that write belongs -//! there rather than in a caller. +//! `store`." Most other functions in this file take `&Store`; the exceptions +//! are [`get_head`] (records the head it just computed; see its own +//! documentation for why that write belongs there rather than in a caller) +//! and `record_block_timeliness`/`update_proposer_boost_root`, which write +//! `store.block_timeliness`/`store.proposer_boost_root` on [`on_block`]'s +//! behalf: [`on_block`] is their only caller. //! //! # Units: one seconds-granularity clock, read out in milliseconds at the edges //! @@ -169,7 +171,7 @@ use crate::beacon::containers::{bellatrix, deneb, electra, fulu, phase0}; use crate::beacon::error::{Error, Result, verify}; use crate::beacon::fork::ForkName; use crate::beacon::helpers::accessors::{ - CommitteeCache, get_active_validator_indices, get_beacon_proposer_index, get_current_epoch, + CommitteeCache, CommitteeCacheExt, get_active_validator_indices, get_current_epoch, get_total_active_balance, }; use crate::beacon::helpers::attestation as phase0_attestation; @@ -683,9 +685,9 @@ pub fn resolve_invalid_block( /// specification's own version, and /// `tests::a_vote_for_a_pruned_block_weighs_nothing_instead_of_failing` pins /// that divergence on purpose. Nothing on this node's paths calls it today -/// ([`get_proposer_head`] and [`should_override_forkchoice_update`] have no -/// callers outside this file), but wiring up a beacon proposer duty would make -/// it reachable, and it should get `compute_weights`' treatment first. +/// ([`get_proposer_head`] has no callers outside this file), but wiring up a +/// beacon proposer duty would make it reachable, and it should get +/// `compute_weights`' treatment first. pub fn invalidate_subtree(store: &mut Store, invalid_root: Root) -> usize { let index = store.block_index(); @@ -1133,34 +1135,32 @@ pub fn get_proposer_score(store: &Store, config: &Config) -> Result { Ok(committee_weight.saturating_mul(config.proposer_score_boost) / 100) } -/// The LMD GHOST weight of `root`: the effective balance of every -/// non-equivocating, active, unslashed validator whose latest vote descends -/// through `root`, plus the proposer boost if it applies. +/// The effective balance of every non-equivocating, active, unslashed +/// validator in `state` whose latest vote descends through `root`: the +/// attestation half of [`get_weight`], without the proposer boost. /// /// Takes `index` rather than building it, the way [`filter_block_tree`] does, /// and reuses it for every [`get_ancestor`] call this makes: one per active -/// validator, plus one for the proposer boost. See [`get_ancestor`]'s -/// documentation for why that matters. +/// validator. See [`get_ancestor`]'s documentation for why that matters. /// -/// The specification's own per-root definition, kept as written. [`get_head`] -/// calls [`compute_weights`] instead, which produces the same numbers for the -/// whole tree at once; this is what that is tested against. -pub fn get_weight( +/// Split out of [`get_weight`] so [`is_head_weak`] and [`is_parent_strong`] +/// can score a root against the justified state without also asking whether +/// the proposer boost applies to it; the specification makes the same split, +/// since neither of those checks the boost. +pub fn get_attestation_score( store: &Store, index: &HashMap, root: Root, - config: &Config, + state: &BeaconState, ) -> Result { - let justified_checkpoint = store.beacon_justified_checkpoint(); - let state = checkpoint_state(store, &justified_checkpoint, config)?; - let current_epoch = get_current_epoch(&state); + let current_epoch = get_current_epoch(state); let block_slot = index .get(&root) .ok_or(Error::SpecAssert("root in store.blocks"))? .0; let mut attestation_score: Gwei = 0; - for validator_index in get_active_validator_indices(&state, current_epoch) { + for validator_index in get_active_validator_indices(state, current_epoch) { let validator = state.validator(validator_index)?; if validator.slashed || store.is_equivocating(validator_index) { continue; @@ -1172,12 +1172,34 @@ pub fn get_weight( attestation_score = attestation_score.saturating_add(validator.effective_balance); } } + Ok(attestation_score) +} + +/// The LMD GHOST weight of `root`: [`get_attestation_score`] against the +/// justified checkpoint's state, plus the proposer boost if it applies. +/// +/// The specification's own per-root definition, kept as written. [`get_head`] +/// calls [`compute_weights`] instead, which produces the same numbers for the +/// whole tree at once; this is what that is tested against. +pub fn get_weight( + store: &Store, + index: &HashMap, + root: Root, + config: &Config, +) -> Result { + let justified_checkpoint = store.beacon_justified_checkpoint(); + let state = checkpoint_state(store, &justified_checkpoint, config)?; + let attestation_score = get_attestation_score(store, index, root, &state)?; let proposer_boost_root = store.proposer_boost_root(); if proposer_boost_root.is_zero() { return Ok(attestation_score); } + let block_slot = index + .get(&root) + .ok_or(Error::SpecAssert("root in store.blocks"))? + .0; let mut proposer_score: Gwei = 0; if get_ancestor(index, proposer_boost_root, block_slot)? == root { proposer_score = get_proposer_score(store, config)?; @@ -1429,27 +1451,28 @@ pub fn get_filtered_block_tree( /// itself: it is the specification's own second whole-`Dict` scan the module /// documentation calls out, but it never costs a further backend round trip. /// -/// Takes `&mut Store`, unlike most functions in this file: it records the head -/// it just found through -/// [`Store::update_checkpoints`](ethlambda_storage::Store::update_checkpoints), -/// the head-and-checkpoint writer both chains share, so a restarted node has -/// something to answer from immediately rather than replaying this whole walk -/// on its first tick. That writer also keeps the canonical `BlockRoots` index -/// in step with the branch fork choice just picked. -/// -/// Written unconditionally on every call, not only when the head changes: a -/// value written once and then left alone is a second source of truth a bug -/// can let drift, and the write is one small metadata row plus an index diff -/// that is empty whenever the head did not move, set against a whole weighted -/// tree walk. -pub fn get_head(store: &mut Store, config: &Config) -> Result { - // One scan for the whole walk: the filtered tree and the weight table are - // both built from it, instead of each rescanning the live chain for itself. - let index = store.block_index(); - let blocks = get_filtered_block_tree(store, &index, config)?; +/// Takes `&Store` rather than `&mut Store`: unlike [`get_head`], which records +/// the head it finds, this is the read-only walk underneath it. [`on_block`] +/// is the other caller, which needs the head *before* the block it is +/// importing joins the store (see `update_proposer_boost_root` in the +/// specification's `on_block`) and must not record that transient answer as +/// the store's own head. +/// +/// Takes `index` rather than building it, matching [`get_filtered_block_tree`] +/// and [`compute_weights`], which this hands the same one to: [`get_head`] +/// has nothing else to build it from, but [`on_block`] already scanned +/// `LiveChain` once for its own finalized-descendant check by the time it +/// gets here, and passes that scan's result on rather than paying for a +/// second one just to compute the head. +fn compute_head( + store: &Store, + index: &HashMap, + config: &Config, +) -> Result { + let blocks = get_filtered_block_tree(store, index, config)?; // Every candidate's weight at once: see `compute_weights` for why the // specification's per-root `get_weight` is not what the descent calls. - let weights = compute_weights(store, &index, config)?; + let weights = compute_weights(store, index, config)?; let mut head = store.beacon_justified_checkpoint().root; loop { let children: Vec = blocks @@ -1477,10 +1500,30 @@ pub fn get_head(store: &mut Store, config: &Config) -> Result { .1; } + Ok(head) +} + +/// [`compute_head`], recorded as the store's own head. +/// +/// Takes `&mut Store`, unlike most functions in this file: it records the head +/// it just found through +/// [`Store::update_checkpoints`](ethlambda_storage::Store::update_checkpoints), +/// the head-and-checkpoint writer both chains share, so a restarted node has +/// something to answer from immediately rather than replaying this whole walk +/// on its first tick. That writer also keeps the canonical `BlockRoots` index +/// in step with the branch fork choice just picked. +/// +/// Written unconditionally on every call, not only when the head changes: a +/// value written once and then left alone is a second source of truth a bug +/// can let drift, and the write is one small metadata row plus an index diff +/// that is empty whenever the head did not move, set against a whole weighted +/// tree walk. +pub fn get_head(store: &mut Store, config: &Config) -> Result { + let index = store.block_index(); + let head = compute_head(store, &index, config)?; store .update_checkpoints(ForkCheckpoints::head_only(head)) .expect("record beacon head"); - Ok(head) } @@ -1633,22 +1676,105 @@ pub fn is_proposing_on_time(store: &Store, config: &Config) -> bool { /// Whether `head_root` has few enough votes to be overpowered by the /// proposer's own boost, i.e. reorging it out would not be fighting an /// already-decisive lead. -pub fn is_head_weak(store: &Store, head_root: Root, config: &Config) -> Result { +/// +/// Counts an equivocating validator's effective balance toward `head_root`'s +/// weight whenever that validator sits in one of the head slot's committees, +/// on top of [`get_attestation_score`]'s own vote-based count. Without this, +/// the weight this reads could only fall as more equivocation evidence +/// arrived (an attester's vote stops counting once it is known to have +/// equivocated), so a head could flip from "weak" to "not weak" to "weak" +/// again as evidence trickled in; adding the equivocators' balance back in +/// keeps the total monotonic, so once a head is not weak it cannot become weak +/// again from something that was already true when it was imported. +/// +/// `committees` derives the head slot's committees off `head_root`'s own +/// post-state, per bci's committee cache (`helpers/accessors.rs`), rather than +/// recomputing the shuffling by hand. +pub fn is_head_weak( + store: &Store, + head_root: Root, + config: &Config, + committees: &CommitteeCache, +) -> Result { let justified_checkpoint = store.beacon_justified_checkpoint(); let justified_state = checkpoint_state(store, &justified_checkpoint, config)?; let reorg_threshold = calculate_committee_fraction(&justified_state, config.reorg_head_weight_threshold)?; - Ok(get_weight(store, &store.block_index(), head_root, config)? < reorg_threshold) + + let index = store.block_index(); + let mut head_weight = get_attestation_score(store, &index, head_root, &justified_state)?; + + let head_state = store + .get_state(&head_root) + .expect("get") + .ok_or(Error::SpecAssert("head_root in store.block_states"))?; + let (head_slot, _) = store + .block_entry(&head_root) + .ok_or(Error::SpecAssert("head_root in store.blocks"))?; + let epoch = compute_epoch_at_slot(head_slot); + let epoch_committees = committees.committees(&head_state, epoch); + for committee_index in 0..epoch_committees.committees_per_slot() { + for &validator_index in epoch_committees.committee(head_slot, committee_index)? { + if store.is_equivocating(validator_index) { + let validator = justified_state.validator(validator_index)?; + head_weight = head_weight.saturating_add(validator.effective_balance); + } + } + } + + Ok(head_weight < reorg_threshold) } -/// Whether `parent_root` already has enough votes of its own that the missing -/// votes are assigned to it rather than being hoarded elsewhere. -pub fn is_parent_strong(store: &Store, parent_root: Root, config: &Config) -> Result { +/// Whether `root`'s parent already has enough votes of its own that the +/// missing votes are assigned to it rather than being hoarded elsewhere. +/// +/// Takes `root`, not the parent directly: the specification derives +/// `parent_root` from `store.blocks[root].parent_root` inside this function, +/// so a caller (`get_proposer_head`) that already looked up the parent for +/// its own purposes and this function's own lookup cannot disagree about +/// which block that is. +pub fn is_parent_strong(store: &Store, root: Root, config: &Config) -> Result { let justified_checkpoint = store.beacon_justified_checkpoint(); let justified_state = checkpoint_state(store, &justified_checkpoint, config)?; let parent_threshold = calculate_committee_fraction(&justified_state, config.reorg_parent_weight_threshold)?; - Ok(get_weight(store, &store.block_index(), parent_root, config)? > parent_threshold) + let (_, parent_root) = store + .block_entry(&root) + .ok_or(Error::SpecAssert("root in store.blocks"))?; + let index = store.block_index(); + let parent_weight = get_attestation_score(store, &index, parent_root, &justified_state)?; + Ok(parent_weight > parent_threshold) +} + +/// Whether `root`'s proposer has published more than one block for its slot. +/// +/// The specification scans every known block for one sharing `root`'s slot +/// and proposer. `index` is not free (it is a `Store::block_index` call, a +/// full `LiveChain` scan), but every caller here has already paid for one for +/// its own purposes and passes that same result on, so this answers the slot +/// half with no *further* scan, and pays for [`Store::get_signed_block`] only +/// on the handful of blocks that actually compete at that one slot, not the +/// whole indexed history. +pub fn is_proposer_equivocation( + store: &Store, + index: &HashMap, + root: Root, +) -> Result { + let block = store + .get_signed_block(&root) + .expect("get") + .ok_or(Error::SpecAssert("root in store.blocks"))?; + let proposer_index = block.proposer_index(); + let slot = block.slot(); + + let matching_roots = index + .iter() + .filter(|&(_, &(candidate_slot, _))| candidate_slot == slot) + .filter_map(|(&candidate_root, _)| store.get_signed_block(&candidate_root).expect("get")) + .filter(|candidate| candidate.proposer_index() == proposer_index) + .count(); + + Ok(matching_roots > 1) } /// The block a proposer at `slot` should build on: `head_root`'s parent @@ -1658,15 +1784,30 @@ pub fn is_parent_strong(store: &Store, parent_root: Root, config: &Config) -> Re /// *Note*: the ordering of conditions here is the specification's suggested /// order, not a requirement; an implementation may reorder or short-circuit /// for performance. +/// +/// Fulu (EIP-7917) drops the `shuffling_stable` requirement: `shuffling_stable` +/// is folded into `true` rather than left out of the `&&` chain, which reads +/// the same as the specification's own two near-identical copies of this +/// function without keeping two Rust copies to drift apart. Every fork this +/// module implements shares one copy for the same reason no other condition +/// here has a per-fork variant. pub fn get_proposer_head( store: &Store, head_root: Root, slot: Slot, config: &Config, + committees: &CommitteeCache, ) -> Result { - let (head_slot, parent_root) = store - .block_entry(&head_root) + // Read once and reused for `head_slot`/`parent_root` below and for the + // fulu gate just after: `Store::block_entry` decodes the same signed + // block internally on a beacon directory anyway (see its own + // documentation), so this pays for one decode rather than two. + let head_block = store + .get_signed_block(&head_root) + .expect("get") .ok_or(Error::SpecAssert("head_root in store.blocks"))?; + let head_slot = head_block.slot(); + let parent_root = head_block.parent_root(); let (parent_slot, _) = store .block_entry(&parent_root) .ok_or(Error::SpecAssert("parent_root in store.blocks"))?; @@ -1675,8 +1816,14 @@ pub fn get_proposer_head( // deadline. let head_late = is_head_late(store, head_root)?; // Do not re-org on an epoch boundary where the proposer shuffling could - // change. - let shuffling_stable = is_shuffling_stable(slot); + // change. [Modified in Fulu:EIP7917] The proposer lookahead fixes + // assignments before the epoch boundary, so this is no longer a + // requirement once the head being reorged is itself a fulu-or-later + // block. Read off `head_block`'s own fork rather than + // `Config::fork_at_epoch`: the config's schedule is the chain's real + // activation epochs, which a small-slot fixture case never reaches, while + // the block in hand already carries the fork that produced it. + let shuffling_stable = head_block.fork_name() >= ForkName::Fulu || is_shuffling_stable(slot); // Ensure that the FFG information of the new head will be competitive // with the current head. let ffg_competitive = is_ffg_competitive(store, head_root, parent_root)?; @@ -1696,11 +1843,16 @@ pub fn get_proposer_head( store.proposer_boost_root() != head_root, "store.proposer_boost_root != head_root", )?; - let head_weak = is_head_weak(store, head_root, config)?; + let head_weak = is_head_weak(store, head_root, config, committees)?; // Check that the missing votes are assigned to the parent and not being // hoarded. - let parent_strong = is_parent_strong(store, parent_root, config)?; + let parent_strong = is_parent_strong(store, head_root, config)?; + + // Re-org more aggressively if there is a proposer equivocation in the + // previous slot. + let index = store.block_index(); + let proposer_equivocation = is_proposer_equivocation(store, &index, head_root)?; if head_late && shuffling_stable @@ -1713,104 +1865,13 @@ pub fn get_proposer_head( { // We can re-org the current head by building upon its parent block. Ok(parent_root) + } else if head_weak && current_time_ok && proposer_equivocation { + Ok(parent_root) } else { Ok(head_root) } } -/// Whether a proposer confident it will build the next block should ask its -/// execution engine to build on `head_root`'s parent instead of `head_root` -/// itself, suppressing the `notify_forkchoice_updated` call bellatrix's -/// `ExecutionEngine` protocol would otherwise make right away. -/// -/// `validator_is_connected` stands in for the specification's own -/// `validator_is_connected(validator_index: ValidatorIndex) -> bool`, "a -/// function that indicates whether the validator ... is connected to the -/// node (e.g. has sent an unexpired proposer preparation message)" -/// (`specs/bellatrix/fork-choice.md`). Every real answer is -/// implementation-specific, so a caller supplies its own policy here rather -/// than this module guessing at one; the fixture suites that exercise this -/// supply a fixed answer directly, the same way [`stf::ExecutionEngine`] -/// stands in for a real execution client elsewhere in this module. -/// -/// Shares [`get_proposer_head`]'s own reorg conditions -/// (`is_head_late`/`is_shuffling_stable`/`is_ffg_competitive`/`is_finalization_ok`), -/// but evaluated against `proposal_slot` (`head_root`'s slot plus one) -/// rather than the caller's own current slot: this asks about the block a -/// confident proposer is *about* to build, one slot ahead of `head_root`, -/// not about reorging a block already received. -pub fn should_override_forkchoice_update( - store: &Store, - head_root: Root, - validator_is_connected: impl Fn(ValidatorIndex) -> bool, - config: &Config, -) -> Result { - let (head_slot, parent_root) = store - .block_entry(&head_root) - .ok_or(Error::SpecAssert("head_root in store.blocks"))?; - let (parent_slot, _) = store - .block_entry(&parent_root) - .ok_or(Error::SpecAssert("parent_root in store.blocks"))?; - let current_slot = get_current_slot(store, config); - let proposal_slot = head_slot.saturating_add(1); - - // Only re-org the head block if it arrived later than the attestation - // deadline. - let head_late = is_head_late(store, head_root)?; - // Shuffling stable. - let shuffling_stable = is_shuffling_stable(proposal_slot); - // FFG information of the new head block will be competitive with the - // current head. - let ffg_competitive = is_ffg_competitive(store, head_root, parent_root)?; - // Do not re-org if the chain is not finalizing with acceptable frequency. - let finalization_ok = is_finalization_ok(store, proposal_slot, config); - - // Only suppress the fork choice update if we are confident that we will - // propose the next block. `get_state` hands back a shared `Arc`, so this - // clones out of it before advancing: matching the specification's own - // `.copy()`, advancing to `proposal_slot` is only how this samples the - // proposer that slot would draw, not a change the store's own cached - // entry for `parent_root` should keep. - let parent_state = store - .get_state(&parent_root) - .expect("get") - .ok_or(Error::SpecAssert("parent_root in store.block_states"))?; - let mut parent_state_advanced = (*parent_state).clone(); - stf::process_slots(&mut parent_state_advanced, proposal_slot, config)?; - let proposer_index = get_beacon_proposer_index(&parent_state_advanced)?; - let proposing_reorg_slot = validator_is_connected(proposer_index); - - // Single slot re-org. - let parent_slot_ok = parent_slot.checked_add(1) == Some(head_slot); - let proposing_on_time = is_proposing_on_time(store, config); - // Note that this condition is different from `get_proposer_head`. - let current_time_ok = - head_slot == current_slot || (proposal_slot == current_slot && proposing_on_time); - let single_slot_reorg = parent_slot_ok && current_time_ok; - - // Check the head weight only if the attestations from the head slot have - // already been applied; before then, both conditions default to true - // rather than judging the head on attestations that have not arrived - // yet. - let (head_weak, parent_strong) = if current_slot > head_slot { - ( - is_head_weak(store, head_root, config)?, - is_parent_strong(store, parent_root, config)?, - ) - } else { - (true, true) - }; - - Ok(head_late - && shuffling_stable - && ffg_competitive - && finalization_ok - && proposing_reorg_slot - && single_slot_reorg - && head_weak - && parent_strong) -} - // --------------------------------------------------------------------------- // Merge transition helpers (bellatrix) // --------------------------------------------------------------------------- @@ -2076,6 +2137,158 @@ pub fn compute_pulled_up_tip( Ok(()) } +// --------------------------------------------------------------------------- +// on_block helpers +// --------------------------------------------------------------------------- + +/// Whether a block at `block_slot` arrived before its slot's attestation +/// deadline: [`record_block_timeliness`]'s own definition of timely, minus +/// the write. +/// +/// Split out so [`on_block`] can gate the (otherwise unconditional) +/// proposer-boost head computation on the same answer *before* the block +/// joins the store, without either recomputing the formula by hand or paying +/// to look the block back up once it has, the way [`record_block_timeliness`] +/// does. +fn is_block_timely(store: &Store, block_slot: Slot, config: &Config) -> bool { + let time_into_slot_ms = store.ms_since_genesis() % config.slot_duration_ms; + let epoch = get_current_store_epoch(store, config); + let attestation_threshold_ms = get_attestation_due_ms(epoch, config); + let is_before_attesting_interval = time_into_slot_ms < attestation_threshold_ms; + get_current_slot(store, config) == block_slot && is_before_attesting_interval +} + +/// Records whether `root`'s block, at `block_slot`, arrived before its slot's +/// attestation deadline: the timeliness [`update_proposer_boost_root`] and +/// [`is_head_late`] both read back out of +/// [`Store::block_timeliness`](ethlambda_storage::Store::block_timeliness). +/// +/// Takes `block_slot` directly rather than reading it back out of the block +/// [`on_block`] just inserted: `Store::block_entry` decodes the whole signed +/// block to answer it on a beacon directory (see its own documentation), and +/// the one caller here always already has the slot, from the block it just +/// inserted. +/// +/// `pub(crate)` rather than `pub`, unlike most functions in this file: unlike +/// [`is_block_timely`], which mirrors a piece of the specification's own +/// `record_block_timeliness`, this one's own signature (`block_slot` as a +/// parameter) is this crate's adaptation of it rather than a transcription, +/// and [`on_block`] is its only caller. +pub(crate) fn record_block_timeliness( + store: &mut Store, + root: Root, + block_slot: Slot, + config: &Config, +) { + let is_timely = is_block_timely(store, block_slot, config); + store.set_block_timeliness(root, is_timely); +} + +/// The first slot of the lookahead window `epoch`'s proposer shuffling opens +/// in: `MIN_SEED_LOOKAHEAD` epochs before `epoch` itself starts. +pub fn compute_shuffling_lookahead_start_slot(epoch: Epoch) -> Slot { + let lookahead_epoch = epoch.saturating_sub(preset::MIN_SEED_LOOKAHEAD); + compute_start_slot_at_epoch(lookahead_epoch) +} + +/// The last slot before `epoch`'s proposer shuffling could still change: one +/// before [`compute_shuffling_lookahead_start_slot`]. +pub fn compute_shuffling_dependent_slot(epoch: Epoch) -> Slot { + compute_shuffling_lookahead_start_slot(epoch).saturating_sub(1) +} + +/// Like [`get_ancestor`], but stops at the lowest indexed ancestor on +/// `root`'s chain instead of failing when the walk would need to step past +/// it. +/// +/// The specification's own `get_ancestor` never needs this: a full node's +/// earliest block is genesis, at slot 0, so a walk toward any real target +/// slot always terminates there. A checkpoint-synced node's anchor is not +/// slot 0, and its `parent_root` is a real historical root this store never +/// held, so [`get_ancestor`] would raise `SpecAssert("root in store.blocks")` +/// stepping past it. [`get_shuffling_dependent_root`] is the one caller that +/// can ask for a slot below the anchor (shortly after sync, the dependent +/// slot for the current epoch can still be that low), so it uses this +/// instead: the lowest indexed ancestor is the anchor itself for every chain +/// this store holds, since everything in it descends from that one root, so +/// two different chains asking this the same question both land on it and +/// answer the same way `get_ancestor` would once genesis, rather than the +/// anchor, is the floor. +fn get_ancestor_or_lowest_indexed( + index: &HashMap, + root: Root, + slot: Slot, +) -> Root { + let mut current = root; + loop { + let Some(&(block_slot, parent_root)) = index.get(¤t) else { + // `current` is not itself indexed. Every caller starts this walk + // from an indexed root, so this only happens after stepping to a + // `parent_root` the loop below already checked is indexed, which + // makes this branch unreachable; kept as a safe fallback rather + // than an `expect`, since returning wherever the walk got to is + // still a sound answer even if that invariant ever slipped. + return current; + }; + if block_slot <= slot || !index.contains_key(&parent_root) { + return current; + } + current = parent_root; + } +} + +/// The block that fixed `epoch`'s proposer shuffling on `root`'s chain: the +/// ancestor of `root` at [`compute_shuffling_dependent_slot`], or the lowest +/// indexed ancestor on `root`'s chain if that slot is not covered; see +/// [`get_ancestor_or_lowest_indexed`] for why the second case is sound. +pub fn get_shuffling_dependent_root( + index: &HashMap, + root: Root, + epoch: Epoch, +) -> Root { + get_ancestor_or_lowest_indexed(index, root, compute_shuffling_dependent_slot(epoch)) +} + +/// Boosts `root`'s weight, if it is the first block seen for its slot, arrived +/// on time, and shares `head`'s proposer shuffling. +/// +/// The shuffling check is what `v1.7.0` added over the boost's original rule +/// (timely and first, full stop): a block whose proposer shuffling has already +/// diverged from the chain fork choice currently follows is not boosted, so +/// the boost cannot itself be the thing that drags the head onto a branch with +/// a different, no-longer-relevant view of who was supposed to propose it. +/// +/// `head` must be the head [`compute_head`] found *before* `root` joined the +/// store: [`on_block`] is this function's only caller, and it passes exactly +/// that. `index` should include `root`'s own entry (`on_block` extends its +/// own pre-insertion index with it rather than re-scanning `LiveChain`), or +/// the walk below falls back to whatever the closest indexed ancestor +/// answers, per [`get_ancestor_or_lowest_indexed`]. +/// +/// `pub(crate)`, not `pub`: like [`record_block_timeliness`], this one's +/// `index` parameter is this crate's own adaptation, not a transcription, and +/// [`on_block`] is its only caller. +pub(crate) fn update_proposer_boost_root( + store: &mut Store, + index: &HashMap, + head: Root, + root: Root, + config: &Config, +) { + let is_first_block = store.proposer_boost_root().is_zero(); + let is_timely = store + .block_timeliness(&root) + .expect("record_block_timeliness always runs first, on the same root"); + let epoch = get_current_store_epoch(store, config); + let head_dependent_root = get_shuffling_dependent_root(index, head, epoch); + let block_dependent_root = get_shuffling_dependent_root(index, root, epoch); + let is_same_dependent_root = head_dependent_root == block_dependent_root; + + if is_timely && is_first_block && is_same_dependent_root { + store.set_proposer_boost_root(root); + } +} + // --------------------------------------------------------------------------- // on_tick helpers // --------------------------------------------------------------------------- @@ -2319,6 +2532,32 @@ pub fn on_block( let block_root = signed_block.message_hash_tree_root(); let parent_root = signed_block.parent_root(); + // Return early if the block already has a post-state: a re-delivery must + // not re-run `record_block_timeliness`/`update_proposer_boost_root` below + // on a block already fully imported. The chain actor's own import cascade + // already deduplicates a known root before ever reaching here + // (`Store::has_state`, which also skips re-running `state_transition`; + // see its own call site's documentation for the cost of not having that + // check), so in practice this rarely fires from that path; it is the + // specification's own guard, for every caller, mapped onto what + // `store.blocks` membership means here. + // + // `Store::has_state`, not `Store::has_block`: the specification's + // `store.blocks` only gains a root at the very end of `on_block`, once a + // post-state has been computed for it, so membership there is really "has + // been imported". This store's own `blocks` table does not line up with + // that: a block the actor is holding for missing data columns is + // persisted *before* import (`holding_a_block_persists_it_and_records_its_root`), + // so `has_block` answers true for a root that has never actually run + // `state_transition`. Guarding on it instead of `has_state` made this + // function answer `Ok(())` for a held block without importing it, which + // starved every later delivery naming it as an ancestor: none of them + // could see a post-state either, so each walked back to this root, found + // it "already known", and gave up without importing anything, forever. + if store.has_state(&block_root).expect("get") { + return Ok(()); + } + // Parent block must be known. `get_state` hands back a shared `Arc`, which // both checks the parent is known and gives the value to clone the copy // `state_transition` below mutates from: `state_transition` must not be @@ -2351,7 +2590,15 @@ pub fn on_block( // finalized slot. A single-call index: see `get_ancestor`'s documentation // for why a per-hop lookup would be the wrong trade, which does not apply // to this one walk. - let index = store.block_index(); + // + // `mut`, and kept alive for the rest of this function: the pre-import + // head computation and the proposer-boost shuffling check further down + // both want this same pre-insertion snapshot of `LiveChain` (`compute_head` + // takes it as a parameter, matching `get_filtered_block_tree`/ + // `compute_weights`), so it is built once here rather than three times + // over. Once `block_root` itself joins the store below, one `insert` + // keeps this index in step with it instead of re-scanning for that alone. + let mut index = store.block_index(); let finalized_checkpoint_block = get_checkpoint_block(&index, parent_root, finalized_checkpoint.epoch)?; verify( @@ -2493,14 +2740,38 @@ pub fn on_block( let current_justified = state.current_justified_checkpoint(); let finalized = state.finalized_checkpoint(); - // Add new block to the store, and the new state for this block to the - // store. `block_slot` is copied out first since `signed_block` moves next. + // [New in v1.7.0] Whether this block can even be a candidate for the + // proposer boost, decided *before* it joins the store and before the one + // fallible step below runs, so that a failure here leaves the store + // exactly as it was. `update_proposer_boost_root`'s own gate is + // `is_timely and is_first_block and `; the first two conditions are cheap and already decide most + // blocks, especially every block a follower receives while syncing, which + // is never timely. Computing that head (`compute_head`: a `LiveChain` + // scan, a filtered-tree walk, and a pass over every latest message; see + // `compute_weights` for why the last of those matters at scale) is not + // worth paying for on a block the shuffling check could not change the + // answer for anyway. let block_slot = signed_block.slot(); + let is_timely = is_block_timely(store, block_slot, config); + let is_first_block = store.proposer_boost_root().is_zero(); + let pre_block_head = if is_timely && is_first_block { + Some(compute_head(store, &index, config)?) + } else { + None + }; + + // Add new block to the store, and the new state for this block to the + // store. let signed_block_el_hash = signed_block.execution_block_hash(); store .insert_signed_block(block_root, signed_block) .expect("insert"); store.insert_state(block_root, state).expect("insert"); + // Keep `index` in step with the one block that changed, rather than + // re-scanning `LiveChain` for it: see this function's earlier comment on + // `index` for who below still needs it. + index.insert(block_root, (block_slot, parent_root)); // Cache this block's own execution hash for `forkchoiceUpdated` and for the // `latestValidHash` walk, and record whether the execution layer has @@ -2529,19 +2800,13 @@ pub fn on_block( PayloadValidity::NotRequired | PayloadValidity::Invalidated { .. } => {} } - // Add block timeliness to the store. - let time_into_slot_ms = store.ms_since_genesis() % config.slot_duration_ms; - let epoch = get_current_store_epoch(store, config); - let attestation_threshold_ms = get_attestation_due_ms(epoch, config); - let is_before_attesting_interval = time_into_slot_ms < attestation_threshold_ms; - let is_timely = get_current_slot(store, config) == block_slot && is_before_attesting_interval; - store.set_block_timeliness(block_root, is_timely); - - // Add proposer score boost if the block is timely and not conflicting - // with an existing block. - let is_first_block = store.proposer_boost_root().is_zero(); - if is_timely && is_first_block { - store.set_proposer_boost_root(block_root); + // Add block timeliness to the store, and boost its score if it is timely, + // first, and shares the pre-import head's proposer shuffling. Both calls + // are infallible: nothing from here to the end of this function can turn + // into an `Err`, and the store has already been mutated above. + record_block_timeliness(store, block_root, block_slot, config); + if let Some(pre_block_head) = pre_block_head { + update_proposer_boost_root(store, &index, pre_block_head, block_root, config); } // Update checkpoints in store if necessary. @@ -2719,8 +2984,12 @@ mod tests { use ethlambda_storage::backend::InMemoryBackend; use super::*; + use crate::beacon::bls; use crate::beacon::containers::BeaconBlockHeader; + use crate::beacon::helpers::accessors; + use crate::beacon::helpers::misc::compute_signing_root; use crate::beacon::helpers::test_state; + use crate::beacon::primitives::BlsSignature; /// A store backed by a fresh in-memory backend, with every checkpoint at /// its default (genesis) value and no anchor block or state written. @@ -2777,6 +3046,20 @@ mod tests { }) } + /// [`block`], with an explicit proposer index rather than the default + /// `0`, for tests that care about which proposer a block names. + fn block_with_proposer( + slot: Slot, + parent_root: Root, + proposer_index: ValidatorIndex, + ) -> SignedBeaconBlock { + let SignedBeaconBlock::Phase0(mut inner) = block(slot, parent_root) else { + unreachable!("`block` builds a phase0 signed block"); + }; + inner.message.proposer_index = proposer_index; + SignedBeaconBlock::Phase0(inner) + } + /// An exact anchor pair: `state` is `block`'s own post-state. /// /// Built as a fixed point, the way the state transition produces one. The @@ -2992,6 +3275,40 @@ mod tests { assert!(get_ancestor(&index, Root::repeat_byte(9), 0).is_err()); } + /// A checkpoint-synced anchor's own `parent_root` names real history this + /// store never held, so it is never indexed. `get_shuffling_dependent_root` + /// asking for a dependent slot below such an anchor (which the import + /// benchmark's `replay.rs` hits on its very first block: the clock is set + /// to that block's own slot, so the current epoch can be as little as one + /// past the anchor's) must not walk into that gap the way plain + /// `get_ancestor` would. + #[test] + fn get_shuffling_dependent_root_stops_at_a_non_genesis_anchor() { + let anchor_root = Root::repeat_byte(1); + let anchor_slot = 100; + let unindexed_parent = Root::repeat_byte(0xff); + + let mut index = HashMap::new(); + index.insert(anchor_root, (anchor_slot, unindexed_parent)); + + // One epoch past the anchor's own: still low enough that the + // dependent slot (`MIN_SEED_LOOKAHEAD` epochs, minus one, before it) + // falls before `anchor_slot`. + let epoch = compute_epoch_at_slot(anchor_slot) + 1; + assert!( + compute_shuffling_dependent_slot(epoch) < anchor_slot, + "the scenario this test exists for requires a dependent slot \ + below the anchor" + ); + + assert_eq!( + get_shuffling_dependent_root(&index, anchor_root, epoch), + anchor_root, + "the walk must stop at the lowest indexed ancestor rather than \ + stepping into `unindexed_parent`" + ); + } + #[test] fn compute_slots_since_epoch_start_counts_from_the_epoch_boundary() { let epoch_start = compute_start_slot_at_epoch(3); @@ -3048,6 +3365,136 @@ mod tests { ); } + #[test] + fn is_proposer_equivocation_true_for_two_blocks_from_the_same_proposer_and_slot() { + let a_root = Root::repeat_byte(2); + let b_root = Root::repeat_byte(3); + let mut store = empty_store(); + store + .insert_signed_block(a_root, block_with_proposer(1, Root::ZERO, 7)) + .unwrap(); + store + .insert_signed_block(b_root, block_with_proposer(1, Root::ZERO, 7)) + .unwrap(); + let index = index(&[(a_root, 1, Root::ZERO), (b_root, 1, Root::ZERO)]); + + assert!( + is_proposer_equivocation(&store, &index, a_root).unwrap(), + "two blocks sharing a slot and a proposer must be an equivocation" + ); + } + + #[test] + fn is_proposer_equivocation_false_for_two_blocks_same_slot_different_proposers() { + let a_root = Root::repeat_byte(2); + let b_root = Root::repeat_byte(3); + let mut store = empty_store(); + store + .insert_signed_block(a_root, block_with_proposer(1, Root::ZERO, 7)) + .unwrap(); + store + .insert_signed_block(b_root, block_with_proposer(1, Root::ZERO, 8)) + .unwrap(); + let index = index(&[(a_root, 1, Root::ZERO), (b_root, 1, Root::ZERO)]); + + assert!( + !is_proposer_equivocation(&store, &index, a_root).unwrap(), + "sharing a slot alone, with different proposers, is not an \ + equivocation" + ); + } + + #[test] + fn is_head_weak_counts_an_equivocating_validators_balance_from_the_head_slot_committees() { + let config = Config::active(); + // Exactly `SLOTS_PER_EPOCH` active validators and one committee per + // slot puts exactly one validator in the anchor's own slot's + // committee 0, so which index the shuffle picks does not matter to + // this test: it asks the same cache `is_head_weak` will. + let (mut store, anchor_root, anchor_slot) = + anchored_store(preset::SLOTS_PER_EPOCH as usize); + let committees = CommitteeCache::default(); + + assert!( + is_head_weak(&store, anchor_root, &config, &committees).unwrap(), + "a head with no votes and no equivocators must be weak" + ); + + let head_state = store + .get_state(&anchor_root) + .expect("get") + .expect("state exists"); + let epoch = compute_epoch_at_slot(anchor_slot); + let equivocator = committees + .committees(&head_state, epoch) + .committee(anchor_slot, 0) + .expect("committee 0 of the anchor's own slot")[0]; + store.insert_equivocating_index(equivocator); + + assert!( + !is_head_weak(&store, anchor_root, &config, &committees).unwrap(), + "the equivocator's own effective balance must count toward the \ + head's weight, clearing the reorg threshold on its own" + ); + } + + #[test] + fn get_proposer_head_reorgs_a_weak_timely_head_on_proposer_equivocation() { + let config = Config::active(); + let genesis_root = Root::repeat_byte(1); + let parent_root = Root::repeat_byte(2); + let head_root = Root::repeat_byte(3); + let twin_root = Root::repeat_byte(4); + let committees = CommitteeCache::default(); + + let mut store = store_anchored_at(genesis_root); + store + .insert_signed_block(genesis_root, block(0, Root::ZERO)) + .unwrap(); + store + .insert_signed_block(parent_root, block(1, genesis_root)) + .unwrap(); + store + .insert_signed_block(head_root, block_with_proposer(2, parent_root, 9)) + .unwrap(); + // A second block for `head_root`'s own slot and proposer: the new + // `is_proposer_equivocation` condition this test exercises. + store + .insert_signed_block(twin_root, block_with_proposer(2, parent_root, 9)) + .unwrap(); + + let state = test_state::with_validators(1); + store.insert_state(genesis_root, state.clone()).unwrap(); + store.insert_state(parent_root, state.clone()).unwrap(); + store.insert_state(head_root, state).unwrap(); + + // Timely: `is_head_late` answers `false`, which alone keeps the main + // reorg branch (`head_late && ...`) from firing, so a `parent_root` + // result below can only come from the new `head_weak && + // current_time_ok && proposer_equivocation` branch. + store.set_block_timeliness(head_root, true); + + // `is_ffg_competitive` needs both roots to already have an + // unrealized justification on record; its value is irrelevant to the + // branch under test as long as both calls succeed. + let checkpoint = Checkpoint { + epoch: 0, + root: genesis_root, + }; + store.set_unrealized_justification(head_root, checkpoint); + store.set_unrealized_justification(parent_root, checkpoint); + + // One slot past `head_root`'s own slot (2): `current_time_ok`. + let proposer_head = get_proposer_head(&store, head_root, 3, &config, &committees) + .expect("every condition this test sets up should let this succeed"); + + assert_eq!( + proposer_head, parent_root, + "a weak, timely head with a second block from its own proposer \ + must still be reorged out" + ); + } + #[test] fn get_voting_source_pulls_up_a_prior_epoch_blocks_vote() { let config = Config::active(); @@ -3173,6 +3620,154 @@ mod tests { assert!(store.is_err()); } + /// [`on_block`]'s early-return guard must read `Store::has_state`, not + /// `Store::has_block`: a block the actor is holding for missing data + /// columns is written with + /// [`Store::insert_pending_block`](ethlambda_storage::Store::insert_pending_block) + /// before it ever imports, which stores the whole signed block in one + /// `BlockHeaders` row (so `has_block` answers true) but writes no + /// `LiveChain` entry (so [`Store::block_index`](ethlambda_storage::Store::block_index), + /// a `LiveChain` scan, does not name it). Guarding on `has_block` instead + /// of `has_state` made `on_block` answer `Ok(())` for a held block + /// without importing it, so every later delivery naming it as an + /// ancestor found it "already known" and gave up without importing + /// anything either -- forever, since nothing ever gave the root a + /// post-state. + /// + /// Delivered timely (the first block for its slot, before the + /// attestation deadline), so this also proves `on_block`'s pre-import + /// `compute_head` call (walking `Store::block_index`, for the new + /// proposer-boost dependent-root gate) tolerates a held block sitting + /// unindexed in the store: that walk never sees it, being a `LiveChain` + /// scan, so it is untouched by the held block's absent post-state. + /// + /// Builds a genuinely valid child block (correct proposer, RANDAO reveal, + /// and state root, all real BLS signatures over + /// [`test_state::secret_key_for`]'s key) rather than the zero-signature + /// [`block`] helper, since this exercises the real, signature-checking + /// [`on_block`], not a fixture harness that can skip verification. + #[test] + fn on_block_imports_a_block_already_persisted_with_no_post_state() { + let config = Config::active(); + let (anchor_state, anchor_block) = anchor_pair_with(1); + let anchor_root = anchor_block.message_hash_tree_root(); + let target_slot = anchor_state.slot() + 1; + + // The proposer and signing domain for `target_slot` only depend on + // advancing the slot clock, not on the block itself, so these are + // derived from their own clone ahead of building the block. + let mut probe_state = anchor_state.clone(); + stf::process_slots(&mut probe_state, target_slot, &config).expect("process_slots"); + let proposer_index = + accessors::get_beacon_proposer_index(&probe_state).expect("get_beacon_proposer_index"); + let proposer_key = test_state::secret_key_for(proposer_index as usize); + let proposer_domain = + accessors::get_domain(&probe_state, constants::DOMAIN_BEACON_PROPOSER, None); + let randao_domain = accessors::get_domain(&probe_state, constants::DOMAIN_RANDAO, None); + let randao_epoch = get_current_epoch(&probe_state); + let randao_reveal = BlsSignature( + proposer_key + .sign( + compute_signing_root(randao_epoch.hash_tree_root(), randao_domain).as_slice(), + bls::DST, + &[], + ) + .to_bytes(), + ); + + let mut draft = block(target_slot, anchor_root); + let SignedBeaconBlock::Phase0(inner) = &mut draft else { + unreachable!("`block` builds a phase0 signed block"); + }; + inner.message.proposer_index = proposer_index; + inner.message.body.randao_reveal = randao_reveal; + + // Learn the real post-state (and so the real state root) by running + // the transition once, unvalidated, the way a proposer decides + // everything but the signature and the state root before signing. + let mut trial_state = anchor_state.clone(); + let trial_committees = CommitteeCache::default(); + stf::state_transition( + &mut trial_state, + &draft, + false, + &config, + &stf::ExecutionEngine::valid(), + &trial_committees, + ) + .expect("the drafted block must transition cleanly"); + let state_root = trial_state.hash_tree_root(); + + let SignedBeaconBlock::Phase0(inner) = &mut draft else { + unreachable!("still phase0"); + }; + inner.message.state_root = state_root; + let signing_root = compute_signing_root(draft.message_hash_tree_root(), proposer_domain); + let SignedBeaconBlock::Phase0(inner) = &mut draft else { + unreachable!("still phase0"); + }; + inner.signature = BlsSignature( + proposer_key + .sign(signing_root.as_slice(), bls::DST, &[]) + .to_bytes(), + ); + let signed_block = draft; + let block_root = signed_block.message_hash_tree_root(); + + // The store holds the child block already, exactly the shape + // `hold_block_for_columns` leaves it in: one `BlockHeaders` row (so + // `has_block` is true and the block is readable back by root) but no + // `LiveChain` entry (so `block_index` does not name it) and no + // post-state (nothing has imported it yet). + let mut store = get_forkchoice_store( + Arc::new(InMemoryBackend::new()), + anchor_state, + anchor_block, + &config, + ) + .expect("the anchor pair matches"); + // `on_block` refuses a block from the future (`get_current_slot(store) + // >= block.slot`), so the store's clock has to have reached the + // child's own slot before this test's `on_block` call below. Set to + // the slot's own start, its first block, so `is_block_timely` is + // true and `on_block` runs its pre-import `compute_head` call too + // (see this test's own doc for why that matters). + store + .set_time_ms(seconds_to_milliseconds( + config.seconds_per_slot * target_slot, + )) + .expect("set time"); + store + .insert_pending_block(block_root, signed_block.clone()) + .expect("insert"); + assert!(store.has_block(&block_root), "persisted above"); + assert!( + !store.has_state(&block_root).expect("get"), + "no import has run yet" + ); + assert!( + !store.block_index().contains_key(&block_root), + "a held block carries no LiveChain entry" + ); + + let committees = CommitteeCache::default(); + on_block( + &mut store, + signed_block, + &config, + &DataAvailability::NotRequired, + &PayloadValidity::NotRequired, + &committees, + ) + .expect("a validly signed, correctly rooted block must still import"); + + assert!( + store.has_state(&block_root).expect("get"), + "on_block must not mistake a persisted-but-unimported (held) \ + block for an already-imported one" + ); + } + #[test] fn the_commitments_subtree_index_is_the_bodys_own_position() { // The generalized index the fixture states, minus the offset of a tree diff --git a/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs b/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs index 6c92e24e7..8fb610278 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs @@ -6,11 +6,9 @@ //! `steps.yaml` in order. A step is one of `tick`, `block`, `attestation`, //! `attester_slashing`, or `on_merge_block` (bellatrix's own `pow_block` //! step), driving the correspondingly named handler, or `checks`, which -//! asserts on the store's own fields and on [`fork_choice::get_head`], -//! [`fork_choice::get_proposer_head`], and -//! [`fork_choice::should_override_forkchoice_update`]. See -//! `tests/formats/fork_choice/README.md` in the pinned specification checkout -//! for the format in full. +//! asserts on the store's own fields and on [`fork_choice::get_head`] and +//! [`fork_choice::get_proposer_head`]. See `tests/formats/fork_choice/README.md` +//! in the pinned specification checkout for the format in full. //! //! No released fixture targets phase0 directly: the earliest suite is //! altair's, built from altair-shaped states because the generator needs a @@ -63,8 +61,12 @@ //! `viable_for_head_roots_and_weights` is part of the format, but no case at //! any implemented fork's `checks` step names it, on either preset, so it is //! not modeled here rather than guessed at. `should_override_forkchoice_update` -//! *is* exercised, once per fork from bellatrix on, and [`apply_checks`] -//! checks it. +//! is in the format's README too, but no `fork-choice.md` at `v1.7.0-beta.2` +//! defines it any more (not even `fast-confirmation.md`, which absorbed most +//! of what bellatrix's `fork-choice.md` dropped): it is gone from the +//! specification outright, not merely relocated, so this crate drops it with +//! it rather than keeping an implementation of a function the pinned release +//! no longer has. //! //! # `columns: []` means "simulate unavailable", not "vacuously available" //! @@ -173,7 +175,7 @@ fn default_valid() -> bool { /// A `checks` step's assertions against the store. /// -/// See the module documentation for the one field of the format this leaves +/// See the module documentation for the two fields of the format this leaves /// out, and why. #[derive(serde::Deserialize)] pub(super) struct Checks { @@ -184,9 +186,6 @@ pub(super) struct Checks { finalized_checkpoint: Option, proposer_boost_root: Option, get_proposer_head: Option, - /// `[New in Bellatrix]` see - /// [`fork_choice::should_override_forkchoice_update`]. - should_override_forkchoice_update: Option, } /// The expected value of [`fork_choice::get_head`], as `checks.head` gives it: @@ -205,16 +204,6 @@ struct CheckpointCheck { root: String, } -/// The expected value of -/// [`fork_choice::should_override_forkchoice_update`]: the fixed -/// `validator_is_connected` answer to call it with, and the result it must -/// then return. -#[derive(serde::Deserialize)] -struct ShouldOverrideForkchoiceUpdateCheck { - validator_is_connected: bool, - result: bool, -} - /// Parses a fixture's `0x`-prefixed hex root. /// /// Duplicated from `ssz_static`'s private helper of the same purpose rather @@ -633,48 +622,21 @@ fn check_get_proposer_head( expected_hex: &str, store: &mut Store, config: &Config, + committees: &CommitteeCache, ) -> Result<(), String> { let head = fork_choice::get_head(store, config).map_err(|err| format!("get_head: {err:?}"))?; let slot = fork_choice::get_current_slot(store, config); - let actual = fork_choice::get_proposer_head(store, head, slot, config) + let actual = fork_choice::get_proposer_head(store, head, slot, config, committees) .map_err(|err| format!("get_proposer_head: {err:?}"))?; check_root("get_proposer_head", expected_hex, actual) } -/// Checks `should_override_forkchoice_update`, the same way -/// [`check_get_proposer_head`] checks `get_proposer_head`: from the current -/// head, not a root the fixture supplies separately. `validator_is_connected` -/// is a fixed answer regardless of which proposer index is asked, matching -/// what the fixture format itself supplies: one bool for the whole call, not -/// a per-validator registry. -fn check_should_override_forkchoice_update( - expected: &ShouldOverrideForkchoiceUpdateCheck, - store: &mut Store, - config: &Config, -) -> Result<(), String> { - let head = fork_choice::get_head(store, config).map_err(|err| format!("get_head: {err:?}"))?; - let actual = fork_choice::should_override_forkchoice_update( - store, - head, - |_| expected.validator_is_connected, - config, - ) - .map_err(|err| format!("should_override_forkchoice_update: {err:?}"))?; - if actual == expected.result { - Ok(()) - } else { - Err(format!( - "should_override_forkchoice_update: expected {}, got {actual}", - expected.result - )) - } -} - /// Applies one `checks` step: every field the fixture sets must match. pub(super) fn apply_checks( store: &mut Store, checks: &Checks, config: &Config, + committees: &CommitteeCache, ) -> Result<(), String> { if let Some(expected) = checks.time { // The fixture's `time` is the specification's seconds; the store keeps @@ -706,10 +668,7 @@ pub(super) fn apply_checks( check_head(expected, store, config)?; } if let Some(expected) = &checks.get_proposer_head { - check_get_proposer_head(expected, store, config)?; - } - if let Some(expected) = &checks.should_override_forkchoice_update { - check_should_override_forkchoice_update(expected, store, config)?; + check_get_proposer_head(expected, store, config, committees)?; } Ok(()) } @@ -747,7 +706,7 @@ fn run_case(case: &Case, config: &Config) -> Result<(), String> { let steps: Vec = case.yaml("steps"); for (index, step) in steps.iter().enumerate() { let outcome = match &step.checks { - Some(checks) => apply_checks(&mut store, checks, config), + Some(checks) => apply_checks(&mut store, checks, config, &committees), None => apply_execution_step(&mut store, case, step, config, &committees), }; outcome.map_err(|err| format!("step {index}: {err}"))?; diff --git a/crates/blockchain/state_transition/tests/beacon_spec/sync.rs b/crates/blockchain/state_transition/tests/beacon_spec/sync.rs index 0b50bddb8..26473b1b3 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/sync.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/sync.rs @@ -241,7 +241,9 @@ fn run_case(case: &Case, config: &Config) -> Result<(), String> { let steps: Vec = case.yaml("steps"); for (index, step) in steps.iter().enumerate() { let outcome = match &step.checks { - Some(checks) => super::fork_choice::apply_checks(&mut store, checks, config), + Some(checks) => { + super::fork_choice::apply_checks(&mut store, checks, config, &committees) + } None => apply_step(&mut store, case, step, config, &committees), }; outcome.map_err(|err| format!("step {index}: {err}"))?; diff --git a/docs/beacon_stf.md b/docs/beacon_stf.md index ac4b74817..c91c7c687 100644 --- a/docs/beacon_stf.md +++ b/docs/beacon_stf.md @@ -472,8 +472,9 @@ forces a decision instead of quietly widening the gap. ### Fork choice is fixture-verified 150 mainnet `fork_choice` cases pass, covering bellatrix's `on_merge_block`/ -terminal-PoW validation, `should_override_forkchoice_update`, deneb's blob -data availability, and fulu's column data availability. +terminal-PoW validation, the `v1.7.0` proposer-boost dependent-root gate and +`get_proposer_head`'s proposer-equivocation branch, deneb's blob data +availability, and fulu's column data availability. The release still ships no phase0 `fork_choice` suite: the earliest is altair's, built from altair-shaped states even though altair changes nothing From c5c0f063987c736b2d19e2e48ba4f1461895a50f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 25 Sep 2026 12:16:01 -0300 Subject: [PATCH 013/132] test(beacon): run the gloas ssz_static vectors Gloas's state transition is not implemented yet, so HIGHEST_IMPLEMENTED_FORK stays at Fulu and every Gloas fixture case is still marked ignored by default. But the fork's containers already exist (containers/gloas.rs), and ssz_static only exercises encoding, hash_tree_root, and decoding, none of which needs a state transition. Turning its Gloas cases on now is free coverage that catches a container mistake (field order, a wrong list kind, a missing progressive_container attribute) long before the state transition that depends on those shapes is written. GLOAS_RUNNERS names the runners allowed to run their Gloas cases ahead of HIGHEST_IMPLEMENTED_FORK reaching Gloas; case_trial consults it alongside Case::in_scope. Only ssz_static is in it for now, so every other runner's Gloas cases stay ignored exactly as before. ssz_static's own handler match gains a Gloas arm for every container whose shape changed (attestations and their aggregates, the block family, the state, the execution payload, its new-payload request, and the builder's execution requests), placed ahead of the >= Electra/>= Deneb range arms they would otherwise fall into, plus unconditional arms for containers gloas introduces outright (the builder registry and its payment queues, payload attestations, the execution payload bid and its envelope, proposer preferences). MatrixEntry and DataColumnsByRootIdentifier are unchanged from fulu, so their existing arms now also match gloas rather than gaining a duplicate. No container needed a fix: every Gloas ssz_static case passes on both presets on the first run. That is 8364 cases on minimal and 340 on mainnet; LightClient* stays ignored, as for every earlier fork. --- .../state_transition/tests/beacon_spec/mod.rs | 19 ++- .../tests/beacon_spec/ssz_static.rs | 128 +++++++++++++++++- 2 files changed, 140 insertions(+), 7 deletions(-) diff --git a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs index 9c03ed7e8..107725589 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs @@ -84,6 +84,16 @@ pub const PRESET: &str = if cfg!(feature = "preset-minimal") { /// is one line. pub const HIGHEST_IMPLEMENTED_FORK: ForkName = ForkName::Fulu; +/// Runners whose Gloas cases already run, while the fork is only partly +/// implemented. +/// +/// [`case_trial`] checks this alongside [`Case::in_scope`], so a runner named +/// here has its Gloas cases run (and counted toward pass/fail) even though +/// [`HIGHEST_IMPLEMENTED_FORK`] has not reached Gloas yet; every other +/// runner's Gloas cases stay ignored. Emptied, and removed along with this +/// check, when [`HIGHEST_IMPLEMENTED_FORK`] becomes `Gloas`. +pub const GLOAS_RUNNERS: &[&str] = &["ssz_static"]; + /// The root of the extracted fixture tree. /// /// Panics rather than skipping when the fixtures are absent. A spec suite that @@ -461,6 +471,12 @@ pub fn fixture_fork_trials() -> Vec { /// itself and print the count; the test harness counts ignored tests already, /// and names each one, which is strictly more than the tally said. /// +/// A Gloas case is the one exception: while Gloas is only partly implemented, +/// [`GLOAS_RUNNERS`] names the runners whose Gloas cases already run, ahead of +/// [`HIGHEST_IMPLEMENTED_FORK`] reaching Gloas, so those cases run (and are +/// counted toward pass/fail) even though [`Case::in_scope`] alone would mark +/// them ignored. +/// /// A panic inside `run` fails this case alone: the harness catches it per test. /// So a fixture that will not decode takes its own case down and no other. pub fn case_trial( @@ -468,7 +484,8 @@ pub fn case_trial( case: Case, run: impl FnOnce(&Case) -> Result<(), String> + Send + 'static, ) -> Trial { - let ignored = !case.in_scope(); + let gloas_enabled = case.fork == ForkName::Gloas && GLOAS_RUNNERS.contains(&runner); + let ignored = !(case.in_scope() || gloas_enabled); let name = format!("{runner}/{}", case.id()); Trial::test(name, move || run(&case).map_err(Failed::from)).with_ignored_flag(ignored) } diff --git a/crates/blockchain/state_transition/tests/beacon_spec/ssz_static.rs b/crates/blockchain/state_transition/tests/beacon_spec/ssz_static.rs index 198ea8058..ae0f338ae 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/ssz_static.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/ssz_static.rs @@ -18,7 +18,7 @@ use ethlambda_state_transition::beacon::ForkName; use ethlambda_state_transition::beacon::containers::{ - altair, bellatrix, capella, deneb, electra, fulu, phase0, shared, + altair, bellatrix, capella, deneb, electra, fulu, gloas, phase0, shared, }; use ethlambda_state_transition::beacon::primitives::{HashTreeRoot, Root}; use libssz::{SszDecode, SszEncode}; @@ -116,6 +116,25 @@ pub fn trials() -> Vec { "VoluntaryExit" => check::(case), "SignedVoluntaryExit" => check::(case), + // Gloas (EIP-7688) widens the attestation containers again, + // from electra's bounded lists to unbounded progressive ones, + // so these arms have to come before the `>= Electra` arms + // below, which would otherwise catch a gloas case first and + // check it against electra's bounded shape. + "Attestation" if fork == ForkName::Gloas => check::(case), + "IndexedAttestation" if fork == ForkName::Gloas => { + check::(case) + } + "AttesterSlashing" if fork == ForkName::Gloas => { + check::(case) + } + "AggregateAndProof" if fork == ForkName::Gloas => { + check::(case) + } + "SignedAggregateAndProof" if fork == ForkName::Gloas => { + check::(case) + } + // Electra reshapes the attestation containers, widening the // aggregation bits and attesting indices from one committee to a // whole slot's worth, so phase0's definitions hold only through @@ -175,6 +194,11 @@ pub fn trials() -> Vec { // Fulu appends proposer_lookahead to electra's state, so unlike // the block family below it needs its own type here. "BeaconState" if fork == ForkName::Fulu => check::(case), + // Gloas (EIP-7732/EIP-7688) reshapes the state again: the + // registry and several other lists become progressive, and + // the builder registry and payload-timeliness bookkeeping are + // new fields, so it needs its own type too. + "BeaconState" if fork == ForkName::Gloas => check::(case), "BeaconBlock" if fork == ForkName::Phase0 => check::(case), "BeaconBlockBody" if fork == ForkName::Phase0 => { @@ -218,6 +242,19 @@ pub fn trials() -> Vec { check::(case) } + // Gloas (EIP-7732) moves the execution payload out of the body + // into a separate envelope and adds the bid and payload + // attestations instead, so it needs its own block family + // rather than electra's. Ahead of the `>= Electra` arms below, + // which would otherwise catch a gloas case first. + "BeaconBlock" if fork == ForkName::Gloas => check::(case), + "BeaconBlockBody" if fork == ForkName::Gloas => { + check::(case) + } + "SignedBeaconBlock" if fork == ForkName::Gloas => { + check::(case) + } + // Fulu does not change the block's shape: its body still carries // exactly the execution_payload and execution_requests electra's // does, so there is no fulu::BeaconBlock, fulu::BeaconBlockBody, @@ -271,6 +308,16 @@ pub fn trials() -> Vec { "ExecutionPayloadHeader" if fork == ForkName::Capella => { check::(case) } + // Gloas (EIP-7732) moves the payload out of the block into a + // signed envelope, so there is no header left to commit to in + // its place; only `ExecutionPayload` itself still exists, now + // progressive (EIP-7688) and carrying `block_access_list` + // (EIP-7928) and `slot_number` (EIP-8061). Ahead of the + // `>= Deneb` arm below, which would otherwise catch a gloas + // case first. + "ExecutionPayload" if fork == ForkName::Gloas => { + check::(case) + } "ExecutionPayload" if fork >= ForkName::Deneb => { check::(case) } @@ -293,6 +340,13 @@ pub fn trials() -> Vec { "NewPayloadRequest" if fork == ForkName::Deneb => { check::(case) } + // Gloas's own execution_payload and execution_requests are + // progressive, so the request built from them needs its own + // type too. Ahead of the `>= Electra` arm below, which would + // otherwise catch a gloas case first. + "NewPayloadRequest" if fork == ForkName::Gloas => { + check::(case) + } "NewPayloadRequest" if fork >= ForkName::Electra => { check::(case) } @@ -330,6 +384,14 @@ pub fn trials() -> Vec { "ConsolidationRequest" if fork >= ForkName::Electra => { check::(case) } + // Gloas (EIP-8282) adds builder deposit and exit requests + // alongside electra's three, and the whole container is + // progressive (EIP-7688), so it needs its own type. Ahead of + // the `>= Electra` arm below, which would otherwise catch a + // gloas case first. + "ExecutionRequests" if fork == ForkName::Gloas => { + check::(case) + } "ExecutionRequests" if fork >= ForkName::Electra => { check::(case) } @@ -344,31 +406,85 @@ pub fn trials() -> Vec { } // Data availability sampling is fulu-only: no earlier fork has - // these containers at all. + // these containers at all. Gloas drops the sidecar's header + // and inclusion proof (its commitments are read from the + // block's bid instead, see `gloas::DataColumnSidecar`'s own + // doc), so it needs its own type there, but `MatrixEntry` and + // `DataColumnsByRootIdentifier` are untouched by that change + // and gloas's `beacon-chain.md`/`p2p-interface.md` do not + // redefine either, so fulu's types cover both forks' cases. "DataColumnSidecar" if fork == ForkName::Fulu => { check::(case) } - "MatrixEntry" if fork == ForkName::Fulu => check::(case), - "DataColumnsByRootIdentifier" if fork == ForkName::Fulu => { + "DataColumnSidecar" if fork == ForkName::Gloas => { + check::(case) + } + "MatrixEntry" if matches!(fork, ForkName::Fulu | ForkName::Gloas) => { + check::(case) + } + "DataColumnsByRootIdentifier" + if matches!(fork, ForkName::Fulu | ForkName::Gloas) => + { check::(case) } // Partial columns (gossipsub's Partial Message Extension) are - // fulu-only too, and transcribed from the same + // fulu-only or later, transcribed from the same // `partial-columns/p2p-interface.md` rather than - // `beacon-chain.md`. + // `beacon-chain.md`. Gloas's own `partial-columns/p2p-interface.md` + // modifies fulu's: `CellsBitList` becomes progressive (so + // every container holding one gets a different merkleization + // even where its field list looks unchanged), + // `PartialDataColumnSidecar` drops `header` entirely, and + // `PartialDataColumnGroupID` gains `slot`; there is no + // `PartialDataColumnHeader` left to define for gloas at all. + // See the gloas module's own doc for the detail. "PartialDataColumnSidecar" if fork == ForkName::Fulu => { check::(case) } + "PartialDataColumnSidecar" if fork == ForkName::Gloas => { + check::(case) + } "PartialDataColumnPartsMetadata" if fork == ForkName::Fulu => { check::(case) } + "PartialDataColumnPartsMetadata" if fork == ForkName::Gloas => { + check::(case) + } "PartialDataColumnHeader" if fork == ForkName::Fulu => { check::(case) } "PartialDataColumnGroupID" if fork == ForkName::Fulu => { check::(case) } + "PartialDataColumnGroupID" if fork == ForkName::Gloas => { + check::(case) + } + + // New in gloas (EIP-7732/EIP-8282): the builder registry and + // its payment/withdrawal queues, the payload timeliness + // committee's attestations, the builder's bid and its + // envelope reveal, and the proposer's advance broadcast of + // its fee recipient and gas limit preference. None of these + // has an earlier-fork shape to fall back to, so each gets an + // unconditional arm rather than a fork guard. + "Builder" => check::(case), + "BuilderPendingPayment" => check::(case), + "BuilderPendingWithdrawal" => check::(case), + "BuilderDepositRequest" => check::(case), + "BuilderExitRequest" => check::(case), + "PayloadAttestationData" => check::(case), + "PayloadAttestation" => check::(case), + "PayloadAttestationMessage" => check::(case), + "IndexedPayloadAttestation" => check::(case), + "ExecutionPayloadBid" => check::(case), + "SignedExecutionPayloadBid" => check::(case), + "ExecutionPayloadEnvelope" => check::(case), + "SignedExecutionPayloadEnvelope" => { + check::(case) + } + "ProposerPreferences" => check::(case), + "SignedProposerPreferences" => check::(case), // No arm matched, so this crate has no container for this // handler/fork pair, and this is not `LightClient*` (that From 2436bea23c4d5ca5ed1bac13004364376e466843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 25 Sep 2026 12:27:23 -0300 Subject: [PATCH 014/132] fix(beacon): use the minimal preset's consolidation churn quotient Config::minimal() had copied mainnet's values for four keys instead of reading configs/minimal.yaml. CONSOLIDATION_CHURN_LIMIT_QUOTIENT is 32 on minimal, not mainnet's 65_536. Nothing read it before gloas (EIP-8061 introduces it), but gloas's consolidation churn divides by it: at 65_536, a minimal state's churn limit rounds down to zero and every consolidation request is dropped. The deposit chain and network ids are minimal's 5 (Goerli's), not mainnet's 1, and the deposit contract address is the file's own placeholder rather than mainnet's contract. Every other field of Config::minimal() was checked against configs/minimal.yaml, and every field of Config::mainnet() against configs/mainnet.yaml; no other value differs. --- crates/common/types/src/beacon/config.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/common/types/src/beacon/config.rs b/crates/common/types/src/beacon/config.rs index 6d6908eaa..9732c99f5 100644 --- a/crates/common/types/src/beacon/config.rs +++ b/crates/common/types/src/beacon/config.rs @@ -917,11 +917,14 @@ impl Config { min_epochs_for_data_column_sidecars_requests: 4_096, subnets_per_node: 2, - deposit_chain_id: 1, - deposit_network_id: 1, + // configs/minimal.yaml: Ethereum Goerli testnet's chain and + // network id, not mainnet's; the contract address is not + // Goerli's real one, just the file's own repeating placeholder. + deposit_chain_id: 5, + deposit_network_id: 5, deposit_contract_address: [ - 0x00, 0x00, 0x00, 0x00, 0x21, 0x9a, 0xb5, 0x40, 0x35, 0x6c, 0xbb, 0x83, 0x9c, 0xbe, - 0x05, 0x30, 0x3d, 0x77, 0x05, 0xfa, + 0x12, 0x34, 0x56, 0x78, 0x90, 0x12, 0x34, 0x56, 0x78, 0x90, 0x12, 0x34, 0x56, 0x78, + 0x90, 0x12, 0x34, 0x56, 0x78, 0x90, ], balance_per_additional_custody_group: 32_000_000_000, @@ -930,7 +933,7 @@ impl Config { samples_per_slot: 8, validator_custody_requirement: 8, - consolidation_churn_limit_quotient: 65_536, + consolidation_churn_limit_quotient: 32, attestation_subnet_prefix_bits: 6, max_request_blob_sidecars: 768, From 633b29471da19b9817530e16aaf06f0e231b3db3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 25 Sep 2026 12:36:08 -0300 Subject: [PATCH 015/132] feat(beacon): add the gloas accessors, predicates and churn Gloas's block and epoch processing need a layer of helpers before any of it can be ported. ePBS (EIP-7732) separates proposers from builders and adds a payload timeliness committee. EIP-8045 limits the proposer draw to unslashed validators. EIP-8061 splits electra's combined activation/exit churn into independent activation, exit and consolidation limits. This commit adds those helpers as plain functions in helpers/gloas.rs; no block or epoch driver calls them yet. Where gloas modifies an inherited function by a single call, the inherited function is shared rather than copied, the way pyspec resolves it. Gloas changes one line of compute_exit_epoch_and_update_churn (its churn limit) and none of initiate_validator_exit, process_withdrawal_request, process_consolidation_request or process_registry_updates. So electra's exit and consolidation cursors pick the churn limit by fork and read gloas's on a gloas state, through a projection that exposes the four cursor fields for electra, fulu and gloas. Electra's operations are then already right on a gloas state as far as churn goes. The proposer lookahead has the same type in fulu and gloas, so both read it through one accessor and one get_beacon_proposer_index. Three shared dispatchers that refused gloas now route it: get_beacon_proposer_index, initiate_validator_exit_for_fork and get_next_sync_committee. The activation churn gate in epoch/registry.rs still refuses gloas, like electra and fulu: it is phase0..deneb's validator-count cap, and gloas's get_activation_churn_limit is read only by gloas's own process_pending_deposits. compute_ptc takes the committee cache: process_ptc_window calls it once per slot of every epoch, and rebuilding each committee from the registry costs a full registry scan per committee. The unit tests check the new draws against the functions the spec says they generalize. compute_balance_weighted_selection with one draw and shuffling picks electra's proposer, and the gloas sync committee draw equals electra's. A gloas exit consumes gloas's exit churn, not electra's. Both of get_ptc's window branches are covered. --- .../src/beacon/helpers/accessors.rs | 19 +- .../src/beacon/helpers/altair.rs | 14 +- .../src/beacon/helpers/electra.rs | 300 ++++- .../src/beacon/helpers/fulu.rs | 49 +- .../src/beacon/helpers/gloas.rs | 1043 +++++++++++++++++ .../src/beacon/helpers/mod.rs | 1 + .../src/beacon/helpers/mutators.rs | 56 +- .../src/beacon/helpers/test_state.rs | 96 +- .../src/beacon/stf/electra.rs | 4 +- .../src/beacon/stf/epoch/capella.rs | 14 +- .../src/beacon/stf/epoch/registry.rs | 9 + .../state_transition/src/beacon/upgrade.rs | 2 +- 12 files changed, 1483 insertions(+), 124 deletions(-) create mode 100644 crates/blockchain/state_transition/src/beacon/helpers/gloas.rs diff --git a/crates/blockchain/state_transition/src/beacon/helpers/accessors.rs b/crates/blockchain/state_transition/src/beacon/helpers/accessors.rs index 94e3c3a96..fd92b0370 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/accessors.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/accessors.rs @@ -16,7 +16,7 @@ pub use ethlambda_storage::{CommitteeCache, Lookup, ShufflingKey}; use crate::beacon::config::Config; use crate::beacon::constants; use crate::beacon::containers::BeaconState; -use crate::beacon::error::{Error, Result}; +use crate::beacon::error::Result; use crate::beacon::fork::ForkName; use crate::beacon::hash::hash; use crate::beacon::lean_state_unreachable; @@ -359,25 +359,16 @@ pub fn get_beacon_proposer_index(state: &BeaconState) -> Result // why a seed, and therefore a proposer, is only ever knowable that far // ahead of time in the first place. match state.fork_name() { - ForkName::Fulu => return super::fulu::get_beacon_proposer_index(state), + // Gloas's own `proposer_lookahead` is unchanged from fulu (see + // `containers::gloas`'s module doc) and gloas does not redefine + // this function, so both forks share fulu's exact same lookup. + ForkName::Fulu | ForkName::Gloas => return super::fulu::get_beacon_proposer_index(state), ForkName::Phase0 | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb | ForkName::Electra => {} - // Gloas's own `proposer_lookahead` is unchanged from fulu (see - // `containers::gloas`'s module doc), so this almost certainly - // reduces to `super::fulu::get_beacon_proposer_index` unmodified. - // Left as an explicit gap rather than assumed, since ePBS separates - // the block proposer from the payload's builder and this repository - // has not yet ported that distinction. - ForkName::Gloas => { - return Err(Error::UnsupportedForFork { - function: "get_beacon_proposer_index", - fork: ForkName::Gloas, - }); - } ForkName::Lean => lean_state_unreachable("get_beacon_proposer_index"), } diff --git a/crates/blockchain/state_transition/src/beacon/helpers/altair.rs b/crates/blockchain/state_transition/src/beacon/helpers/altair.rs index b48b77ed1..6f4a2a453 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/altair.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/altair.rs @@ -163,15 +163,11 @@ pub fn get_next_sync_committee(state: &BeaconState) -> Result get_next_sync_committee_indices(state)?, - // Almost certainly reuses electra's indices function, the way fulu - // does, since gloas keeps electra's effective-balance ceiling. Left - // as an explicit gap rather than assumed; a later task ports it. - ForkName::Gloas => { - return Err(Error::UnsupportedForFork { - function: "get_next_sync_committee", - fork: ForkName::Gloas, - }); - } + // Modified in gloas (EIP-7732): the draw is + // `compute_balance_weighted_selection` rather than electra's inline + // rejection-sampling loop, though it keeps electra's effective-balance + // ceiling. See `crate::beacon::helpers::gloas`'s own module doc. + ForkName::Gloas => crate::beacon::helpers::gloas::get_next_sync_committee_indices(state)?, ForkName::Lean => lean_state_unreachable("get_next_sync_committee"), }; let mut pubkeys = Vec::with_capacity(indices.len()); diff --git a/crates/blockchain/state_transition/src/beacon/helpers/electra.rs b/crates/blockchain/state_transition/src/beacon/helpers/electra.rs index 6a25ed7d5..efdbf9e14 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/electra.rs @@ -125,9 +125,11 @@ use crate::beacon::bls; use crate::beacon::config::Config; use crate::beacon::constants::{self, FAR_FUTURE_EPOCH}; use crate::beacon::containers::shared::Validator; -use crate::beacon::containers::{BeaconState, electra, fulu}; +use crate::beacon::containers::{BeaconState, electra, fulu, gloas}; use crate::beacon::error::{Error, Result}; +use crate::beacon::fork::ForkName; use crate::beacon::hash::hash; +use crate::beacon::lean_state_unreachable; use crate::beacon::preset; use crate::beacon::primitives::{ BLS_SIGNATURE_SIZE, BlsSignature, Bytes32, CommitteeIndex, Epoch, Gwei, HashTreeRoot as _, @@ -562,26 +564,53 @@ pub fn initiate_validator_exit( Ok(()) } -/// Advances electra's exit-queue cursor for an exit of `exit_balance`, -/// returning the epoch it may take effect at. +/// Advances the exit-queue cursor for an exit of `exit_balance`, returning +/// the epoch it may take effect at. /// /// This is where the "balance to consume" cursor this module's doc describes /// actually lives: `earliest_exit_epoch` is the earliest epoch that still has /// unspent churn, and `exit_balance_to_consume` is how much of that epoch's /// budget remains. A new epoch's budget is only opened (refilled to a full -/// [`get_activation_exit_churn_limit`]) once the cursor actually needs to move -/// past the epoch it currently sits on; until then, a later, smaller exit in -/// the same epoch spends whatever an earlier one left over instead of always -/// waiting for a fresh epoch. +/// per-epoch churn limit) once the cursor actually needs to move past the +/// epoch it currently sits on; until then, a later, smaller exit in the same +/// epoch spends whatever an earlier one left over instead of always waiting +/// for a fresh epoch. +/// +/// Gloas modifies this function by exactly the one line the specification's +/// own diff shows (EIP-8061, beacon-chain.md): the per-epoch churn limit is +/// [`crate::beacon::helpers::gloas::get_exit_churn_limit`]'s own uncapped +/// budget rather than [`get_activation_exit_churn_limit`]'s combined one. +/// Nothing past that line differs, so gloas does not get a copy of its own: +/// this picks the limit by fork and otherwise runs the one shared cursor +/// update, through [`ChurnCursorsMut`] rather than [`ElectraOrFuluMut`] (see +/// that enum's own doc for why). Gloas modifies neither +/// `initiate_validator_exit` (so neither voluntary exits nor registry +/// ejections) nor `process_withdrawal_request`, so calling this from any +/// of them already picks up gloas's churn automatically, the same way +/// pyspec's unmodified callers do. pub fn compute_exit_epoch_and_update_churn( state: &mut BeaconState, exit_balance: Gwei, config: &Config, ) -> Result { let current_epoch = get_current_epoch(state); - let per_epoch_churn = get_activation_exit_churn_limit(state, config)?; + let per_epoch_churn = match state.fork_name() { + ForkName::Electra | ForkName::Fulu => get_activation_exit_churn_limit(state, config)?, + ForkName::Gloas => crate::beacon::helpers::gloas::get_exit_churn_limit(state, config)?, + fork @ (ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb) => { + return Err(Error::UnsupportedForFork { + function: "compute_exit_epoch_and_update_churn", + fork, + }); + } + ForkName::Lean => lean_state_unreachable("compute_exit_epoch_and_update_churn"), + }; - let mut fields = electra_state(state, "compute_exit_epoch_and_update_churn")?; + let mut fields = churn_cursors_mut(state, "compute_exit_epoch_and_update_churn")?; let mut earliest_exit_epoch = fields .earliest_exit_epoch() @@ -636,24 +665,64 @@ pub fn compute_exit_epoch_and_update_churn( Ok(earliest_exit_epoch) } -/// Advances electra's consolidation-queue cursor for a consolidation moving +/// The per-epoch consolidation churn limit for `state`'s own fork, +/// dispatching between electra's [`get_consolidation_churn_limit`] (also +/// fulu's, unmodified) and gloas's own +/// [`crate::beacon::helpers::gloas::get_consolidation_churn_limit`] +/// (EIP-8061, independently derived from total active balance rather than +/// left over from the activation/exit split). +/// +/// The one call site every consolidation-churn reader reaches, so a gloas +/// state is never charged electra's formula by accident: +/// [`compute_consolidation_epoch_and_update_churn`] and +/// [`crate::beacon::stf::electra::process_consolidation_request`] both go +/// through this rather than either raw formula directly. +pub fn get_consolidation_churn_limit_for_fork( + state: &BeaconState, + config: &Config, +) -> Result { + match state.fork_name() { + ForkName::Electra | ForkName::Fulu => get_consolidation_churn_limit(state, config), + ForkName::Gloas => { + crate::beacon::helpers::gloas::get_consolidation_churn_limit(state, config) + } + fork @ (ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb) => Err(Error::UnsupportedForFork { + function: "get_consolidation_churn_limit_for_fork", + fork, + }), + ForkName::Lean => lean_state_unreachable("get_consolidation_churn_limit_for_fork"), + } +} + +/// Advances the consolidation-queue cursor for a consolidation moving /// `consolidation_balance`, returning the epoch it may take effect at. /// /// The consolidation-side counterpart of /// [`compute_exit_epoch_and_update_churn`], carrying the exact same /// `(earliest_epoch, balance_to_consume)` cursor shape but drawing from -/// [`get_consolidation_churn_limit`]'s separate budget instead of the -/// activation/exit one, so a burst of consolidations cannot also drain the -/// budget an unrelated exit needs. +/// [`get_consolidation_churn_limit_for_fork`]'s separate budget instead of +/// the activation/exit one, so a burst of consolidations cannot also drain +/// the budget an unrelated exit needs. +/// +/// Unlike [`compute_exit_epoch_and_update_churn`], gloas does not redefine +/// this function itself: only its callee (electra's +/// [`get_consolidation_churn_limit`]) is gloas-modified. So the cursor +/// read/write below stays the one copy every fork through gloas shares, +/// fed by [`get_consolidation_churn_limit_for_fork`] rather than either raw +/// formula. pub fn compute_consolidation_epoch_and_update_churn( state: &mut BeaconState, consolidation_balance: Gwei, config: &Config, ) -> Result { let current_epoch = get_current_epoch(state); - let per_epoch_churn = get_consolidation_churn_limit(state, config)?; + let per_epoch_churn = get_consolidation_churn_limit_for_fork(state, config)?; - let mut fields = electra_state(state, "compute_consolidation_epoch_and_update_churn")?; + let mut fields = churn_cursors_mut(state, "compute_consolidation_epoch_and_update_churn")?; let mut earliest_consolidation_epoch = fields .earliest_consolidation_epoch() @@ -838,98 +907,148 @@ pub(crate) enum ElectraOrFuluMut<'a> { } impl<'a> ElectraOrFuluMut<'a> { + /// Deposits queued but not yet credited to the validator registry, pushed + /// to by [`queue_excess_active_balance`] and + /// [`queue_entire_balance_and_reset_validator`]. + pub(crate) fn pending_deposits_mut(&mut self) -> &mut electra::PendingDeposits { + match self { + ElectraOrFuluMut::Electra(state) => &mut state.pending_deposits, + ElectraOrFuluMut::Fulu(state) => &mut state.pending_deposits, + } + } +} + +/// The electra-or-fulu state, mutably, or an error naming the function that +/// needs one. See [`ElectraOrFuluMut`] and this module's doc for why both +/// forks are accepted. +pub(crate) fn electra_state<'a>( + state: &'a mut BeaconState, + function: &'static str, +) -> Result> { + match state { + BeaconState::Electra(state) => Ok(ElectraOrFuluMut::Electra(state)), + BeaconState::Fulu(state) => Ok(ElectraOrFuluMut::Fulu(state)), + other => Err(Error::UnsupportedForFork { + function, + fork: other.fork_name(), + }), + } +} + +/// The electra-or-fulu state, immutably. See [`electra_state`]. +pub(crate) fn electra_state_ref<'a>( + state: &'a BeaconState, + function: &'static str, +) -> Result> { + match state { + BeaconState::Electra(state) => Ok(ElectraOrFulu::Electra(state)), + BeaconState::Fulu(state) => Ok(ElectraOrFulu::Fulu(state)), + other => Err(Error::UnsupportedForFork { + function, + fork: other.fork_name(), + }), + } +} + +/// Either fork whose state carries the four balance-churn cursor fields +/// (`earliest_exit_epoch`, `exit_balance_to_consume`, +/// `earliest_consolidation_epoch`, `consolidation_balance_to_consume`, +/// EIP-7251) unchanged: electra, fulu, or gloas, which keeps every one of +/// them at the same type (see `containers::gloas`'s module doc) even though +/// gloas redefines which churn limit feeds them +/// (`compute_exit_epoch_and_update_churn`, +/// [`get_consolidation_churn_limit_for_fork`]). +/// +/// Deliberately narrower than [`ElectraOrFuluMut`]: that enum also exposes +/// `pending_deposits_mut`, whose gloas type +/// ([`crate::beacon::containers::gloas::PendingDeposits`], a +/// [`libssz_types::ProgressiveList`], EIP-7688) is not electra's and fulu's +/// bounded [`electra::PendingDeposits`], so widening `ElectraOrFuluMut` +/// itself to gloas would leave that one method with no honest gloas arm to +/// give it. This projection carries only the four fields every one of the +/// three forks actually shares. +pub(crate) enum ChurnCursorsMut<'a> { + Electra(&'a mut electra::BeaconState), + Fulu(&'a mut fulu::BeaconState), + Gloas(&'a mut gloas::BeaconState), +} + +impl<'a> ChurnCursorsMut<'a> { pub(crate) fn earliest_exit_epoch(&self) -> Epoch { match self { - ElectraOrFuluMut::Electra(state) => state.earliest_exit_epoch, - ElectraOrFuluMut::Fulu(state) => state.earliest_exit_epoch, + ChurnCursorsMut::Electra(state) => state.earliest_exit_epoch, + ChurnCursorsMut::Fulu(state) => state.earliest_exit_epoch, + ChurnCursorsMut::Gloas(state) => state.earliest_exit_epoch, } } pub(crate) fn earliest_exit_epoch_mut(&mut self) -> &mut Epoch { match self { - ElectraOrFuluMut::Electra(state) => &mut state.earliest_exit_epoch, - ElectraOrFuluMut::Fulu(state) => &mut state.earliest_exit_epoch, + ChurnCursorsMut::Electra(state) => &mut state.earliest_exit_epoch, + ChurnCursorsMut::Fulu(state) => &mut state.earliest_exit_epoch, + ChurnCursorsMut::Gloas(state) => &mut state.earliest_exit_epoch, } } pub(crate) fn exit_balance_to_consume(&self) -> Gwei { match self { - ElectraOrFuluMut::Electra(state) => state.exit_balance_to_consume, - ElectraOrFuluMut::Fulu(state) => state.exit_balance_to_consume, + ChurnCursorsMut::Electra(state) => state.exit_balance_to_consume, + ChurnCursorsMut::Fulu(state) => state.exit_balance_to_consume, + ChurnCursorsMut::Gloas(state) => state.exit_balance_to_consume, } } pub(crate) fn exit_balance_to_consume_mut(&mut self) -> &mut Gwei { match self { - ElectraOrFuluMut::Electra(state) => &mut state.exit_balance_to_consume, - ElectraOrFuluMut::Fulu(state) => &mut state.exit_balance_to_consume, + ChurnCursorsMut::Electra(state) => &mut state.exit_balance_to_consume, + ChurnCursorsMut::Fulu(state) => &mut state.exit_balance_to_consume, + ChurnCursorsMut::Gloas(state) => &mut state.exit_balance_to_consume, } } pub(crate) fn earliest_consolidation_epoch(&self) -> Epoch { match self { - ElectraOrFuluMut::Electra(state) => state.earliest_consolidation_epoch, - ElectraOrFuluMut::Fulu(state) => state.earliest_consolidation_epoch, + ChurnCursorsMut::Electra(state) => state.earliest_consolidation_epoch, + ChurnCursorsMut::Fulu(state) => state.earliest_consolidation_epoch, + ChurnCursorsMut::Gloas(state) => state.earliest_consolidation_epoch, } } pub(crate) fn earliest_consolidation_epoch_mut(&mut self) -> &mut Epoch { match self { - ElectraOrFuluMut::Electra(state) => &mut state.earliest_consolidation_epoch, - ElectraOrFuluMut::Fulu(state) => &mut state.earliest_consolidation_epoch, + ChurnCursorsMut::Electra(state) => &mut state.earliest_consolidation_epoch, + ChurnCursorsMut::Fulu(state) => &mut state.earliest_consolidation_epoch, + ChurnCursorsMut::Gloas(state) => &mut state.earliest_consolidation_epoch, } } pub(crate) fn consolidation_balance_to_consume(&self) -> Gwei { match self { - ElectraOrFuluMut::Electra(state) => state.consolidation_balance_to_consume, - ElectraOrFuluMut::Fulu(state) => state.consolidation_balance_to_consume, + ChurnCursorsMut::Electra(state) => state.consolidation_balance_to_consume, + ChurnCursorsMut::Fulu(state) => state.consolidation_balance_to_consume, + ChurnCursorsMut::Gloas(state) => state.consolidation_balance_to_consume, } } pub(crate) fn consolidation_balance_to_consume_mut(&mut self) -> &mut Gwei { match self { - ElectraOrFuluMut::Electra(state) => &mut state.consolidation_balance_to_consume, - ElectraOrFuluMut::Fulu(state) => &mut state.consolidation_balance_to_consume, - } - } - - /// Deposits queued but not yet credited to the validator registry, pushed - /// to by [`queue_excess_active_balance`] and - /// [`queue_entire_balance_and_reset_validator`]. - pub(crate) fn pending_deposits_mut(&mut self) -> &mut electra::PendingDeposits { - match self { - ElectraOrFuluMut::Electra(state) => &mut state.pending_deposits, - ElectraOrFuluMut::Fulu(state) => &mut state.pending_deposits, + ChurnCursorsMut::Electra(state) => &mut state.consolidation_balance_to_consume, + ChurnCursorsMut::Fulu(state) => &mut state.consolidation_balance_to_consume, + ChurnCursorsMut::Gloas(state) => &mut state.consolidation_balance_to_consume, } } } -/// The electra-or-fulu state, mutably, or an error naming the function that -/// needs one. See [`ElectraOrFuluMut`] and this module's doc for why both -/// forks are accepted. -pub(crate) fn electra_state<'a>( +/// The electra-or-fulu-or-gloas state, mutably, through [`ChurnCursorsMut`]. +/// See that enum's own doc for why it exists apart from [`electra_state`]. +pub(crate) fn churn_cursors_mut<'a>( state: &'a mut BeaconState, function: &'static str, -) -> Result> { - match state { - BeaconState::Electra(state) => Ok(ElectraOrFuluMut::Electra(state)), - BeaconState::Fulu(state) => Ok(ElectraOrFuluMut::Fulu(state)), - other => Err(Error::UnsupportedForFork { - function, - fork: other.fork_name(), - }), - } -} - -/// The electra-or-fulu state, immutably. See [`electra_state`]. -pub(crate) fn electra_state_ref<'a>( - state: &'a BeaconState, - function: &'static str, -) -> Result> { +) -> Result> { match state { - BeaconState::Electra(state) => Ok(ElectraOrFulu::Electra(state)), - BeaconState::Fulu(state) => Ok(ElectraOrFulu::Fulu(state)), + BeaconState::Electra(state) => Ok(ChurnCursorsMut::Electra(state)), + BeaconState::Fulu(state) => Ok(ChurnCursorsMut::Fulu(state)), + BeaconState::Gloas(state) => Ok(ChurnCursorsMut::Gloas(state)), other => Err(Error::UnsupportedForFork { function, fork: other.fork_name(), @@ -1417,4 +1536,59 @@ mod tests { let exit_epoch = compute_exit_epoch_and_update_churn(&mut state, 1, &config).unwrap(); assert_eq!(exit_epoch, baseline); } + + /// Gloas does not redefine `compute_consolidation_epoch_and_update_churn` + /// itself: only its churn limit is gloas-modified + /// ([`get_consolidation_churn_limit_for_fork`]). A gloas state must still + /// go through this shared cursor update, fed gloas's own churn limit + /// rather than electra's. + /// + /// 2048 validators: with mainnet's `CONSOLIDATION_CHURN_LIMIT_QUOTIENT` + /// (the wider of the two configs this runs under, `Config::mainnet()` + /// and `Config::minimal()`), that is exactly the active balance needed + /// for gloas's independent, from-total-active-balance formula to clear + /// one whole `EFFECTIVE_BALANCE_INCREMENT` rather than round down to + /// zero. + #[test] + fn compute_consolidation_epoch_and_update_churn_uses_gloas_own_limit_on_a_gloas_state() { + let config = Config::mainnet(); + let mut state = + crate::beacon::helpers::test_state::with_validators_at(ForkName::Gloas, 2_048); + let gloas_limit = + crate::beacon::helpers::gloas::get_consolidation_churn_limit(&state, &config).unwrap(); + assert!(gloas_limit > 0, "the fixture must exercise a real budget"); + + compute_consolidation_epoch_and_update_churn(&mut state, 1, &config).unwrap(); + + let fields = churn_cursors_mut(&mut state, "test").unwrap(); + assert_eq!(fields.consolidation_balance_to_consume(), gloas_limit - 1); + } + + /// Proves [`get_consolidation_churn_limit_for_fork`] actually + /// discriminates: `process_consolidation_request` + /// (`crate::beacon::stf::electra`) reads it too, so if it ever collapsed + /// to always answering with electra's formula, a gloas state would + /// silently mis-gate consolidation requests against the wrong budget. + #[test] + fn get_consolidation_churn_limit_for_fork_differs_between_electra_and_gloas() { + let config = Config::minimal(); + let electra_state = electra_state_with_validators(64); + let gloas_state = + crate::beacon::helpers::test_state::with_validators_at(ForkName::Gloas, 64); + + let electra_limit = + get_consolidation_churn_limit_for_fork(&electra_state, &config).unwrap(); + let gloas_limit = get_consolidation_churn_limit_for_fork(&gloas_state, &config).unwrap(); + + assert_eq!( + electra_limit, + get_consolidation_churn_limit(&electra_state, &config).unwrap() + ); + assert_eq!( + gloas_limit, + crate::beacon::helpers::gloas::get_consolidation_churn_limit(&gloas_state, &config) + .unwrap() + ); + assert_ne!(electra_limit, gloas_limit); + } } diff --git a/crates/blockchain/state_transition/src/beacon/helpers/fulu.rs b/crates/blockchain/state_transition/src/beacon/helpers/fulu.rs index d488a0f7d..7c138c64b 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/fulu.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/fulu.rs @@ -23,14 +23,16 @@ //! //! [`get_beacon_proposer_index`] here does not replace //! [`super::accessors::get_beacon_proposer_index`] in place: the two coexist, -//! fulu's reading the precomputed window and every earlier fork's computing -//! on demand, and that accessor dispatches between them by fork itself, -//! rather than each of *its* own callers doing so, the same split +//! this one reading the precomputed window (for fulu and, unchanged, gloas +//! too; see `containers::gloas`'s module doc) and every earlier fork +//! computing on demand, and that accessor dispatches between them by fork +//! itself, rather than each of *its* own callers doing so, the same split //! [`super::altair::altair_state_ref`] documents for altair's -//! participation-flag fields. A state older than fulu has no -//! `proposer_lookahead` to read, so [`get_beacon_proposer_index`] fails -//! through [`fulu_state_ref`] rather than falling back to the on-demand -//! computation. +//! participation-flag fields. A state with no `proposer_lookahead` to read +//! fails through the narrower `proposer_lookahead` projection rather than +//! falling back to the on-demand computation; that projection is scoped to +//! this one field, not [`fulu_state_ref`], since gloas does not carry every +//! field that one gates (`latest_execution_payload_header` among them). //! //! [`compute_proposer_indices`] and [`get_beacon_proposer_indices`] are the //! building blocks the window is filled from. Both take a [`BeaconState`] of @@ -125,22 +127,47 @@ pub fn get_beacon_proposer_indices( compute_proposer_indices(state, epoch, seed, &indices) } +/// The `proposer_lookahead` field, for whichever fork's state carries it. +/// +/// Fulu introduces it; gloas keeps it at the identical type unchanged (see +/// `containers::gloas`'s module doc), so both variants read through here +/// rather than gloas carrying a duplicate accessor of its own. Scoped to +/// this one field rather than widening [`fulu_state_ref`], which also gates +/// `latest_execution_payload_header`, a field gloas does not carry at all, +/// so that projection has nothing true to answer for a gloas state. +fn proposer_lookahead<'a>( + state: &'a BeaconState, + function: &'static str, +) -> Result<&'a fulu::ProposerLookahead> { + match state { + BeaconState::Fulu(inner) => Ok(&inner.proposer_lookahead), + BeaconState::Gloas(inner) => Ok(&inner.proposer_lookahead), + other => Err(Error::UnsupportedForFork { + function, + fork: other.fork_name(), + }), + } +} + /// The proposer for the state's current slot. /// /// Fulu's replacement for [`super::accessors::get_beacon_proposer_index`]: a /// lookup into the current epoch's slice of `proposer_lookahead` rather than /// a shuffle computed on demand. See the module docs for why that lookup is /// possible at all and why the on-demand version is not simply reused here. +/// +/// Also gloas's: gloas does not redefine this function (`containers::gloas`'s +/// module doc), so [`super::accessors::get_beacon_proposer_index`] routes +/// both `Fulu` and `Gloas` here rather than gloas carrying a copy of its own. pub fn get_beacon_proposer_index(state: &BeaconState) -> Result { - let fulu_state = fulu_state_ref(state, "get_beacon_proposer_index")?; + let lookahead = proposer_lookahead(state, "get_beacon_proposer_index")?; let index = (state.slot() % preset::SLOTS_PER_EPOCH) as usize; - fulu_state - .proposer_lookahead + lookahead .get(index) .copied() .ok_or(Error::IndexOutOfBounds { index, - len: fulu_state.proposer_lookahead.len(), + len: lookahead.len(), }) } diff --git a/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs b/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs new file mode 100644 index 000000000..195a207ab --- /dev/null +++ b/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs @@ -0,0 +1,1043 @@ +//! Gloas's new and changed helper functions. +//! +//! Gloas bundles several EIPs (see `containers::gloas`'s own module doc for +//! the full list); this file carries the *behaviour* each one adds to the +//! specification's "Helpers" section, not the containers it reads and +//! writes. +//! +//! **EIP-7732 (ePBS)** is the largest of them, and most of what is below +//! belongs to it. A block no longer carries its own execution payload: a +//! proposer instead commits to a builder's [`gloas::ExecutionPayloadBid`], +//! and the builder reveals the payload afterward. That split introduces a +//! second registry ([`is_builder_index`], [`is_active_builder`], +//! [`is_builder_withdrawal_credential`], [`convert_builder_index_to_validator_index`], +//! [`convert_validator_index_to_builder_index`] all exist only because +//! builders and validators now share one index space, distinguished by a +//! single flag bit), a second escrow-and-payment accounting +//! ([`get_pending_balance_to_withdraw_for_builder`], [`can_builder_cover_bid`], +//! [`get_builder_payment_quorum_threshold`], [`initiate_builder_exit`], +//! [`settle_builder_payment`]), and a payload timeliness committee that votes +//! on whether the revealed payload showed up on time +//! ([`compute_ptc`], [`get_ptc`], [`get_indexed_payload_attestation`], +//! [`is_valid_indexed_payload_attestation`]). Because a payload can now be +//! revealed a slot late, [`is_attestation_same_slot`] and the payload-index +//! bit [`get_attestation_participation_flag_indices`] reads off +//! `execution_payload_availability` are what let an attester's head vote +//! still name which payload it is voting for. +//! +//! **EIP-8045** narrows the proposer draw +//! ([`get_beacon_proposer_indices`]) to unslashed validators only, and +//! **EIP-8061** replaces the combined activation/exit churn budget electra +//! shares between the two with independent ones +//! ([`get_activation_churn_limit`], [`get_exit_churn_limit`], both read +//! straight from `Config` rather than derived from one another the way +//! electra's [`super::electra::get_activation_exit_churn_limit`] and +//! [`super::electra::get_consolidation_churn_limit`] are). +//! [`super::electra::compute_exit_epoch_and_update_churn`] draws from +//! [`get_exit_churn_limit`]'s uncapped budget on a gloas state rather than +//! electra's combined one: gloas modifies that function by exactly the +//! per-epoch-churn line the specification's own diff shows, so it lives +//! there as a fork dispatch rather than as a copy of its own here; see that +//! function's own doc. [`get_consolidation_churn_limit`] is modified too, +//! independently derived from total active balance rather than left over +//! from the activation/exit split, and reached the same way, through +//! [`super::electra::get_consolidation_churn_limit_for_fork`]. +//! +//! **[`compute_balance_weighted_selection`]** is the new sampling primitive +//! behind nearly every draw in this file: [`compute_proposer_indices`] (one +//! draw per slot), [`get_next_sync_committee_indices`] (`SYNC_COMMITTEE_SIZE` +//! draws), and [`compute_ptc`] (`PTC_SIZE` draws) all become thin callers of +//! it rather than each repeating electra's rejection-sampling loop +//! ([`super::electra::compute_proposer_index`], +//! [`super::electra::get_next_sync_committee_indices`]) with a different +//! candidate pool. It reuses a random 32-byte hash across up to 16 +//! consecutive draws (recomputing only when a new hash is needed, exactly +//! where the specification's own `if offset == 0` guards it) rather than +//! hashing once per draw the way electra's loop does; ported statement for +//! statement, not simplified into always rehashing, since the two only +//! produce the same bytes because the guard is there. +//! +//! # The fork projection +//! +//! [`gloas_state`] and [`gloas_state_ref`] project a [`BeaconState`] down to +//! gloas's own concrete struct, for functions that mix a gloas-only field +//! with others reached through fork-invariant accessors ([`get_ptc`]'s +//! `ptc_window`, [`get_attestation_participation_flag_indices`]'s +//! `execution_payload_availability`). Functions whose *every* read is +//! gloas-only instead take a [`gloas::BeaconState`] directly +//! ([`is_active_builder`], [`get_pending_balance_to_withdraw_for_builder`], +//! [`can_builder_cover_bid`], [`initiate_builder_exit`], +//! [`settle_builder_payment`]): nothing in them needs a fork-invariant +//! accessor at all, so the caller (already holding a state it knows is +//! gloas) passes the concrete type straight through rather than this module +//! projecting it back out again. Functions that only ever read fields +//! shared with every fork (the registry, `finalized_checkpoint`, block +//! roots, ...) take a [`BeaconState`] and reach those fields through its +//! own fork-invariant accessors, the same split [`super::fulu`]'s module +//! doc draws for its own two kinds of function. +//! +//! The four balance-churn cursor fields are none of these three: their type +//! is shared with electra and fulu, so +//! [`super::electra::compute_exit_epoch_and_update_churn`] and +//! [`super::electra::compute_consolidation_epoch_and_update_churn`] reach +//! them through `helpers::electra`'s own `ChurnCursorsMut` instead of a +//! projection here; see that enum's own doc. `proposer_lookahead` is +//! likewise shared with fulu (unchanged, see `containers::gloas`'s module +//! doc), read through [`super::fulu::get_beacon_proposer_index`] rather +//! than a copy of it here. + +use crate::beacon::bls; +use crate::beacon::config::Config; +use crate::beacon::constants; +use crate::beacon::containers::shared::AttestationData; +use crate::beacon::containers::{BeaconState, electra, gloas}; +use crate::beacon::error::{Error, Result}; +use crate::beacon::hash::hash; +use crate::beacon::preset; +use crate::beacon::primitives::{ + BlsPubkey, Bytes32, Epoch, Gwei, HashTreeRoot as _, Slot, ValidatorIndex, +}; + +use super::accessors::{ + CommitteeCache, CommitteeCacheExt, get_active_validator_indices, get_block_root, + get_block_root_at_slot, get_current_epoch, get_domain, get_seed, get_total_active_balance, +}; +use super::math::{bytes_to_uint64, integer_squareroot}; +use super::misc::{compute_epoch_at_slot, compute_signing_root, compute_start_slot_at_epoch}; +use super::predicates::are_indices_sorted_and_unique; +use super::shuffling::compute_shuffled_index; + +// --------------------------------------------------------------------------- +// Predicates +// --------------------------------------------------------------------------- + +/// Whether an indexed attestation names a valid attester set and carries +/// their aggregate signature. +/// +/// Modified from [`super::electra::is_valid_indexed_attestation`] (EIP-7688): +/// [`gloas::IndexedAttestation::attesting_indices`] is now the unbounded +/// [`gloas::AttestingIndices`], so the length that used to be enforced by the +/// container's own SSZ bound (`MAX_VALIDATORS_PER_SLOT`) is checked here +/// explicitly instead. +pub fn is_valid_indexed_attestation( + state: &BeaconState, + indexed_attestation: &gloas::IndexedAttestation, +) -> bool { + let indices: &[ValidatorIndex] = &indexed_attestation.attesting_indices; + if indices.is_empty() + || indices.len() > preset::MAX_VALIDATORS_PER_SLOT + || !are_indices_sorted_and_unique(indices) + { + return false; + } + + let mut pubkeys = Vec::with_capacity(indices.len()); + for index in indices { + match state.validator(*index) { + Ok(validator) => pubkeys.push(validator.pubkey), + Err(_) => return false, + } + } + + let domain = get_domain( + state, + constants::DOMAIN_BEACON_ATTESTER, + Some(indexed_attestation.data.target.epoch), + ); + let signing_root = compute_signing_root(indexed_attestation.data.hash_tree_root(), domain); + bls::fast_aggregate_verify(&pubkeys, signing_root, &indexed_attestation.signature) +} + +/// Whether `validator_index` actually names a [`gloas::BuilderIndex`] +/// (EIP-7732): builders and validators share one index space, distinguished +/// by this one bit. +pub fn is_builder_index(validator_index: ValidatorIndex) -> bool { + (validator_index & constants::BUILDER_INDEX_FLAG) != 0 +} + +/// Whether the builder at `builder_index` is active: its placement in the +/// registry is finalized, and it has not initiated an exit. +pub fn is_active_builder( + state: &gloas::BeaconState, + builder_index: gloas::BuilderIndex, +) -> Result { + let builder = state + .builders + .get(builder_index as usize) + .ok_or(Error::IndexOutOfBounds { + index: builder_index as usize, + len: state.builders.len(), + })?; + Ok( + // Placement in builder list is finalized. + builder.deposit_epoch < state.finalized_checkpoint.epoch + // Has not initiated exit. + && builder.withdrawable_epoch == constants::FAR_FUTURE_EPOCH, + ) +} + +/// Whether `withdrawal_credentials` is builder-prefixed +/// ([`constants::BUILDER_WITHDRAWAL_PREFIX`]). +pub fn is_builder_withdrawal_credential(withdrawal_credentials: Bytes32) -> bool { + withdrawal_credentials.0[0] == constants::BUILDER_WITHDRAWAL_PREFIX +} + +/// Whether `data` attests to the block proposed at its own slot, rather than +/// to a slot skipped since. +/// +/// A payload can be revealed a slot late (EIP-7732), so +/// [`get_attestation_participation_flag_indices`] needs to know whether an +/// attester's payload vote (`data.index`) is about *this* slot's block or an +/// earlier one's, which is what this decides: slot 0 always counts (there is +/// no earlier slot to have skipped from), and otherwise the attested root +/// must both match the block actually proposed at `data.slot` and differ +/// from the one at the slot before it, i.e. `data.slot` was not itself +/// skipped. +pub fn is_attestation_same_slot(state: &BeaconState, data: &AttestationData) -> Result { + if data.slot == 0 { + return Ok(true); + } + + let block_root = data.beacon_block_root; + let slot_block_root = get_block_root_at_slot(state, data.slot)?; + let prev_block_root = get_block_root_at_slot(state, data.slot - 1)?; + + Ok(block_root == slot_block_root && block_root != prev_block_root) +} + +/// Whether an indexed payload attestation names a non-empty, sorted attester +/// set and carries their aggregate signature. +/// +/// Unlike [`is_valid_indexed_attestation`], a payload timeliness committee's +/// attesting indices need only be sorted, not deduplicated: the payload +/// timeliness committee ([`compute_ptc`]) is drawn with replacement, so the +/// same validator can legitimately hold more than one seat and so appear +/// more than once. +pub fn is_valid_indexed_payload_attestation( + state: &BeaconState, + attestation: &gloas::IndexedPayloadAttestation, +) -> bool { + let indices: &[ValidatorIndex] = &attestation.attesting_indices; + if indices.is_empty() || !indices.windows(2).all(|pair| pair[0] <= pair[1]) { + return false; + } + + let mut pubkeys = Vec::with_capacity(indices.len()); + for index in indices { + match state.validator(*index) { + Ok(validator) => pubkeys.push(validator.pubkey), + Err(_) => return false, + } + } + + let domain = get_domain( + state, + constants::DOMAIN_PTC_ATTESTER, + Some(compute_epoch_at_slot(attestation.data.slot)), + ); + let signing_root = compute_signing_root(attestation.data.hash_tree_root(), domain); + bls::fast_aggregate_verify(&pubkeys, signing_root, &attestation.signature) +} + +/// Whether a pending deposit with a valid signature is already queued for +/// `pubkey`. +/// +/// *Note (from the specification):* this naively reverifies a deposit +/// signature on every call; a caller iterating many pubkeys should cache the +/// verification instead of calling this in a loop. +pub fn is_pending_validator( + pending_deposits: &[electra::PendingDeposit], + pubkey: BlsPubkey, + config: &Config, +) -> bool { + for pending_deposit in pending_deposits { + if pending_deposit.pubkey != pubkey { + continue; + } + if crate::beacon::stf::electra::is_valid_deposit_signature( + pending_deposit.pubkey, + pending_deposit.withdrawal_credentials, + pending_deposit.amount, + &pending_deposit.signature, + config, + ) { + return true; + } + } + false +} + +// --------------------------------------------------------------------------- +// Misc +// --------------------------------------------------------------------------- + +/// The [`ValidatorIndex`] a [`gloas::BuilderIndex`] is addressed as in a +/// context (like a `Withdrawal`) that only ever names validators. +pub fn convert_builder_index_to_validator_index( + builder_index: gloas::BuilderIndex, +) -> ValidatorIndex { + builder_index | constants::BUILDER_INDEX_FLAG +} + +/// The inverse of [`convert_builder_index_to_validator_index`]. +pub fn convert_validator_index_to_builder_index( + validator_index: ValidatorIndex, +) -> gloas::BuilderIndex { + validator_index & !constants::BUILDER_INDEX_FLAG +} + +/// The balance still owed to `builder_index` by withdrawals already queued, +/// whether settled into [`gloas::BeaconState::builder_pending_withdrawals`] +/// or still sitting in an unsettled +/// [`gloas::BeaconState::builder_pending_payments`] entry. +/// +/// [`can_builder_cover_bid`] subtracts this from a builder's balance before +/// checking it can cover a new bid, so a builder cannot bid its way into +/// double-spending a balance a withdrawal has already claimed. +pub fn get_pending_balance_to_withdraw_for_builder( + state: &gloas::BeaconState, + builder_index: gloas::BuilderIndex, +) -> Result { + let mut balance: Gwei = 0; + for withdrawal in state.builder_pending_withdrawals.iter() { + if withdrawal.builder_index == builder_index { + balance = balance + .checked_add(withdrawal.amount) + .ok_or(Error::ArithmeticOverflow( + "get_pending_balance_to_withdraw_for_builder: balance + withdrawal.amount", + ))?; + } + } + for payment in state.builder_pending_payments.iter() { + if payment.withdrawal.builder_index == builder_index { + balance = balance.checked_add(payment.withdrawal.amount).ok_or( + Error::ArithmeticOverflow( + "get_pending_balance_to_withdraw_for_builder: balance + payment.withdrawal.amount", + ), + )?; + } + } + Ok(balance) +} + +/// Whether the builder at `builder_index` can cover a bid of `bid_amount`, +/// after keeping [`preset::MIN_DEPOSIT_AMOUNT`] and every balance it already +/// owes ([`get_pending_balance_to_withdraw_for_builder`]) in reserve. +pub fn can_builder_cover_bid( + state: &gloas::BeaconState, + builder_index: gloas::BuilderIndex, + bid_amount: Gwei, +) -> Result { + let builder_balance = state + .builders + .get(builder_index as usize) + .ok_or(Error::IndexOutOfBounds { + index: builder_index as usize, + len: state.builders.len(), + })? + .balance; + let pending_withdrawals_amount = + get_pending_balance_to_withdraw_for_builder(state, builder_index)?; + let min_balance = preset::MIN_DEPOSIT_AMOUNT + .checked_add(pending_withdrawals_amount) + .ok_or(Error::ArithmeticOverflow( + "can_builder_cover_bid: MIN_DEPOSIT_AMOUNT + pending_withdrawals_amount", + ))?; + if builder_balance < min_balance { + return Ok(false); + } + Ok(builder_balance - min_balance >= bid_amount) +} + +/// `size` indices sampled from `indices` by effective balance, with possible +/// duplicates. +/// +/// The rejection-sampling primitive behind [`compute_proposer_indices`], +/// [`get_next_sync_committee_indices`], and [`compute_ptc`]: a candidate is +/// drawn (shuffled from `indices` when `shuffle_indices`, or taken in order +/// otherwise) and accepted with probability proportional to its effective +/// balance, exactly [`super::electra::compute_proposer_index`]'s acceptance +/// test, generalised to keep drawing until `size` candidates are accepted +/// rather than stopping at the first. +/// +/// Reuses one 32-byte random hash across up to 16 consecutive draws +/// (`sha256(seed + i / 16)` has 16 two-byte slices), recomputing only when +/// `i % 16 == 0`, which is why `random_bytes` is threaded through the loop +/// rather than recomputed every iteration the way +/// [`super::electra::compute_proposer_index`]'s does: both give the same +/// bytes for the same `i`, since the hash input only depends on `i / 16`, but +/// this follows the specification's own guard rather than always rehashing. +pub fn compute_balance_weighted_selection( + state: &BeaconState, + indices: &[ValidatorIndex], + seed: Bytes32, + size: u64, + shuffle_indices: bool, +) -> Result> { + // `2**16 - 1`, the largest value a two-byte little-endian draw can take. + const MAX_RANDOM_VALUE: u64 = u16::MAX as u64; + + let total = indices.len() as u64; + crate::beacon::verify(total > 0, "compute_balance_weighted_selection: total > 0")?; + + let mut effective_balances = Vec::with_capacity(indices.len()); + for index in indices { + effective_balances.push(state.validator(*index)?.effective_balance); + } + + let mut selected = Vec::new(); + let mut random_bytes = Bytes32::ZERO; + let mut i: u64 = 0; + while (selected.len() as u64) < size { + let offset = (i % 16) * 2; + if offset == 0 { + let mut input = Vec::with_capacity(32 + 8); + input.extend_from_slice(&seed.0); + input.extend_from_slice(&(i / 16).to_le_bytes()); + random_bytes = hash(&input); + } + + let mut next_index = i % total; + if shuffle_indices { + next_index = compute_shuffled_index(next_index, total, seed)?; + } + + let weight = effective_balances[next_index as usize] + .checked_mul(MAX_RANDOM_VALUE) + .ok_or(Error::ArithmeticOverflow( + "compute_balance_weighted_selection: effective_balance * MAX_RANDOM_VALUE", + ))?; + let random_value = bytes_to_uint64(&random_bytes.0[offset as usize..offset as usize + 2]); + let threshold = preset::MAX_EFFECTIVE_BALANCE_ELECTRA * random_value; + if weight >= threshold { + selected.push(indices[next_index as usize]); + } + i += 1; + } + Ok(selected) +} + +/// The proposer for every slot of `epoch`, drawn from `indices` under `seed`. +/// +/// Modified from [`super::fulu::compute_proposer_indices`] (EIP-7732): each +/// slot's single proposer now comes from [`compute_balance_weighted_selection`] +/// (`size = 1`, shuffled) rather than +/// [`super::electra::compute_proposer_index`]'s own rejection-sampling loop, +/// otherwise identical, including reusing that per-slot hash (`slot_seed`, +/// the specification's shadowed `seed`) as the selection's own seed rather +/// than the epoch seed passed in. +pub fn compute_proposer_indices( + state: &BeaconState, + epoch: Epoch, + seed: Bytes32, + indices: &[ValidatorIndex], +) -> Result> { + let start_slot = compute_start_slot_at_epoch(epoch); + + let mut proposer_indices = Vec::with_capacity(preset::SLOTS_PER_EPOCH as usize); + for offset in 0..preset::SLOTS_PER_EPOCH { + let slot = start_slot + .checked_add(offset) + .ok_or(Error::ArithmeticOverflow( + "compute_proposer_indices: start_slot + offset", + ))?; + let mut input = Vec::with_capacity(32 + 8); + input.extend_from_slice(&seed.0); + input.extend_from_slice(&slot.to_le_bytes()); + let slot_seed = hash(&input); + + let selected = compute_balance_weighted_selection(state, indices, slot_seed, 1, true)?; + proposer_indices.push( + *selected + .first() + .expect("compute_balance_weighted_selection(size=1) returns exactly one index"), + ); + } + Ok(proposer_indices) +} + +/// The payload timeliness committee, with possible duplicates, for `slot`. +/// +/// Concatenates every one of `slot`'s own committees (`0..committees_per_slot`) +/// and samples [`preset::PTC_SIZE`] members from that pool by effective +/// balance. +/// `shuffle_indices=false`: each committee is already a shuffled slice of +/// the active set (the same split [`CommitteeCache`] draws attester +/// committees from), so walking the concatenated pool in order still draws +/// from a shuffled ordering; a second shuffle here would just reorder +/// already-shuffled input. +/// +/// Takes `committees` rather than deriving each committee itself +/// (`get_beacon_committee`, one active-set scan and one per-member shuffle +/// apiece): `process_ptc_window` (a later task) calls this once per slot of +/// an epoch, and at mainnet scale that is `SLOTS_PER_EPOCH` calls against a +/// multi-million-validator registry, so sharing one epoch-wide shuffle +/// across all of them (see [`CommitteeCache`]'s own doc) is what keeps that +/// affordable; [`super::electra::get_attesting_indices`] takes the same +/// parameter for the same reason. +pub fn compute_ptc( + state: &BeaconState, + slot: Slot, + committees: &CommitteeCache, +) -> Result { + let epoch = compute_epoch_at_slot(slot); + let mut input = Vec::with_capacity(32 + 8); + input.extend_from_slice(&get_seed(state, epoch, constants::DOMAIN_PTC_ATTESTER).0); + input.extend_from_slice(&slot.to_le_bytes()); + let seed = hash(&input); + + let epoch_committees = committees.committees(state, epoch); + let mut indices = Vec::new(); + for committee_index in 0..epoch_committees.committees_per_slot() { + indices.extend(epoch_committees.committee(slot, committee_index)?); + } + + let selected = + compute_balance_weighted_selection(state, &indices, seed, preset::PTC_SIZE as u64, false)?; + Ok(selected.try_into()?) +} + +// --------------------------------------------------------------------------- +// Beacon state accessors +// --------------------------------------------------------------------------- + +/// The proposer indices for `epoch`. +/// +/// Modified from [`super::fulu::get_beacon_proposer_indices`] (EIP-8045): the +/// candidate pool excludes slashed validators before the draw, rather than +/// only after, so this call's own draw can never select an already-slashed +/// validator. It does not reach back into a window slot this same call did +/// not compute: a validator slashed after its lookahead entry was already +/// filled still stands in that entry, unrevised, until the window slides +/// forward and recomputes it. +pub fn get_beacon_proposer_indices( + state: &BeaconState, + epoch: Epoch, +) -> Result> { + let mut indices = Vec::new(); + for index in get_active_validator_indices(state, epoch) { + if !state.validator(index)?.slashed { + indices.push(index); + } + } + let seed = get_seed(state, epoch, constants::DOMAIN_BEACON_PROPOSER); + compute_proposer_indices(state, epoch, seed, &indices) +} + +/// The sync committee indices, with possible duplicates, for the sync +/// committee period starting next epoch. +/// +/// Modified from [`super::electra::get_next_sync_committee_indices`] +/// (EIP-7732): the draw is now [`compute_balance_weighted_selection`] rather +/// than a repeated inline rejection-sampling loop, otherwise identical +/// (same seed, same candidate pool, same ceiling). +pub fn get_next_sync_committee_indices(state: &BeaconState) -> Result> { + let epoch = get_current_epoch(state) + 1; + let seed = get_seed(state, epoch, constants::DOMAIN_SYNC_COMMITTEE); + let indices = get_active_validator_indices(state, epoch); + compute_balance_weighted_selection( + state, + &indices, + seed, + preset::SYNC_COMMITTEE_SIZE as u64, + true, + ) +} + +/// Which of the three participation flags an attestation with `data`, +/// included after `inclusion_delay` slots against a block whose parent was +/// at `parent_slot`, satisfies. +/// +/// Modified from [`super::altair::get_attestation_participation_flag_indices`] +/// (EIP-7732): matching the head now also requires the attested payload +/// status to match. A same-slot attestation (`is_attestation_same_slot`) +/// must vote `data.index == 0`; otherwise the vote is checked against +/// `execution_payload_availability` at *`parent_slot`*, not `data.slot`, +/// since the timely-head flag's own minimum-inclusion-delay requirement +/// already ties it to the parent block, and a skipped `data.slot` has no +/// payload availability of its own to read. The timely-target flag also +/// carries no `inclusion_delay <= SLOTS_PER_EPOCH` gate. That is not a +/// gloas change: the gate was already dropped at deneb (EIP-7045, which +/// widened the attestation inclusion window; see +/// `crate::beacon::stf::deneb::attestation_participation_flag_indices`), so +/// its absence here is inherited from deneb onward, unlike altair's own +/// version, which still carries it. +pub fn get_attestation_participation_flag_indices( + state: &BeaconState, + data: &AttestationData, + inclusion_delay: u64, + parent_slot: Slot, +) -> Result> { + // Matching source. + let justified_checkpoint = if data.target.epoch == get_current_epoch(state) { + state.current_justified_checkpoint() + } else { + state.previous_justified_checkpoint() + }; + let is_matching_source = data.source == justified_checkpoint; + + // Matching target. + let target_root = get_block_root(state, data.target.epoch)?; + let target_root_matches = data.target.root == target_root; + let is_matching_target = is_matching_source && target_root_matches; + + // Matching payload. + let payload_matches = if is_attestation_same_slot(state, data)? { + crate::beacon::verify( + data.index == 0, + "get_attestation_participation_flag_indices: data.index == 0", + )?; + true + } else { + let inner = gloas_state_ref(state, "get_attestation_participation_flag_indices")?; + let slot_index = (parent_slot % preset::SLOTS_PER_HISTORICAL_ROOT as u64) as usize; + let payload_index = inner.execution_payload_availability.get(slot_index).ok_or( + Error::IndexOutOfBounds { + index: slot_index, + len: inner.execution_payload_availability.len(), + }, + )?; + data.index == payload_index as u64 + }; + + // Matching head. + let head_root = get_block_root_at_slot(state, data.slot)?; + let head_root_matches = data.beacon_block_root == head_root; + let is_matching_head = is_matching_target && head_root_matches && payload_matches; + + crate::beacon::verify( + is_matching_source, + "get_attestation_participation_flag_indices: is_matching_source", + )?; + + let mut participation_flag_indices = Vec::new(); + if is_matching_source && inclusion_delay <= integer_squareroot(preset::SLOTS_PER_EPOCH) { + participation_flag_indices.push(constants::TIMELY_SOURCE_FLAG_INDEX); + } + if is_matching_target { + participation_flag_indices.push(constants::TIMELY_TARGET_FLAG_INDEX); + } + if is_matching_head && inclusion_delay == preset::MIN_ATTESTATION_INCLUSION_DELAY { + participation_flag_indices.push(constants::TIMELY_HEAD_FLAG_INDEX); + } + + Ok(participation_flag_indices) +} + +/// The payload timeliness committee for `slot`. +/// +/// Unlike [`compute_ptc`] (which always derives one fresh), this reads the +/// cached window [`gloas::BeaconState::ptc_window`] +/// `process_ptc_window` (gloas epoch processing) +/// refreshes each epoch: `slot`'s epoch must be the state's own, the one +/// before it, or within [`preset::MIN_SEED_LOOKAHEAD`] epochs ahead, which is +/// exactly the window's own span. +pub fn get_ptc( + state: &BeaconState, + slot: Slot, + config: &Config, +) -> Result { + let epoch = compute_epoch_at_slot(slot); + crate::beacon::verify( + epoch >= config.gloas_fork_epoch, + "get_ptc: epoch >= GLOAS_FORK_EPOCH", + )?; + let state_epoch = get_current_epoch(state); + let inner = gloas_state_ref(state, "get_ptc")?; + + let index = + if epoch < state_epoch { + let next_epoch = epoch + .checked_add(1) + .ok_or(Error::ArithmeticOverflow("get_ptc: epoch + 1"))?; + crate::beacon::verify( + next_epoch == state_epoch, + "get_ptc: epoch + 1 == state_epoch", + )?; + (slot % preset::SLOTS_PER_EPOCH) as usize + } else { + let lookahead_bound = state_epoch.checked_add(preset::MIN_SEED_LOOKAHEAD).ok_or( + Error::ArithmeticOverflow("get_ptc: state_epoch + MIN_SEED_LOOKAHEAD"), + )?; + crate::beacon::verify( + epoch <= lookahead_bound, + "get_ptc: epoch <= state_epoch + MIN_SEED_LOOKAHEAD", + )?; + let epoch_offset = epoch + .checked_sub(state_epoch) + .and_then(|delta| delta.checked_add(1)) + .ok_or(Error::ArithmeticOverflow( + "get_ptc: epoch - state_epoch + 1", + ))?; + let offset = epoch_offset.checked_mul(preset::SLOTS_PER_EPOCH).ok_or( + Error::ArithmeticOverflow("get_ptc: (epoch - state_epoch + 1) * SLOTS_PER_EPOCH"), + )?; + offset + .checked_add(slot % preset::SLOTS_PER_EPOCH) + .ok_or(Error::ArithmeticOverflow( + "get_ptc: offset + slot % SLOTS_PER_EPOCH", + ))? as usize + }; + + inner + .ptc_window + .get(index) + .cloned() + .ok_or(Error::IndexOutOfBounds { + index, + len: inner.ptc_window.len(), + }) +} + +/// The same payload attestation with its attesters named rather than +/// bit-encoded, the payload-attestation counterpart of +/// [`super::electra::get_indexed_attestation`]. +pub fn get_indexed_payload_attestation( + state: &BeaconState, + payload_attestation: &gloas::PayloadAttestation, + config: &Config, +) -> Result { + let slot = payload_attestation.data.slot; + let ptc = get_ptc(state, slot, config)?; + let bits = &payload_attestation.aggregation_bits; + + let mut attesting_indices = Vec::new(); + for (index, validator_index) in ptc.iter().enumerate() { + if bits.get(index).unwrap_or(false) { + attesting_indices.push(*validator_index); + } + } + attesting_indices.sort_unstable(); + + Ok(gloas::IndexedPayloadAttestation { + attesting_indices: attesting_indices.try_into()?, + data: payload_attestation.data, + signature: payload_attestation.signature, + }) +} + +/// The attesting weight a builder payment needs before +/// `process_builder_pending_payments` (a later task) settles it in full +/// rather than dropping it: [`constants::BUILDER_PAYMENT_THRESHOLD_NUMERATOR`] +/// `/` [`constants::BUILDER_PAYMENT_THRESHOLD_DENOMINATOR`] of one slot's +/// share of the total active balance. +pub fn get_builder_payment_quorum_threshold(state: &BeaconState) -> Result { + let per_slot_balance = get_total_active_balance(state)? / preset::SLOTS_PER_EPOCH; + let quorum = per_slot_balance + .checked_mul(constants::BUILDER_PAYMENT_THRESHOLD_NUMERATOR) + .ok_or(Error::ArithmeticOverflow( + "get_builder_payment_quorum_threshold: per_slot_balance * BUILDER_PAYMENT_THRESHOLD_NUMERATOR", + ))?; + Ok(quorum / constants::BUILDER_PAYMENT_THRESHOLD_DENOMINATOR) +} + +/// The per-epoch churn limit for activations, in Gwei. +/// +/// New in gloas (EIP-8061): unlike electra's combined +/// [`super::electra::get_activation_exit_churn_limit`] (a share of one +/// budget split with exits), this is its own independent proportion of total +/// active balance, floored at [`Config::min_per_epoch_churn_limit_electra`] +/// and capped at [`Config::max_per_epoch_activation_churn_limit_gloas`]. +pub fn get_activation_churn_limit(state: &BeaconState, config: &Config) -> Result { + let total_active_balance = get_total_active_balance(state)?; + let churn = config + .min_per_epoch_churn_limit_electra + .max(total_active_balance / config.churn_limit_quotient_gloas); + let churn = churn - churn % preset::EFFECTIVE_BALANCE_INCREMENT; + Ok(config.max_per_epoch_activation_churn_limit_gloas.min(churn)) +} + +/// The per-epoch churn limit for exits, in Gwei. +/// +/// New in gloas (EIP-8061): the same proportion and floor as +/// [`get_activation_churn_limit`], but with no upper cap of its own, unlike +/// activations. electra's +/// [`compute_exit_epoch_and_update_churn`](super::electra::compute_exit_epoch_and_update_churn) +/// draws from this on a gloas state +/// rather than electra's combined budget. +pub fn get_exit_churn_limit(state: &BeaconState, config: &Config) -> Result { + let total_active_balance = get_total_active_balance(state)?; + let churn = config + .min_per_epoch_churn_limit_electra + .max(total_active_balance / config.churn_limit_quotient_gloas); + Ok(churn - churn % preset::EFFECTIVE_BALANCE_INCREMENT) +} + +/// The per-epoch churn limit reserved for consolidations, in Gwei. +/// +/// Modified from [`super::electra::get_consolidation_churn_limit`]: no longer +/// left over from the activation/exit split, but its own independent +/// proportion of total active balance. [`Config::consolidation_churn_limit_quotient`] +/// is itself new in gloas: electra's own consolidation limit has no +/// quotient of its own to read, only the combined-budget split this +/// replaces. Floored at zero rather than at +/// [`Config::min_per_epoch_churn_limit_electra`] the way the activation and +/// exit limits are. +pub fn get_consolidation_churn_limit(state: &BeaconState, config: &Config) -> Result { + let total_active_balance = get_total_active_balance(state)?; + let churn = total_active_balance / config.consolidation_churn_limit_quotient; + Ok(churn - churn % preset::EFFECTIVE_BALANCE_INCREMENT) +} + +// --------------------------------------------------------------------------- +// Beacon state mutators +// --------------------------------------------------------------------------- + +/// Starts the exit of the builder at `builder_index`: sets its withdrawable +/// epoch [`Config::min_builder_withdrawability_delay`] epochs out, the +/// builder-registry counterpart of +/// [`crate::beacon::helpers::mutators::initiate_validator_exit`]. +pub fn initiate_builder_exit( + state: &mut gloas::BeaconState, + builder_index: gloas::BuilderIndex, + config: &Config, +) -> Result<()> { + let current_epoch = compute_epoch_at_slot(state.slot); + let len = state.builders.len(); + let builder = + state + .builders + .get_mut(builder_index as usize) + .ok_or(Error::IndexOutOfBounds { + index: builder_index as usize, + len, + })?; + builder.withdrawable_epoch = current_epoch + .checked_add(config.min_builder_withdrawability_delay) + .ok_or(Error::ArithmeticOverflow( + "initiate_builder_exit: current_epoch + MIN_BUILDER_WITHDRAWABILITY_DELAY", + ))?; + Ok(()) +} + +/// Settles (or drops) one builder payment: pushes its withdrawal onto +/// [`gloas::BeaconState::builder_pending_withdrawals`] if it carries a real +/// amount, then clears the slot. +/// +/// Called by `process_builder_pending_payments` (a later task) once a +/// payment's accumulated attesting weight has been compared against +/// [`get_builder_payment_quorum_threshold`]. +pub fn settle_builder_payment(state: &mut gloas::BeaconState, payment_index: u64) -> Result<()> { + let len = state.builder_pending_payments.len() as u64; + crate::beacon::verify( + payment_index < len, + "settle_builder_payment: payment index in range", + )?; + let payment = state.builder_pending_payments[payment_index as usize].clone(); + if payment.withdrawal.amount > 0 { + state.builder_pending_withdrawals.push(payment.withdrawal); + } + state.builder_pending_payments[payment_index as usize] = + gloas::BuilderPendingPayment::default(); + Ok(()) +} + +// --------------------------------------------------------------------------- +// Fork projection +// --------------------------------------------------------------------------- + +/// The gloas state, mutably, or an error naming the function that needs one. +/// +/// Kept alongside [`gloas_state_ref`] for whichever later task first needs to +/// mutate a gloas-only field (`builders`, the payload/PTC bookkeeping) +/// through a generic [`BeaconState`] rather than one already known to be +/// [`BeaconState::Gloas`] the way [`initiate_builder_exit`] and +/// [`settle_builder_payment`] are always called with; nothing in this file +/// calls it yet. +#[allow(dead_code)] +pub(crate) fn gloas_state<'a>( + state: &'a mut BeaconState, + function: &'static str, +) -> Result<&'a mut gloas::BeaconState> { + match state { + BeaconState::Gloas(inner) => Ok(inner), + other => Err(Error::UnsupportedForFork { + function, + fork: other.fork_name(), + }), + } +} + +/// The gloas state, immutably. See [`gloas_state`]. +pub(crate) fn gloas_state_ref<'a>( + state: &'a BeaconState, + function: &'static str, +) -> Result<&'a gloas::BeaconState> { + match state { + BeaconState::Gloas(inner) => Ok(inner), + other => Err(Error::UnsupportedForFork { + function, + fork: other.fork_name(), + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::beacon::fork::ForkName; + + /// A gloas state with `count` fully active, full-balance validators, + /// positioned the same way `crate::beacon::helpers::test_state::with_validators` + /// positions its phase0 state. + fn gloas_state_with_validators(count: usize) -> BeaconState { + crate::beacon::helpers::test_state::with_validators_at(ForkName::Gloas, count) + } + + #[test] + fn compute_balance_weighted_selection_returns_exactly_size_indices_from_the_pool() { + let state = gloas_state_with_validators(16); + let indices: Vec = (0..16).collect(); + let selected = + compute_balance_weighted_selection(&state, &indices, Bytes32::repeat_byte(7), 20, true) + .unwrap(); + assert_eq!(selected.len(), 20); + assert!(selected.iter().all(|index| *index < 16)); + } + + #[test] + fn compute_balance_weighted_selection_rejects_an_empty_pool() { + let state = gloas_state_with_validators(16); + assert!(compute_balance_weighted_selection(&state, &[], Bytes32::ZERO, 1, true).is_err()); + } + + #[test] + fn compute_balance_weighted_selection_is_deterministic_for_a_fixed_seed() { + let state = gloas_state_with_validators(16); + let indices: Vec = (0..16).collect(); + let seed = Bytes32::repeat_byte(3); + let first = compute_balance_weighted_selection(&state, &indices, seed, 10, true).unwrap(); + let second = compute_balance_weighted_selection(&state, &indices, seed, 10, true).unwrap(); + assert_eq!(first, second); + } + + #[test] + fn compute_ptc_returns_ptc_size_active_indices_deterministically() { + // 64, not 16: with too few validators relative to `SLOTS_PER_EPOCH`, + // a committee slices down to zero members (the whole active set + // split `SLOTS_PER_EPOCH` ways leaves some slices empty), and the + // current slot lands on exactly one of those under the mainnet + // preset. 64 safely clears that under both presets: + // `helpers::accessors`'s own committee tests use the same count for + // the same reason. + let count = 64; + let state = gloas_state_with_validators(count); + let slot = state.slot(); + let committees = CommitteeCache::default(); + + let first = compute_ptc(&state, slot, &committees).unwrap(); + let second = compute_ptc(&state, slot, &committees).unwrap(); + + assert_eq!(first.len(), preset::PTC_SIZE); + assert_eq!(&first[..], &second[..]); + assert!(first.iter().all(|index| (*index as usize) < count)); + } + + /// The module doc's claim that gloas's proposer draw is otherwise + /// identical to electra's rejection-sampling loop, checked directly: + /// `compute_balance_weighted_selection(.., size=1, shuffle_indices=true)` + /// must select the exact same candidate `electra::compute_proposer_index` + /// does, for the same indices, seed, and effective balances. Varied + /// effective balances (not all equal), so the acceptance test actually + /// discriminates between candidates instead of every candidate passing + /// (or failing) the same way regardless of which byte a bug reads. + /// Checked across many seeds, since the two loops could still coincide + /// on any one seed by chance even with a subtly wrong offset. + #[test] + fn compute_balance_weighted_selection_size_one_shuffled_matches_electras_compute_proposer_index() + { + let count = 32; + let mut state = gloas_state_with_validators(count); + for index in 0..count as ValidatorIndex { + state.validator_mut(index).unwrap().effective_balance = + preset::EFFECTIVE_BALANCE_INCREMENT * (1 + index % 8); + } + let indices: Vec = (0..count as u64).collect(); + + for seed_byte in 0..32u8 { + let seed = Bytes32::repeat_byte(seed_byte); + let expected = + crate::beacon::helpers::electra::compute_proposer_index(&indices, seed, |index| { + Ok(state.validator(index)?.effective_balance) + }) + .unwrap(); + let selected = + compute_balance_weighted_selection(&state, &indices, seed, 1, true).unwrap(); + assert_eq!(selected, vec![expected], "seed byte {seed_byte}"); + } + } + + /// The module doc's claim that gloas's sync committee draw is otherwise + /// identical to electra's, checked directly: both functions read the + /// exact same state (active indices, seed domain, effective balances), + /// so they must return the exact same committee. Varied effective + /// balances for the same reason the proposer-index equivalence test + /// above uses them. + #[test] + fn get_next_sync_committee_indices_matches_electras_own_function_on_a_gloas_state() { + let count = 32; + let mut state = gloas_state_with_validators(count); + for index in 0..count as ValidatorIndex { + state.validator_mut(index).unwrap().effective_balance = + preset::MAX_EFFECTIVE_BALANCE * (1 + index % 8); + } + + let expected = + crate::beacon::helpers::electra::get_next_sync_committee_indices(&state).unwrap(); + let actual = get_next_sync_committee_indices(&state).unwrap(); + assert_eq!(actual, expected); + } + + /// `get_ptc`'s `epoch < state_epoch` branch: the only earlier epoch it + /// ever accepts is `state_epoch - 1`, read from the window's first + /// `SLOTS_PER_EPOCH` slice (index `0..SLOTS_PER_EPOCH`). + #[test] + fn get_ptc_reads_the_previous_epoch_slice_of_the_window() { + let mut state = gloas_state_with_validators(4); + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 0); + mark_ptc_window(&mut state); + + let state_epoch = get_current_epoch(&state); + let previous_epoch_slot = compute_start_slot_at_epoch(state_epoch - 1); + + let committee = get_ptc(&state, previous_epoch_slot, &config).unwrap(); + assert_eq!( + committee[0], 0, + "window index 0 starts the previous-epoch slice" + ); + } + + /// `get_ptc`'s `epoch >= state_epoch` branch, at its simplest offset + /// (`epoch == state_epoch`): the window's second `SLOTS_PER_EPOCH` slice + /// (index `SLOTS_PER_EPOCH..2*SLOTS_PER_EPOCH`). + #[test] + fn get_ptc_reads_the_current_epoch_slice_of_the_window() { + let mut state = gloas_state_with_validators(4); + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 0); + mark_ptc_window(&mut state); + + let state_epoch = get_current_epoch(&state); + let current_epoch_slot = compute_start_slot_at_epoch(state_epoch); + + let committee = get_ptc(&state, current_epoch_slot, &config).unwrap(); + assert_eq!( + committee[0], + preset::SLOTS_PER_EPOCH, + "window index SLOTS_PER_EPOCH starts the current-epoch slice" + ); + } + + /// Fills every `ptc_window` entry with its own window index, repeated + /// across the committee, so a test can tell which slice `get_ptc` read + /// back out just by looking at the first element. + fn mark_ptc_window(state: &mut BeaconState) { + let BeaconState::Gloas(inner) = state else { + unreachable!("built as Gloas"); + }; + for window_index in 0..preset::PTC_WINDOW_LENGTH { + inner.ptc_window[window_index] = vec![window_index as u64; preset::PTC_SIZE] + .try_into() + .expect("built at exactly PTC_SIZE"); + } + } +} diff --git a/crates/blockchain/state_transition/src/beacon/helpers/mod.rs b/crates/blockchain/state_transition/src/beacon/helpers/mod.rs index 3449b26aa..fe79b80bb 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/mod.rs @@ -18,6 +18,7 @@ pub mod capella; pub mod electra; pub mod finality; pub mod fulu; +pub mod gloas; pub mod math; pub mod misc; pub mod mutators; diff --git a/crates/blockchain/state_transition/src/beacon/helpers/mutators.rs b/crates/blockchain/state_transition/src/beacon/helpers/mutators.rs index 131706411..077591fc4 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/mutators.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/mutators.rs @@ -132,7 +132,14 @@ fn initiate_validator_exit_for_fork( config: &Config, ) -> Result<()> { match state.fork_name() { - ForkName::Electra | ForkName::Fulu => { + // Gloas is not a redefinition of `initiate_validator_exit` (EIP-8061 + // touches only `compute_exit_epoch_and_update_churn`'s own churn + // limit, by fork dispatch there), so it reaches electra's copy + // unchanged, the same way pyspec's own unmodified + // `initiate_validator_exit` does on a gloas state: the churn it + // draws through `compute_exit_epoch_and_update_churn` is already + // gloas's own by the time it gets here. + ForkName::Electra | ForkName::Fulu | ForkName::Gloas => { crate::beacon::helpers::electra::initiate_validator_exit(state, index, config) } ForkName::Phase0 @@ -140,15 +147,6 @@ fn initiate_validator_exit_for_fork( | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb => initiate_validator_exit(state, index, config), - // Gloas (EIP-8061) redefines the validator exit churn itself - // (`churn_limit_quotient_gloas`/`max_per_epoch_activation_churn_limit_gloas` - // in `Config`, read by `get_activation_churn_limit`/ - // `get_exit_churn_limit`), which is more than a retuned constant; - // left as an explicit gap for the STF port rather than assumed. - ForkName::Gloas => Err(Error::UnsupportedForFork { - function: "initiate_validator_exit_for_fork", - fork: ForkName::Gloas, - }), ForkName::Lean => lean_state_unreachable("initiate_validator_exit_for_fork"), } } @@ -274,4 +272,42 @@ mod tests { assert!(state.balance(3).unwrap() < balance_before); assert!(state.balance(3).unwrap() > balance_before / 2); } + + /// `initiate_validator_exit_for_fork` routes a gloas state to electra's + /// `initiate_validator_exit` unchanged, which in turn calls + /// `compute_exit_epoch_and_update_churn`, which is where the gloas and + /// electra churn split lives (a per-epoch limit chosen by fork, not a + /// separate gloas copy of the whole cursor update). This proves the + /// split reaches all the way through: with `Config::minimal()` and 64 + /// validators, gloas's exit churn limit differs from electra's combined + /// activation/exit limit (the test asserts they differ before relying + /// on it), so the two leave a different `exit_balance_to_consume` + /// behind for the same single exit. + #[test] + fn initiate_validator_exit_for_fork_uses_gloas_own_exit_churn_on_a_gloas_state() { + let config = Config::minimal(); + let mut state = crate::beacon::helpers::test_state::with_validators_at(ForkName::Gloas, 64); + + let gloas_limit = + crate::beacon::helpers::gloas::get_exit_churn_limit(&state, &config).unwrap(); + let electra_limit = + crate::beacon::helpers::electra::get_activation_exit_churn_limit(&state, &config) + .unwrap(); + assert_ne!( + gloas_limit, electra_limit, + "the fixture must actually discriminate between the two churn formulas" + ); + + let effective_balance = state.validator(0).unwrap().effective_balance; + initiate_validator_exit_for_fork(&mut state, 0, &config).unwrap(); + + let BeaconState::Gloas(inner) = &state else { + unreachable!("built as Gloas"); + }; + assert_eq!( + inner.exit_balance_to_consume, + gloas_limit - effective_balance, + "the exit must have consumed gloas's own churn budget, not electra's" + ); + } } diff --git a/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs b/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs index 8b31a8093..9050ef21f 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/test_state.rs @@ -30,7 +30,7 @@ use crate::beacon::containers::shared::{ }; use crate::beacon::containers::{ BeaconState, BlockRoots, RandaoMixes, Slashings, altair, bellatrix, capella, deneb, electra, - fulu, phase0, + fulu, gloas, phase0, }; use crate::beacon::fork::ForkName; use crate::beacon::lean_fork_unreachable; @@ -140,13 +140,7 @@ pub fn with_validators_at(fork: ForkName, count: usize) -> BeaconState { ForkName::Deneb => BeaconState::Deneb(deneb_state(count)), ForkName::Electra => BeaconState::Electra(electra_state(count)), ForkName::Fulu => BeaconState::Fulu(fulu_state(count)), - // No gloas literal builder exists yet: its registry is progressive - // rather than bounded (see `containers::gloas`'s module doc), so this - // cannot reuse `fulu_state`'s placeholder fields unchanged. A later - // task adds `gloas_state` alongside the others above. - ForkName::Gloas => { - unimplemented!("with_validators_at: gloas has no test-state builder yet") - } + ForkName::Gloas => BeaconState::Gloas(gloas_state(count)), // A test asking for a lean state from a beacon builder, which no fixture // fork name can produce; the `fork:` form, since the argument is the // whole input. @@ -546,3 +540,89 @@ fn fulu_state(count: usize) -> fulu::BeaconState { .expect("the vector is built at its exact length"), } } + +/// The [`with_validators_at`] counterpart for gloas. +/// +/// A separate literal rather than a small patch over [`fulu_state`]: gloas's +/// registry ([`crate::beacon::containers::shared::ProgressiveValidators`]/ +/// [`crate::beacon::containers::shared::ProgressiveBalances`]) is tree-backed +/// rather than bounded (EIP-7688, see `containers::gloas`'s module doc), so +/// [`full_validators`]/[`full_balances`]'s `SszList`-typed output does not +/// fit gloas's fields; both convert from a plain `Vec` through `.into()` +/// instead of `full_validators`/`full_balances`'s `.try_into()`. Every field +/// from `builders` onward is new in gloas and has no meaningful default +/// beyond an empty registry and an all-zero window, the same placeholder +/// `crate::beacon::containers::gloas`'s own `zero_state` test helper uses. +fn gloas_state(count: usize) -> gloas::BeaconState { + let validators: Vec = (0..count) + .map(|_| Validator { + effective_balance: preset::MAX_EFFECTIVE_BALANCE, + activation_eligibility_epoch: 0, + activation_epoch: 0, + exit_epoch: constants::FAR_FUTURE_EPOCH, + withdrawable_epoch: constants::FAR_FUTURE_EPOCH, + ..Default::default() + }) + .collect(); + let empty_ptc: gloas::PayloadTimelinessCommittee = vec![0u64; preset::PTC_SIZE] + .try_into() + .expect("built at exactly PTC_SIZE"); + + gloas::BeaconState { + genesis_time: 0, + genesis_validators_root: Root::ZERO, + slot: preset::SLOTS_PER_EPOCH, + fork: Default::default(), + latest_block_header: Default::default(), + block_roots: zero_root_vector(), + state_roots: zero_root_vector(), + historical_roots: Default::default(), + eth1_data: Default::default(), + eth1_data_votes: Default::default(), + eth1_deposit_index: 0, + validators: validators.into(), + balances: vec![preset::MAX_EFFECTIVE_BALANCE; count].into(), + randao_mixes: zero_randao_mixes(), + slashings: zero_slashings(), + previous_epoch_participation: vec![0; count].into(), + current_epoch_participation: vec![0; count].into(), + justification_bits: Default::default(), + previous_justified_checkpoint: Default::default(), + current_justified_checkpoint: Default::default(), + finalized_checkpoint: Default::default(), + inactivity_scores: vec![0; count].into(), + current_sync_committee: empty_sync_committee(), + next_sync_committee: empty_sync_committee(), + latest_block_hash: ExecutionBlockHash::ZERO, + next_withdrawal_index: 0, + next_withdrawal_validator_index: 0, + historical_summaries: Default::default(), + deposit_requests_start_index: constants::UNSET_DEPOSIT_REQUESTS_START_INDEX, + deposit_balance_to_consume: 0, + exit_balance_to_consume: 0, + earliest_exit_epoch: 0, + consolidation_balance_to_consume: 0, + earliest_consolidation_epoch: 0, + pending_deposits: Default::default(), + pending_partial_withdrawals: Default::default(), + pending_consolidations: Default::default(), + proposer_lookahead: vec![0; preset::PROPOSER_LOOKAHEAD_LENGTH] + .try_into() + .expect("the vector is built at its exact length"), + builders: Default::default(), + next_withdrawal_builder_index: 0, + execution_payload_availability: Default::default(), + builder_pending_payments: vec![ + gloas::BuilderPendingPayment::default(); + preset::BUILDER_PENDING_PAYMENTS_LENGTH + ] + .try_into() + .expect("the vector is built at its exact length"), + builder_pending_withdrawals: Default::default(), + latest_execution_payload_bid: Default::default(), + payload_expected_withdrawals: Default::default(), + ptc_window: vec![empty_ptc; preset::PTC_WINDOW_LENGTH] + .try_into() + .expect("the vector is built at its exact length"), + } +} diff --git a/crates/blockchain/state_transition/src/beacon/stf/electra.rs b/crates/blockchain/state_transition/src/beacon/stf/electra.rs index 6a6a2dcfd..e4e872da4 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/electra.rs @@ -62,7 +62,7 @@ use crate::beacon::helpers::accessors::{ use crate::beacon::helpers::altair::{add_flag, get_base_reward_per_increment, has_flag}; use crate::beacon::helpers::electra::{ compute_exit_epoch_and_update_churn, electra_state, get_committee_indices, - get_consolidation_churn_limit, get_indexed_attestation, get_max_effective_balance, + get_consolidation_churn_limit_for_fork, get_indexed_attestation, get_max_effective_balance, get_pending_balance_to_withdraw, has_compounding_withdrawal_credential, has_eth1_withdrawal_credential, has_execution_withdrawal_credential, initiate_validator_exit as electra_initiate_validator_exit, is_fully_withdrawable_validator, @@ -1438,7 +1438,7 @@ pub fn process_consolidation_request( if pending_consolidations_len == preset::PENDING_CONSOLIDATIONS_LIMIT { return Ok(()); } - if get_consolidation_churn_limit(state, config)? <= preset::MIN_ACTIVATION_BALANCE { + if get_consolidation_churn_limit_for_fork(state, config)? <= preset::MIN_ACTIVATION_BALANCE { return Ok(()); } diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/capella.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/capella.rs index 6a93c68b7..61ae4fa0d 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/capella.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/capella.rs @@ -81,12 +81,13 @@ pub fn process_historical_summaries_update(state: &mut BeaconState) -> Result<() /// /// `historical_summaries` enters the state at capella and every later fork /// keeps the identical field: deneb reuses this whole driver unchanged (see -/// this module's own documentation), and electra and fulu each fold this same -/// step into their own, larger driver rather than redefining it, since -/// neither fork's specification says anything about historical summaries at -/// all. Matching every variant that actually has the field, rather than only -/// [`BeaconState::Capella`], is what lets each of those reuse this function -/// instead of a copy of it. +/// this module's own documentation), electra and fulu each fold this same +/// step into their own, larger driver rather than redefining it, and gloas +/// keeps the field unchanged too (see `containers::gloas`'s module doc), +/// since neither fork's specification says anything about historical +/// summaries at all. Matching every variant that actually has the field, +/// rather than only [`BeaconState::Capella`], is what lets each of those +/// reuse this function instead of a copy of it. fn historical_summaries_mut<'a>( state: &'a mut BeaconState, function: &'static str, @@ -96,6 +97,7 @@ fn historical_summaries_mut<'a>( BeaconState::Deneb(state) => Ok(&mut state.historical_summaries), BeaconState::Electra(state) => Ok(&mut state.historical_summaries), BeaconState::Fulu(state) => Ok(&mut state.historical_summaries), + BeaconState::Gloas(state) => Ok(&mut state.historical_summaries), other => Err(Error::UnsupportedForFork { function, fork: other.fork_name(), diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/registry.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/registry.rs index f6985d73c..0d115f157 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/registry.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/registry.rs @@ -37,6 +37,15 @@ use crate::beacon::primitives::{Epoch, Gwei, ValidatorIndex}; /// where the specification no longer has one at all, so both are refused /// outright rather than guessed at; see [`process_registry_updates`]'s own /// documentation for exactly which forks that leaves this serving. +/// +/// Gloas is refused for the same reason, not routed to +/// [`crate::beacon::helpers::gloas::get_activation_churn_limit`]: gloas +/// inherits electra's balance-denominated `process_registry_updates` +/// wholesale too (it has no validator-count activation cap of its own for +/// this function to pick between), so this gate has nothing to serve gloas +/// either. `get_activation_churn_limit` belongs to gloas's own +/// `process_pending_deposits` (EIP-8061, a later task), which caps *deposit* +/// processing, not registry activation. fn activation_churn_limit(state: &BeaconState, config: &Config) -> Result { let churn_limit = get_validator_churn_limit(state, config); match state.fork_name() { diff --git a/crates/blockchain/state_transition/src/beacon/upgrade.rs b/crates/blockchain/state_transition/src/beacon/upgrade.rs index b9730601e..bedda7b5d 100644 --- a/crates/blockchain/state_transition/src/beacon/upgrade.rs +++ b/crates/blockchain/state_transition/src/beacon/upgrade.rs @@ -685,7 +685,7 @@ pub fn upgrade_to_electra(pre: &BeaconState, config: &Config) -> Result Date: Fri, 25 Sep 2026 16:48:44 -0300 Subject: [PATCH 016/132] feat(beacon): upgrade a fulu state to gloas Ports upgrade_to_gloas, initialize_ptc_window and onboard_builders_from_pending_deposits (gloas fork.md), the irregular state change that turns a fulu state into gloas's shape at GLOAS_FORK_EPOCH. EIP-7688 makes the registry-shaped fields progressive; EIP-7732 replaces the execution payload header with a synthetic self-build bid and a builder registry, and the payload timeliness committee window is filled from the fork epoch onward. onboard_builders_from_pending_deposits is the one path through the validator deposit contract that creates builders. Its helpers, add_builder_to_registry and get_index_for_new_builder, are pub in helpers/gloas.rs because builder deposit requests (EIP-8282) reuse them. Onboarding runs over the whole pending deposit queue, and every state that crosses the fork pays for it: the chain actor's checkpoint states and the column gossip checks, not only block import. The spec's membership tests read as list scans (a validator pubkey per deposit, a builder pubkey list recomputed per iteration, is_pending_validator re-verifying signatures), which is quadratic on a mainnet-sized registry. Here they are a set built from the queue's own pubkeys, a pubkey-to-index map kept in step with the builder registry, and a per-entry memo of signature verdicts; each gives the spec's answer, and the pass is linear. process_slot gains gloas's one new step: it clears the next slot's execution payload availability bit, after the state root is cached. The fork runner now runs for gloas. transition stays off until gloas block processing exists, since its cases cross into gloas blocks. The binary's genesis-anchor-block test used to stop at gloas because upgrade_to_gloas did not exist; it now chains through gloas like every other fork. --- bin/ethlambda/src/main.rs | 28 +- .../src/beacon/helpers/gloas.rs | 195 +++++++- .../state_transition/src/beacon/stf/mod.rs | 95 ++++ .../state_transition/src/beacon/upgrade.rs | 456 +++++++++++++++++- .../state_transition/tests/beacon_spec/mod.rs | 6 +- 5 files changed, 732 insertions(+), 48 deletions(-) diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index e0c9f782b..1cab37343 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -2806,37 +2806,15 @@ validators: /// to each new fork's own empty body below, exactly as a genesis /// generator targeting that fork directly would have to. /// - /// Stops at fulu: `upgrade_state` has no `upgrade_to_gloas` yet (it - /// returns `Error::UnsupportedForFork` for `ForkName::Gloas` until a - /// later task adds one), so this loop cannot chain a gloas state the way - /// it does every earlier fork. `genesis_anchor_block`'s own gloas arm - /// therefore has no coverage here until that lands. + /// Chains all the way through gloas: `upgrade_state` handles every fork + /// from altair to gloas, so this loop exercises `genesis_anchor_block`'s + /// gloas arm the same way it does every earlier fork's. #[test] fn a_genesis_anchor_block_hashes_to_the_states_own_header_at_every_fork() { let config = Config::mainnet(); let mut state = beacon::mainnet_genesis_state().unwrap(); for fork in ForkName::ALL { - if fork == ForkName::Gloas { - // Pinned so this test starts failing, loudly, the moment - // `upgrade_to_gloas` lands: that failure is the cue to - // delete this early break and extend the loop past it. - let err = ethlambda_state_transition::beacon::upgrade::upgrade_state( - &state, fork, &config, - ) - .expect_err("upgrade_to_gloas does not exist yet"); - assert!( - matches!( - err, - ethlambda_state_transition::beacon::Error::UnsupportedForFork { - fork: ForkName::Gloas, - .. - } - ), - "expected UnsupportedForFork, got {err:?}" - ); - break; - } if fork != ForkName::Phase0 { state = ethlambda_state_transition::beacon::upgrade::upgrade_state( &state, fork, &config, diff --git a/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs b/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs index 195a207ab..d169fad72 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs @@ -95,7 +95,7 @@ use crate::beacon::error::{Error, Result}; use crate::beacon::hash::hash; use crate::beacon::preset; use crate::beacon::primitives::{ - BlsPubkey, Bytes32, Epoch, Gwei, HashTreeRoot as _, Slot, ValidatorIndex, + BlsPubkey, Bytes32, Epoch, ExecutionAddress, Gwei, HashTreeRoot as _, Slot, ValidatorIndex, }; use super::accessors::{ @@ -832,19 +832,74 @@ pub fn settle_builder_payment(state: &mut gloas::BeaconState, payment_index: u64 Ok(()) } +// --------------------------------------------------------------------------- +// Builder registry (EIP-8282) +// --------------------------------------------------------------------------- + +/// `get_index_for_new_builder` (gloas `beacon-chain.md`). +/// +/// The registry slot a new builder record can reuse: an already-exited, +/// fully swept builder (`withdrawable_epoch` past, balance zero), or, absent +/// one, the registry's own length, so [`add_builder_to_registry`]'s +/// `set_or_append_list` appends a fresh entry instead of overwriting one. +/// Builder indices are reusable this way, unlike the validator registry, +/// which never removes an entry. +pub fn get_index_for_new_builder(state: &gloas::BeaconState) -> gloas::BuilderIndex { + let current_epoch = compute_epoch_at_slot(state.slot); + for (index, builder) in state.builders.iter().enumerate() { + if builder.withdrawable_epoch <= current_epoch && builder.balance == 0 { + return index as gloas::BuilderIndex; + } + } + state.builders.len() as gloas::BuilderIndex +} + +/// `add_builder_to_registry` (gloas `beacon-chain.md`). +/// +/// Registers a new builder record at [`get_index_for_new_builder`]'s slot +/// (the specification's `set_or_append_list`: an append when that slot is +/// the registry's own length, an overwrite of a reused, exited slot +/// otherwise), and returns the index the record now lives at, so a caller +/// keeping its own index of the registry (`onboard_builders_from_pending_deposits`'s +/// pubkey-to-index map, `crate::beacon::upgrade`) can update it without a +/// second lookup. Called from that one-time fork-boundary onboarding pass +/// and from builder deposit requests (EIP-8282's ongoing onboarding path). +pub fn add_builder_to_registry( + state: &mut gloas::BeaconState, + pubkey: BlsPubkey, + version: u8, + execution_address: ExecutionAddress, + amount: Gwei, + slot: Slot, +) -> gloas::BuilderIndex { + let index = get_index_for_new_builder(state); + let builder = gloas::Builder { + pubkey, + version, + execution_address, + balance: amount, + deposit_epoch: compute_epoch_at_slot(slot), + withdrawable_epoch: constants::FAR_FUTURE_EPOCH, + }; + if index == state.builders.len() as gloas::BuilderIndex { + state.builders.push(builder); + } else { + state.builders[index as usize] = builder; + } + index +} + // --------------------------------------------------------------------------- // Fork projection // --------------------------------------------------------------------------- /// The gloas state, mutably, or an error naming the function that needs one. /// -/// Kept alongside [`gloas_state_ref`] for whichever later task first needs to -/// mutate a gloas-only field (`builders`, the payload/PTC bookkeeping) -/// through a generic [`BeaconState`] rather than one already known to be -/// [`BeaconState::Gloas`] the way [`initiate_builder_exit`] and -/// [`settle_builder_payment`] are always called with; nothing in this file -/// calls it yet. -#[allow(dead_code)] +/// Kept alongside [`gloas_state_ref`] for a function that mixes a gloas-only +/// field with others reached through fork-invariant accessors, the way +/// [`gloas_state_ref`]'s own doc describes; `crate::beacon::upgrade::upgrade_to_gloas` +/// is the first caller, reaching `ptc_window` and `pending_deposits` through +/// this projection after building `post` as the full [`BeaconState`] enum. pub(crate) fn gloas_state<'a>( state: &'a mut BeaconState, function: &'static str, @@ -1040,4 +1095,128 @@ mod tests { .expect("built at exactly PTC_SIZE"); } } + + /// An arbitrary builder record, active (not exited) at `state`'s current + /// epoch: `get_index_for_new_builder`'s never-reusable case. + fn active_builder() -> gloas::Builder { + gloas::Builder { + withdrawable_epoch: constants::FAR_FUTURE_EPOCH, + balance: 1, + ..Default::default() + } + } + + /// A builder exited and fully swept as of `current_epoch`: the one + /// condition `get_index_for_new_builder` reuses a slot for. + fn exited_swept_builder(current_epoch: Epoch) -> gloas::Builder { + gloas::Builder { + withdrawable_epoch: current_epoch, + balance: 0, + ..Default::default() + } + } + + #[test] + fn get_index_for_new_builder_appends_past_an_empty_registry() { + let BeaconState::Gloas(state) = gloas_state_with_validators(4) else { + panic!("gloas_state_with_validators returns a gloas state"); + }; + assert_eq!(get_index_for_new_builder(&state), 0); + } + + #[test] + fn get_index_for_new_builder_appends_when_every_builder_is_still_active() { + let BeaconState::Gloas(mut state) = gloas_state_with_validators(4) else { + panic!("gloas_state_with_validators returns a gloas state"); + }; + state.builders.push(active_builder()); + state.builders.push(active_builder()); + assert_eq!(get_index_for_new_builder(&state), 2); + } + + #[test] + fn get_index_for_new_builder_reuses_an_exited_swept_slot() { + let BeaconState::Gloas(mut state) = gloas_state_with_validators(4) else { + panic!("gloas_state_with_validators returns a gloas state"); + }; + let current_epoch = compute_epoch_at_slot(state.slot); + state.builders.push(active_builder()); + // The one reusable slot: exited (`withdrawable_epoch` already past) + // and fully swept (balance zero). + state.builders.push(exited_swept_builder(current_epoch)); + state.builders.push(active_builder()); + assert_eq!(get_index_for_new_builder(&state), 1); + } + + #[test] + fn get_index_for_new_builder_does_not_reuse_an_exited_but_unswept_slot() { + let BeaconState::Gloas(mut state) = gloas_state_with_validators(4) else { + panic!("gloas_state_with_validators returns a gloas state"); + }; + let current_epoch = compute_epoch_at_slot(state.slot); + // Withdrawable, but balance still nonzero: not yet swept, so not + // reusable, even though the epoch condition alone holds. + state.builders.push(gloas::Builder { + withdrawable_epoch: current_epoch, + balance: 1, + ..Default::default() + }); + assert_eq!(get_index_for_new_builder(&state), 1); + } + + #[test] + fn add_builder_to_registry_appends_and_returns_the_new_index() { + let BeaconState::Gloas(mut state) = gloas_state_with_validators(4) else { + panic!("gloas_state_with_validators returns a gloas state"); + }; + let pubkey = BlsPubkey([9; crate::beacon::primitives::BLS_PUBKEY_SIZE]); + let len_before = state.builders.len(); + let slot = state.slot; + + let index = add_builder_to_registry( + &mut state, + pubkey, + constants::PAYLOAD_BUILDER_VERSION, + ExecutionAddress::ZERO, + preset::MIN_DEPOSIT_AMOUNT, + slot, + ); + + assert_eq!(index, len_before as gloas::BuilderIndex); + assert_eq!(state.builders.len(), len_before + 1); + assert_eq!(state.builders[index as usize].pubkey, pubkey); + } + + #[test] + fn add_builder_to_registry_overwrites_a_reused_slot_without_growing_the_registry() { + let BeaconState::Gloas(mut state) = gloas_state_with_validators(4) else { + panic!("gloas_state_with_validators returns a gloas state"); + }; + let current_epoch = compute_epoch_at_slot(state.slot); + state.builders.push(exited_swept_builder(current_epoch)); + let len_before = state.builders.len(); + let pubkey = BlsPubkey([3; crate::beacon::primitives::BLS_PUBKEY_SIZE]); + let slot = state.slot; + + let index = add_builder_to_registry( + &mut state, + pubkey, + constants::PAYLOAD_BUILDER_VERSION, + ExecutionAddress::ZERO, + preset::MIN_DEPOSIT_AMOUNT, + slot, + ); + + assert_eq!(index, 0); + assert_eq!( + state.builders.len(), + len_before, + "reuse must not grow the registry" + ); + assert_eq!(state.builders[0].pubkey, pubkey); + assert_eq!( + state.builders[0].withdrawable_epoch, + constants::FAR_FUTURE_EPOCH + ); + } } diff --git a/crates/blockchain/state_transition/src/beacon/stf/mod.rs b/crates/blockchain/state_transition/src/beacon/stf/mod.rs index fabe1aeac..5b17097cd 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/mod.rs @@ -68,6 +68,7 @@ use crate::beacon::containers::{BeaconState, phase0}; use crate::beacon::error::{Error, Result, verify}; use crate::beacon::helpers::accessors::{CommitteeCache, get_domain}; use crate::beacon::helpers::misc::compute_signing_root; +use crate::beacon::lean_state_unreachable; use crate::beacon::preset; use crate::beacon::primitives::{HashTreeRoot as _, Slot}; use crate::beacon::{bls, config::Config, constants}; @@ -280,6 +281,33 @@ pub fn process_slot(state: &mut BeaconState) -> Result<()> { let previous_block_root = state.latest_block_header().hash_tree_root(); state.block_roots_mut()[position] = previous_block_root; + // [New in Gloas:EIP7732] Unset `slot + 1`'s payload availability bit: + // that next slot has not built (or revealed) a payload yet, so nothing + // has attested to one being available for it. + // `process_parent_execution_payload` sets the bit back once a payload + // for that slot is actually revealed by the following block. + match state { + BeaconState::Phase0(_) + | BeaconState::Altair(_) + | BeaconState::Bellatrix(_) + | BeaconState::Capella(_) + | BeaconState::Deneb(_) + | BeaconState::Electra(_) + | BeaconState::Fulu(_) => {} + BeaconState::Gloas(inner) => { + let next_slot = inner + .slot + .checked_add(1) + .ok_or(Error::ArithmeticOverflow("process_slot: slot + 1"))?; + let next_position = (next_slot % preset::SLOTS_PER_HISTORICAL_ROOT as u64) as usize; + inner + .execution_payload_availability + .set(next_position, false) + .expect("next_position is in [0, SLOTS_PER_HISTORICAL_ROOT)"); + } + BeaconState::Lean(_) => lean_state_unreachable("process_slot"), + } + Ok(()) } @@ -317,8 +345,75 @@ pub(crate) fn phase0_state_ref<'a>( #[cfg(test)] mod tests { use super::*; + use crate::beacon::fork::ForkName; use crate::beacon::helpers::test_state; + #[test] + fn process_slot_unsets_next_slots_payload_availability_on_gloas() { + let mut state = test_state::with_validators_at(ForkName::Gloas, 4); + let position = state.slot() as usize % preset::SLOTS_PER_HISTORICAL_ROOT; + let next_position = (position + 1) % preset::SLOTS_PER_HISTORICAL_ROOT; + let other_position = (next_position + 1) % preset::SLOTS_PER_HISTORICAL_ROOT; + let BeaconState::Gloas(inner) = &mut state else { + panic!("with_validators_at(Gloas) returns a gloas state"); + }; + // Every bit starts `false` (`Default::default()`); set both so the + // test can tell "process_slot cleared it" from "it was already + // clear". + inner + .execution_payload_availability + .set(next_position, true) + .unwrap(); + inner + .execution_payload_availability + .set(other_position, true) + .unwrap(); + + // Taken from a clone before `process_slot` runs, so it still carries + // `next_position`'s bit `true`. `state_roots[position]` must cache + // exactly this root: if the implementation cleared the bit before + // caching that root instead of after, the cached root would reflect + // the bit already `false`, and the comparison below would catch it. + let expected_root = state.clone().hash_tree_root(); + + process_slot(&mut state).unwrap(); + + assert_eq!(state.state_roots()[position], expected_root); + + let BeaconState::Gloas(inner) = &state else { + panic!("process_slot does not change the state's fork"); + }; + assert_eq!( + inner.execution_payload_availability.get(next_position), + Some(false) + ); + // Only `slot + 1`'s bit is touched; every other bit is left alone. + assert_eq!( + inner.execution_payload_availability.get(other_position), + Some(true) + ); + } + + /// `next_position` wraps from `SLOTS_PER_HISTORICAL_ROOT - 1` back to + /// `0`, the same ring-buffer indexing `state_roots`/`block_roots` already + /// rely on. + #[test] + fn process_slot_wraps_the_payload_availability_index_at_the_end_of_the_window() { + let mut state = test_state::with_validators_at(ForkName::Gloas, 4); + *state.slot_mut() = preset::SLOTS_PER_HISTORICAL_ROOT as Slot - 1; + let BeaconState::Gloas(inner) = &mut state else { + panic!("with_validators_at(Gloas) returns a gloas state"); + }; + inner.execution_payload_availability.set(0, true).unwrap(); + + process_slot(&mut state).unwrap(); + + let BeaconState::Gloas(inner) = &state else { + panic!("process_slot does not change the state's fork"); + }; + assert_eq!(inner.execution_payload_availability.get(0), Some(false)); + } + #[test] fn process_slot_leaves_no_buffered_registry_writes() { let mut state = test_state::with_validators(4); diff --git a/crates/blockchain/state_transition/src/beacon/upgrade.rs b/crates/blockchain/state_transition/src/beacon/upgrade.rs index bedda7b5d..a2aca6d2f 100644 --- a/crates/blockchain/state_transition/src/beacon/upgrade.rs +++ b/crates/blockchain/state_transition/src/beacon/upgrade.rs @@ -16,22 +16,26 @@ //! out once, in one place, checkable against the specification's own //! constructor line by line. +use std::collections::{HashMap, HashSet}; + use crate::beacon::config::Config; use crate::beacon::constants; use crate::beacon::containers::{ BeaconState, EpochParticipation, Fork, InactivityScores, altair, bellatrix, capella, deneb, - electra, fulu, phase0, + electra, fulu, gloas, phase0, }; use crate::beacon::error::{Error, Result}; use crate::beacon::fork::ForkName; -use crate::beacon::helpers::accessors::CommitteeCache; +use crate::beacon::helpers::accessors::{CommitteeCache, get_current_epoch}; use crate::beacon::helpers::attestation::get_attesting_indices; -use crate::beacon::helpers::misc::{compute_activation_exit_epoch, compute_epoch_at_slot}; +use crate::beacon::helpers::misc::{ + compute_activation_exit_epoch, compute_epoch_at_slot, compute_start_slot_at_epoch, +}; use crate::beacon::lean_fork_unreachable; use crate::beacon::preset; use crate::beacon::primitives::{ - BlsPubkey, BlsSignature, Bytes32, ExecutionAddress, ExecutionBlockHash, Root, Uint256, - ValidatorIndex, + BlsPubkey, BlsSignature, Bytes32, ExecutionAddress, ExecutionBlockHash, HashTreeRoot as _, + Root, Uint256, ValidatorIndex, }; /// Replays `pending_attestations` into `post`'s `previous_epoch_participation`. @@ -811,6 +815,438 @@ pub fn upgrade_to_fulu(pre: &BeaconState, config: &Config) -> Result gloas::PayloadTimelinessCommittee { + vec![0 as ValidatorIndex; preset::PTC_SIZE] + .try_into() + .expect("built at exactly PTC_SIZE") +} + +/// The cached payload timeliness committee window [`upgrade_to_gloas`] seeds +/// `post.ptc_window` with. +/// +/// Transcribed from `specs/gloas/fork.md`'s `initialize_ptc_window`: the +/// previous epoch's slice starts all-zero (there is no earlier payload +/// timeliness committee to have cached at the fork boundary), and the current +/// epoch through [`preset::MIN_SEED_LOOKAHEAD`] epochs ahead are each computed +/// fresh via [`crate::beacon::helpers::gloas::compute_ptc`]. `state` is the +/// [`BeaconState`] enum, not the bare `gloas::BeaconState` `post` is still +/// being assembled as when this runs: `compute_ptc` reads through +/// fork-invariant accessors (`get_seed`, the epoch's committees), which need +/// the enum to dispatch through. +/// +/// Shares one [`CommitteeCache`] across every one of the +/// `(1 + MIN_SEED_LOOKAHEAD) * SLOTS_PER_EPOCH` [`compute_ptc`](crate::beacon::helpers::gloas::compute_ptc) +/// calls this makes, the same reason that function's own doc gives for taking +/// one in: an epoch's committees are the same shuffle no matter which slot of +/// it is asked about, so computing that shuffle once and reusing it is what +/// keeps filling the whole window affordable. +fn initialize_ptc_window(state: &BeaconState) -> Result { + let mut window = vec![empty_ptc(); preset::SLOTS_PER_EPOCH as usize]; + + let committees = CommitteeCache::default(); + let current_epoch = get_current_epoch(state); + for offset in 0..=preset::MIN_SEED_LOOKAHEAD { + let epoch = current_epoch + .checked_add(offset) + .ok_or(Error::ArithmeticOverflow( + "initialize_ptc_window: current_epoch + offset", + ))?; + let start_slot = compute_start_slot_at_epoch(epoch); + for slot_offset in 0..preset::SLOTS_PER_EPOCH { + let slot = start_slot + .checked_add(slot_offset) + .ok_or(Error::ArithmeticOverflow( + "initialize_ptc_window: start_slot + slot_offset", + ))?; + window.push(crate::beacon::helpers::gloas::compute_ptc( + state, + slot, + &committees, + )?); + } + } + + Ok(window.try_into()?) +} + +/// An incremental index over [`onboard_builders_from_pending_deposits`]'s +/// growing `pending_deposits` list, by pubkey, with each entry's deposit +/// signature verdict memoized the first time it is needed. +/// +/// [`is_pending_validator`](crate::beacon::helpers::gloas::is_pending_validator) +/// is the specification's own literal transcription (kept as-is: a general, +/// spec-named function, not tied to this one caller's bookkeeping), and its +/// own doc already carries the specification's note that a caller iterating +/// many pubkeys should cache verification results rather than calling it in +/// a loop. This is that cache: the specification notes the pending deposit +/// queue might be large at the fork, and this index turns a query that would otherwise +/// rescan the whole list and re-verify a signature per matching entry into a +/// pubkey lookup plus, at most once per entry, one signature check. +#[derive(Default)] +struct PendingValidatorIndex { + by_pubkey: HashMap>, + verdicts: Vec>, +} + +impl PendingValidatorIndex { + /// Records that `pending_deposits[index]` (just pushed) carries `pubkey`. + fn push(&mut self, index: usize, pubkey: BlsPubkey) { + self.by_pubkey.entry(pubkey).or_default().push(index); + debug_assert_eq!(self.verdicts.len(), index); + self.verdicts.push(None); + } + + /// [`is_pending_validator`](crate::beacon::helpers::gloas::is_pending_validator)'s + /// exact result for `pubkey`, reached through the pubkey index instead of + /// a linear scan, with each candidate's signature verdict computed at + /// most once no matter how many later deposits ask the same question. + fn is_pending_validator( + &mut self, + pending_deposits: &[electra::PendingDeposit], + pubkey: BlsPubkey, + config: &Config, + ) -> bool { + let Some(indices) = self.by_pubkey.get(&pubkey) else { + return false; + }; + for &index in indices { + let verdict = *self.verdicts[index].get_or_insert_with(|| { + let deposit = &pending_deposits[index]; + crate::beacon::stf::electra::is_valid_deposit_signature( + deposit.pubkey, + deposit.withdrawal_credentials, + deposit.amount, + &deposit.signature, + config, + ) + }); + if verdict { + return true; + } + } + false + } +} + +/// Pushes `deposit` back onto the rebuilt `pending_deposits` queue and +/// records it in `pending_index` under its own pubkey, keeping the two in +/// step: every push is one of the specification's own `pending_deposits.append(deposit)` +/// calls, never bypassed. +fn requeue( + pending_deposits: &mut Vec, + pending_index: &mut PendingValidatorIndex, + deposit: electra::PendingDeposit, +) { + let pubkey = deposit.pubkey; + pending_deposits.push(deposit); + pending_index.push(pending_deposits.len() - 1, pubkey); +} + +/// Applies every pending deposit the fulu pre-state was still holding, +/// onboarding a builder for any whose withdrawal credential and signature +/// qualify. +/// +/// Transcribed from `specs/gloas/fork.md`'s `onboard_builders_from_pending_deposits`: +/// the only path through the validator deposit contract that ever creates a +/// builder. From the fork onward, a builder is created or topped up only +/// through a `BuilderDepositRequest` (EIP-8282). +/// +/// A deposit for an already-registered validator stays queued untouched. A +/// deposit whose pubkey neither registry recognizes yet is onboarded as a +/// builder only if its withdrawal credential is builder-prefixed, no deposit +/// already re-queued in this same pass could still turn that pubkey into a +/// validator instead ([`is_pending_validator`](crate::beacon::helpers::gloas::is_pending_validator)), +/// and its signature checks out. A deposit that fails the credential or +/// pending-validator check stays queued instead; one that fails the +/// signature check is dropped outright, matching phase0's own "an invalid +/// signature just means the deposit is not credited" rule one level up. A +/// deposit for an already-registered builder tops up its balance directly +/// instead of registering a second record. +/// +/// The pending deposit queue might be large at the fork, so every +/// membership question this asks per deposit is kept O(1) amortized rather +/// than a rescan: `validator_pubkeys` is a set built from the *deposit* +/// queue's own pubkeys (at most `D` of them), narrowed against the registry +/// in one O(V) pass rather than a `Vec` scanned once per deposit; +/// `builder_indices` is a map kept in step with the registry, updated the +/// moment [`add_builder_to_registry`](crate::beacon::helpers::gloas::add_builder_to_registry) +/// returns the index it used, so a lookup a few iterations later still sees +/// a builder this same pass already onboarded. The specification's own +/// comment says `builder_pubkeys` must be recomputed every iteration for +/// exactly that reason, and a map kept current does the same job without the +/// recomputation. Finally, [`PendingValidatorIndex`] answers +/// `is_pending_validator`'s own question without a rescan or a repeated +/// signature check. +fn onboard_builders_from_pending_deposits( + state: &mut gloas::BeaconState, + config: &Config, +) -> Result<()> { + // Narrowed to the deposit queue's own pubkeys rather than every + // validator: this only ever answers "is this pubkey already a + // validator's" for a pubkey a deposit actually names, so it need not + // hold more than `D` entries even when the registry itself holds `V`. + let deposit_pubkeys: HashSet = state + .pending_deposits + .iter() + .map(|deposit| deposit.pubkey) + .collect(); + let validator_pubkeys: HashSet = state + .validators + .iter() + .map(|validator| validator.pubkey) + .filter(|pubkey| deposit_pubkeys.contains(pubkey)) + .collect(); + + let mut builder_indices: HashMap = state + .builders + .iter() + .enumerate() + .map(|(index, builder)| (builder.pubkey, index as gloas::BuilderIndex)) + .collect(); + + let mut pending_deposits: Vec = Vec::new(); + let mut pending_index = PendingValidatorIndex::default(); + + // Moved out rather than cloned: nothing below reads the old + // `state.pending_deposits` (the loop below rebuilds it from scratch), + // and taking ownership here is what lets the loop body hold `state` + // mutably (`add_builder_to_registry`, the balance top-up) without a + // lingering borrow of one of its own fields. + for deposit in std::mem::take(&mut state.pending_deposits) { + if validator_pubkeys.contains(&deposit.pubkey) { + requeue(&mut pending_deposits, &mut pending_index, deposit); + continue; + } + + match builder_indices.get(&deposit.pubkey).copied() { + None => { + if !crate::beacon::helpers::gloas::is_builder_withdrawal_credential( + deposit.withdrawal_credentials, + ) { + requeue(&mut pending_deposits, &mut pending_index, deposit); + continue; + } + if pending_index.is_pending_validator(&pending_deposits, deposit.pubkey, config) { + requeue(&mut pending_deposits, &mut pending_index, deposit); + continue; + } + if !crate::beacon::stf::electra::is_valid_deposit_signature( + deposit.pubkey, + deposit.withdrawal_credentials, + deposit.amount, + &deposit.signature, + config, + ) { + continue; + } + + let index = crate::beacon::helpers::gloas::add_builder_to_registry( + state, + deposit.pubkey, + constants::PAYLOAD_BUILDER_VERSION, + ExecutionAddress::from_slice(&deposit.withdrawal_credentials.0[12..]), + deposit.amount, + deposit.slot, + ); + builder_indices.insert(deposit.pubkey, index); + } + Some(builder_index) => { + let len = state.builders.len(); + let builder = state.builders.get_mut(builder_index as usize).ok_or( + Error::IndexOutOfBounds { + index: builder_index as usize, + len, + }, + )?; + builder.balance = builder.balance.checked_add(deposit.amount).ok_or( + Error::ArithmeticOverflow( + "onboard_builders_from_pending_deposits: builder.balance + deposit.amount", + ), + )?; + } + } + } + + state.pending_deposits = pending_deposits.into(); + Ok(()) +} + +/// Upgrades a fulu state to gloas's shape. +/// +/// Transcribed from `specs/gloas/fork.md`'s `upgrade_to_gloas`. Fields +/// through `pending_consolidations` are fulu's, field for field, with the +/// eight registry-shaped ones (`validators`, `balances`, +/// `previous_epoch_participation`, `current_epoch_participation`, +/// `inactivity_scores`, `pending_deposits`, `pending_partial_withdrawals`, +/// `pending_consolidations`) rebuilt as their progressive counterparts +/// (EIP-7688): one full rehash at the fork, the same cost every earlier +/// fork's `upgrade_to_*` pays whenever a list's *contents* change, except here +/// it is the list's *kind* that changes while the contents do not. +/// `proposer_lookahead` carries over unchanged (fulu's own shape survives +/// into gloas; see `containers::gloas`'s module doc). +/// +/// `latest_execution_payload_header` has no gloas counterpart at all +/// (EIP-7732 moves the payload itself out of the state): only its +/// `block_hash` survives, as the new `latest_block_hash` field, and its other +/// fields seed `latest_execution_payload_bid` instead, a synthetic +/// self-build bid ([`constants::BUILDER_INDEX_SELF_BUILD`]) describing the +/// payload the pre-state had already applied. Its `slot` is +/// `pre.latest_block_header.slot`, the last applied block's own slot, which +/// can be earlier than the fork's own slot when slots were missed right +/// before the fork: the bid describes that already-applied payload, not a +/// claim about the fork slot itself. +/// +/// Every field from `builders` onward is new. `execution_payload_availability` +/// starts every bit set (every slot in the window is "available": there is no +/// missing payload to report yet) rather than at its `Default`, whose +/// all-zero value would claim the opposite. `builder_pending_payments` and +/// `ptc_window` are `SszVector`s, so each starts at its own fixed length +/// ([`preset::BUILDER_PENDING_PAYMENTS_LENGTH`]/[`preset::PTC_WINDOW_LENGTH`] +/// entries) built explicitly rather than through `Default`, which a fixed +/// vector has none of; `ptc_window`'s placeholder entries are then +/// overwritten by [`initialize_ptc_window`] once `post` exists as a full +/// [`BeaconState`], the same reason [`upgrade_to_altair`] computes its sync +/// committees only after `post` exists as `BeaconState::Altair`. +/// [`onboard_builders_from_pending_deposits`] runs last, after `ptc_window`, +/// matching the specification's own ordering. +pub fn upgrade_to_gloas(pre: &BeaconState, config: &Config) -> Result { + let pre = crate::beacon::helpers::fulu::fulu_state_ref(pre, "upgrade_to_gloas")?; + let epoch = compute_epoch_at_slot(pre.slot); + + let fork = Fork { + previous_version: pre.fork.current_version, + current_version: config.gloas_fork_version, + epoch, + }; + + // Seeds `latest_execution_payload_bid` from the pre-state's already- + // applied payload; see this function's own doc for why. + let execution_header = &pre.latest_execution_payload_header; + let latest_execution_payload_bid = gloas::ExecutionPayloadBid { + parent_block_hash: execution_header.parent_hash, + parent_block_root: pre.latest_block_header.parent_root, + block_hash: execution_header.block_hash, + prev_randao: execution_header.prev_randao, + fee_recipient: ExecutionAddress::ZERO, + gas_limit: execution_header.gas_limit, + builder_index: constants::BUILDER_INDEX_SELF_BUILD, + slot: pre.latest_block_header.slot, + value: 0, + execution_payment: 0, + blob_kzg_commitments: Default::default(), + execution_requests_root: gloas::ExecutionRequests::default().hash_tree_root(), + }; + + // Every slot starts "available"; see this function's own doc. + let mut execution_payload_availability = gloas::ExecutionPayloadAvailability::new(); + for index in 0..preset::SLOTS_PER_HISTORICAL_ROOT { + execution_payload_availability + .set(index, true) + .expect("index is in [0, SLOTS_PER_HISTORICAL_ROOT)"); + } + + // A placeholder, overwritten by `initialize_ptc_window` below once `post` + // exists as a full `BeaconState`; see this function's own doc for why + // that has to wait. + let placeholder_ptc_window: gloas::PayloadTimelinessCommitteeWindow = + vec![empty_ptc(); preset::PTC_WINDOW_LENGTH] + .try_into() + .expect("built at exactly PTC_WINDOW_LENGTH"); + + let post = gloas::BeaconState { + genesis_time: pre.genesis_time, + genesis_validators_root: pre.genesis_validators_root, + slot: pre.slot, + fork, + latest_block_header: pre.latest_block_header.clone(), + block_roots: pre.block_roots.clone(), + state_roots: pre.state_roots.clone(), + historical_roots: pre.historical_roots.clone(), + eth1_data: pre.eth1_data.clone(), + eth1_data_votes: pre.eth1_data_votes.clone(), + eth1_deposit_index: pre.eth1_deposit_index, + // [Modified in Gloas:EIP7688] + validators: pre.validators.to_vec().into(), + // [Modified in Gloas:EIP7688] + balances: pre.balances.to_vec().into(), + randao_mixes: pre.randao_mixes.clone(), + slashings: pre.slashings.clone(), + // [Modified in Gloas:EIP7688] + previous_epoch_participation: pre.previous_epoch_participation.to_vec().into(), + // [Modified in Gloas:EIP7688] + current_epoch_participation: pre.current_epoch_participation.to_vec().into(), + justification_bits: pre.justification_bits.clone(), + previous_justified_checkpoint: pre.previous_justified_checkpoint, + current_justified_checkpoint: pre.current_justified_checkpoint, + finalized_checkpoint: pre.finalized_checkpoint, + // [Modified in Gloas:EIP7688] + inactivity_scores: pre.inactivity_scores.to_vec().into(), + current_sync_committee: pre.current_sync_committee.clone(), + next_sync_committee: pre.next_sync_committee.clone(), + // [New in Gloas:EIP7732] Removed `latest_execution_payload_header`; + // see this function's own doc. + latest_block_hash: execution_header.block_hash, + next_withdrawal_index: pre.next_withdrawal_index, + next_withdrawal_validator_index: pre.next_withdrawal_validator_index, + historical_summaries: pre.historical_summaries.clone(), + deposit_requests_start_index: pre.deposit_requests_start_index, + deposit_balance_to_consume: pre.deposit_balance_to_consume, + exit_balance_to_consume: pre.exit_balance_to_consume, + earliest_exit_epoch: pre.earliest_exit_epoch, + consolidation_balance_to_consume: pre.consolidation_balance_to_consume, + earliest_consolidation_epoch: pre.earliest_consolidation_epoch, + // [Modified in Gloas:EIP7688] + pending_deposits: pre.pending_deposits.to_vec().into(), + // [Modified in Gloas:EIP7688] + pending_partial_withdrawals: pre.pending_partial_withdrawals.to_vec().into(), + // [Modified in Gloas:EIP7688] + pending_consolidations: pre.pending_consolidations.to_vec().into(), + proposer_lookahead: pre.proposer_lookahead.clone(), + // [New in Gloas:EIP7732] + builders: Default::default(), + // [New in Gloas:EIP7732] + next_withdrawal_builder_index: 0, + // [New in Gloas:EIP7732] + execution_payload_availability, + // [New in Gloas:EIP7732] + builder_pending_payments: vec![ + gloas::BuilderPendingPayment::default(); + preset::BUILDER_PENDING_PAYMENTS_LENGTH + ] + .try_into() + .expect("built at exactly BUILDER_PENDING_PAYMENTS_LENGTH"), + // [New in Gloas:EIP7732] + builder_pending_withdrawals: Default::default(), + // [New in Gloas:EIP7732] + latest_execution_payload_bid, + // [New in Gloas:EIP7732] + payload_expected_withdrawals: Default::default(), + // [New in Gloas:EIP7732] Overwritten below; see this function's own + // doc. + ptc_window: placeholder_ptc_window, + }; + let mut post = BeaconState::Gloas(post); + + // [New in Gloas:EIP7732] + let ptc_window = initialize_ptc_window(&post)?; + crate::beacon::helpers::gloas::gloas_state(&mut post, "upgrade_to_gloas")?.ptc_window = + ptc_window; + + // [New in Gloas:EIP7732] + onboard_builders_from_pending_deposits( + crate::beacon::helpers::gloas::gloas_state(&mut post, "upgrade_to_gloas")?, + config, + )?; + + Ok(post) +} + /// Applies the fork upgrade that produces `to`'s state shape from `state`. /// /// Dispatches over the per-fork upgrade functions by [`ForkName`] so a caller @@ -841,15 +1277,7 @@ pub fn upgrade_state(state: &BeaconState, to: ForkName, config: &Config) -> Resu ForkName::Deneb => upgrade_to_deneb(state, config), ForkName::Electra => upgrade_to_electra(state, config), ForkName::Fulu => upgrade_to_fulu(state, config), - // `upgrade_to_gloas` does not exist yet: EIP-7688 replaces the - // registry's list kind and EIP-7732 removes - // `latest_execution_payload_header` outright, so this is not a - // field-by-field carry-over the way `upgrade_to_fulu` is. A later - // task writes it. - ForkName::Gloas => Err(Error::UnsupportedForFork { - function: "upgrade_state", - fork: to, - }), + ForkName::Gloas => upgrade_to_gloas(state, config), // The `fork:` form, not `state:`: this dispatches on the requested // target, so it is the argument that is wrong, not what `state` holds. ForkName::Lean => lean_fork_unreachable("upgrade_state"), diff --git a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs index 107725589..c21afbd2b 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs @@ -92,7 +92,11 @@ pub const HIGHEST_IMPLEMENTED_FORK: ForkName = ForkName::Fulu; /// [`HIGHEST_IMPLEMENTED_FORK`] has not reached Gloas yet; every other /// runner's Gloas cases stay ignored. Emptied, and removed along with this /// check, when [`HIGHEST_IMPLEMENTED_FORK`] becomes `Gloas`. -pub const GLOAS_RUNNERS: &[&str] = &["ssz_static"]; +/// +/// `"fork"` (not `"transition"`): `upgrade_to_gloas` is done, so the `fork` +/// suite's own upgrade-only cases pass, but `transition`'s Gloas cases also +/// exercise Gloas block processing, which does not exist yet. +pub const GLOAS_RUNNERS: &[&str] = &["ssz_static", "fork"]; /// The root of the extracted fixture tree. /// From 07c1bb6f9b65725cc4e1a0ee24e512189f91958b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:24:53 -0300 Subject: [PATCH 017/132] feat(beacon): add gloas epoch processing Ports gloas's process_epoch (EIP-7732, EIP-8061) in the specification's step order and turns on the gloas epoch_processing and rewards suites. Most gloas epoch steps are textually unmodified, but EIP-7688 moves the participation lists, the inactivity scores and the pending queues onto progressive lists, a different Rust type from every earlier fork's bounded SszList. Both list kinds deref to a plain slice with the same element type, so the shared implementations serve gloas through narrow views instead of gloas copies: - altair_validator_lists returns slices and includes gloas, so altair's justification and finalization, rewards and penalties and their deltas run unchanged on a gloas state; inactivity_scores_mut gives the element writes process_inactivity_updates needs. - PendingQueueFields gains a gloas arm and moves the consolidation queue in and out as a Vec, so electra's process_pending_consolidations and apply_pending_deposit serve gloas. - fulu's process_proposer_lookahead shifts the window through an accessor both forks share and picks the proposer draw by fork, since gloas's get_beacon_proposer_indices excludes slashed validators (EIP-8045). Gloas keeps its own process_pending_deposits, which the specification modifies (the activation-only churn budget of EIP-8061), and its own process_participation_flag_updates, which replaces the whole list and needs the fork's concrete list type. process_builder_pending_payments and process_ptc_window are new. The justification and finalization dispatcher that fork choice's pulled-up tip shares gains the gloas arm too. Gloas ships a second pending_deposits_churn handler, which runs the same process_pending_deposits against the new churn cap; both handlers dispatch to it. --- .../src/beacon/helpers/fulu.rs | 20 + .../src/beacon/helpers/gloas.rs | 46 +- .../src/beacon/stf/epoch/altair.rs | 26 +- .../src/beacon/stf/epoch/electra.rs | 157 +++- .../src/beacon/stf/epoch/fulu.rs | 50 +- .../src/beacon/stf/epoch/gloas.rs | 861 ++++++++++++++++++ .../src/beacon/stf/epoch/mod.rs | 33 +- .../tests/beacon_spec/epoch_processing.rs | 178 +++- .../state_transition/tests/beacon_spec/mod.rs | 7 +- .../tests/beacon_spec/rewards.rs | 17 +- .../common/types/src/beacon/containers/mod.rs | 84 +- 11 files changed, 1319 insertions(+), 160 deletions(-) create mode 100644 crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs diff --git a/crates/blockchain/state_transition/src/beacon/helpers/fulu.rs b/crates/blockchain/state_transition/src/beacon/helpers/fulu.rs index 7c138c64b..3ecbd7cf3 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/fulu.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/fulu.rs @@ -149,6 +149,26 @@ fn proposer_lookahead<'a>( } } +/// The `proposer_lookahead` field, mutably. See [`proposer_lookahead`] for +/// why this is scoped to fulu and gloas rather than widening +/// [`fulu_state`]/[`fulu_state_ref`]. +/// +/// `pub(crate)`: `crate::beacon::stf::epoch::fulu::process_proposer_lookahead`, +/// shared by both forks (see that module's own doc), is the one caller. +pub(crate) fn proposer_lookahead_mut<'a>( + state: &'a mut BeaconState, + function: &'static str, +) -> Result<&'a mut fulu::ProposerLookahead> { + match state { + BeaconState::Fulu(inner) => Ok(&mut inner.proposer_lookahead), + BeaconState::Gloas(inner) => Ok(&mut inner.proposer_lookahead), + other => Err(Error::UnsupportedForFork { + function, + fork: other.fork_name(), + }), + } +} + /// The proposer for the state's current slot. /// /// Fulu's replacement for [`super::accessors::get_beacon_proposer_index`]: a diff --git a/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs b/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs index d169fad72..36a3fe453 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs @@ -469,12 +469,12 @@ pub fn compute_proposer_indices( /// /// Takes `committees` rather than deriving each committee itself /// (`get_beacon_committee`, one active-set scan and one per-member shuffle -/// apiece): `process_ptc_window` (a later task) calls this once per slot of -/// an epoch, and at mainnet scale that is `SLOTS_PER_EPOCH` calls against a -/// multi-million-validator registry, so sharing one epoch-wide shuffle -/// across all of them (see [`CommitteeCache`]'s own doc) is what keeps that -/// affordable; [`super::electra::get_attesting_indices`] takes the same -/// parameter for the same reason. +/// apiece): [`crate::beacon::stf::epoch::gloas::process_ptc_window`] calls this +/// once per slot of an epoch, and at mainnet scale that is `SLOTS_PER_EPOCH` +/// calls against a multi-million-validator registry, so sharing one +/// epoch-wide shuffle across all of them (see [`CommitteeCache`]'s own doc) is +/// what keeps that affordable; [`super::electra::get_attesting_indices`] takes +/// the same parameter for the same reason. pub fn compute_ptc( state: &BeaconState, slot: Slot, @@ -628,10 +628,10 @@ pub fn get_attestation_participation_flag_indices( /// /// Unlike [`compute_ptc`] (which always derives one fresh), this reads the /// cached window [`gloas::BeaconState::ptc_window`] -/// `process_ptc_window` (gloas epoch processing) -/// refreshes each epoch: `slot`'s epoch must be the state's own, the one -/// before it, or within [`preset::MIN_SEED_LOOKAHEAD`] epochs ahead, which is -/// exactly the window's own span. +/// [`crate::beacon::stf::epoch::gloas::process_ptc_window`] refreshes each epoch: +/// `slot`'s epoch must be the state's own, the one before it, or within +/// [`preset::MIN_SEED_LOOKAHEAD`] epochs ahead, which is exactly the window's +/// own span. pub fn get_ptc( state: &BeaconState, slot: Slot, @@ -717,8 +717,8 @@ pub fn get_indexed_payload_attestation( } /// The attesting weight a builder payment needs before -/// `process_builder_pending_payments` (a later task) settles it in full -/// rather than dropping it: [`constants::BUILDER_PAYMENT_THRESHOLD_NUMERATOR`] +/// [`crate::beacon::stf::epoch::gloas::process_builder_pending_payments`] settles +/// it in full rather than dropping it: [`constants::BUILDER_PAYMENT_THRESHOLD_NUMERATOR`] /// `/` [`constants::BUILDER_PAYMENT_THRESHOLD_DENOMINATOR`] of one slot's /// share of the total active balance. pub fn get_builder_payment_quorum_threshold(state: &BeaconState) -> Result { @@ -814,9 +814,14 @@ pub fn initiate_builder_exit( /// [`gloas::BeaconState::builder_pending_withdrawals`] if it carries a real /// amount, then clears the slot. /// -/// Called by `process_builder_pending_payments` (a later task) once a -/// payment's accumulated attesting weight has been compared against -/// [`get_builder_payment_quorum_threshold`]. +/// Called by `apply_parent_execution_payload` (parent-payload block +/// processing, a later task; SPEC `beacon-chain.md`'s "Settle the builder +/// payment", ~1767-1780), not by +/// [`crate::beacon::stf::epoch::gloas::process_builder_pending_payments`]: that +/// epoch step evicts and settles the *older* half of `builder_pending_payments` +/// by weight, against [`get_builder_payment_quorum_threshold`]; this settles +/// one parent block's still-live entry unconditionally, the moment that +/// block's payload is applied. pub fn settle_builder_payment(state: &mut gloas::BeaconState, payment_index: u64) -> Result<()> { let len = state.builder_pending_payments.len() as u64; crate::beacon::verify( @@ -895,11 +900,12 @@ pub fn add_builder_to_registry( /// The gloas state, mutably, or an error naming the function that needs one. /// -/// Kept alongside [`gloas_state_ref`] for a function that mixes a gloas-only -/// field with others reached through fork-invariant accessors, the way -/// [`gloas_state_ref`]'s own doc describes; `crate::beacon::upgrade::upgrade_to_gloas` -/// is the first caller, reaching `ptc_window` and `pending_deposits` through -/// this projection after building `post` as the full [`BeaconState`] enum. +/// Kept alongside [`gloas_state_ref`] for mutating a gloas-only field +/// through a generic [`BeaconState`] rather than one already known to be +/// [`BeaconState::Gloas`] the way [`initiate_builder_exit`] and +/// [`settle_builder_payment`] are always called with. The fork upgrade +/// (`crate::beacon::upgrade::upgrade_to_gloas`) and +/// [`crate::beacon::stf::epoch::gloas`] are its callers. pub(crate) fn gloas_state<'a>( state: &'a mut BeaconState, function: &'static str, diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/altair.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/altair.rs index 09d7869eb..b70cc3eb4 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/altair.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/altair.rs @@ -111,16 +111,24 @@ pub fn process_justification_and_finalization(state: &mut BeaconState) -> Result /// /// Skipped at the genesis epoch, since the score update reads the previous /// epoch's participation and genesis has none. +/// +/// Shared by every fork from altair on, gloas included: reading +/// [`crate::beacon::containers::BeaconState::inactivity_scores_mut`] rather than +/// [`crate::beacon::containers::BeaconState::altair_validator_lists_mut`] is what +/// makes that so, since this only ever writes a score in place and never +/// grows or replaces the list, and that narrower accessor extends to a gloas +/// state's progressive `inactivity_scores` where the wider one cannot (see +/// its own doc). pub fn process_inactivity_updates(state: &mut BeaconState, config: &Config) -> Result<()> { if get_current_epoch(state) == constants::GENESIS_EPOCH { return Ok(()); } // Every read below needs `&BeaconState`, so they all run before this takes - // the mutable borrow `inactivity_scores` requires: `altair_validator_lists_mut` - // borrows the whole state, and there is no way to hold that mutably while - // also calling `get_eligible_validator_indices`, `get_unslashed_participating_indices`, - // or `is_in_inactivity_leak`, each of which needs its own `&BeaconState`. + // the mutable borrow `inactivity_scores_mut` requires, and there is no way + // to hold that mutably while also calling `get_eligible_validator_indices`, + // `get_unslashed_participating_indices`, or `is_in_inactivity_leak`, each + // of which needs its own `&BeaconState`. // `process_effective_balance_updates` in the parent module resolves the // identical conflict the same way: decide everything in one pass over // immutable state, then apply it in a second pass over a mutable borrow. @@ -133,7 +141,7 @@ pub fn process_inactivity_updates(state: &mut BeaconState, config: &Config) -> R )?; let leaking = is_in_inactivity_leak(state); - let (_, _, inactivity_scores) = state.altair_validator_lists_mut()?; + let inactivity_scores = state.inactivity_scores_mut()?; let score_count = inactivity_scores.len(); for index in eligible_indices { let score = inactivity_scores @@ -344,13 +352,17 @@ mod tests { fn inactivity_updates_are_a_no_op_at_genesis() { let mut state = altair_state_with_validators(4); *state.slot_mut() = 0; + // Owned, not `.clone()`: `altair_validator_lists` now hands back a + // slice borrowed from `state`, and `<&[u64]>::clone()` would just + // copy the reference, not the contents, which could not outlive the + // `&mut state` call below anyway. let (_, _, scores) = state.altair_validator_lists().unwrap(); - let before = scores.clone(); + let before = scores.to_vec(); process_inactivity_updates(&mut state, &Config::mainnet()).unwrap(); let (_, _, scores) = state.altair_validator_lists().unwrap(); - assert_eq!(*scores, before); + assert_eq!(scores, before); } #[test] diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs index 6608119c6..7432f781c 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs @@ -35,8 +35,9 @@ use crate::beacon::bls; use crate::beacon::config::Config; use crate::beacon::constants::{self, FAR_FUTURE_EPOCH}; use crate::beacon::containers::shared::{DepositMessage, Validator}; -use crate::beacon::containers::{BeaconState, electra, fulu}; +use crate::beacon::containers::{BeaconState, electra, fulu, gloas}; use crate::beacon::error::{Error, Result}; +use crate::beacon::fork::ForkName; use crate::beacon::helpers::accessors::{get_current_epoch, get_total_active_balance}; use crate::beacon::helpers::electra::{ get_activation_exit_churn_limit, get_max_effective_balance, initiate_validator_exit, @@ -333,7 +334,7 @@ pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Res "deposit_balance_to_consume + get_activation_exit_churn_limit", ))?; let deposits: Vec = - core::mem::take(fields.pending_deposits_mut()).into_inner(); + core::mem::take(fields.pending_deposits_mut()?).into_inner(); ( deposit_requests_start_index, available_for_processing, @@ -423,7 +424,7 @@ pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Res }; let mut fields = pending_queue_fields(state, "process_pending_deposits")?; - *fields.pending_deposits_mut() = electra::PendingDeposits::try_from(remaining)?; + *fields.pending_deposits_mut()? = electra::PendingDeposits::try_from(remaining)?; *fields.deposit_balance_to_consume_mut() = deposit_balance_to_consume; Ok(()) @@ -579,12 +580,22 @@ fn add_validator_from_pending_deposit( /// behind it, is left in place (not postponed to the back, unlike /// [`process_pending_deposits`]'s exited-validator case) to be retried once /// it clears. +/// +/// Also served, unmodified, by gloas: nothing in gloas's own +/// `beacon-chain.md` touches this function, and the only thing that keeps a +/// gloas state from calling this exact copy is the queue's own type +/// (`ProgressiveList` rather than `SszList`, EIP-7688), which +/// [`PendingQueueFields::take_pending_consolidations`]/[`PendingQueueFields::set_pending_consolidations`] +/// already abstract over at the `Vec` level. Contrast +/// [`process_pending_deposits`], which gloas's own `beacon-chain.md` modifies +/// outright and which therefore keeps its own copy +/// (`crate::beacon::stf::epoch::gloas::process_pending_deposits`). pub fn process_pending_consolidations(state: &mut BeaconState, _config: &Config) -> Result<()> { let next_epoch = get_current_epoch(state) + 1; let consolidations: Vec = { let mut fields = pending_queue_fields(state, "process_pending_consolidations")?; - core::mem::take(fields.pending_consolidations_mut()).into_inner() + fields.take_pending_consolidations() }; let mut next_pending_consolidation = 0usize; @@ -619,7 +630,7 @@ pub fn process_pending_consolidations(state: &mut BeaconState, _config: &Config) .skip(next_pending_consolidation) .collect(); let mut fields = pending_queue_fields(state, "process_pending_consolidations")?; - *fields.pending_consolidations_mut() = electra::PendingConsolidations::try_from(remaining)?; + fields.set_pending_consolidations(remaining)?; Ok(()) } @@ -697,9 +708,18 @@ pub fn process_effective_balance_updates(state: &mut BeaconState) -> Result<()> /// the same way this module's own [`process_pending_deposits`] does; it never /// reads [`Self::deposit_requests_start_index`], the one field that function /// stops consulting. +/// +/// Gloas joins the enum too (EIP-8061 leaves `deposit_requests_start_index` +/// and `deposit_balance_to_consume` unchanged, and `beacon-chain.md` does not +/// modify [`process_pending_consolidations`] at all), even though gloas's own +/// [`process_pending_deposits`] is a separate function +/// (`crate::beacon::stf::epoch::gloas::process_pending_deposits`, SPEC marks it +/// Modified) that reaches its own queue and churn field directly through +/// `gloas_state` rather than through here. pub(crate) enum PendingQueueFields<'a> { Electra(&'a mut electra::BeaconState), Fulu(&'a mut fulu::BeaconState), + Gloas(&'a mut gloas::BeaconState), } impl<'a> PendingQueueFields<'a> { @@ -713,11 +733,14 @@ impl<'a> PendingQueueFields<'a> { /// [`crate::beacon::constants::UNSET_DEPOSIT_REQUESTS_START_INDEX`], the /// precondition [`super::fulu::process_pending_deposits`]'s own doc /// explains. [`super::fulu::process_pending_deposits`] itself never reads - /// this field. + /// this field, and no gloas caller reads it through here at all (gloas's + /// own `process_pending_deposits` does not read it either; see this + /// enum's own doc). pub(crate) fn deposit_requests_start_index(&self) -> u64 { match self { PendingQueueFields::Electra(state) => state.deposit_requests_start_index, PendingQueueFields::Fulu(state) => state.deposit_requests_start_index, + PendingQueueFields::Gloas(state) => state.deposit_requests_start_index, } } @@ -726,6 +749,7 @@ impl<'a> PendingQueueFields<'a> { match self { PendingQueueFields::Electra(state) => state.deposit_balance_to_consume, PendingQueueFields::Fulu(state) => state.deposit_balance_to_consume, + PendingQueueFields::Gloas(state) => state.deposit_balance_to_consume, } } @@ -733,29 +757,88 @@ impl<'a> PendingQueueFields<'a> { match self { PendingQueueFields::Electra(state) => &mut state.deposit_balance_to_consume, PendingQueueFields::Fulu(state) => &mut state.deposit_balance_to_consume, + PendingQueueFields::Gloas(state) => &mut state.deposit_balance_to_consume, } } /// Deposits known but not yet credited to the validator registry. - pub(crate) fn pending_deposits_mut(&mut self) -> &mut electra::PendingDeposits { + /// + /// Electra and fulu only, even though this enum now spans gloas too (for + /// [`Self::take_pending_consolidations`]'s sake): gloas's own + /// `pending_deposits` is `ProgressiveList` (EIP-7688), a + /// different Rust type from electra's and fulu's bounded `SszList` this + /// returns, and gloas's own `process_pending_deposits` reads its queue + /// directly through `gloas_state` instead of through here (see this + /// enum's own doc). Fallible, unlike the pre-gloas version of this + /// method, purely so a gloas-backed value answers `UnsupportedForFork` + /// here instead of failing to compile a shared return type; no caller + /// ever actually reaches that arm. + pub(crate) fn pending_deposits_mut(&mut self) -> Result<&mut electra::PendingDeposits> { + match self { + PendingQueueFields::Electra(state) => Ok(&mut state.pending_deposits), + PendingQueueFields::Fulu(state) => Ok(&mut state.pending_deposits), + PendingQueueFields::Gloas(_) => Err(Error::UnsupportedForFork { + function: "PendingQueueFields::pending_deposits_mut", + fork: ForkName::Gloas, + }), + } + } + + /// Takes the whole pending-consolidations queue out, leaving an empty one + /// behind; [`Self::set_pending_consolidations`] puts a (possibly + /// shorter) one back. Vec-level rather than the container-typed accessor + /// [`Self::pending_deposits_mut`] uses: gloas's own queue is + /// `ProgressiveList` (EIP-7688), a different Rust + /// type from electra's and fulu's bounded `SszList`, and unlike + /// `process_pending_deposits`, [`process_pending_consolidations`] is + /// meant to serve a gloas state directly (gloas's `beacon-chain.md` does + /// not modify this function at all), which a shared Vec return type is + /// what makes possible. + pub(crate) fn take_pending_consolidations(&mut self) -> Vec { match self { - PendingQueueFields::Electra(state) => &mut state.pending_deposits, - PendingQueueFields::Fulu(state) => &mut state.pending_deposits, + PendingQueueFields::Electra(state) => { + core::mem::take(&mut state.pending_consolidations).into_inner() + } + PendingQueueFields::Fulu(state) => { + core::mem::take(&mut state.pending_consolidations).into_inner() + } + PendingQueueFields::Gloas(state) => { + core::mem::take(&mut state.pending_consolidations).into_inner() + } } } - /// Consolidations known but not yet applied. - fn pending_consolidations_mut(&mut self) -> &mut electra::PendingConsolidations { + /// See [`Self::take_pending_consolidations`]. Fallible only for electra's + /// and fulu's bounded `SszList`; gloas's progressive one never rejects a + /// length. + pub(crate) fn set_pending_consolidations( + &mut self, + consolidations: Vec, + ) -> Result<()> { match self { - PendingQueueFields::Electra(state) => &mut state.pending_consolidations, - PendingQueueFields::Fulu(state) => &mut state.pending_consolidations, + PendingQueueFields::Electra(state) => { + state.pending_consolidations = + electra::PendingConsolidations::try_from(consolidations)?; + } + PendingQueueFields::Fulu(state) => { + state.pending_consolidations = + electra::PendingConsolidations::try_from(consolidations)?; + } + PendingQueueFields::Gloas(state) => { + state.pending_consolidations = gloas::PendingConsolidations::from(consolidations); + } } + Ok(()) } /// Extends `previous_epoch_participation`, `current_epoch_participation`, /// and `inactivity_scores` by one all-zero entry each, keeping them /// exactly as long as the registry after /// [`add_validator_from_pending_deposit`] appends a validator. + /// + /// Gloas's own three lists are progressive (`libssz_types::ProgressiveList`), + /// so their `push` is infallible, unlike electra's and fulu's bounded + /// `SszList`. fn push_empty_participation_and_inactivity(&mut self) -> Result<()> { match self { PendingQueueFields::Electra(state) => { @@ -768,14 +851,20 @@ impl<'a> PendingQueueFields<'a> { state.current_epoch_participation.push(0)?; state.inactivity_scores.push(0)?; } + PendingQueueFields::Gloas(state) => { + state.previous_epoch_participation.push(0); + state.current_epoch_participation.push(0); + state.inactivity_scores.push(0); + } } Ok(()) } } -/// The electra-or-fulu state, mutably, through [`PendingQueueFields`]. See -/// its own doc for why this is a second projection rather than a call into -/// [`crate::beacon::helpers::electra::electra_state`]. +/// The electra, fulu, or gloas state, mutably, through [`PendingQueueFields`]. +/// See its own doc for why this is a second projection rather than a call +/// into [`crate::beacon::helpers::electra::electra_state`], and for why gloas +/// joins electra and fulu here. pub(crate) fn pending_queue_fields<'a>( state: &'a mut BeaconState, function: &'static str, @@ -783,6 +872,7 @@ pub(crate) fn pending_queue_fields<'a>( match state { BeaconState::Electra(state) => Ok(PendingQueueFields::Electra(state)), BeaconState::Fulu(state) => Ok(PendingQueueFields::Fulu(state)), + BeaconState::Gloas(state) => Ok(PendingQueueFields::Gloas(state)), other => Err(Error::UnsupportedForFork { function, fork: other.fork_name(), @@ -841,6 +931,7 @@ mod tests { pending_queue_fields(&mut state, "test setup") .unwrap() .pending_deposits_mut() + .unwrap() .push(deposit) .unwrap(); @@ -850,7 +941,7 @@ mod tests { assert_eq!(state.balance(0).unwrap(), balance_before + deposit_amount); let mut fields = pending_queue_fields(&mut state, "test assertion").unwrap(); assert_eq!(fields.deposit_balance_to_consume(), 0); - assert!(fields.pending_deposits_mut().is_empty()); + assert!(fields.pending_deposits_mut().unwrap().is_empty()); } #[test] @@ -878,6 +969,7 @@ mod tests { pending_queue_fields(&mut state, "test setup") .unwrap() .pending_deposits_mut() + .unwrap() .push(deposit.clone()) .unwrap(); @@ -888,7 +980,7 @@ mod tests { assert_eq!(state.balance(0).unwrap(), balance_before); // ...and not lost: it comes back out the other end of the queue. let mut fields = pending_queue_fields(&mut state, "test assertion").unwrap(); - let remaining = fields.pending_deposits_mut(); + let remaining = fields.pending_deposits_mut().unwrap(); assert_eq!(remaining.len(), 1); assert_eq!(remaining[0], deposit); } @@ -926,8 +1018,12 @@ mod tests { }; { let mut fields = pending_queue_fields(&mut state, "test setup").unwrap(); - fields.pending_deposits_mut().push(first).unwrap(); - fields.pending_deposits_mut().push(second.clone()).unwrap(); + fields.pending_deposits_mut().unwrap().push(first).unwrap(); + fields + .pending_deposits_mut() + .unwrap() + .push(second.clone()) + .unwrap(); } let balance_0_before = state.balance(0).unwrap(); @@ -946,7 +1042,7 @@ mod tests { fields.deposit_balance_to_consume(), churn_limit - first_amount ); - let remaining = fields.pending_deposits_mut(); + let remaining = fields.pending_deposits_mut().unwrap(); assert_eq!(remaining.len(), 1); assert_eq!(remaining[0], second); } @@ -990,6 +1086,7 @@ mod tests { pending_queue_fields(&mut state, "test setup") .unwrap() .pending_deposits_mut() + .unwrap() .push(deposit) .unwrap(); @@ -1041,6 +1138,7 @@ mod tests { pending_queue_fields(&mut state, "test setup") .unwrap() .pending_deposits_mut() + .unwrap() .push(deposit) .unwrap(); @@ -1054,7 +1152,7 @@ mod tests { ); let mut fields = pending_queue_fields(&mut state, "test assertion").unwrap(); assert!( - fields.pending_deposits_mut().is_empty(), + fields.pending_deposits_mut().unwrap().is_empty(), "the entry is still consumed from the queue, just never credited" ); } @@ -1075,8 +1173,7 @@ mod tests { }; pending_queue_fields(&mut state, "test setup") .unwrap() - .pending_consolidations_mut() - .push(consolidation) + .set_pending_consolidations(vec![consolidation]) .unwrap(); let source_balance_before = state.balance(0).unwrap(); @@ -1086,7 +1183,7 @@ mod tests { assert_eq!(state.balance(0).unwrap(), source_balance_before); assert_eq!(state.balance(1).unwrap(), target_balance_before); let mut fields = pending_queue_fields(&mut state, "test assertion").unwrap(); - assert!(fields.pending_consolidations_mut().is_empty()); + assert!(fields.take_pending_consolidations().is_empty()); } #[test] @@ -1102,14 +1199,13 @@ mod tests { }; pending_queue_fields(&mut state, "test setup") .unwrap() - .pending_consolidations_mut() - .push(consolidation.clone()) + .set_pending_consolidations(vec![consolidation.clone()]) .unwrap(); process_pending_consolidations(&mut state, &config).unwrap(); let mut fields = pending_queue_fields(&mut state, "test assertion").unwrap(); - let remaining = fields.pending_consolidations_mut(); + let remaining = fields.take_pending_consolidations(); assert_eq!(remaining.len(), 1); assert_eq!(remaining[0], consolidation); } @@ -1126,8 +1222,7 @@ mod tests { }; pending_queue_fields(&mut state, "test setup") .unwrap() - .pending_consolidations_mut() - .push(consolidation) + .set_pending_consolidations(vec![consolidation]) .unwrap(); let source_effective_balance = state.validator(0).unwrap().effective_balance; @@ -1140,7 +1235,7 @@ mod tests { assert_eq!(state.balance(0).unwrap(), source_balance_before - moved); assert_eq!(state.balance(1).unwrap(), target_balance_before + moved); let mut fields = pending_queue_fields(&mut state, "test assertion").unwrap(); - assert!(fields.pending_consolidations_mut().is_empty()); + assert!(fields.take_pending_consolidations().is_empty()); } // ----------------------------------------------------------------------- diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/fulu.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/fulu.rs index 802a8d9d3..d0bdad0bc 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/fulu.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/fulu.rs @@ -21,6 +21,13 @@ //! a seed, and therefore a proposer, is only ever knowable that far ahead and //! no further. //! +//! Gloas shares this exact function: `proposer_lookahead` keeps the identical +//! field and type from fulu on (`containers::gloas`'s own module doc), and +//! gloas's `beacon-chain.md` does not redefine this step either. Only the +//! callee that draws the newly-visible epoch's proposers differs by fork +//! (EIP-8045 excludes slashed validators for gloas), so +//! [`process_proposer_lookahead`] picks it by fork rather than being copied. +//! //! # Why this step runs last //! //! The newly-visible epoch's proposers come from @@ -50,10 +57,12 @@ use crate::beacon::constants::FAR_FUTURE_EPOCH; use crate::beacon::containers::BeaconState; use crate::beacon::containers::electra as electra_containers; use crate::beacon::error::{Error, Result}; +use crate::beacon::fork::ForkName; use crate::beacon::helpers::accessors::get_current_epoch; use crate::beacon::helpers::electra::get_activation_exit_churn_limit; -use crate::beacon::helpers::fulu::{fulu_state, get_beacon_proposer_indices}; +use crate::beacon::helpers::fulu::{get_beacon_proposer_indices, proposer_lookahead_mut}; use crate::beacon::helpers::misc::compute_start_slot_at_epoch; +use crate::beacon::lean_state_unreachable; use crate::beacon::preset; use crate::beacon::primitives::Gwei; @@ -133,7 +142,7 @@ pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Res "deposit_balance_to_consume + get_activation_exit_churn_limit", ))?; let deposits: Vec = - core::mem::take(fields.pending_deposits_mut()).into_inner(); + core::mem::take(fields.pending_deposits_mut()?).into_inner(); (available_for_processing, deposits) }; @@ -210,7 +219,7 @@ pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Res }; let mut fields = electra::pending_queue_fields(state, "process_pending_deposits")?; - *fields.pending_deposits_mut() = electra_containers::PendingDeposits::try_from(remaining)?; + *fields.pending_deposits_mut()? = electra_containers::PendingDeposits::try_from(remaining)?; *fields.deposit_balance_to_consume_mut() = deposit_balance_to_consume; Ok(()) @@ -234,16 +243,37 @@ pub fn process_proposer_lookahead(state: &mut BeaconState) -> Result<()> { // The seed for this epoch is only just now fixed, per the module docs, so // this is the earliest moment its proposers could have been computed. let new_epoch = get_current_epoch(state) + preset::MIN_SEED_LOOKAHEAD + 1; - let new_epoch_proposers = get_beacon_proposer_indices(state, new_epoch)?; + let new_epoch_proposers = match state.fork_name() { + ForkName::Fulu => get_beacon_proposer_indices(state, new_epoch)?, + // EIP-8045: gloas's own draw excludes slashed validators before + // sampling; see `crate::beacon::helpers::gloas::get_beacon_proposer_indices`'s + // own doc. This is the one part of the step gloas cannot share + // unchanged; see this module's own module doc. + ForkName::Gloas => { + crate::beacon::helpers::gloas::get_beacon_proposer_indices(state, new_epoch)? + } + fork @ (ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra) => { + return Err(Error::UnsupportedForFork { + function: "process_proposer_lookahead", + fork, + }); + } + ForkName::Lean => lean_state_unreachable("process_proposer_lookahead"), + }; - let fulu_state = fulu_state(state, "process_proposer_lookahead")?; let slots_per_epoch = preset::SLOTS_PER_EPOCH as usize; + let lookahead = proposer_lookahead_mut(state, "process_proposer_lookahead")?; let mut window = Vec::with_capacity(preset::PROPOSER_LOOKAHEAD_LENGTH); - window.extend_from_slice(&fulu_state.proposer_lookahead[slots_per_epoch..]); + window.extend_from_slice(&lookahead[slots_per_epoch..]); window.extend(new_epoch_proposers); - fulu_state.proposer_lookahead = window.try_into().expect( + *lookahead = window.try_into().expect( "dropping SLOTS_PER_EPOCH entries and appending SLOTS_PER_EPOCH more preserves \ PROPOSER_LOOKAHEAD_LENGTH", ); @@ -383,6 +413,7 @@ mod tests { electra::pending_queue_fields(&mut state, "test setup") .unwrap() .pending_deposits_mut() + .unwrap() .push(deposit) .unwrap(); @@ -396,7 +427,7 @@ mod tests { balance_before + preset::EFFECTIVE_BALANCE_INCREMENT ); let mut fields = electra::pending_queue_fields(&mut state, "test assertion").unwrap(); - assert!(fields.pending_deposits_mut().is_empty()); + assert!(fields.pending_deposits_mut().unwrap().is_empty()); } #[test] @@ -410,6 +441,7 @@ mod tests { electra::pending_queue_fields(&mut state, "test setup") .unwrap() .pending_deposits_mut() + .unwrap() .push(deposit.clone()) .unwrap(); @@ -426,7 +458,7 @@ mod tests { // reaching it. assert_eq!(state.balance(0).unwrap(), balance_before); let mut fields = electra::pending_queue_fields(&mut state, "test assertion").unwrap(); - let remaining = fields.pending_deposits_mut(); + let remaining = fields.pending_deposits_mut().unwrap(); assert_eq!(remaining.len(), 1); assert_eq!(remaining[0], deposit); } diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs new file mode 100644 index 000000000..7a8daef22 --- /dev/null +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs @@ -0,0 +1,861 @@ +//! Gloas-specific epoch processing. +//! +//! [`process_epoch`] is transcribed from `beacon-chain.md`'s own list, in the +//! order it gives them. Gloas inserts one new step, +//! [`process_builder_pending_payments`] (EIP-7732), between fulu's own +//! `process_pending_consolidations` and `process_effective_balance_updates`, +//! and appends another, [`process_ptc_window`] (EIP-7732), after +//! `process_proposer_lookahead`. [`process_pending_deposits`] is also modified +//! in place (EIP-8061: gloas's own activation-only churn budget, see its own +//! doc below); every other step is fulu's own step list, unmodified by +//! gloas's `beacon-chain.md`. +//! +//! Being unmodified in the specification text is not the same as being +//! callable unchanged, though. EIP-7688 makes `previous_epoch_participation`, +//! `current_epoch_participation`, `inactivity_scores`, `pending_deposits`, +//! and `pending_consolidations` progressive lists +//! (`libssz_types::ProgressiveList`) on a gloas state, a different Rust type +//! from every earlier fork's bounded `SszList`. Each step was checked against +//! the real code rather than assumed, and falls into one of three groups: +//! +//! - **Registry and balance steps** ([`super::electra::process_registry_updates`], +//! [`super::electra::process_slashings`], +//! [`super::electra::process_effective_balance_updates`]) touch only +//! `validators` and `balances`, which [`crate::beacon::containers::BeaconState`] +//! already reaches through element accessors generic over the bounded/tree-backed +//! split (`iter_validators`, `validator_mut`, `push_validator`, ...). These +//! call straight through, unchanged, on a gloas state. +//! - **Participation and inactivity steps** +//! ([`super::altair::process_justification_and_finalization`], +//! [`super::altair::process_inactivity_updates`], +//! [`super::altair::process_rewards_and_penalties`]) reach +//! `previous_epoch_participation`/`current_epoch_participation`/`inactivity_scores` +//! as plain slices, through [`crate::beacon::containers::BeaconState::altair_validator_lists`] +//! and [`crate::beacon::containers::BeaconState::inactivity_scores_mut`]: both +//! `SszList` and `ProgressiveList` `Deref` to a plain `[T]`. +//! - **[`super::electra::process_pending_consolidations`]** takes the queue +//! as a `Vec` and writes it back, through +//! [`super::electra::PendingQueueFields::take_pending_consolidations`] and +//! [`super::electra::PendingQueueFields::set_pending_consolidations`], which +//! accept a gloas state. +//! - **[`super::fulu::process_proposer_lookahead`]** shifts the window +//! through an accessor both forks share, and chooses the proposer draw by +//! fork: gloas's `get_beacon_proposer_indices` excludes slashed validators +//! (EIP-8045). +//! - Resets and other +//! fixed-shape fields ([`super::process_eth1_data_reset`], +//! [`super::process_slashings_reset`], [`super::process_randao_mixes_reset`], +//! [`super::capella::process_historical_summaries_update`], +//! [`super::altair::process_sync_committee_updates`]) call straight through, +//! through accessors that already list gloas among the forks they serve. +//! - **[`process_pending_deposits`] and [`process_participation_flag_updates`] +//! are gloas's own copy**, over [`gloas::BeaconState`] directly (through +//! `helpers::gloas::gloas_state`). `process_pending_deposits` is +//! spec-modified (EIP-8061: gloas's own activation-only churn budget) and +//! still calls the shared [`super::electra::apply_pending_deposit`] for the +//! "credit one dequeued deposit" sub-step. +//! `process_participation_flag_updates` is unmodified in the specification, +//! but replaces the whole list (a new length, not an element write), which +//! only the fork's own concrete container type can do: electra's and +//! fulu's `EpochParticipation::try_from` is fallible where gloas's +//! `ProgressiveList::from` is not, so there is no shared return type a +//! `dispatch_state_from!` body could give both. + +use crate::beacon::config::Config; +use crate::beacon::constants::FAR_FUTURE_EPOCH; +use crate::beacon::containers::{BeaconState, electra, gloas}; +use crate::beacon::error::{Error, Result}; +use crate::beacon::helpers::accessors::{CommitteeCache, get_current_epoch}; +use crate::beacon::helpers::gloas::{ + compute_ptc, get_activation_churn_limit, get_builder_payment_quorum_threshold, gloas_state, +}; +use crate::beacon::helpers::misc::compute_start_slot_at_epoch; +use crate::beacon::preset; +use crate::beacon::primitives::Gwei; + +/// Gloas's epoch-boundary driver, in the specification's own order +/// (`beacon-chain.md`'s "Modified `process_epoch`"): fulu's own step list, +/// with [`process_builder_pending_payments`] inserted right after +/// [`process_pending_consolidations`](super::electra::process_pending_consolidations) +/// and [`process_ptc_window`] appended after `process_proposer_lookahead`. See +/// this module's own doc for which steps below are shared and which are +/// gloas's own copy. +pub fn process_epoch(state: &mut BeaconState, config: &Config) -> Result<()> { + super::altair::process_justification_and_finalization(state)?; + super::altair::process_inactivity_updates(state, config)?; + super::altair::process_rewards_and_penalties(state, config)?; + super::electra::process_registry_updates(state, config)?; + super::electra::process_slashings(state, config)?; + super::process_eth1_data_reset(state)?; + // [Modified in Gloas:EIP8061] + process_pending_deposits(state, config)?; + super::electra::process_pending_consolidations(state, config)?; + // [New in Gloas:EIP7732] + process_builder_pending_payments(state)?; + super::electra::process_effective_balance_updates(state)?; + super::process_slashings_reset(state)?; + super::process_randao_mixes_reset(state)?; + super::capella::process_historical_summaries_update(state)?; + process_participation_flag_updates(state)?; + super::altair::process_sync_committee_updates(state)?; + super::fulu::process_proposer_lookahead(state)?; + // [New in Gloas:EIP7732] + process_ptc_window(state)?; + Ok(()) +} + +// --------------------------------------------------------------------------- +// Participation flag updates +// --------------------------------------------------------------------------- + +/// Rotates the current epoch's participation flags into the previous slot and +/// installs a fresh, all-zero current list, gloas's own copy of +/// [`crate::beacon::stf::epoch::altair::process_participation_flag_updates`]: +/// identical logic, writing gloas's own progressive lists through +/// [`gloas_state`] (an infallible `From>` conversion, unlike the +/// bounded `EpochParticipation`'s fallible `TryFrom` altair's version needs). +/// See this module's own doc for why that difference is what keeps this a +/// separate copy rather than a shared, `Vec`-level function the way +/// [`super::electra::process_pending_consolidations`] is. +pub fn process_participation_flag_updates(state: &mut BeaconState) -> Result<()> { + let validator_count = state.validator_count(); + let inner = gloas_state(state, "process_participation_flag_updates")?; + + inner.previous_epoch_participation = core::mem::take(&mut inner.current_epoch_participation); + inner.current_epoch_participation = vec![0; validator_count].into(); + + Ok(()) +} + +// --------------------------------------------------------------------------- +// Pending deposits +// --------------------------------------------------------------------------- + +/// `process_pending_deposits` (gloas `beacon-chain.md`). +/// +/// Modified from electra's and fulu's (EIP-8061): the churn budget is +/// [`get_activation_churn_limit`]'s own, independent activation-only budget, +/// not electra's combined activation/exit one +/// (`crate::beacon::helpers::electra::get_activation_exit_churn_limit`); like fulu's +/// own copy, the eth1-bridge-ahead-of-requests gate is gone outright (see +/// `crate::beacon::stf::epoch::fulu::process_pending_deposits`'s own doc for why +/// dropping it changes nothing observable once a chain has reached this far). +/// The queue is gloas's own progressive one (EIP-7688), read and written +/// directly through [`gloas_state`] rather than through +/// [`super::electra::pending_queue_fields`]'s [`PendingQueueFields`](super::electra::PendingQueueFields), +/// which refuses that field for a gloas state (see its own doc). Once a +/// deposit is dequeued and cleared to apply, though, +/// [`super::electra::apply_pending_deposit`] is the identical function electra +/// and fulu use: nothing about crediting one already-dequeued deposit differs +/// for gloas. +pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Result<()> { + let next_epoch = get_current_epoch(state) + 1; + let churn_limit = get_activation_churn_limit(state, config)?; + let finalized_slot = compute_start_slot_at_epoch(state.finalized_checkpoint().epoch); + + // See `crate::beacon::stf::epoch::electra::process_pending_deposits`'s own doc + // for why the queue is taken by value here rather than iterated in place. + let (available_for_processing, deposits) = { + let inner = gloas_state(state, "process_pending_deposits")?; + let available_for_processing = inner + .deposit_balance_to_consume + .checked_add(churn_limit) + .ok_or(Error::ArithmeticOverflow( + "deposit_balance_to_consume + get_activation_churn_limit", + ))?; + let deposits: Vec = + core::mem::take(&mut inner.pending_deposits).into_inner(); + (available_for_processing, deposits) + }; + + let mut processed_amount: Gwei = 0; + let mut next_deposit_index = 0usize; + let mut deposits_to_postpone: Vec = Vec::new(); + let mut is_churn_limit_reached = false; + + for deposit in &deposits { + // A deposit whose queue position could still be reorged out must + // wait: crediting it now and reverting later is not an option, since + // nothing else in the state transition undoes a balance change. + if deposit.slot > finalized_slot { + break; + } + + if next_deposit_index >= preset::MAX_PENDING_DEPOSITS_PER_EPOCH as usize { + break; + } + + let (is_validator_exited, is_validator_withdrawn) = state + .iter_validators() + .find(|validator| validator.pubkey == deposit.pubkey) + .map(|validator| { + ( + validator.exit_epoch < FAR_FUTURE_EPOCH, + validator.withdrawable_epoch < next_epoch, + ) + }) + .unwrap_or((false, false)); + + if is_validator_withdrawn { + super::electra::apply_pending_deposit(state, deposit, config)?; + } else if is_validator_exited { + deposits_to_postpone.push(deposit.clone()); + } else { + match processed_amount.checked_add(deposit.amount) { + Some(sum) if sum <= available_for_processing => { + processed_amount = sum; + super::electra::apply_pending_deposit(state, deposit, config)?; + } + // Either the sum overflowed (certainly too much) or it fit in + // a `u64` but still exceeded the budget: both mean this + // epoch's processing stops here. + _ => { + is_churn_limit_reached = true; + break; + } + } + } + + next_deposit_index += 1; + } + + let remaining: Vec = deposits + .into_iter() + .skip(next_deposit_index) + .chain(deposits_to_postpone) + .collect(); + + let deposit_balance_to_consume = if is_churn_limit_reached { + available_for_processing + .checked_sub(processed_amount) + .ok_or(Error::ArithmeticOverflow( + "available_for_processing - processed_amount", + ))? + } else { + 0 + }; + + let inner = gloas_state(state, "process_pending_deposits")?; + inner.pending_deposits = gloas::PendingDeposits::from(remaining); + inner.deposit_balance_to_consume = deposit_balance_to_consume; + + Ok(()) +} + +// --------------------------------------------------------------------------- +// Builder pending payments (EIP-7732) +// --------------------------------------------------------------------------- + +/// `process_builder_pending_payments` (gloas `beacon-chain.md`). +/// +/// New in gloas (EIP-7732). Settles or drops each of the previous epoch's +/// [`gloas::BuilderPendingPayment`]s, weighed against +/// [`get_builder_payment_quorum_threshold`], then shifts the fixed two-epoch +/// window (`builder_pending_payments`, `preset::BUILDER_PENDING_PAYMENTS_LENGTH` +/// long) down by one epoch, the builder-payment counterpart of +/// [`process_pending_deposits`]'s deposit queue. +/// +/// The loop over `builder_pending_payments` borrows it immutably while +/// pushing to `builder_pending_withdrawals`; those are different fields of +/// the same `&mut gloas::BeaconState`, so this compiles with direct field +/// access rather than through methods that would each take `&mut *inner` +/// whole. +pub fn process_builder_pending_payments(state: &mut BeaconState) -> Result<()> { + let quorum = get_builder_payment_quorum_threshold(state)?; + let inner = gloas_state(state, "process_builder_pending_payments")?; + let epoch_len = preset::SLOTS_PER_EPOCH as usize; + + for payment in &inner.builder_pending_payments[..epoch_len] { + if payment.weight >= quorum { + inner + .builder_pending_withdrawals + .push(payment.withdrawal.clone()); + } + } + + for index in 0..epoch_len { + inner.builder_pending_payments[index] = + inner.builder_pending_payments[epoch_len + index].clone(); + inner.builder_pending_payments[epoch_len + index] = gloas::BuilderPendingPayment::default(); + } + + Ok(()) +} + +// --------------------------------------------------------------------------- +// PTC window (EIP-7732) +// --------------------------------------------------------------------------- + +/// `process_ptc_window` (gloas `beacon-chain.md`). +/// +/// New in gloas (EIP-7732). Shifts the cached `ptc_window` +/// (`preset::PTC_WINDOW_LENGTH` slots long: the previous, current, and +/// `MIN_SEED_LOOKAHEAD`-epochs-ahead committees) down by one epoch, the same +/// fixed-window shift `super::fulu::process_proposer_lookahead` performs, and +/// fills the newly-visible epoch's slice with [`compute_ptc`], one committee +/// per slot. Draws every one of that epoch's committees through one shared +/// [`CommitteeCache`], the same reason [`compute_ptc`]'s own doc gives for why +/// its caller should hold one across a whole epoch's worth of calls rather +/// than letting each shuffle the active set on its own. +pub fn process_ptc_window(state: &mut BeaconState) -> Result<()> { + let next_epoch = get_current_epoch(state) + preset::MIN_SEED_LOOKAHEAD + 1; + let start_slot = compute_start_slot_at_epoch(next_epoch); + let slots_per_epoch = preset::SLOTS_PER_EPOCH as usize; + + let committees = CommitteeCache::default(); + let mut new_slice = Vec::with_capacity(slots_per_epoch); + for offset in 0..preset::SLOTS_PER_EPOCH { + let slot = start_slot + .checked_add(offset) + .ok_or(Error::ArithmeticOverflow( + "process_ptc_window: start_slot + offset", + ))?; + new_slice.push(compute_ptc(state, slot, &committees)?); + } + + let inner = gloas_state(state, "process_ptc_window")?; + let mut window = Vec::with_capacity(preset::PTC_WINDOW_LENGTH); + window.extend_from_slice(&inner.ptc_window[slots_per_epoch..]); + window.extend(new_slice); + inner.ptc_window = window.try_into().expect( + "dropping SLOTS_PER_EPOCH entries and appending SLOTS_PER_EPOCH more preserves \ + PTC_WINDOW_LENGTH", + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::beacon::constants; + use crate::beacon::fork::ForkName; + use crate::beacon::helpers::accessors::get_total_active_balance; + use crate::beacon::helpers::gloas::{get_ptc, gloas_state_ref}; + use crate::beacon::primitives::ValidatorIndex; + + /// A gloas state with `count` fully active, full-balance validators, + /// positioned one epoch in, the same way + /// `crate::beacon::helpers::test_state::with_validators` positions its phase0 + /// state. + fn gloas_state_with_validators(count: usize) -> BeaconState { + crate::beacon::helpers::test_state::with_validators_at(ForkName::Gloas, count) + } + + fn gloas_inner(state: &BeaconState) -> &gloas::BeaconState { + gloas_state_ref(state, "test").unwrap() + } + + fn gloas_inner_mut(state: &mut BeaconState) -> &mut gloas::BeaconState { + gloas_state(state, "test").unwrap() + } + + // ----------------------------------------------------------------------- + // process_epoch + // ----------------------------------------------------------------------- + + #[test] + fn process_epoch_runs_every_step_without_error_on_a_fresh_state() { + // 64, not fewer: `process_ptc_window` (a step of `process_epoch`) + // needs enough validators that every slot's committee is non-empty; + // see `compute_ptc_returns_ptc_size_active_indices_deterministically` + // in `helpers::gloas`'s own tests for the same floor and why. + let config = Config::mainnet(); + let mut state = gloas_state_with_validators(64); + process_epoch(&mut state, &config).unwrap(); + } + + // ----------------------------------------------------------------------- + // process_participation_flag_updates + // ----------------------------------------------------------------------- + + #[test] + fn participation_flag_updates_rotate_current_into_previous_and_reset_current() { + let mut state = gloas_state_with_validators(4); + gloas_inner_mut(&mut state).current_epoch_participation[0] = 5; + gloas_inner_mut(&mut state).current_epoch_participation[1] = 7; + let current_before = gloas_inner(&state).current_epoch_participation.to_vec(); + + process_participation_flag_updates(&mut state).unwrap(); + + let inner = gloas_inner(&state); + assert_eq!(inner.previous_epoch_participation.to_vec(), current_before); + assert_eq!(inner.current_epoch_participation.len(), 4); + assert!( + inner + .current_epoch_participation + .iter() + .all(|&flags| flags == 0), + "the fresh current list must start all-zero" + ); + } + + // ----------------------------------------------------------------------- + // process_pending_deposits + // ----------------------------------------------------------------------- + + #[test] + fn a_pending_deposit_for_an_already_withdrawn_validator_bypasses_churn() { + let config = Config::mainnet(); + let mut state = gloas_state_with_validators(2); + let next_epoch = get_current_epoch(&state) + 1; + + let (pubkey, withdrawal_credentials) = { + let validator = state.validator_mut(0).unwrap(); + validator.exit_epoch = 0; + validator.withdrawable_epoch = 0; + (validator.pubkey, validator.withdrawal_credentials) + }; + assert!(state.validator(0).unwrap().withdrawable_epoch < next_epoch); + + let churn_limit = get_activation_churn_limit(&state, &config).unwrap(); + // Deliberately larger than the whole churn budget: a withdrawn + // validator's deposit must go through regardless of budget. + let deposit_amount = churn_limit + preset::EFFECTIVE_BALANCE_INCREMENT; + let deposit = electra::PendingDeposit { + pubkey, + withdrawal_credentials, + amount: deposit_amount, + signature: Default::default(), + slot: constants::GENESIS_SLOT, + }; + gloas_inner_mut(&mut state).pending_deposits.push(deposit); + + let balance_before = state.balance(0).unwrap(); + process_pending_deposits(&mut state, &config).unwrap(); + + assert_eq!(state.balance(0).unwrap(), balance_before + deposit_amount); + let inner = gloas_inner(&state); + assert_eq!(inner.deposit_balance_to_consume, 0); + assert!(inner.pending_deposits.is_empty()); + } + + #[test] + fn a_pending_deposit_for_an_exited_but_not_yet_withdrawn_validator_is_postponed_not_dropped() { + let config = Config::mainnet(); + let mut state = gloas_state_with_validators(2); + let next_epoch = get_current_epoch(&state) + 1; + + let (pubkey, withdrawal_credentials) = { + let validator = state.validator_mut(0).unwrap(); + validator.exit_epoch = 0; + validator.withdrawable_epoch = FAR_FUTURE_EPOCH; + (validator.pubkey, validator.withdrawal_credentials) + }; + assert!(state.validator(0).unwrap().withdrawable_epoch >= next_epoch); + + let deposit = electra::PendingDeposit { + pubkey, + withdrawal_credentials, + amount: preset::EFFECTIVE_BALANCE_INCREMENT, + signature: Default::default(), + slot: constants::GENESIS_SLOT, + }; + gloas_inner_mut(&mut state) + .pending_deposits + .push(deposit.clone()); + + let balance_before = state.balance(0).unwrap(); + process_pending_deposits(&mut state, &config).unwrap(); + + assert_eq!(state.balance(0).unwrap(), balance_before); + let inner = gloas_inner(&state); + assert_eq!(inner.pending_deposits.len(), 1); + assert_eq!(inner.pending_deposits[0], deposit); + } + + #[test] + fn pending_deposits_stop_at_the_activation_only_churn_limit() { + // A config whose gloas activation-churn quotient is deliberately + // different from electra's combined one, with the floor zeroed so a + // small registry does not flatten both to the same constant the way + // `Config::mainnet()`'s much larger floor would: see the `assert_ne!` + // below, which is what makes this test actually gloas-specific + // rather than exercising whatever churn `process_pending_deposits` + // happens to be handed. + let config = Config { + min_per_epoch_churn_limit_electra: 0, + churn_limit_quotient: 4, + churn_limit_quotient_gloas: 2, + max_per_epoch_activation_exit_churn_limit: u64::MAX, + max_per_epoch_activation_churn_limit_gloas: u64::MAX, + ..Config::mainnet() + }; + let mut state = gloas_state_with_validators(4); + + let total_active_balance = get_total_active_balance(&state).unwrap(); + let electra_combined_churn = + crate::beacon::helpers::electra::get_balance_churn_limit(&state, &config).unwrap(); + let churn_limit = get_activation_churn_limit(&state, &config).unwrap(); + assert_ne!( + churn_limit, electra_combined_churn, + "the two quotients must actually diverge, or excluding electra's \ + combined churn below proves nothing gloas-specific" + ); + // Computed independently of `get_activation_churn_limit`, straight + // from its own formula (`beacon-chain.md`'s New `get_activation_churn_limit`): + // floor at `min_per_epoch_churn_limit_electra`, round down to an + // increment, cap at `max_per_epoch_activation_churn_limit_gloas`. + let expected_churn_limit = { + let churn = config + .min_per_epoch_churn_limit_electra + .max(total_active_balance / config.churn_limit_quotient_gloas); + let churn = churn - churn % preset::EFFECTIVE_BALANCE_INCREMENT; + config.max_per_epoch_activation_churn_limit_gloas.min(churn) + }; + assert_eq!(churn_limit, expected_churn_limit); + + let (pubkey_0, credentials_0) = { + let v = state.validator(0).unwrap(); + (v.pubkey, v.withdrawal_credentials) + }; + let (pubkey_1, credentials_1) = { + let v = state.validator(1).unwrap(); + (v.pubkey, v.withdrawal_credentials) + }; + + let first_amount = churn_limit / 2; + let second_amount = churn_limit; + let first = electra::PendingDeposit { + pubkey: pubkey_0, + withdrawal_credentials: credentials_0, + amount: first_amount, + signature: Default::default(), + slot: constants::GENESIS_SLOT, + }; + let second = electra::PendingDeposit { + pubkey: pubkey_1, + withdrawal_credentials: credentials_1, + amount: second_amount, + signature: Default::default(), + slot: constants::GENESIS_SLOT, + }; + { + let inner = gloas_inner_mut(&mut state); + inner.pending_deposits.push(first); + inner.pending_deposits.push(second.clone()); + } + + let balance_0_before = state.balance(0).unwrap(); + let balance_1_before = state.balance(1).unwrap(); + process_pending_deposits(&mut state, &config).unwrap(); + + assert_eq!(state.balance(0).unwrap(), balance_0_before + first_amount); + assert_eq!( + state.balance(1).unwrap(), + balance_1_before, + "over budget: must not apply" + ); + + let inner = gloas_inner(&state); + assert_eq!(inner.deposit_balance_to_consume, churn_limit - first_amount); + assert_eq!(inner.pending_deposits.len(), 1); + assert_eq!(inner.pending_deposits[0], second); + } + + // ----------------------------------------------------------------------- + // process_pending_consolidations (shared: super::electra's own copy, + // exercised here against a gloas state) + // ----------------------------------------------------------------------- + + #[test] + fn an_eligible_pending_consolidation_moves_the_source_balance_to_the_target() { + let config = Config::mainnet(); + let mut state = gloas_state_with_validators(2); + state.validator_mut(0).unwrap().withdrawable_epoch = 0; + + let consolidation = electra::PendingConsolidation { + source_index: 0, + target_index: 1, + }; + gloas_inner_mut(&mut state) + .pending_consolidations + .push(consolidation); + + let source_effective_balance = state.validator(0).unwrap().effective_balance; + let source_balance_before = state.balance(0).unwrap(); + let target_balance_before = state.balance(1).unwrap(); + + super::super::electra::process_pending_consolidations(&mut state, &config).unwrap(); + + let moved = source_balance_before.min(source_effective_balance); + assert_eq!(state.balance(0).unwrap(), source_balance_before - moved); + assert_eq!(state.balance(1).unwrap(), target_balance_before + moved); + assert!(gloas_inner(&state).pending_consolidations.is_empty()); + } + + #[test] + fn a_pending_consolidation_from_a_slashed_source_is_dropped_from_the_queue() { + let config = Config::mainnet(); + let mut state = gloas_state_with_validators(2); + state.validator_mut(0).unwrap().slashed = true; + + let consolidation = electra::PendingConsolidation { + source_index: 0, + target_index: 1, + }; + gloas_inner_mut(&mut state) + .pending_consolidations + .push(consolidation); + + let source_balance_before = state.balance(0).unwrap(); + let target_balance_before = state.balance(1).unwrap(); + super::super::electra::process_pending_consolidations(&mut state, &config).unwrap(); + + assert_eq!(state.balance(0).unwrap(), source_balance_before); + assert_eq!(state.balance(1).unwrap(), target_balance_before); + assert!(gloas_inner(&state).pending_consolidations.is_empty()); + } + + // ----------------------------------------------------------------------- + // process_builder_pending_payments + // ----------------------------------------------------------------------- + + #[test] + fn a_payment_at_or_above_quorum_settles_into_a_withdrawal_and_below_quorum_is_dropped() { + let mut state = gloas_state_with_validators(64); + let quorum = get_builder_payment_quorum_threshold(&state).unwrap(); + + { + let inner = gloas_inner_mut(&mut state); + inner.builder_pending_payments[0] = gloas::BuilderPendingPayment { + weight: quorum, + withdrawal: gloas::BuilderPendingWithdrawal { + builder_index: 7, + amount: 100, + ..Default::default() + }, + proposer_index: 0, + }; + inner.builder_pending_payments[1] = gloas::BuilderPendingPayment { + weight: quorum - 1, + withdrawal: gloas::BuilderPendingWithdrawal { + builder_index: 9, + amount: 200, + ..Default::default() + }, + proposer_index: 0, + }; + } + + process_builder_pending_payments(&mut state).unwrap(); + + let inner = gloas_inner(&state); + assert_eq!(inner.builder_pending_withdrawals.len(), 1); + assert_eq!(inner.builder_pending_withdrawals[0].builder_index, 7); + } + + /// Unlike `settle_builder_payment`, which only pushes a withdrawal when + /// `payment.withdrawal.amount > 0`, `process_builder_pending_payments`'s + /// own gate is `payment.weight >= quorum` alone (`beacon-chain.md`'s own + /// `if payment.weight >= quorum: ... append(payment.withdrawal)`, no + /// amount check at all): a zero-amount payment that still cleared quorum + /// is appended anyway. + #[test] + fn a_zero_amount_payment_at_quorum_is_still_appended() { + use crate::beacon::helpers::gloas::settle_builder_payment; + + let mut state = gloas_state_with_validators(64); + let quorum = get_builder_payment_quorum_threshold(&state).unwrap(); + + let payment = gloas::BuilderPendingPayment { + weight: quorum, + withdrawal: gloas::BuilderPendingWithdrawal { + builder_index: 7, + amount: 0, + ..Default::default() + }, + proposer_index: 0, + }; + gloas_inner_mut(&mut state).builder_pending_payments[0] = payment.clone(); + + process_builder_pending_payments(&mut state).unwrap(); + + let inner = gloas_inner(&state); + assert_eq!( + inner.builder_pending_withdrawals.len(), + 1, + "process_builder_pending_payments must append even a zero-amount \ + payment once it clears quorum" + ); + assert_eq!(inner.builder_pending_withdrawals[0].builder_index, 7); + + // The contrast: `settle_builder_payment` itself would have dropped + // this same payment silently, since its own amount check gates on + // `> 0` regardless of weight. + let mut settle_state = gloas_state_with_validators(64); + gloas_inner_mut(&mut settle_state).builder_pending_payments[0] = payment; + settle_builder_payment(gloas_inner_mut(&mut settle_state), 0).unwrap(); + assert!( + gloas_inner(&settle_state) + .builder_pending_withdrawals + .is_empty(), + "settle_builder_payment drops a zero-amount withdrawal instead" + ); + } + + #[test] + fn process_builder_pending_payments_shifts_the_second_epoch_into_the_first_and_zeroes_the_rest() + { + let mut state = gloas_state_with_validators(4); + let epoch_len = preset::SLOTS_PER_EPOCH as usize; + + { + let inner = gloas_inner_mut(&mut state); + inner.builder_pending_payments[epoch_len] = gloas::BuilderPendingPayment { + weight: 0, + withdrawal: gloas::BuilderPendingWithdrawal { + builder_index: 3, + amount: 55, + ..Default::default() + }, + proposer_index: 0, + }; + } + + process_builder_pending_payments(&mut state).unwrap(); + + let inner = gloas_inner(&state); + assert_eq!( + inner.builder_pending_payments[0].withdrawal.builder_index, + 3 + ); + assert_eq!(inner.builder_pending_payments[0].withdrawal.amount, 55); + for payment in &inner.builder_pending_payments[epoch_len..] { + assert_eq!(*payment, gloas::BuilderPendingPayment::default()); + } + } + + // ----------------------------------------------------------------------- + // process_proposer_lookahead (shared: super::fulu's own copy, exercised + // here against a gloas state) + // ----------------------------------------------------------------------- + + #[test] + fn process_proposer_lookahead_shifts_the_window_and_excludes_a_slashed_validator() { + let mut state = gloas_state_with_validators(64); + let slots_per_epoch = preset::SLOTS_PER_EPOCH as usize; + + // Distinct values in the existing window, so a shift that dropped or + // reordered the carried-over slice (rather than moving it down by + // exactly one epoch) would be caught: every slot in the default, + // all-zero test state otherwise looks the same before and after. + { + let inner = gloas_inner_mut(&mut state); + for (index, value) in inner.proposer_lookahead.iter_mut().enumerate() { + *value = index as ValidatorIndex; + } + } + let before = gloas_inner(&state).proposer_lookahead.to_vec(); + + // Control: fulu's own, unfiltered draw (no EIP-8045 exclusion) on + // this exact state, computed before anyone is slashed. Picking the + // validator it actually names, rather than an arbitrary index, + // guarantees the exclusion check below is not vacuous: an unfiltered + // draw that happened to skip whichever index the test slashed would + // let a broken filter (or none at all) pass just as easily. + let new_epoch = get_current_epoch(&state) + preset::MIN_SEED_LOOKAHEAD + 1; + let unfiltered = + crate::beacon::helpers::fulu::get_beacon_proposer_indices(&state, new_epoch).unwrap(); + let slashed_index = *unfiltered + .first() + .expect("SLOTS_PER_EPOCH draws always name at least one proposer"); + state.validator_mut(slashed_index).unwrap().slashed = true; + + super::super::fulu::process_proposer_lookahead(&mut state).unwrap(); + + let inner = gloas_inner(&state); + assert_eq!( + inner.proposer_lookahead.len(), + preset::PROPOSER_LOOKAHEAD_LENGTH + ); + assert_eq!( + inner.proposer_lookahead[..inner.proposer_lookahead.len() - slots_per_epoch], + before[slots_per_epoch..], + "the carried-over slice must shift down by exactly one epoch" + ); + let new_slice = + &inner.proposer_lookahead[inner.proposer_lookahead.len() - slots_per_epoch..]; + assert!( + !new_slice.contains(&slashed_index), + "the slashed validator must never be drawn into the new slice, even though \ + the unfiltered (fulu) draw above proves it would otherwise be" + ); + } + + // ----------------------------------------------------------------------- + // process_ptc_window + // ----------------------------------------------------------------------- + + #[test] + fn process_ptc_window_shifts_and_fills_a_fresh_lookahead_slice() { + let mut state = gloas_state_with_validators(64); + let window_len = preset::PTC_WINDOW_LENGTH; + let slots_per_epoch = preset::SLOTS_PER_EPOCH as usize; + + // Distinct values per window index, so a shift that scrambled order + // (rather than moving each slice down by exactly one epoch) would be + // caught: every slice in the default, all-zero test state is + // otherwise identical and could not tell a correct shift from a + // wrong one. Mirrors `helpers::gloas::tests::mark_ptc_window`. + { + let inner = gloas_inner_mut(&mut state); + for window_index in 0..window_len { + inner.ptc_window[window_index] = vec![window_index as u64; preset::PTC_SIZE] + .try_into() + .expect("built at exactly PTC_SIZE"); + } + } + let before = gloas_inner(&state).ptc_window.to_vec(); + + let epoch_before = get_current_epoch(&state); + process_ptc_window(&mut state).unwrap(); + + let inner = gloas_inner(&state); + assert_eq!(inner.ptc_window.len(), window_len); + assert_eq!( + inner.ptc_window[..window_len - slots_per_epoch], + before[slots_per_epoch..], + "the carried-over slices must shift down by exactly one epoch" + ); + + // The freshly-filled tail: one `compute_ptc` per slot of the + // newly-visible epoch, matching `process_ptc_window`'s own fill loop. + let next_epoch = epoch_before + preset::MIN_SEED_LOOKAHEAD + 1; + let start_slot = compute_start_slot_at_epoch(next_epoch); + let committees = CommitteeCache::default(); + for offset in 0..preset::SLOTS_PER_EPOCH { + let slot = start_slot + offset; + let expected = compute_ptc(&state, slot, &committees).unwrap(); + let actual = &inner.ptc_window[window_len - slots_per_epoch + offset as usize]; + assert_eq!(&actual[..], &expected[..], "slot {slot}"); + } + } + + #[test] + fn get_ptc_reads_back_what_process_ptc_window_just_filled() { + let mut state = gloas_state_with_validators(64); + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 0); + + let epoch_before = get_current_epoch(&state); + process_ptc_window(&mut state).unwrap(); + + // `get_ptc`'s future-epoch branch only accepts up to + // `state_epoch + MIN_SEED_LOOKAHEAD`, so read the freshly-filled + // slice back the way `process_epoch` actually leaves it for: after + // the state's own slot has advanced into the next epoch, which is + // what `process_slots` does once epoch processing finishes and is + // what turns the slice this call just filled + // (`epoch_before + MIN_SEED_LOOKAHEAD + 1`) into the new + // `state_epoch + MIN_SEED_LOOKAHEAD` boundary. + *state.slot_mut() += preset::SLOTS_PER_EPOCH; + + let next_epoch = epoch_before + preset::MIN_SEED_LOOKAHEAD + 1; + let slot = compute_start_slot_at_epoch(next_epoch); + let expected = { + let committees = CommitteeCache::default(); + compute_ptc(&state, slot, &committees).unwrap() + }; + let read_back = get_ptc(&state, slot, &config).unwrap(); + assert_eq!(&read_back[..], &expected[..]); + } +} diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs index b8e50f47c..a68fcc1c1 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/mod.rs @@ -18,13 +18,14 @@ pub mod altair; pub mod capella; pub mod electra; pub mod fulu; +pub mod gloas; pub mod justification; pub mod registry; pub mod rewards; use crate::beacon::containers::phase0::PendingAttestation; use crate::beacon::containers::{BeaconState, HistoricalBatch}; -use crate::beacon::error::{Error, Result, verify}; +use crate::beacon::error::{Result, verify}; use crate::beacon::fork::ForkName; use crate::beacon::helpers::accessors::{ CommitteeCache, CommitteeCacheExt, get_block_root, get_block_root_at_slot, get_current_epoch, @@ -92,12 +93,12 @@ pub fn process_epoch(state: &mut BeaconState, config: &Config) -> Result<()> { ForkName::Electra => electra::process_epoch(state, config), ForkName::Fulu => fulu::process_epoch(state, config), // EIP-7732's builder registry, payment queues, and payload - // availability each need their own epoch-boundary steps; a later - // task writes gloas's own driver. - ForkName::Gloas => Err(Error::UnsupportedForFork { - function: "process_epoch", - fork: ForkName::Gloas, - }), + // availability each need their own epoch-boundary steps + // (`process_builder_pending_payments`, `process_ptc_window`), and + // EIP-7688's progressive participation, inactivity, and pending-queue + // lists need their own copies of several more (see `gloas`'s own + // module doc for which), so gloas gets its own driver too. + ForkName::Gloas => gloas::process_epoch(state, config), ForkName::Lean => lean_state_unreachable("process_epoch"), } } @@ -113,7 +114,13 @@ pub fn process_epoch(state: &mut BeaconState, config: &Config) -> Result<()> { /// /// Altair rewrote the step to read participation flags instead of replaying /// stored attestations, and no later fork changes it again, so altair's version -/// serves everything from altair on. +/// serves everything from altair on, gloas included: nothing in gloas's own +/// `beacon-chain.md` touches this step either, and +/// [`crate::beacon::containers::BeaconState::altair_validator_lists`] now reaches +/// a gloas state's participation lists too (as plain slices, since gloas's are +/// progressive rather than `SszList`; see that accessor's own doc), which is +/// what lets this one copy serve every fork from altair on without a +/// gloas-specific one. pub fn process_justification_and_finalization( state: &mut BeaconState, config: &Config, @@ -125,14 +132,8 @@ pub fn process_justification_and_finalization( | ForkName::Capella | ForkName::Deneb | ForkName::Electra - | ForkName::Fulu => altair::process_justification_and_finalization(state), - // Almost certainly unchanged from altair (nothing in gloas's own - // `beacon-chain.md` section touches justification), but left as an - // explicit gap rather than assumed until confirmed against the spec. - ForkName::Gloas => Err(Error::UnsupportedForFork { - function: "process_justification_and_finalization", - fork: ForkName::Gloas, - }), + | ForkName::Fulu + | ForkName::Gloas => altair::process_justification_and_finalization(state), ForkName::Lean => lean_state_unreachable("process_justification_and_finalization"), } } diff --git a/crates/blockchain/state_transition/tests/beacon_spec/epoch_processing.rs b/crates/blockchain/state_transition/tests/beacon_spec/epoch_processing.rs index 4be921334..ebe40f15d 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/epoch_processing.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/epoch_processing.rs @@ -81,18 +81,18 @@ fn apply( // its own step. "rewards_and_penalties" => match fork { ForkName::Phase0 => epoch::rewards::process_rewards_and_penalties(state, config), + // `BeaconState::altair_validator_lists` now reaches a gloas + // state's participation and inactivity lists too, as plain + // slices (progressive rather than `SszList`, EIP-7688, but both + // `Deref` to `[T]`), so this one copy serves gloas as well; see + // that accessor's own doc. ForkName::Altair | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb | ForkName::Electra - | ForkName::Fulu => epoch::altair::process_rewards_and_penalties(state, config), - ForkName::Gloas => Err( - ethlambda_state_transition::beacon::Error::UnsupportedForFork { - function: "rewards_and_penalties", - fork: ForkName::Gloas, - }, - ), + | ForkName::Fulu + | ForkName::Gloas => epoch::altair::process_rewards_and_penalties(state, config), ForkName::Lean => lean_is_not_a_fixture_fork("rewards_and_penalties"), }, // Deneb's own change (EIP-7514's activation-churn cap) is selected @@ -105,7 +105,13 @@ fn apply( // deneb's superseded rule, so routing them here would fail loudly, not // incorrectly. "registry_updates" => match fork { - ForkName::Electra | ForkName::Fulu => { + // Gloas does not redefine this function either: it stays + // electra's balance-denominated version, with no activation-churn + // cap, and it only touches `validators`/`balances`, which + // `BeaconState` already reaches through element accessors generic + // over the bounded/progressive split, so a gloas state runs the + // identical function unchanged. + ForkName::Electra | ForkName::Fulu | ForkName::Gloas => { epoch::electra::process_registry_updates(state, config) } ForkName::Phase0 @@ -113,12 +119,6 @@ fn apply( | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb => epoch::registry::process_registry_updates(state, config), - ForkName::Gloas => Err( - ethlambda_state_transition::beacon::Error::UnsupportedForFork { - function: "registry_updates", - fork: ForkName::Gloas, - }, - ), ForkName::Lean => lean_is_not_a_fixture_fork("registry_updates"), }, // Altair's and bellatrix's own changes here are scoped to the @@ -131,18 +131,17 @@ fn apply( // rounding, so electra and fulu need their own function, not just their // own constant. "slashings" => match fork { - ForkName::Electra | ForkName::Fulu => epoch::electra::process_slashings(state, config), + // Unmodified in gloas too, and, like `registry_updates` above, + // only touches `validators`/`balances` and `state.slashings()` + // (a fork-invariant field), so it runs unchanged on a gloas state. + ForkName::Electra | ForkName::Fulu | ForkName::Gloas => { + epoch::electra::process_slashings(state, config) + } ForkName::Phase0 | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb => epoch::registry::process_slashings(state, config), - ForkName::Gloas => Err( - ethlambda_state_transition::beacon::Error::UnsupportedForFork { - function: "slashings", - fork: ForkName::Gloas, - }, - ), ForkName::Lean => lean_is_not_a_fixture_fork("slashings"), }, "eth1_data_reset" => epoch::process_eth1_data_reset(state), @@ -152,7 +151,9 @@ fn apply( // compounding validator's can be far higher, and fulu never reverts // that, so both route to electra's. "effective_balance_updates" => match fork { - ForkName::Electra | ForkName::Fulu => { + // Same reasoning as `registry_updates`/`slashings` above: unmodified + // in gloas, and reaches only `validators`/`balances`. + ForkName::Electra | ForkName::Fulu | ForkName::Gloas => { epoch::electra::process_effective_balance_updates(state) } ForkName::Phase0 @@ -160,12 +161,6 @@ fn apply( | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb => epoch::process_effective_balance_updates(state), - ForkName::Gloas => Err( - ethlambda_state_transition::beacon::Error::UnsupportedForFork { - function: "effective_balance_updates", - fork: ForkName::Gloas, - }, - ), ForkName::Lean => lean_is_not_a_fixture_fork("effective_balance_updates"), }, "slashings_reset" => epoch::process_slashings_reset(state), @@ -177,46 +172,138 @@ fn apply( "historical_roots_update" => epoch::process_historical_roots_update(state), // New in capella, replacing `historical_roots_update` above, and // carried unchanged through every later fork: `historical_summaries_mut` - // itself accepts capella, deneb, electra, and fulu states alike (see - // its own doc for why), so one call here serves all four. + // itself accepts capella, deneb, electra, fulu, and gloas states alike + // (see its own doc for why), so one call here serves all five. "historical_summaries_update" => epoch::capella::process_historical_summaries_update(state), // Phase0 only: altair replaces the backlog this replays with // participation flags, which have no epoch-end "roll the backlog // over" step of their own to speak of here (see // `participation_flag_updates` below). "participation_record_updates" => epoch::process_participation_record_updates(state), - // New in altair; no phase0 case ever reaches these, and no later - // fork's own `process_epoch` redefines any of the three (each calls - // straight through to `epoch::altair`, per its own module doc), so one - // call per handler serves every fork that has it. - "inactivity_updates" => epoch::altair::process_inactivity_updates(state, config), - "participation_flag_updates" => epoch::altair::process_participation_flag_updates(state), + // New in altair; no phase0 case ever reaches these. Bellatrix through + // fulu each call straight through to `epoch::altair`, per each fork + // module's own doc. `inactivity_updates` reads gloas too, below: its + // participation and inactivity lists are progressive (EIP-7688), a + // different Rust type from the bounded `SszList` altair's own copy + // used to need, but `BeaconState::inactivity_scores_mut` and + // `altair_validator_lists` both now reach either list kind through a + // shared slice type. `participation_flag_updates`, just below, is the + // one exception: it *replaces* the whole list rather than writing an + // element, which only the fork's own concrete container type can do, + // so gloas keeps its own copy of that one step; see + // `epoch::gloas`'s own module doc. + "inactivity_updates" => match fork { + // `BeaconState::inactivity_scores_mut` reaches a gloas state's + // scores too (element writes only, which both list kinds allow + // through `DerefMut`; see that accessor's own doc), so this one + // copy serves gloas as well. + ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Fulu + | ForkName::Gloas => epoch::altair::process_inactivity_updates(state, config), + ForkName::Phase0 => Err( + ethlambda_state_transition::beacon::Error::UnsupportedForFork { + function: "inactivity_updates", + fork: ForkName::Phase0, + }, + ), + ForkName::Lean => lean_is_not_a_fixture_fork("inactivity_updates"), + }, + "participation_flag_updates" => match fork { + ForkName::Gloas => epoch::gloas::process_participation_flag_updates(state), + ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Fulu => epoch::altair::process_participation_flag_updates(state), + ForkName::Phase0 => Err( + ethlambda_state_transition::beacon::Error::UnsupportedForFork { + function: "participation_flag_updates", + fork: ForkName::Phase0, + }, + ), + ForkName::Lean => lean_is_not_a_fixture_fork("participation_flag_updates"), + }, + // Unmodified in gloas (both sync committees are fork-invariant + // fields, `BeaconState::sync_committees`/`_mut` already list gloas + // among the forks they serve), so one call still serves every fork. "sync_committee_updates" => epoch::altair::process_sync_committee_updates(state), // New in electra (EIP-7251). Fulu's driver keeps calling the very // same `process_pending_consolidations` (see `electra::process_epoch`'s // own doc for why that one needs no rewrite), but `pending_deposits` // has its own fulu-specific function: `epoch::fulu::process_pending_deposits`'s // own doc explains why running electra's version under fulu would be - // wrong, not merely redundant. + // wrong, not merely redundant. Gloas modifies `pending_deposits` + // again (EIP-8061: its own activation-only churn budget) and needs + // its own copy of `pending_consolidations` too, purely to reach the + // progressive queue fields `electra::pending_queue_fields` refuses a + // gloas state for; see `epoch::gloas`'s own module doc. "pending_deposits" => match fork { ForkName::Fulu => epoch::fulu::process_pending_deposits(state, config), + ForkName::Gloas => epoch::gloas::process_pending_deposits(state, config), ForkName::Phase0 | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb | ForkName::Electra => epoch::electra::process_pending_deposits(state, config), - ForkName::Gloas => Err( + ForkName::Lean => lean_is_not_a_fixture_fork("pending_deposits"), + }, + // Gloas-only: a second handler over the very same `process_pending_deposits` + // (its cases are named `activation_churn__*`), split out from + // `pending_deposits` above purely to cover EIP-8061's new cap + // (`max_per_epoch_activation_churn_limit_gloas`) in isolation. No + // `process_pending_deposits_churn` exists in `beacon-chain.md`; this + // is not a new sub-transition. + "pending_deposits_churn" => epoch::gloas::process_pending_deposits(state, config), + // Unmodified in gloas too (`beacon-chain.md` does not touch this + // function), and `PendingQueueFields::take_pending_consolidations`/ + // `set_pending_consolidations` reach a gloas state's queue at the + // `Vec` level, so this one copy serves it as well; see + // `epoch::electra::process_pending_consolidations`'s own doc. + "pending_consolidations" => match fork { + ForkName::Electra | ForkName::Fulu | ForkName::Gloas => { + epoch::electra::process_pending_consolidations(state, config) + } + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb => Err( ethlambda_state_transition::beacon::Error::UnsupportedForFork { - function: "pending_deposits", - fork: ForkName::Gloas, + function: "pending_consolidations", + fork, }, ), - ForkName::Lean => lean_is_not_a_fixture_fork("pending_deposits"), + ForkName::Lean => lean_is_not_a_fixture_fork("pending_consolidations"), + }, + // New in fulu (EIP-7917). Gloas keeps `proposer_lookahead` itself + // unchanged from fulu too, and `epoch::fulu::process_proposer_lookahead` + // now picks the proposer-drawing callee by fork internally (EIP-8045 + // excludes slashed validators for gloas), so this one copy serves + // both; see that function's own doc. + "proposer_lookahead" => match fork { + ForkName::Fulu | ForkName::Gloas => epoch::fulu::process_proposer_lookahead(state), + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra => Err( + ethlambda_state_transition::beacon::Error::UnsupportedForFork { + function: "proposer_lookahead", + fork, + }, + ), + ForkName::Lean => lean_is_not_a_fixture_fork("proposer_lookahead"), }, - "pending_consolidations" => epoch::electra::process_pending_consolidations(state, config), - // New in fulu (EIP-7917); no earlier fork has this handler at all. - "proposer_lookahead" => epoch::fulu::process_proposer_lookahead(state), + // New in gloas (EIP-7732); no earlier fork has either handler at all. + "builder_pending_payments" => epoch::gloas::process_builder_pending_payments(state), + "ptc_window" => epoch::gloas::process_ptc_window(state), other => return Err(format!("unhandled epoch step `{other}`")), }; @@ -277,8 +364,11 @@ fn every_shipped_handler_is_dispatched() -> Trial { "participation_flag_updates", "sync_committee_updates", "pending_deposits", + "pending_deposits_churn", "pending_consolidations", "proposer_lookahead", + "builder_pending_payments", + "ptc_window", ]; let mut unknown: Vec = collect_all_handlers(PRESET, "epoch_processing") diff --git a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs index c21afbd2b..0bc2946c7 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs @@ -93,10 +93,9 @@ pub const HIGHEST_IMPLEMENTED_FORK: ForkName = ForkName::Fulu; /// runner's Gloas cases stay ignored. Emptied, and removed along with this /// check, when [`HIGHEST_IMPLEMENTED_FORK`] becomes `Gloas`. /// -/// `"fork"` (not `"transition"`): `upgrade_to_gloas` is done, so the `fork` -/// suite's own upgrade-only cases pass, but `transition`'s Gloas cases also -/// exercise Gloas block processing, which does not exist yet. -pub const GLOAS_RUNNERS: &[&str] = &["ssz_static", "fork"]; +/// `"transition"` is not listed: its Gloas cases also exercise Gloas block +/// processing, which does not exist yet. +pub const GLOAS_RUNNERS: &[&str] = &["ssz_static", "fork", "epoch_processing", "rewards"]; /// The root of the extracted fixture tree. /// diff --git a/crates/blockchain/state_transition/tests/beacon_spec/rewards.rs b/crates/blockchain/state_transition/tests/beacon_spec/rewards.rs index fef71f3ab..b2eecb833 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/rewards.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/rewards.rs @@ -129,13 +129,20 @@ fn components( // No `inclusion_delay_deltas` here: see this module's doc for why // altair has nothing left for that component to compute. Bellatrix // through fulu reuse this same list; see this module's doc for why - // none of them changes it. + // none of them changes it. Gloas reuses it too: + // `BeaconState::altair_validator_lists` now reaches a gloas state's + // participation and inactivity lists as plain slices (progressive + // rather than `SszList`, EIP-7688, but both `Deref` to `[T]`), which + // is all `altair_helpers::get_flag_index_deltas`/ + // `get_inactivity_penalty_deltas` ever need; see that accessor's own + // doc. ForkName::Altair | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb | ForkName::Electra - | ForkName::Fulu => vec![ + | ForkName::Fulu + | ForkName::Gloas => vec![ ( "source_deltas", altair_helpers::get_flag_index_deltas(state, TIMELY_SOURCE_FLAG_INDEX), @@ -153,12 +160,6 @@ fn components( altair_helpers::get_inactivity_penalty_deltas(state, config), ), ], - // Gloas's `previous_epoch_participation`/`current_epoch_participation` - // are progressive lists (EIP-7688), a different Rust type from the - // `SszList` `altair_helpers::get_flag_index_deltas` reads, so this is - // not altair's list unchanged even though the reward rule itself is; - // a later task writes gloas's own copy over the progressive type. - ForkName::Gloas => return Err("gloas not implemented".into()), ForkName::Lean => lean_is_not_a_fixture_fork("rewards"), }; diff --git a/crates/common/types/src/beacon/containers/mod.rs b/crates/common/types/src/beacon/containers/mod.rs index aa4cca202..fe6139668 100644 --- a/crates/common/types/src/beacon/containers/mod.rs +++ b/crates/common/types/src/beacon/containers/mod.rs @@ -57,7 +57,7 @@ use crate::beacon::error::{Error, Result}; use crate::beacon::fork::ForkName; use crate::beacon::primitives::{ BlsSignature, Bytes32, CommitteeIndex, Epoch, ExecutionBlockHash, Gwei, HashTreeRoot as _, - Root, Slot, ValidatorIndex, WithdrawalIndex, + ParticipationFlags, Root, Slot, ValidatorIndex, WithdrawalIndex, }; use crate::beacon::{beacon_value_unreachable, lean_block_unreachable, lean_state_unreachable}; @@ -870,24 +870,28 @@ impl BeaconState { } /// The three per-validator lists that exist from altair on, by reference and - /// all at once. + /// all at once, as plain slices. /// /// These cannot join `shared_state_accessors`' lists, since phase0 has /// none of them, and a per-fork projection to a concrete state struct (the /// way the beacon STF's `helpers::altair::altair_state_ref` reaches them) /// cannot serve every fork that carries them: bellatrix, capella, deneb, electra, - /// and fulu all keep the identical three fields, but each is a distinct - /// Rust type, so a projection typed to return `&altair::BeaconState` can - /// only ever answer for an altair state. + /// fulu, and gloas all keep the identical three fields, but each fork's own + /// struct is a distinct Rust type, so a projection typed to return + /// `&altair::BeaconState` can only ever answer for an altair state. /// - /// Gloas is in `absent_from` here, not `carried_by`: EIP-7688 makes all - /// three progressive lists, so `gloas::BeaconState`'s own - /// `previous_epoch_participation`, `current_epoch_participation` and - /// `inactivity_scores` are a different Rust type from every other fork's - /// (`ProgressiveList` rather than `SszList`), and this accessor's return - /// type is fixed to the pre-gloas one. A gloas caller reaches them through - /// a per-fork projection instead, the way `withdrawal_cursor`'s doc above - /// explains a projection cannot serve *this* accessor. + /// Gloas joins `carried_by` here, unlike [`Self::altair_validator_lists_mut`]: + /// EIP-7688 makes all three progressive lists on a gloas state + /// (`ProgressiveList` rather than `SszList`), a different concrete Rust + /// type from every earlier fork's, but both list kinds `Deref` to a plain + /// `[T]`, and a slice is all a *read* ever needs (`.get`, `.len`, + /// `.binary_search`, ...). Returning slices rather than the list types + /// themselves is what lets one `dispatch_state_from!` body serve both + /// kinds: the write side still needs the concrete container type, to grow + /// or replace the whole list, which is why [`Self::altair_validator_lists_mut`] + /// stays bounded-only. On gloas, element writes go through + /// [`Self::inactivity_scores_mut`], and growing or replacing a list goes + /// through a per-fork projection. /// /// Handed back together rather than one accessor per field for the same /// reason [`Self::altair_validator_lists_mut`] does: the fork condition @@ -896,23 +900,25 @@ impl BeaconState { /// the rest away with `_`. pub fn altair_validator_lists( &self, - ) -> Result<(&EpochParticipation, &EpochParticipation, &InactivityScores)> { + ) -> Result<(&[ParticipationFlags], &[ParticipationFlags], &[u64])> { dispatch_state_from!( self, "BeaconState::altair_validator_lists", |state| ( - &state.previous_epoch_participation, - &state.current_epoch_participation, - &state.inactivity_scores, + &state.previous_epoch_participation[..], + &state.current_epoch_participation[..], + &state.inactivity_scores[..], ), - carried_by: [Altair, Bellatrix, Capella, Deneb, Electra, Fulu], - absent_from: [Phase0, Gloas], + carried_by: [Altair, Bellatrix, Capella, Deneb, Electra, Fulu, Gloas], + absent_from: [Phase0], ) } /// The three per-validator lists that exist from altair on, mutably and all - /// at once. See [`Self::altair_validator_lists`] for why this cannot be a - /// per-fork projection instead. + /// at once, as the fork's own concrete container type. See + /// [`Self::altair_validator_lists`] for why this cannot be a per-fork + /// projection instead, and for why, unlike that read-only accessor, this + /// one does not extend to gloas. /// /// Handed back together for two reasons that stack: /// the beacon STF's `stf::operations::add_validator_to_registry` genuinely @@ -927,6 +933,18 @@ impl BeaconState { /// Borrowing three fields of one struct at once is what the tuple is for. /// Rust permits it because the fields are disjoint, whereas three successive /// accessor calls would each borrow the whole enum. + /// + /// Gloas is in `absent_from` here, not `carried_by`: the container type + /// this returns (`EpochParticipation`/`InactivityScores`, both `SszList`) + /// is fixed to the pre-gloas one, and `gloas::BeaconState`'s own three + /// lists are the progressive `ProgressiveList` instead, a different Rust + /// type a shared return type cannot name. A gloas caller either grows the + /// registry through a per-fork projection + /// (`PendingQueueFields::push_empty_participation_and_inactivity`), writes + /// one score in place through + /// [`Self::inactivity_scores_mut`], or replaces a whole list outright + /// through its own state's own field, the way + /// `stf::epoch::gloas::process_participation_flag_updates` does. pub fn altair_validator_lists_mut( &mut self, ) -> Result<( @@ -947,6 +965,30 @@ impl BeaconState { ) } + /// `inactivity_scores`, mutably, as a plain slice: element writes only, no + /// whole-list replace or length change, which is what lets this include + /// gloas where [`Self::altair_validator_lists_mut`] cannot (see that + /// accessor's own doc). A `&mut [u64]` cannot grow or shrink, so handing + /// one out cannot break either list kind's own length invariant, the same + /// reasoning [`libssz_types::ProgressiveList`]'s own `DerefMut` doc gives + /// for allowing element mutation but not resizing through a slice. + /// + /// `stf::epoch::altair::process_inactivity_updates` (shared by every fork + /// from altair on, gloas included) is the one caller: it only ever writes + /// scores in place, never grows or replaces the list, which + /// `add_validator_to_registry` and `process_participation_flag_updates` + /// do instead (see [`Self::altair_validator_lists_mut`]'s own doc for + /// where each of those goes). + pub fn inactivity_scores_mut(&mut self) -> Result<&mut [u64]> { + dispatch_state_from!( + self, + "BeaconState::inactivity_scores_mut", + |state| &mut state.inactivity_scores[..], + carried_by: [Altair, Bellatrix, Capella, Deneb, Electra, Fulu, Gloas], + absent_from: [Phase0], + ) + } + /// The current and next sync committee, by reference. /// /// Both exist from altair on, byte-for-byte the same field in every later From 5b396db22f75fad5ab5a8eb414fed298b211c489 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 28 Sep 2026 11:40:20 -0300 Subject: [PATCH 018/132] feat(beacon): add gloas withdrawals, bids and parent payloads Ports the withdrawal sweeps, the execution payload bid and the parent execution payload processing EIP-7732 introduces. A gloas block carries no payload: it commits to a builder's bid (process_execution_payload_bid), and the previous slot's payload is applied only once the next block shows whether it arrived (process_parent_execution_payload and apply_parent_execution_payload). Withdrawals gain a builder payout sweep and a builder registry sweep ahead of the validator ones, and process_withdrawals takes no payload, since the sweep now follows from the state alone. apply_parent_execution_payload is where gloas applies the execution-layer requests: deposits, withdrawals and consolidations, which gloas's process_operations no longer calls, and the new builder deposit and exit requests. Gloas leaves the first three unmodified, so electra's and fulu's processors serve it. They differ only in the list type behind the pending queues, which is progressive in gloas, and one PendingQueueFields projection now covers electra, fulu and gloas: slice views and element pushes for the block-level processors, take and set pairs for epoch processing. Electra's process_voluntary_exit works on a gloas state the same way. The pending deposit take and set refuse a gloas state, since gloas modifies process_pending_deposits and drains that queue itself. Electra's get_expected_withdrawals is split into get_pending_partial_withdrawals and get_validators_sweep_withdrawals, along the specification's own function boundaries, so gloas's modified get_expected_withdrawals calls them unchanged. withdrawal_address has one definition, in helpers::capella. GLOAS_RUNNERS accepts "runner/handler" entries, so a runner can turn its gloas handlers on one at a time. Enabled here: operations' execution_payload_bid, withdrawals, parent_execution_payload, deposit_request, withdrawal_request, consolidation_request, builder_deposit_request and builder_exit_request. --- .../src/beacon/helpers/capella.rs | 15 +- .../src/beacon/helpers/electra.rs | 455 ++++++-- .../src/beacon/stf/capella.rs | 19 +- .../src/beacon/stf/electra.rs | 337 +++--- .../src/beacon/stf/epoch/electra.rs | 270 +---- .../src/beacon/stf/epoch/fulu.rs | 40 +- .../src/beacon/stf/epoch/gloas.rs | 6 +- .../state_transition/src/beacon/stf/fulu.rs | 16 +- .../state_transition/src/beacon/stf/gloas.rs | 967 ++++++++++++++++++ .../state_transition/src/beacon/stf/mod.rs | 1 + .../state_transition/src/beacon/upgrade.rs | 11 +- .../state_transition/tests/beacon_spec/mod.rs | 60 +- .../tests/beacon_spec/operations.rs | 83 +- 13 files changed, 1757 insertions(+), 523 deletions(-) create mode 100644 crates/blockchain/state_transition/src/beacon/stf/gloas.rs diff --git a/crates/blockchain/state_transition/src/beacon/helpers/capella.rs b/crates/blockchain/state_transition/src/beacon/helpers/capella.rs index da190912f..a8399a06e 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/capella.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/capella.rs @@ -30,7 +30,20 @@ use crate::beacon::constants; use crate::beacon::containers::shared::Validator; use crate::beacon::preset; -use crate::beacon::primitives::{Epoch, Gwei}; +use crate::beacon::primitives::{Epoch, ExecutionAddress, Gwei}; + +/// The execution address a validator's payout is sent to: the low bytes of +/// its withdrawal credentials, present regardless of whether those +/// credentials are eth1 or (from electra on) compounding, both of which are +/// execution-form. +/// +/// `pub(crate)`, not private: capella's, electra's, and gloas's own +/// withdrawal sweeps (`crate::beacon::stf::capella`, `crate::beacon::stf::electra`, +/// `crate::beacon::stf::gloas`) all read a withdrawable validator's payout +/// address the same way, so this is the one copy rather than three. +pub(crate) fn withdrawal_address(validator: &Validator) -> ExecutionAddress { + ExecutionAddress::from_slice(&validator.withdrawal_credentials.0[12..]) +} /// Whether `validator`'s withdrawal credentials have been upgraded to an /// execution address. diff --git a/crates/blockchain/state_transition/src/beacon/helpers/electra.rs b/crates/blockchain/state_transition/src/beacon/helpers/electra.rs index efdbf9e14..4e14498a7 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/electra.rs @@ -113,9 +113,11 @@ //! accessor on [`BeaconState`]. Unlike altair's participation flags (see //! [`crate::beacon::helpers::altair::altair_state`]'s doc), these fields are not //! electra-only: fulu keeps every one of them unchanged (see the [`fulu`] -//! module doc), so [`electra_state`] and [`electra_state_ref`] match both -//! `BeaconState::Electra` and `BeaconState::Fulu` rather than only the -//! former. A projection that matched only `Electra`, the way +//! module doc), and gloas keeps the churn cursors and (behind a slice view; +//! see [`PendingQueueFields`]'s own doc) the pending queues unchanged too, so +//! [`churn_cursors_mut`] and [`pending_queue_fields`]/[`pending_queue_fields_ref`] +//! match `BeaconState::Electra`, `BeaconState::Fulu`, and `BeaconState::Gloas` +//! rather than only the first. A projection that matched only `Electra`, the way //! [`crate::beacon::helpers::altair::altair_state`] matches only `Altair`, would //! silently break every one of these functions on a fulu state, which is the //! single easiest mistake to make copying that shape without also copying @@ -387,14 +389,23 @@ pub fn get_consolidation_churn_limit(state: &BeaconState, config: &Config) -> Re /// The total amount queued in [`electra::PendingPartialWithdrawal`]s for /// `index`, not yet paid out. /// -/// Read by electra's execution layer consolidation request processing (not -/// implemented in this file) to refuse consolidating a validator that still -/// has a partial withdrawal in flight: consolidating it out from under that -/// withdrawal would leave nothing left to pay the withdrawal from. +/// Read by [`crate::beacon::stf::electra::process_withdrawal_request`] and +/// [`crate::beacon::stf::electra::process_consolidation_request`] to refuse consolidating, +/// or fully exiting, a validator that still has a partial withdrawal in +/// flight: doing either out from under that withdrawal would leave nothing +/// left to pay it from. +/// +/// Also served, unmodified, by gloas: nothing in gloas's own `beacon-chain.md` +/// touches this function, and the only thing that kept a gloas state from +/// calling this exact copy was the queue's own type +/// ([`crate::beacon::containers::gloas::PendingPartialWithdrawals`], a +/// [`libssz_types::ProgressiveList`], EIP-7688), which +/// [`PendingQueueFieldsRef::pending_partial_withdrawals`] already abstracts +/// over as a slice. pub fn get_pending_balance_to_withdraw(state: &BeaconState, index: ValidatorIndex) -> Result { - let fields = electra_state_ref(state, "get_pending_balance_to_withdraw")?; + let fields = pending_queue_fields_ref(state, "get_pending_balance_to_withdraw")?; let mut total: Gwei = 0; - for withdrawal in fields.pending_partial_withdrawals().iter() { + for withdrawal in fields.pending_partial_withdrawals() { if withdrawal.validator_index == index { total = total.saturating_add(withdrawal.amount); } @@ -582,7 +593,7 @@ pub fn initiate_validator_exit( /// budget rather than [`get_activation_exit_churn_limit`]'s combined one. /// Nothing past that line differs, so gloas does not get a copy of its own: /// this picks the limit by fork and otherwise runs the one shared cursor -/// update, through [`ChurnCursorsMut`] rather than [`ElectraOrFuluMut`] (see +/// update, through [`ChurnCursorsMut`] rather than [`PendingQueueFields`] (see /// that enum's own doc for why). Gloas modifies neither /// `initiate_validator_exit` (so neither voluntary exits nor registry /// ejections) nor `process_withdrawal_request`, so calling this from any @@ -804,10 +815,7 @@ pub fn queue_excess_active_balance(state: &mut BeaconState, index: ValidatorInde *state.balance_mut(index)? = preset::MIN_ACTIVATION_BALANCE; let deposit = placeholder_pending_deposit(state.validator(index)?, excess_balance); - electra_state(state, "queue_excess_active_balance")? - .pending_deposits_mut() - .push(deposit)?; - Ok(()) + pending_queue_fields(state, "queue_excess_active_balance")?.push_pending_deposit(deposit) } /// Zeroes a validator's balance and effective balance, resets its activation @@ -836,10 +844,8 @@ pub fn queue_entire_balance_and_reset_validator( validator.activation_eligibility_epoch = FAR_FUTURE_EPOCH; let deposit = placeholder_pending_deposit(state.validator(index)?, balance); - electra_state(state, "queue_entire_balance_and_reset_validator")? - .pending_deposits_mut() - .push(deposit)?; - Ok(()) + pending_queue_fields(state, "queue_entire_balance_and_reset_validator")? + .push_pending_deposit(deposit) } // --------------------------------------------------------------------------- @@ -855,7 +861,12 @@ pub fn queue_entire_balance_and_reset_validator( /// `eth_fast_aggregate_verify`'s empty-committee case, but does not export /// it, so this file builds its own rather than reaching into that module's /// internals. -fn g2_point_at_infinity() -> BlsSignature { +/// +/// `pub(crate)`: gloas's +/// [`crate::beacon::stf::gloas::process_execution_payload_bid`] requires a +/// self-built bid to carry exactly this signature, since a self-build is +/// not signed by a builder. +pub(crate) fn g2_point_at_infinity() -> BlsSignature { let mut bytes = [0u8; BLS_SIGNATURE_SIZE]; // The top two bits are the compression and infinity flags; setting both // and leaving every other bit zero is the point at infinity's compressed @@ -879,55 +890,283 @@ fn placeholder_pending_deposit(validator: &Validator, amount: Gwei) -> electra:: } } -/// Either fork whose state carries electra's balance-churn accounting and -/// pending-deposit/withdrawal queues (EIP-7251) unchanged: electra itself, or -/// fulu, which never redefines any of the fields read through this. See this -/// module's doc for why both are accepted, and [`electra_state`] for the -/// mutable counterpart. -pub(crate) enum ElectraOrFulu<'a> { - Electra(&'a electra::BeaconState), - Fulu(&'a fulu::BeaconState), +/// Fields the execution-layer-triggered request processors gloas leaves +/// unmodified need, gathered from wherever they sit on electra's, fulu's, or +/// gloas's own state: `deposit_requests_start_index`, the +/// `deposit_balance_to_consume` churn cursor, a new validator's empty +/// participation and inactivity entries +/// ([`Self::push_empty_participation_and_inactivity`]), and the +/// three pending queues (`pending_deposits`, `pending_partial_withdrawals`, +/// `pending_consolidations`), both as element-level slice views (`len()`, +/// `iter()`/`get()` through the returned `&[T]`, one-at-a-time `push_*`) and +/// as a whole (`take_*`/`set_*`, for epoch processing's own whole-queue +/// drains). +/// +/// Electra's, fulu's, and gloas's `pending_deposits` (and the other two +/// queues) share one element type each but not one container: electra's and +/// fulu's is the bounded `SszList` (`electra::PendingDeposits` and friends), +/// gloas's is EIP-7688's unbounded [`libssz_types::ProgressiveList`] +/// (`crate::beacon::containers::gloas::PendingDeposits` and friends). Both +/// deref to `[T]` with the same element type, which is what lets the +/// slice-view and element-`push_*` methods below serve all three forks +/// through one match rather than needing a gloas copy of every caller. +/// +/// Epoch processing's whole-queue drains and the block-level request +/// processors (`crate::beacon::stf::electra`'s `process_deposit_request` +/// and friends) share this one projection rather than each keeping a +/// near-identical one. +/// +/// Deliberately separate from [`ChurnCursorsMut`]: that one carries the four +/// exit/consolidation churn cursor fields, which nothing here reads or +/// writes, and every gloas caller that needs those reaches them through +/// `ChurnCursorsMut` directly, not through this projection. +pub(crate) enum PendingQueueFields<'a> { + Electra(&'a mut electra::BeaconState), + Fulu(&'a mut fulu::BeaconState), + Gloas(&'a mut gloas::BeaconState), } -impl<'a> ElectraOrFulu<'a> { - /// Partial withdrawals queued but not yet paid out, read by - /// [`get_pending_balance_to_withdraw`]. - pub(crate) fn pending_partial_withdrawals(&self) -> &electra::PendingPartialWithdrawals { +impl<'a> PendingQueueFields<'a> { + /// The execution-layer deposit request index at which the state switched + /// from crediting deposits off `Eth1Data` votes to crediting them off + /// `DepositRequest`s directly, read by + /// `crate::beacon::stf::epoch::electra::process_pending_deposits` to know + /// whether any eth1-bridge deposit is still outstanding. + /// + /// `pub(crate)`: fulu's own tests read this to assert a test state starts + /// with it at [`crate::beacon::constants::UNSET_DEPOSIT_REQUESTS_START_INDEX`]. + /// Fulu's and gloas's own `process_pending_deposits` never read this + /// field at all. + pub(crate) fn deposit_requests_start_index(&self) -> u64 { match self { - ElectraOrFulu::Electra(state) => &state.pending_partial_withdrawals, - ElectraOrFulu::Fulu(state) => &state.pending_partial_withdrawals, + PendingQueueFields::Electra(state) => state.deposit_requests_start_index, + PendingQueueFields::Fulu(state) => state.deposit_requests_start_index, + PendingQueueFields::Gloas(state) => state.deposit_requests_start_index, } } -} -/// The mutable counterpart of [`ElectraOrFulu`]. See [`electra_state`]. -pub(crate) enum ElectraOrFuluMut<'a> { - Electra(&'a mut electra::BeaconState), - Fulu(&'a mut fulu::BeaconState), -} + /// How much of this epoch's deposit balance churn limit remains unused. + pub(crate) fn deposit_balance_to_consume(&self) -> Gwei { + match self { + PendingQueueFields::Electra(state) => state.deposit_balance_to_consume, + PendingQueueFields::Fulu(state) => state.deposit_balance_to_consume, + PendingQueueFields::Gloas(state) => state.deposit_balance_to_consume, + } + } + + pub(crate) fn deposit_balance_to_consume_mut(&mut self) -> &mut Gwei { + match self { + PendingQueueFields::Electra(state) => &mut state.deposit_balance_to_consume, + PendingQueueFields::Fulu(state) => &mut state.deposit_balance_to_consume, + PendingQueueFields::Gloas(state) => &mut state.deposit_balance_to_consume, + } + } + + /// Queues one more [`electra::PendingDeposit`], pushed by + /// [`queue_excess_active_balance`], [`queue_entire_balance_and_reset_validator`], + /// and the deposit-request processors. Infallible on gloas's unbounded + /// queue; electra's and fulu's bounded one can still reject a length over + /// its own SSZ limit. + pub(crate) fn push_pending_deposit(&mut self, deposit: electra::PendingDeposit) -> Result<()> { + match self { + PendingQueueFields::Electra(state) => state.pending_deposits.push(deposit)?, + PendingQueueFields::Fulu(state) => state.pending_deposits.push(deposit)?, + PendingQueueFields::Gloas(state) => state.pending_deposits.push(deposit), + } + Ok(()) + } + + /// Takes the whole pending-deposits queue out, leaving an empty one + /// behind; [`Self::set_pending_deposits`] puts a (possibly shorter) one + /// back. `Vec`-level, the same shape + /// [`Self::take_pending_consolidations`]/[`Self::set_pending_consolidations`] + /// already use, for the same reason: + /// `crate::beacon::stf::epoch::electra::process_pending_deposits` needs + /// to drain the whole queue into a `Vec` it can freely mutate `state` + /// around, not a reference still borrowing it. + /// + /// Refuses gloas: gloas modifies `process_pending_deposits` and drains + /// the queue through its own state, so a gloas state reaching electra's + /// or fulu's copy is a dispatch mistake, not a caller to serve. + pub(crate) fn take_pending_deposits(&mut self) -> Result> { + match self { + PendingQueueFields::Electra(state) => { + Ok(core::mem::take(&mut state.pending_deposits).into_inner()) + } + PendingQueueFields::Fulu(state) => { + Ok(core::mem::take(&mut state.pending_deposits).into_inner()) + } + PendingQueueFields::Gloas(_) => Err(Error::UnsupportedForFork { + function: "process_pending_deposits", + fork: ForkName::Gloas, + }), + } + } + + /// See [`Self::take_pending_deposits`], including why gloas is refused. + /// Also fails when the queue outgrows electra's and fulu's bounded + /// `SszList`. + pub(crate) fn set_pending_deposits( + &mut self, + deposits: Vec, + ) -> Result<()> { + match self { + PendingQueueFields::Electra(state) => { + state.pending_deposits = electra::PendingDeposits::try_from(deposits)?; + } + PendingQueueFields::Fulu(state) => { + state.pending_deposits = electra::PendingDeposits::try_from(deposits)?; + } + PendingQueueFields::Gloas(_) => { + return Err(Error::UnsupportedForFork { + function: "process_pending_deposits", + fork: ForkName::Gloas, + }); + } + } + Ok(()) + } + + /// Partial withdrawals queued but not yet paid out, as a slice. Read by + /// [`get_pending_balance_to_withdraw`] and by + /// [`crate::beacon::stf::electra::process_withdrawal_request`]'s own queue-length + /// check. + pub(crate) fn pending_partial_withdrawals(&self) -> &[electra::PendingPartialWithdrawal] { + match self { + PendingQueueFields::Electra(state) => &state.pending_partial_withdrawals, + PendingQueueFields::Fulu(state) => &state.pending_partial_withdrawals, + PendingQueueFields::Gloas(state) => &state.pending_partial_withdrawals, + } + } + + /// Queues one more [`electra::PendingPartialWithdrawal`], pushed by + /// [`crate::beacon::stf::electra::process_withdrawal_request`]. + pub(crate) fn push_pending_partial_withdrawal( + &mut self, + withdrawal: electra::PendingPartialWithdrawal, + ) -> Result<()> { + match self { + PendingQueueFields::Electra(state) => { + state.pending_partial_withdrawals.push(withdrawal)? + } + PendingQueueFields::Fulu(state) => { + state.pending_partial_withdrawals.push(withdrawal)? + } + PendingQueueFields::Gloas(state) => state.pending_partial_withdrawals.push(withdrawal), + } + Ok(()) + } + + /// Queued consolidations, as a slice. Read by + /// [`crate::beacon::stf::electra::process_consolidation_request`]'s own queue-length + /// check. + pub(crate) fn pending_consolidations(&self) -> &[electra::PendingConsolidation] { + match self { + PendingQueueFields::Electra(state) => &state.pending_consolidations, + PendingQueueFields::Fulu(state) => &state.pending_consolidations, + PendingQueueFields::Gloas(state) => &state.pending_consolidations, + } + } + + /// Queues one more [`electra::PendingConsolidation`], pushed by + /// [`crate::beacon::stf::electra::process_consolidation_request`]. + pub(crate) fn push_pending_consolidation( + &mut self, + consolidation: electra::PendingConsolidation, + ) -> Result<()> { + match self { + PendingQueueFields::Electra(state) => { + state.pending_consolidations.push(consolidation)? + } + PendingQueueFields::Fulu(state) => state.pending_consolidations.push(consolidation)?, + PendingQueueFields::Gloas(state) => state.pending_consolidations.push(consolidation), + } + Ok(()) + } -impl<'a> ElectraOrFuluMut<'a> { - /// Deposits queued but not yet credited to the validator registry, pushed - /// to by [`queue_excess_active_balance`] and - /// [`queue_entire_balance_and_reset_validator`]. - pub(crate) fn pending_deposits_mut(&mut self) -> &mut electra::PendingDeposits { + /// Takes the whole pending-consolidations queue out, leaving an empty one + /// behind; [`Self::set_pending_consolidations`] puts a (possibly + /// shorter) one back. `Vec`-level, alongside the element-level + /// [`Self::pending_consolidations`]/[`Self::push_pending_consolidation`], + /// because `crate::beacon::stf::epoch::electra::process_pending_consolidations` + /// needs the whole queue in a `Vec` it can freely mutate `state` around + /// while draining it, not a reference still borrowing it. + pub(crate) fn take_pending_consolidations(&mut self) -> Vec { match self { - ElectraOrFuluMut::Electra(state) => &mut state.pending_deposits, - ElectraOrFuluMut::Fulu(state) => &mut state.pending_deposits, + PendingQueueFields::Electra(state) => { + core::mem::take(&mut state.pending_consolidations).into_inner() + } + PendingQueueFields::Fulu(state) => { + core::mem::take(&mut state.pending_consolidations).into_inner() + } + PendingQueueFields::Gloas(state) => { + core::mem::take(&mut state.pending_consolidations).into_inner() + } } } + + /// See [`Self::take_pending_consolidations`]. Fallible only for electra's + /// and fulu's bounded `SszList`; gloas's progressive one never rejects a + /// length. + pub(crate) fn set_pending_consolidations( + &mut self, + consolidations: Vec, + ) -> Result<()> { + match self { + PendingQueueFields::Electra(state) => { + state.pending_consolidations = + electra::PendingConsolidations::try_from(consolidations)?; + } + PendingQueueFields::Fulu(state) => { + state.pending_consolidations = + electra::PendingConsolidations::try_from(consolidations)?; + } + PendingQueueFields::Gloas(state) => { + state.pending_consolidations = gloas::PendingConsolidations::from(consolidations); + } + } + Ok(()) + } + + /// Extends `previous_epoch_participation`, `current_epoch_participation`, + /// and `inactivity_scores` by one all-zero entry each, keeping them + /// exactly as long as the registry after a new validator is appended. + /// + /// Gloas's own three lists are progressive (`libssz_types::ProgressiveList`), + /// so their `push` is infallible, unlike electra's and fulu's bounded + /// `SszList`. + pub(crate) fn push_empty_participation_and_inactivity(&mut self) -> Result<()> { + match self { + PendingQueueFields::Electra(state) => { + state.previous_epoch_participation.push(0)?; + state.current_epoch_participation.push(0)?; + state.inactivity_scores.push(0)?; + } + PendingQueueFields::Fulu(state) => { + state.previous_epoch_participation.push(0)?; + state.current_epoch_participation.push(0)?; + state.inactivity_scores.push(0)?; + } + PendingQueueFields::Gloas(state) => { + state.previous_epoch_participation.push(0); + state.current_epoch_participation.push(0); + state.inactivity_scores.push(0); + } + } + Ok(()) + } } -/// The electra-or-fulu state, mutably, or an error naming the function that -/// needs one. See [`ElectraOrFuluMut`] and this module's doc for why both -/// forks are accepted. -pub(crate) fn electra_state<'a>( +/// The electra, fulu, or gloas state, mutably, through [`PendingQueueFields`]. +/// See that enum's own doc for why gloas joins electra and fulu here. +pub(crate) fn pending_queue_fields<'a>( state: &'a mut BeaconState, function: &'static str, -) -> Result> { +) -> Result> { match state { - BeaconState::Electra(state) => Ok(ElectraOrFuluMut::Electra(state)), - BeaconState::Fulu(state) => Ok(ElectraOrFuluMut::Fulu(state)), + BeaconState::Electra(state) => Ok(PendingQueueFields::Electra(state)), + BeaconState::Fulu(state) => Ok(PendingQueueFields::Fulu(state)), + BeaconState::Gloas(state) => Ok(PendingQueueFields::Gloas(state)), other => Err(Error::UnsupportedForFork { function, fork: other.fork_name(), @@ -935,14 +1174,34 @@ pub(crate) fn electra_state<'a>( } } -/// The electra-or-fulu state, immutably. See [`electra_state`]. -pub(crate) fn electra_state_ref<'a>( +/// The read-only counterpart of [`PendingQueueFields`], for a caller (like +/// [`get_pending_balance_to_withdraw`]) that only ever reads a pending queue +/// and so only ever holds a `&BeaconState`. +pub(crate) enum PendingQueueFieldsRef<'a> { + Electra(&'a electra::BeaconState), + Fulu(&'a fulu::BeaconState), + Gloas(&'a gloas::BeaconState), +} + +impl<'a> PendingQueueFieldsRef<'a> { + pub(crate) fn pending_partial_withdrawals(&self) -> &[electra::PendingPartialWithdrawal] { + match self { + PendingQueueFieldsRef::Electra(state) => &state.pending_partial_withdrawals, + PendingQueueFieldsRef::Fulu(state) => &state.pending_partial_withdrawals, + PendingQueueFieldsRef::Gloas(state) => &state.pending_partial_withdrawals, + } + } +} + +/// The electra, fulu, or gloas state, immutably. See [`pending_queue_fields`]. +pub(crate) fn pending_queue_fields_ref<'a>( state: &'a BeaconState, function: &'static str, -) -> Result> { +) -> Result> { match state { - BeaconState::Electra(state) => Ok(ElectraOrFulu::Electra(state)), - BeaconState::Fulu(state) => Ok(ElectraOrFulu::Fulu(state)), + BeaconState::Electra(state) => Ok(PendingQueueFieldsRef::Electra(state)), + BeaconState::Fulu(state) => Ok(PendingQueueFieldsRef::Fulu(state)), + BeaconState::Gloas(state) => Ok(PendingQueueFieldsRef::Gloas(state)), other => Err(Error::UnsupportedForFork { function, fork: other.fork_name(), @@ -959,14 +1218,10 @@ pub(crate) fn electra_state_ref<'a>( /// (`compute_exit_epoch_and_update_churn`, /// [`get_consolidation_churn_limit_for_fork`]). /// -/// Deliberately narrower than [`ElectraOrFuluMut`]: that enum also exposes -/// `pending_deposits_mut`, whose gloas type -/// ([`crate::beacon::containers::gloas::PendingDeposits`], a -/// [`libssz_types::ProgressiveList`], EIP-7688) is not electra's and fulu's -/// bounded [`electra::PendingDeposits`], so widening `ElectraOrFuluMut` -/// itself to gloas would leave that one method with no honest gloas arm to -/// give it. This projection carries only the four fields every one of the -/// three forks actually shares. +/// Deliberately separate from [`PendingQueueFields`]: that one carries the +/// deposit-balance churn cursor and the three pending queues, none of which +/// this projection reads or writes. This one carries only the four +/// exit/consolidation churn cursor fields. pub(crate) enum ChurnCursorsMut<'a> { Electra(&'a mut electra::BeaconState), Fulu(&'a mut fulu::BeaconState), @@ -1040,7 +1295,7 @@ impl<'a> ChurnCursorsMut<'a> { } /// The electra-or-fulu-or-gloas state, mutably, through [`ChurnCursorsMut`]. -/// See that enum's own doc for why it exists apart from [`electra_state`]. +/// See that enum's own doc for why it exists apart from [`pending_queue_fields`]. pub(crate) fn churn_cursors_mut<'a>( state: &'a mut BeaconState, function: &'static str, @@ -1076,12 +1331,20 @@ mod tests { } /// A fulu state, otherwise identical to [`electra_state_with_validators`], - /// used only to prove [`electra_state`]/[`electra_state_ref`] accept fulu - /// too and are not accidentally scoped to `BeaconState::Electra` alone. + /// used only to prove [`pending_queue_fields`]/[`pending_queue_fields_ref`] + /// accept fulu too and are not accidentally scoped to `BeaconState::Electra` + /// alone. fn fulu_state_with_validators(count: usize) -> BeaconState { crate::beacon::helpers::test_state::with_validators_at(ForkName::Fulu, count) } + /// A gloas state, otherwise identical to [`electra_state_with_validators`], + /// used to prove [`pending_queue_fields`]/[`pending_queue_fields_ref`] + /// accept gloas too. + fn gloas_state_with_validators(count: usize) -> BeaconState { + crate::beacon::helpers::test_state::with_validators_at(ForkName::Gloas, count) + } + // -- Predicates --------------------------------------------------------- #[test] @@ -1504,27 +1767,65 @@ mod tests { // -- Fork projection ------------------------------------------------------ #[test] - fn electra_state_ref_accepts_both_electra_and_fulu_but_not_phase0() { + fn pending_queue_fields_ref_accepts_electra_fulu_and_gloas_but_not_phase0() { let electra_state_value = electra_state_with_validators(4); - assert!(electra_state_ref(&electra_state_value, "test").is_ok()); + assert!(pending_queue_fields_ref(&electra_state_value, "test").is_ok()); let fulu_state_value = fulu_state_with_validators(4); - assert!(electra_state_ref(&fulu_state_value, "test").is_ok()); + assert!(pending_queue_fields_ref(&fulu_state_value, "test").is_ok()); + + let gloas_state_value = gloas_state_with_validators(4); + assert!(pending_queue_fields_ref(&gloas_state_value, "test").is_ok()); let phase0_state_value = crate::beacon::helpers::test_state::with_validators(4); - assert!(electra_state_ref(&phase0_state_value, "test").is_err()); + assert!(pending_queue_fields_ref(&phase0_state_value, "test").is_err()); } #[test] - fn electra_state_mut_accepts_both_electra_and_fulu_but_not_phase0() { + fn pending_queue_fields_accepts_electra_fulu_and_gloas_but_not_phase0() { let mut electra_state_value = electra_state_with_validators(4); - assert!(electra_state(&mut electra_state_value, "test").is_ok()); + assert!(pending_queue_fields(&mut electra_state_value, "test").is_ok()); let mut fulu_state_value = fulu_state_with_validators(4); - assert!(electra_state(&mut fulu_state_value, "test").is_ok()); + assert!(pending_queue_fields(&mut fulu_state_value, "test").is_ok()); + + let mut gloas_state_value = gloas_state_with_validators(4); + assert!(pending_queue_fields(&mut gloas_state_value, "test").is_ok()); let mut phase0_state_value = crate::beacon::helpers::test_state::with_validators(4); - assert!(electra_state(&mut phase0_state_value, "test").is_err()); + assert!(pending_queue_fields(&mut phase0_state_value, "test").is_err()); + } + + #[test] + fn get_pending_balance_to_withdraw_works_on_a_gloas_state() { + let mut state = gloas_state_with_validators(4); + { + let mut fields = pending_queue_fields(&mut state, "test").unwrap(); + fields + .push_pending_partial_withdrawal(electra::PendingPartialWithdrawal { + validator_index: 2, + amount: 7, + withdrawable_epoch: 0, + }) + .unwrap(); + } + assert_eq!(get_pending_balance_to_withdraw(&state, 2).unwrap(), 7); + assert_eq!(get_pending_balance_to_withdraw(&state, 0).unwrap(), 0); + } + + #[test] + fn queue_excess_active_balance_works_on_a_gloas_state() { + let mut state = gloas_state_with_validators(4); + *state.balance_mut(0).unwrap() = preset::MIN_ACTIVATION_BALANCE + 5; + + queue_excess_active_balance(&mut state, 0).unwrap(); + + assert_eq!(state.balance(0).unwrap(), preset::MIN_ACTIVATION_BALANCE); + let BeaconState::Gloas(inner) = &state else { + unreachable!("just built as Gloas"); + }; + assert_eq!(inner.pending_deposits.len(), 1); + assert_eq!(inner.pending_deposits[0].amount, 5); } #[test] diff --git a/crates/blockchain/state_transition/src/beacon/stf/capella.rs b/crates/blockchain/state_transition/src/beacon/stf/capella.rs index 6c42d739a..d5821ed80 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/capella.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/capella.rs @@ -45,20 +45,18 @@ use crate::beacon::bls; use crate::beacon::config::Config; use crate::beacon::constants; -use crate::beacon::containers::shared::{ - Deposit, ProposerSlashing, SignedVoluntaryExit, Validator, -}; +use crate::beacon::containers::shared::{Deposit, ProposerSlashing, SignedVoluntaryExit}; use crate::beacon::containers::{BeaconState, capella, phase0}; use crate::beacon::error::{Error, Result, verify}; use crate::beacon::hash::hash; use crate::beacon::helpers::accessors::{CommitteeCache, get_current_epoch, get_randao_mix}; use crate::beacon::helpers::capella::{ - is_fully_withdrawable_validator, is_partially_withdrawable_validator, + is_fully_withdrawable_validator, is_partially_withdrawable_validator, withdrawal_address, }; use crate::beacon::helpers::misc::{compute_domain, compute_signing_root}; use crate::beacon::helpers::mutators::decrease_balance; use crate::beacon::preset; -use crate::beacon::primitives::{ExecutionAddress, H256, HashTreeRoot as _}; +use crate::beacon::primitives::{H256, HashTreeRoot as _}; use super::ExecutionEngine; @@ -110,14 +108,6 @@ pub fn process_block( // Withdrawals // --------------------------------------------------------------------------- -/// The execution address a validator's payout is sent to: the low bytes of -/// its eth1 withdrawal credentials, the same bytes -/// [`process_bls_to_execution_change`] writes when a validator upgrades into -/// this form. -fn withdrawal_address(validator: &Validator) -> ExecutionAddress { - ExecutionAddress::from_slice(&validator.withdrawal_credentials.0[12..]) -} - /// The withdrawals this block's sweep owes, without applying them. /// /// A bounded walk of the validator registry starting at @@ -509,7 +499,8 @@ mod tests { use super::*; use crate::beacon::fork::ForkName; use crate::beacon::primitives::{ - BlsPubkey, BlsSignature, ExecutionBlockHash, Gwei, Root, Uint256, ValidatorIndex, + BlsPubkey, BlsSignature, ExecutionAddress, ExecutionBlockHash, Gwei, Root, Uint256, + ValidatorIndex, }; use libssz_types::SszVector; diff --git a/crates/blockchain/state_transition/src/beacon/stf/electra.rs b/crates/blockchain/state_transition/src/beacon/stf/electra.rs index e4e872da4..0ec62ee86 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/electra.rs @@ -60,13 +60,15 @@ use crate::beacon::helpers::accessors::{ get_block_root_at_slot, get_current_epoch, get_previous_epoch, get_randao_mix, }; use crate::beacon::helpers::altair::{add_flag, get_base_reward_per_increment, has_flag}; +use crate::beacon::helpers::capella::withdrawal_address; use crate::beacon::helpers::electra::{ - compute_exit_epoch_and_update_churn, electra_state, get_committee_indices, + compute_exit_epoch_and_update_churn, get_committee_indices, get_consolidation_churn_limit_for_fork, get_indexed_attestation, get_max_effective_balance, get_pending_balance_to_withdraw, has_compounding_withdrawal_credential, has_eth1_withdrawal_credential, has_execution_withdrawal_credential, initiate_validator_exit as electra_initiate_validator_exit, is_fully_withdrawable_validator, - is_partially_withdrawable_validator, is_valid_indexed_attestation, + is_partially_withdrawable_validator, is_valid_indexed_attestation, pending_queue_fields, + pending_queue_fields_ref, }; use crate::beacon::helpers::math::integer_squareroot; use crate::beacon::helpers::misc::{ @@ -79,8 +81,8 @@ use crate::beacon::helpers::predicates::{ }; use crate::beacon::preset; use crate::beacon::primitives::{ - BlsPubkey, BlsSignature, Bytes32, ExecutionAddress, Gwei, HashTreeRoot as _, - ParticipationFlags, ValidatorIndex, WithdrawalIndex, + BlsPubkey, BlsSignature, Bytes32, Gwei, HashTreeRoot as _, ParticipationFlags, ValidatorIndex, + WithdrawalIndex, }; use super::ExecutionEngine; @@ -89,22 +91,34 @@ use super::ExecutionEngine; // Local state projection // --------------------------------------------------------------------------- // -// `crate::beacon::helpers::electra::{electra_state, electra_state_ref}` already -// project a `BeaconState` down to electra's (or fulu's) concrete struct, but -// only far enough to reach the balance-churn fields their own module needs -// (`pending_deposits` and the four churn cursors; see that module's doc). This -// file's block-processing steps also need `latest_execution_payload_header`, +// `crate::beacon::helpers::electra::{pending_queue_fields, pending_queue_fields_ref}` +// already project a `BeaconState` down to electra's, fulu's, or gloas's +// concrete struct, but only far enough to reach the deposit-balance churn +// cursor and the three pending queues (see that module's doc). This file's +// other block-processing steps also need `latest_execution_payload_header`, // the withdrawal-sweep cursor (`next_withdrawal_index`, -// `next_withdrawal_validator_index`), `pending_partial_withdrawals` and -// `pending_consolidations` mutably, `deposit_requests_start_index`, and -// (introduced at altair, not electra) `previous_epoch_participation`, -// `current_epoch_participation`, and `inactivity_scores`. None of those are -// fork-invariant (`crate::beacon::containers::mod`'s `shared_state_accessors!` macro -// does not cover any of them), and reaching them would mean adding methods to -// `helpers::electra`'s projection, which is outside this file's ownership. -// This is a second, narrower projection kept local to this module instead, -// for exactly the fields block processing (as opposed to balance-churn -// accounting) needs. +// `next_withdrawal_validator_index`), `deposit_requests_start_index` (this +// fork's own `process_deposit_request`, unlike fulu's and gloas's, still +// writes it), and (introduced at altair, not electra) +// `previous_epoch_participation`, `current_epoch_participation`, and +// `inactivity_scores`. None of those are fork-invariant +// (`crate::beacon::containers::mod`'s `shared_state_accessors!` macro does +// not cover any of them), gloas cannot serve every one of them (it has no +// `latest_execution_payload_header` at all), and reaching them would mean +// adding methods to `helpers::electra`'s own projection that only make sense +// for two of its three forks, which is outside that module's ownership. This +// is a second, narrower, electra-and-fulu-only projection kept local to this +// module instead, for exactly the fields block processing (as opposed to +// balance-churn accounting, or the pending queues) needs. +// +// `pending_partial_withdrawals_mut` still lives on this local projection too, +// for `process_withdrawals`'s own sweep-drain (which, like the fields above, +// gloas does not share: its own withdrawal sweep lives in +// `crate::beacon::stf::gloas`). `process_withdrawal_request` and +// `process_consolidation_request` reach that same queue, and +// `pending_consolidations`, through `pending_queue_fields` instead, +// specifically so gloas can share those two functions unmodified; see their +// own docs. // // Scoped to `BeaconState::Electra` *and* `BeaconState::Fulu`, the same way // `helpers::electra`'s own projection is, rather than to `Electra` alone the @@ -209,13 +223,6 @@ impl<'a> BlockMut<'a> { } } - fn pending_consolidations_mut(&mut self) -> &mut electra::PendingConsolidations { - match self { - BlockMut::Electra(state) => &mut state.pending_consolidations, - BlockMut::Fulu(state) => &mut state.pending_consolidations, - } - } - fn deposit_requests_start_index_mut(&mut self) -> &mut u64 { match self { BlockMut::Electra(state) => &mut state.deposit_requests_start_index, @@ -422,10 +429,7 @@ pub fn apply_deposit( signature: *signature, slot: constants::GENESIS_SLOT, }; - electra_state(state, "apply_deposit")? - .pending_deposits_mut() - .push(deposit)?; - Ok(()) + pending_queue_fields(state, "apply_deposit")?.push_pending_deposit(deposit) } /// Verifies a deposit's merkle proof, then applies it. @@ -883,76 +887,66 @@ fn attestation_participation_flag_indices( // Withdrawals // --------------------------------------------------------------------------- -/// The execution address a validator's payout is sent to: the low bytes of -/// its withdrawal credentials, present regardless of whether those -/// credentials are eth1 or compounding (both are execution-form; see -/// `crate::beacon::helpers::electra::has_execution_withdrawal_credential`). -/// -/// A duplicate of capella's own private `withdrawal_address` -/// (`crate::beacon::stf::capella`): that function is not `pub`, so it is not visible -/// here even within the same crate. -fn withdrawal_address(validator: &Validator) -> ExecutionAddress { - ExecutionAddress::from_slice(&validator.withdrawal_credentials.0[12..]) -} - -/// The withdrawals this block's sweep owes, without applying them, and how -/// many entries of `pending_partial_withdrawals` it consumed while deciding -/// that. -/// -/// Two sweeps, in the specification's order, both drawing from the same -/// `withdrawals` list and the same `withdrawal_index` cursor so a partial -/// withdrawal already collected by the first sweep is visible to the -/// second's own `total_withdrawn` accounting for the same validator. +/// Drains [`electra::PendingPartialWithdrawal`]s (EIP-7251's own queue, +/// absent before this fork), oldest first: each entry is either paid (if the +/// validator is still active, sitting on enough effective and excess +/// balance) or simply dropped, but either way it counts toward the returned +/// processed count, which [`process_withdrawals`] needs to know how much of +/// the queue to drop afterward: a withdrawal request can be consumed by this +/// sweep without ever producing an actual [`capella::Withdrawal`], and the +/// queue still has to advance past it regardless. /// -/// The first sweep drains [`electra::PendingPartialWithdrawal`]s -/// (EIP-7251's own queue, absent before this fork): each entry is either -/// paid (if the validator is still active, sitting on enough effective and -/// excess balance) or simply dropped, but either way it counts toward -/// `processed_partial_withdrawals_count`, which is the second return value -/// [`process_withdrawals`] needs to know how much of the queue to drop -/// afterward: a withdrawal request can be consumed by this sweep without -/// ever producing an actual [`capella::Withdrawal`], and the queue still has -/// to advance past it regardless. +/// `prior_withdrawals` and the returned `withdrawal_index` let a caller chain +/// several sweeps (gloas's own builder-payout and builder-registry sweeps run +/// ahead of this one; see `crate::beacon::stf::gloas::get_expected_withdrawals`) +/// without double-paying a balance an earlier sweep already claimed: the +/// bound below counts `prior_withdrawals` too, and `total_withdrawn` searches +/// both lists. /// -/// The second sweep is capella's own registry walk -/// (`crate::beacon::stf::capella::get_expected_withdrawals`), unchanged in shape, -/// except that both withdrawability predicates are electra's -/// ([`is_fully_withdrawable_validator`], [`is_partially_withdrawable_validator`]) -/// and a partial withdrawal's amount is capped against -/// [`get_max_effective_balance`] (a validator's own ceiling) rather than the -/// single fixed `MAX_EFFECTIVE_BALANCE`. -pub fn get_expected_withdrawals(state: &BeaconState) -> Result<(Vec, usize)> { +/// Also served, unmodified, by gloas: nothing in gloas's own +/// `beacon-chain.md` touches this function, and the only thing that kept a +/// gloas state from calling this exact copy was the queue's own type +/// ([`crate::beacon::containers::gloas::PendingPartialWithdrawals`], a +/// [`libssz_types::ProgressiveList`], EIP-7688), which +/// [`crate::beacon::helpers::electra::PendingQueueFieldsRef::pending_partial_withdrawals`] +/// already abstracts over as a slice. +pub(crate) fn get_pending_partial_withdrawals( + state: &BeaconState, + mut withdrawal_index: WithdrawalIndex, + prior_withdrawals: &[capella::Withdrawal], +) -> Result<(Vec, WithdrawalIndex, u64)> { let epoch = get_current_epoch(state); - let fields = block_ref(state, "get_expected_withdrawals")?; - let mut withdrawal_index = fields.next_withdrawal_index(); - let mut validator_index = fields.next_withdrawal_validator_index(); - - let mut withdrawals: Vec = Vec::new(); - let mut processed_partial_withdrawals_count: usize = 0; - - // [New in Electra:EIP7251]: consume pending partial withdrawals. - let electra_ref = - crate::beacon::helpers::electra::electra_state_ref(state, "get_expected_withdrawals")?; - let pending_partial_withdrawals = electra_ref.pending_partial_withdrawals(); - for withdrawal in pending_partial_withdrawals.iter() { - if withdrawal.withdrawable_epoch > epoch - || withdrawals.len() == preset::MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP as usize - { + let withdrawals_limit = (prior_withdrawals.len() + + preset::MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP as usize) + .min(preset::MAX_WITHDRAWALS_PER_PAYLOAD - 1); + verify( + prior_withdrawals.len() <= withdrawals_limit, + "get_pending_partial_withdrawals: len(prior_withdrawals) <= withdrawals_limit", + )?; + + let mut processed_count: u64 = 0; + let mut withdrawals = Vec::new(); + let fields = pending_queue_fields_ref(state, "get_pending_partial_withdrawals")?; + for withdrawal in fields.pending_partial_withdrawals() { + let is_withdrawable = withdrawal.withdrawable_epoch <= epoch; + if !is_withdrawable || prior_withdrawals.len() + withdrawals.len() >= withdrawals_limit { break; } - let validator = state.validator(withdrawal.validator_index)?; + let validator_index = withdrawal.validator_index; + let validator = state.validator(validator_index)?; let has_sufficient_effective_balance = validator.effective_balance >= preset::MIN_ACTIVATION_BALANCE; - let total_withdrawn: Gwei = withdrawals + let total_withdrawn: Gwei = prior_withdrawals .iter() - .filter(|paid| paid.validator_index == withdrawal.validator_index) + .chain(withdrawals.iter()) + .filter(|paid| paid.validator_index == validator_index) .fold(0, |total, paid| total.saturating_add(paid.amount)); let balance = state - .balance(withdrawal.validator_index)? + .balance(validator_index)? .checked_sub(total_withdrawn) .ok_or(Error::ArithmeticOverflow( - "state.balances[withdrawal.validator_index] - total_withdrawn", + "get_pending_partial_withdrawals: balance - total_withdrawn", ))?; let has_excess_balance = balance > preset::MIN_ACTIVATION_BALANCE; @@ -964,36 +958,72 @@ pub fn get_expected_withdrawals(state: &BeaconState) -> Result<(Vec Result<(Vec, WithdrawalIndex, u64)> { + let epoch = get_current_epoch(state); let validator_count = state.validator_count() as u64; - let bound = validator_count.min(preset::MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP); - for _ in 0..bound { + let validators_limit = validator_count.min(preset::MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP); + let withdrawals_limit = preset::MAX_WITHDRAWALS_PER_PAYLOAD; + verify( + prior_withdrawals.len() < withdrawals_limit, + "get_validators_sweep_withdrawals: len(prior_withdrawals) < withdrawals_limit", + )?; + + let mut processed_count: u64 = 0; + let mut withdrawals = Vec::new(); + let (_, mut validator_index) = state.withdrawal_cursor()?; + for _ in 0..validators_limit { + if prior_withdrawals.len() + withdrawals.len() >= withdrawals_limit { + break; + } + let validator = state.validator(validator_index)?; - let total_withdrawn: Gwei = withdrawals + let total_withdrawn: Gwei = prior_withdrawals .iter() + .chain(withdrawals.iter()) .filter(|paid| paid.validator_index == validator_index) .fold(0, |total, paid| total.saturating_add(paid.amount)); let balance = state .balance(validator_index)? .checked_sub(total_withdrawn) .ok_or(Error::ArithmeticOverflow( - "state.balances[validator_index] - total_withdrawn", + "get_validators_sweep_withdrawals: balance - total_withdrawn", ))?; if is_fully_withdrawable_validator(validator, balance, epoch) { @@ -1005,14 +1035,16 @@ pub fn get_expected_withdrawals(state: &BeaconState) -> Result<(Vec Result<(Vec 0`, so `validator_count` is never zero + // here. validator_index = validator_index .checked_add(1) - .ok_or(Error::ArithmeticOverflow("validator_index + 1"))? + .ok_or(Error::ArithmeticOverflow( + "get_validators_sweep_withdrawals: validator_index + 1", + ))? % validator_count; + processed_count += 1; } - Ok((withdrawals, processed_partial_withdrawals_count)) + Ok((withdrawals, withdrawal_index, processed_count)) +} + +/// The withdrawals this block's sweep owes, without applying them: pending +/// partial withdrawals first ([`get_pending_partial_withdrawals`]), then the +/// bounded validator registry walk ([`get_validators_sweep_withdrawals`]), +/// the second sweep's own `prior_withdrawals` carrying forward every +/// withdrawal the first already produced so neither double-pays a balance +/// the other already claimed. +pub fn get_expected_withdrawals(state: &BeaconState) -> Result<(Vec, usize)> { + let fields = block_ref(state, "get_expected_withdrawals")?; + let withdrawal_index = fields.next_withdrawal_index(); + + let (partial_withdrawals, withdrawal_index, processed_partial_withdrawals_count) = + get_pending_partial_withdrawals(state, withdrawal_index, &[])?; + + let (validators_sweep_withdrawals, _, _) = + get_validators_sweep_withdrawals(state, withdrawal_index, &partial_withdrawals)?; + + let mut withdrawals = partial_withdrawals; + withdrawals.extend(validators_sweep_withdrawals); + + Ok((withdrawals, processed_partial_withdrawals_count as usize)) } /// Applies this block's withdrawal sweep: checks the block's declared @@ -1202,10 +1261,7 @@ pub fn process_deposit_request( signature: request.signature, slot: state.slot(), }; - electra_state(state, "process_deposit_request")? - .pending_deposits_mut() - .push(deposit)?; - Ok(()) + pending_queue_fields(state, "process_deposit_request")?.push_pending_deposit(deposit) } // --------------------------------------------------------------------------- @@ -1244,6 +1300,15 @@ pub fn process_deposit_request( /// means `activation_epoch` itself is corrupt state, not that this request is /// stale, so it is checked and propagated with `?` rather than folded into /// the silent-return chain above it. +/// +/// Also served, unmodified, by gloas: nothing in gloas's own +/// `beacon-chain.md` touches this function, and the only thing that kept a +/// gloas state from calling this exact copy was the pending-partial-withdrawals +/// queue's own type, which +/// [`crate::beacon::helpers::electra::PendingQueueFields`] abstracts over as +/// a slice (the length check) and a one-at-a-time push (queuing a new +/// withdrawal), rather than this file's own [`block_mut`], which cannot +/// serve gloas (see this file's own module doc). pub fn process_withdrawal_request( state: &mut BeaconState, request: &electra::WithdrawalRequest, @@ -1252,9 +1317,10 @@ pub fn process_withdrawal_request( let amount = request.amount; let is_full_exit_request = amount == constants::FULL_EXIT_REQUEST_AMOUNT; - let pending_partial_withdrawals_len = block_mut(state, "process_withdrawal_request")? - .pending_partial_withdrawals_mut() - .len(); + let pending_partial_withdrawals_len = + pending_queue_fields(state, "process_withdrawal_request")? + .pending_partial_withdrawals() + .len(); if pending_partial_withdrawals_len == preset::PENDING_PARTIAL_WITHDRAWALS_LIMIT && !is_full_exit_request { @@ -1339,9 +1405,8 @@ pub fn process_withdrawal_request( amount: to_withdraw, withdrawable_epoch, }; - block_mut(state, "process_withdrawal_request")? - .pending_partial_withdrawals_mut() - .push(withdrawal)?; + pending_queue_fields(state, "process_withdrawal_request")? + .push_pending_partial_withdrawal(withdrawal)?; } Ok(()) @@ -1406,6 +1471,13 @@ pub fn is_valid_switch_to_compounding_request( /// in that function, is `source_validator.activation_epoch + /// SHARD_COMMITTEE_PERIOD`: an overflow there is checked and propagated, /// since it would mean corrupt state rather than a stale request. +/// +/// Also served, unmodified, by gloas, the same reason and the same way +/// [`process_withdrawal_request`] is: nothing in gloas's own +/// `beacon-chain.md` touches this function, and +/// [`crate::beacon::helpers::electra::PendingQueueFields`] abstracts over the +/// pending-consolidations queue's own type change instead of this file's own +/// [`block_mut`], which cannot serve gloas. pub fn process_consolidation_request( state: &mut BeaconState, request: &electra::ConsolidationRequest, @@ -1432,8 +1504,8 @@ pub fn process_consolidation_request( if request.source_pubkey == request.target_pubkey { return Ok(()); } - let pending_consolidations_len = block_mut(state, "process_consolidation_request")? - .pending_consolidations_mut() + let pending_consolidations_len = pending_queue_fields(state, "process_consolidation_request")? + .pending_consolidations() .len(); if pending_consolidations_len == preset::PENDING_CONSOLIDATIONS_LIMIT { return Ok(()); @@ -1519,9 +1591,8 @@ pub fn process_consolidation_request( source_index, target_index, }; - block_mut(state, "process_consolidation_request")? - .pending_consolidations_mut() - .push(consolidation)?; + pending_queue_fields(state, "process_consolidation_request")? + .push_pending_consolidation(consolidation)?; Ok(()) } @@ -1806,6 +1877,7 @@ pub fn process_block( mod tests { use super::*; use crate::beacon::fork::ForkName; + use crate::beacon::primitives::ExecutionAddress; /// An electra state with `count` fully active validators, each with /// [`preset::MIN_ACTIVATION_BALANCE`] and an eth1 withdrawal credential, @@ -2104,9 +2176,9 @@ mod tests { state.validator(0).unwrap() )); assert!( - block_mut(&mut state, "test assertion") + pending_queue_fields(&mut state, "test assertion") .unwrap() - .pending_consolidations_mut() + .pending_consolidations() .is_empty(), "a switch-to-compounding request is not a real consolidation and must not queue one" ); @@ -2137,11 +2209,10 @@ mod tests { let mut state = electra_state_with_validators(2); let config = Config::mainnet(); { - let mut fields = block_mut(&mut state, "test setup").unwrap(); - let queue = fields.pending_consolidations_mut(); + let mut fields = pending_queue_fields(&mut state, "test setup").unwrap(); for _ in 0..preset::PENDING_CONSOLIDATIONS_LIMIT { - queue - .push(electra::PendingConsolidation { + fields + .push_pending_consolidation(electra::PendingConsolidation { source_index: 0, target_index: 0, }) @@ -2154,9 +2225,9 @@ mod tests { } let source_pubkey = state.validator(0).unwrap().pubkey; let target_pubkey = state.validator(1).unwrap().pubkey; - let before_len = block_mut(&mut state, "test assertion") + let before_len = pending_queue_fields(&mut state, "test assertion") .unwrap() - .pending_consolidations_mut() + .pending_consolidations() .len(); let request = electra::ConsolidationRequest { @@ -2168,9 +2239,9 @@ mod tests { .expect("a full queue drops the request silently, not with an error"); assert_eq!( - block_mut(&mut state, "test assertion") + pending_queue_fields(&mut state, "test assertion") .unwrap() - .pending_consolidations_mut() + .pending_consolidations() .len(), before_len, "the queue must not have grown past its limit" @@ -2219,9 +2290,9 @@ mod tests { FAR_FUTURE_EPOCH, "a genuine consolidation must start the source validator's exit" ); - let queue_len = block_mut(&mut state, "test assertion") + let queue_len = pending_queue_fields(&mut state, "test assertion") .unwrap() - .pending_consolidations_mut() + .pending_consolidations() .len(); assert_eq!(queue_len, 1, "the consolidation must have been queued"); } @@ -2257,7 +2328,7 @@ mod tests { // validator at all: see `apply_deposit`'s own documentation. assert_eq!(state.validator_count(), count_before); assert!( - crate::beacon::helpers::electra::electra_state_ref(&state, "test assertion") + crate::beacon::helpers::electra::pending_queue_fields_ref(&state, "test assertion") .unwrap() .pending_partial_withdrawals() .is_empty(), diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs index 7432f781c..701fed5ab 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs @@ -35,13 +35,12 @@ use crate::beacon::bls; use crate::beacon::config::Config; use crate::beacon::constants::{self, FAR_FUTURE_EPOCH}; use crate::beacon::containers::shared::{DepositMessage, Validator}; -use crate::beacon::containers::{BeaconState, electra, fulu, gloas}; +use crate::beacon::containers::{BeaconState, electra}; use crate::beacon::error::{Error, Result}; -use crate::beacon::fork::ForkName; use crate::beacon::helpers::accessors::{get_current_epoch, get_total_active_balance}; use crate::beacon::helpers::electra::{ get_activation_exit_churn_limit, get_max_effective_balance, initiate_validator_exit, - is_eligible_for_activation_queue, + is_eligible_for_activation_queue, pending_queue_fields, }; use crate::beacon::helpers::misc::{ compute_activation_exit_epoch, compute_deposit_domain, compute_signing_root, @@ -333,8 +332,7 @@ pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Res .ok_or(Error::ArithmeticOverflow( "deposit_balance_to_consume + get_activation_exit_churn_limit", ))?; - let deposits: Vec = - core::mem::take(fields.pending_deposits_mut()?).into_inner(); + let deposits: Vec = fields.take_pending_deposits()?; ( deposit_requests_start_index, available_for_processing, @@ -424,7 +422,7 @@ pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Res }; let mut fields = pending_queue_fields(state, "process_pending_deposits")?; - *fields.pending_deposits_mut()? = electra::PendingDeposits::try_from(remaining)?; + fields.set_pending_deposits(remaining)?; *fields.deposit_balance_to_consume_mut() = deposit_balance_to_consume; Ok(()) @@ -585,7 +583,7 @@ fn add_validator_from_pending_deposit( /// `beacon-chain.md` touches this function, and the only thing that keeps a /// gloas state from calling this exact copy is the queue's own type /// (`ProgressiveList` rather than `SszList`, EIP-7688), which -/// [`PendingQueueFields::take_pending_consolidations`]/[`PendingQueueFields::set_pending_consolidations`] +/// [`PendingQueueFields::take_pending_consolidations`](crate::beacon::helpers::electra::PendingQueueFields::take_pending_consolidations)/[`PendingQueueFields::set_pending_consolidations`](crate::beacon::helpers::electra::PendingQueueFields::set_pending_consolidations) /// already abstract over at the `Vec` level. Contrast /// [`process_pending_deposits`], which gloas's own `beacon-chain.md` modifies /// outright and which therefore keeps its own copy @@ -687,199 +685,6 @@ pub fn process_effective_balance_updates(state: &mut BeaconState) -> Result<()> Ok(()) } -// --------------------------------------------------------------------------- -// Fork projection -// --------------------------------------------------------------------------- - -/// Fields [`process_pending_deposits`] and [`process_pending_consolidations`] -/// need that [`crate::beacon::helpers::electra::electra_state`] does not expose: that -/// projection only covers what its own module's functions need (the exit and -/// consolidation churn cursors, and `pending_deposits_mut` for -/// `queue_excess_active_balance` and its neighbors). This crate keeps one -/// file per fork's own state-transition concerns, so a second, file-local -/// projection lives here rather than as an addition to that module, even -/// though its shape, an electra-or-fulu match, is identical: fulu keeps every -/// field this covers unchanged (see `crate::beacon::helpers::electra`'s own module -/// doc for why that module's projection accepts fulu too). -/// -/// `pub(crate)`: fulu's own [`super::fulu::process_pending_deposits`] reads -/// [`Self::deposit_balance_to_consume`] and writes through -/// [`Self::pending_deposits_mut`] and [`Self::deposit_balance_to_consume_mut`] -/// the same way this module's own [`process_pending_deposits`] does; it never -/// reads [`Self::deposit_requests_start_index`], the one field that function -/// stops consulting. -/// -/// Gloas joins the enum too (EIP-8061 leaves `deposit_requests_start_index` -/// and `deposit_balance_to_consume` unchanged, and `beacon-chain.md` does not -/// modify [`process_pending_consolidations`] at all), even though gloas's own -/// [`process_pending_deposits`] is a separate function -/// (`crate::beacon::stf::epoch::gloas::process_pending_deposits`, SPEC marks it -/// Modified) that reaches its own queue and churn field directly through -/// `gloas_state` rather than through here. -pub(crate) enum PendingQueueFields<'a> { - Electra(&'a mut electra::BeaconState), - Fulu(&'a mut fulu::BeaconState), - Gloas(&'a mut gloas::BeaconState), -} - -impl<'a> PendingQueueFields<'a> { - /// The execution-layer deposit request index at which the state switched - /// from crediting deposits off `Eth1Data` votes to crediting them off - /// `DepositRequest`s directly, read by [`process_pending_deposits`] to - /// know whether any eth1-bridge deposit is still outstanding. - /// - /// `pub(crate)`: fulu's own tests (`super::fulu`) read this to assert a - /// test state starts with it at - /// [`crate::beacon::constants::UNSET_DEPOSIT_REQUESTS_START_INDEX`], the - /// precondition [`super::fulu::process_pending_deposits`]'s own doc - /// explains. [`super::fulu::process_pending_deposits`] itself never reads - /// this field, and no gloas caller reads it through here at all (gloas's - /// own `process_pending_deposits` does not read it either; see this - /// enum's own doc). - pub(crate) fn deposit_requests_start_index(&self) -> u64 { - match self { - PendingQueueFields::Electra(state) => state.deposit_requests_start_index, - PendingQueueFields::Fulu(state) => state.deposit_requests_start_index, - PendingQueueFields::Gloas(state) => state.deposit_requests_start_index, - } - } - - /// How much of this epoch's deposit balance churn limit remains unused. - pub(crate) fn deposit_balance_to_consume(&self) -> Gwei { - match self { - PendingQueueFields::Electra(state) => state.deposit_balance_to_consume, - PendingQueueFields::Fulu(state) => state.deposit_balance_to_consume, - PendingQueueFields::Gloas(state) => state.deposit_balance_to_consume, - } - } - - pub(crate) fn deposit_balance_to_consume_mut(&mut self) -> &mut Gwei { - match self { - PendingQueueFields::Electra(state) => &mut state.deposit_balance_to_consume, - PendingQueueFields::Fulu(state) => &mut state.deposit_balance_to_consume, - PendingQueueFields::Gloas(state) => &mut state.deposit_balance_to_consume, - } - } - - /// Deposits known but not yet credited to the validator registry. - /// - /// Electra and fulu only, even though this enum now spans gloas too (for - /// [`Self::take_pending_consolidations`]'s sake): gloas's own - /// `pending_deposits` is `ProgressiveList` (EIP-7688), a - /// different Rust type from electra's and fulu's bounded `SszList` this - /// returns, and gloas's own `process_pending_deposits` reads its queue - /// directly through `gloas_state` instead of through here (see this - /// enum's own doc). Fallible, unlike the pre-gloas version of this - /// method, purely so a gloas-backed value answers `UnsupportedForFork` - /// here instead of failing to compile a shared return type; no caller - /// ever actually reaches that arm. - pub(crate) fn pending_deposits_mut(&mut self) -> Result<&mut electra::PendingDeposits> { - match self { - PendingQueueFields::Electra(state) => Ok(&mut state.pending_deposits), - PendingQueueFields::Fulu(state) => Ok(&mut state.pending_deposits), - PendingQueueFields::Gloas(_) => Err(Error::UnsupportedForFork { - function: "PendingQueueFields::pending_deposits_mut", - fork: ForkName::Gloas, - }), - } - } - - /// Takes the whole pending-consolidations queue out, leaving an empty one - /// behind; [`Self::set_pending_consolidations`] puts a (possibly - /// shorter) one back. Vec-level rather than the container-typed accessor - /// [`Self::pending_deposits_mut`] uses: gloas's own queue is - /// `ProgressiveList` (EIP-7688), a different Rust - /// type from electra's and fulu's bounded `SszList`, and unlike - /// `process_pending_deposits`, [`process_pending_consolidations`] is - /// meant to serve a gloas state directly (gloas's `beacon-chain.md` does - /// not modify this function at all), which a shared Vec return type is - /// what makes possible. - pub(crate) fn take_pending_consolidations(&mut self) -> Vec { - match self { - PendingQueueFields::Electra(state) => { - core::mem::take(&mut state.pending_consolidations).into_inner() - } - PendingQueueFields::Fulu(state) => { - core::mem::take(&mut state.pending_consolidations).into_inner() - } - PendingQueueFields::Gloas(state) => { - core::mem::take(&mut state.pending_consolidations).into_inner() - } - } - } - - /// See [`Self::take_pending_consolidations`]. Fallible only for electra's - /// and fulu's bounded `SszList`; gloas's progressive one never rejects a - /// length. - pub(crate) fn set_pending_consolidations( - &mut self, - consolidations: Vec, - ) -> Result<()> { - match self { - PendingQueueFields::Electra(state) => { - state.pending_consolidations = - electra::PendingConsolidations::try_from(consolidations)?; - } - PendingQueueFields::Fulu(state) => { - state.pending_consolidations = - electra::PendingConsolidations::try_from(consolidations)?; - } - PendingQueueFields::Gloas(state) => { - state.pending_consolidations = gloas::PendingConsolidations::from(consolidations); - } - } - Ok(()) - } - - /// Extends `previous_epoch_participation`, `current_epoch_participation`, - /// and `inactivity_scores` by one all-zero entry each, keeping them - /// exactly as long as the registry after - /// [`add_validator_from_pending_deposit`] appends a validator. - /// - /// Gloas's own three lists are progressive (`libssz_types::ProgressiveList`), - /// so their `push` is infallible, unlike electra's and fulu's bounded - /// `SszList`. - fn push_empty_participation_and_inactivity(&mut self) -> Result<()> { - match self { - PendingQueueFields::Electra(state) => { - state.previous_epoch_participation.push(0)?; - state.current_epoch_participation.push(0)?; - state.inactivity_scores.push(0)?; - } - PendingQueueFields::Fulu(state) => { - state.previous_epoch_participation.push(0)?; - state.current_epoch_participation.push(0)?; - state.inactivity_scores.push(0)?; - } - PendingQueueFields::Gloas(state) => { - state.previous_epoch_participation.push(0); - state.current_epoch_participation.push(0); - state.inactivity_scores.push(0); - } - } - Ok(()) - } -} - -/// The electra, fulu, or gloas state, mutably, through [`PendingQueueFields`]. -/// See its own doc for why this is a second projection rather than a call -/// into [`crate::beacon::helpers::electra::electra_state`], and for why gloas -/// joins electra and fulu here. -pub(crate) fn pending_queue_fields<'a>( - state: &'a mut BeaconState, - function: &'static str, -) -> Result> { - match state { - BeaconState::Electra(state) => Ok(PendingQueueFields::Electra(state)), - BeaconState::Fulu(state) => Ok(PendingQueueFields::Fulu(state)), - BeaconState::Gloas(state) => Ok(PendingQueueFields::Gloas(state)), - other => Err(Error::UnsupportedForFork { - function, - fork: other.fork_name(), - }), - } -} - #[cfg(test)] mod tests { use super::*; @@ -930,18 +735,23 @@ mod tests { }; pending_queue_fields(&mut state, "test setup") .unwrap() - .pending_deposits_mut() - .unwrap() - .push(deposit) + .push_pending_deposit(deposit) .unwrap(); let balance_before = state.balance(0).unwrap(); process_pending_deposits(&mut state, &config).unwrap(); assert_eq!(state.balance(0).unwrap(), balance_before + deposit_amount); - let mut fields = pending_queue_fields(&mut state, "test assertion").unwrap(); - assert_eq!(fields.deposit_balance_to_consume(), 0); - assert!(fields.pending_deposits_mut().unwrap().is_empty()); + assert_eq!( + pending_queue_fields(&mut state, "test assertion") + .unwrap() + .deposit_balance_to_consume(), + 0 + ); + let BeaconState::Electra(inner) = &state else { + unreachable!("built as Electra"); + }; + assert!(inner.pending_deposits.is_empty()); } #[test] @@ -968,9 +778,7 @@ mod tests { }; pending_queue_fields(&mut state, "test setup") .unwrap() - .pending_deposits_mut() - .unwrap() - .push(deposit.clone()) + .push_pending_deposit(deposit.clone()) .unwrap(); let balance_before = state.balance(0).unwrap(); @@ -979,10 +787,11 @@ mod tests { // Not applied... assert_eq!(state.balance(0).unwrap(), balance_before); // ...and not lost: it comes back out the other end of the queue. - let mut fields = pending_queue_fields(&mut state, "test assertion").unwrap(); - let remaining = fields.pending_deposits_mut().unwrap(); - assert_eq!(remaining.len(), 1); - assert_eq!(remaining[0], deposit); + let BeaconState::Electra(inner) = &state else { + unreachable!("built as Electra"); + }; + assert_eq!(inner.pending_deposits.len(), 1); + assert_eq!(inner.pending_deposits[0], deposit); } #[test] @@ -1018,12 +827,8 @@ mod tests { }; { let mut fields = pending_queue_fields(&mut state, "test setup").unwrap(); - fields.pending_deposits_mut().unwrap().push(first).unwrap(); - fields - .pending_deposits_mut() - .unwrap() - .push(second.clone()) - .unwrap(); + fields.push_pending_deposit(first).unwrap(); + fields.push_pending_deposit(second.clone()).unwrap(); } let balance_0_before = state.balance(0).unwrap(); @@ -1037,14 +842,17 @@ mod tests { "over budget: must not apply" ); - let mut fields = pending_queue_fields(&mut state, "test assertion").unwrap(); assert_eq!( - fields.deposit_balance_to_consume(), + pending_queue_fields(&mut state, "test assertion") + .unwrap() + .deposit_balance_to_consume(), churn_limit - first_amount ); - let remaining = fields.pending_deposits_mut().unwrap(); - assert_eq!(remaining.len(), 1); - assert_eq!(remaining[0], second); + let BeaconState::Electra(inner) = &state else { + unreachable!("built as Electra"); + }; + assert_eq!(inner.pending_deposits.len(), 1); + assert_eq!(inner.pending_deposits[0], second); } #[test] @@ -1085,9 +893,7 @@ mod tests { }; pending_queue_fields(&mut state, "test setup") .unwrap() - .pending_deposits_mut() - .unwrap() - .push(deposit) + .push_pending_deposit(deposit) .unwrap(); let validators_before = state.validator_count(); @@ -1137,9 +943,7 @@ mod tests { }; pending_queue_fields(&mut state, "test setup") .unwrap() - .pending_deposits_mut() - .unwrap() - .push(deposit) + .push_pending_deposit(deposit) .unwrap(); let validators_before = state.validator_count(); @@ -1150,9 +954,11 @@ mod tests { validators_before, "an invalid signature must not create a validator" ); - let mut fields = pending_queue_fields(&mut state, "test assertion").unwrap(); + let BeaconState::Electra(inner) = &state else { + unreachable!("built as Electra"); + }; assert!( - fields.pending_deposits_mut().unwrap().is_empty(), + inner.pending_deposits.is_empty(), "the entry is still consumed from the queue, just never credited" ); } diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/fulu.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/fulu.rs index d0bdad0bc..39ddc6061 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/fulu.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/fulu.rs @@ -59,7 +59,7 @@ use crate::beacon::containers::electra as electra_containers; use crate::beacon::error::{Error, Result}; use crate::beacon::fork::ForkName; use crate::beacon::helpers::accessors::get_current_epoch; -use crate::beacon::helpers::electra::get_activation_exit_churn_limit; +use crate::beacon::helpers::electra::{get_activation_exit_churn_limit, pending_queue_fields}; use crate::beacon::helpers::fulu::{get_beacon_proposer_indices, proposer_lookahead_mut}; use crate::beacon::helpers::misc::compute_start_slot_at_epoch; use crate::beacon::lean_state_unreachable; @@ -134,15 +134,14 @@ pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Res // See `electra::process_pending_deposits`'s own doc for why the queue is // taken by value here rather than iterated in place. let (available_for_processing, deposits) = { - let mut fields = electra::pending_queue_fields(state, "process_pending_deposits")?; + let mut fields = pending_queue_fields(state, "process_pending_deposits")?; let available_for_processing = fields .deposit_balance_to_consume() .checked_add(churn_limit) .ok_or(Error::ArithmeticOverflow( "deposit_balance_to_consume + get_activation_exit_churn_limit", ))?; - let deposits: Vec = - core::mem::take(fields.pending_deposits_mut()?).into_inner(); + let deposits: Vec = fields.take_pending_deposits()?; (available_for_processing, deposits) }; @@ -218,8 +217,8 @@ pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Res 0 }; - let mut fields = electra::pending_queue_fields(state, "process_pending_deposits")?; - *fields.pending_deposits_mut()? = electra_containers::PendingDeposits::try_from(remaining)?; + let mut fields = pending_queue_fields(state, "process_pending_deposits")?; + fields.set_pending_deposits(remaining)?; *fields.deposit_balance_to_consume_mut() = deposit_balance_to_consume; Ok(()) @@ -394,7 +393,7 @@ mod tests { fn a_test_state_starts_with_deposit_requests_start_index_unset() { let mut state = fulu_state_with_validators(2); assert_eq!( - electra::pending_queue_fields(&mut state, "test assertion") + pending_queue_fields(&mut state, "test assertion") .unwrap() .deposit_requests_start_index(), constants::UNSET_DEPOSIT_REQUESTS_START_INDEX @@ -410,11 +409,9 @@ mod tests { // this deposit through; only the eth1-bridge gate is under test. state.finalized_checkpoint_mut().epoch = 1; let deposit = request_sourced_pending_deposit(&state, 0); - electra::pending_queue_fields(&mut state, "test setup") + pending_queue_fields(&mut state, "test setup") .unwrap() - .pending_deposits_mut() - .unwrap() - .push(deposit) + .push_pending_deposit(deposit) .unwrap(); let balance_before = state.balance(0).unwrap(); @@ -426,8 +423,10 @@ mod tests { state.balance(0).unwrap(), balance_before + preset::EFFECTIVE_BALANCE_INCREMENT ); - let mut fields = electra::pending_queue_fields(&mut state, "test assertion").unwrap(); - assert!(fields.pending_deposits_mut().unwrap().is_empty()); + let BeaconState::Fulu(inner) = &state else { + unreachable!("built as Fulu"); + }; + assert!(inner.pending_deposits.is_empty()); } #[test] @@ -438,11 +437,9 @@ mod tests { // eth1-bridge gate, not the unrelated finality gate, is under test. state.finalized_checkpoint_mut().epoch = 1; let deposit = request_sourced_pending_deposit(&state, 0); - electra::pending_queue_fields(&mut state, "test setup") - .unwrap() - .pending_deposits_mut() + pending_queue_fields(&mut state, "test setup") .unwrap() - .push(deposit.clone()) + .push_pending_deposit(deposit.clone()) .unwrap(); let balance_before = state.balance(0).unwrap(); @@ -457,9 +454,10 @@ mod tests { // this request-sourced entry and the whole pass breaks before ever // reaching it. assert_eq!(state.balance(0).unwrap(), balance_before); - let mut fields = electra::pending_queue_fields(&mut state, "test assertion").unwrap(); - let remaining = fields.pending_deposits_mut().unwrap(); - assert_eq!(remaining.len(), 1); - assert_eq!(remaining[0], deposit); + let BeaconState::Fulu(inner) = &state else { + unreachable!("built as Fulu"); + }; + assert_eq!(inner.pending_deposits.len(), 1); + assert_eq!(inner.pending_deposits[0], deposit); } } diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs index 7a8daef22..57e3dd7d8 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs @@ -35,8 +35,8 @@ //! `SszList` and `ProgressiveList` `Deref` to a plain `[T]`. //! - **[`super::electra::process_pending_consolidations`]** takes the queue //! as a `Vec` and writes it back, through -//! [`super::electra::PendingQueueFields::take_pending_consolidations`] and -//! [`super::electra::PendingQueueFields::set_pending_consolidations`], which +//! [`crate::beacon::helpers::electra::PendingQueueFields::take_pending_consolidations`] and +//! [`crate::beacon::helpers::electra::PendingQueueFields::set_pending_consolidations`], which //! accept a gloas state. //! - **[`super::fulu::process_proposer_lookahead`]** shifts the window //! through an accessor both forks share, and chooses the proposer draw by @@ -142,7 +142,7 @@ pub fn process_participation_flag_updates(state: &mut BeaconState) -> Result<()> /// dropping it changes nothing observable once a chain has reached this far). /// The queue is gloas's own progressive one (EIP-7688), read and written /// directly through [`gloas_state`] rather than through -/// [`super::electra::pending_queue_fields`]'s [`PendingQueueFields`](super::electra::PendingQueueFields), +/// [`crate::beacon::helpers::electra::pending_queue_fields`]'s [`PendingQueueFields`](crate::beacon::helpers::electra::PendingQueueFields), /// which refuses that field for a gloas state (see its own doc). Once a /// deposit is dequeued and cleared to apply, though, /// [`super::electra::apply_pending_deposit`] is the identical function electra diff --git a/crates/blockchain/state_transition/src/beacon/stf/fulu.rs b/crates/blockchain/state_transition/src/beacon/stf/fulu.rs index 9b81dd6ef..3d720ac33 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/fulu.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/fulu.rs @@ -44,7 +44,7 @@ use crate::beacon::config::Config; use crate::beacon::containers::{BeaconState, deneb, electra}; use crate::beacon::error::{Result, verify}; use crate::beacon::helpers::accessors::{CommitteeCache, get_current_epoch, get_randao_mix}; -use crate::beacon::helpers::electra::electra_state; +use crate::beacon::helpers::electra::pending_queue_fields; use crate::beacon::helpers::fulu::{fulu_state, fulu_state_ref}; use crate::beacon::primitives::{Bytes32, HashTreeRoot as _}; @@ -287,6 +287,15 @@ pub fn process_execution_requests( /// which is exactly the case [`super::epoch::fulu::process_pending_deposits`]'s /// own doc explains this fork's specification means to prevent: the field /// stays unset for good, not merely until this fork's own first request. +/// +/// Also served, unmodified, by gloas: gloas's own `beacon-chain.md` does not +/// redefine `process_deposit_request` either, so it inherits this fork's +/// version rather than electra's, and the only thing that kept a gloas state +/// from calling this exact copy was the queue's own type +/// ([`crate::beacon::containers::gloas::PendingDeposits`], a +/// [`libssz_types::ProgressiveList`], EIP-7688), which +/// [`crate::beacon::helpers::electra::PendingQueueFields::push_pending_deposit`] +/// already abstracts over. pub fn process_deposit_request( state: &mut BeaconState, request: &electra::DepositRequest, @@ -298,10 +307,7 @@ pub fn process_deposit_request( signature: request.signature, slot: state.slot(), }; - electra_state(state, "process_deposit_request")? - .pending_deposits_mut() - .push(deposit)?; - Ok(()) + pending_queue_fields(state, "process_deposit_request")?.push_pending_deposit(deposit) } #[cfg(test)] diff --git a/crates/blockchain/state_transition/src/beacon/stf/gloas.rs b/crates/blockchain/state_transition/src/beacon/stf/gloas.rs new file mode 100644 index 000000000..84c220c73 --- /dev/null +++ b/crates/blockchain/state_transition/src/beacon/stf/gloas.rs @@ -0,0 +1,967 @@ +//! Gloas's withdrawals, execution payload bid, and parent execution payload +//! processing (EIP-7732). +//! +//! Withdrawals gain two new sweeps of their own: builders can be paid out +//! ([`get_builder_withdrawals`], draining +//! [`gloas::BeaconState::builder_pending_withdrawals`]) and swept the same +//! way validators are ([`get_builders_sweep_withdrawals`]), both ahead of the +//! validator-side sweeps `crate::beacon::stf::electra::get_pending_partial_withdrawals` +//! and `crate::beacon::stf::electra::get_validators_sweep_withdrawals` carry +//! over from electra unmodified (see [`get_expected_withdrawals`] for the +//! order all four run in). [`process_withdrawals`] itself drops its +//! `payload` parameter entirely: withdrawals are now deterministic from the +//! state alone, since there is no payload for a block to carry them in. +//! +//! A block no longer embeds its own payload at all. It commits to a builder's +//! bid instead ([`process_execution_payload_bid`]), and the previous slot's +//! payload is only processed once the *next* block reveals whether it showed +//! up ([`process_parent_execution_payload`]): [`apply_parent_execution_payload`] +//! is where a "full" parent's execution-layer-triggered requests are finally +//! run and its builder paid. +//! +//! # Execution-layer-triggered requests live here, not with block operations +//! +//! Deposit, withdrawal, and consolidation requests, and the two new builder +//! registry requests ([`process_builder_deposit_request`], +//! [`process_builder_exit_request`]), are reached exactly one way: through +//! [`apply_parent_execution_payload`]'s `requests` parameter. Gloas's own +//! `process_operations` *removes* the calls to the first three outright, and +//! neither builder request is ever on that list to begin with, so none of +//! the five belong with block operations at all. +//! +//! The first three are not *behaviorally* new: no fork after fulu lists a +//! modified version of any of them, so [`apply_parent_execution_payload`] +//! calls straight into `crate::beacon::stf::fulu::process_deposit_request` +//! and `crate::beacon::stf::electra::{process_withdrawal_request, +//! process_consolidation_request}` rather than keeping copies. What changed +//! is only the Rust type of the lists those functions touch: +//! [`gloas::BeaconState::pending_deposits`], +//! [`gloas::BeaconState::pending_partial_withdrawals`], and +//! [`gloas::BeaconState::pending_consolidations`] are EIP-7688's progressive +//! lists, a different Rust type from electra's and fulu's bounded ones, so +//! those shared functions reach them through +//! [`crate::beacon::helpers::electra::PendingQueueFields`] (and its +//! read-only counterpart), which abstracts over exactly that type change, +//! rather than through a gloas-only copy. + +use crate::beacon::bls; +use crate::beacon::config::Config; +use crate::beacon::constants; +use crate::beacon::containers::shared::DepositMessage; +use crate::beacon::containers::{BeaconState, capella, gloas}; +use crate::beacon::error::{Error, Result, verify}; +use crate::beacon::helpers::accessors::{ + get_beacon_proposer_index, get_block_root_at_slot, get_current_epoch, get_domain, + get_previous_epoch, get_randao_mix, +}; +use crate::beacon::helpers::electra::g2_point_at_infinity; +use crate::beacon::helpers::gloas::{ + add_builder_to_registry, can_builder_cover_bid, convert_builder_index_to_validator_index, + convert_validator_index_to_builder_index, gloas_state, gloas_state_ref, initiate_builder_exit, + is_active_builder, is_builder_index, is_builder_withdrawal_credential, settle_builder_payment, +}; +use crate::beacon::helpers::misc::{compute_domain, compute_epoch_at_slot, compute_signing_root}; +use crate::beacon::helpers::mutators::decrease_balance; +use crate::beacon::preset; +use crate::beacon::primitives::{ExecutionAddress, HashTreeRoot as _, Root, WithdrawalIndex}; + +// --------------------------------------------------------------------------- +// Withdrawals +// --------------------------------------------------------------------------- + +/// `ExpectedWithdrawals` (gloas `beacon-chain.md`): a plain Rust struct, not +/// an SSZ container, since the specification's dataclass never crosses the +/// wire itself, only [`Self::withdrawals`] does, as +/// [`gloas::BeaconState::payload_expected_withdrawals`]. +pub struct ExpectedWithdrawals { + pub withdrawals: Vec, + pub processed_builder_withdrawals_count: u64, + pub processed_partial_withdrawals_count: u64, + pub processed_builders_sweep_count: u64, + pub processed_sweep_withdrawals_count: u64, +} + +/// `get_builder_withdrawals` (gloas `beacon-chain.md`): the withdrawals owed +/// to builders queued in [`gloas::BeaconState::builder_pending_withdrawals`], +/// oldest first, up to `MAX_WITHDRAWALS_PER_PAYLOAD - 1` (one slot is always +/// reserved for the validator sweep further down [`get_expected_withdrawals`]'s +/// own order). +pub fn get_builder_withdrawals( + state: &BeaconState, + mut withdrawal_index: WithdrawalIndex, + prior_withdrawals: &[capella::Withdrawal], +) -> Result<(Vec, WithdrawalIndex, u64)> { + let withdrawals_limit = preset::MAX_WITHDRAWALS_PER_PAYLOAD - 1; + verify( + prior_withdrawals.len() <= withdrawals_limit, + "get_builder_withdrawals: len(prior_withdrawals) <= withdrawals_limit", + )?; + + let mut processed_count: u64 = 0; + let mut withdrawals = Vec::new(); + let inner = gloas_state_ref(state, "get_builder_withdrawals")?; + for withdrawal in inner.builder_pending_withdrawals.iter() { + if prior_withdrawals.len() + withdrawals.len() >= withdrawals_limit { + break; + } + + withdrawals.push(capella::Withdrawal { + index: withdrawal_index, + validator_index: convert_builder_index_to_validator_index(withdrawal.builder_index), + address: withdrawal.fee_recipient, + amount: withdrawal.amount, + }); + withdrawal_index = withdrawal_index + .checked_add(1) + .ok_or(Error::ArithmeticOverflow( + "get_builder_withdrawals: withdrawal_index + 1", + ))?; + processed_count += 1; + } + + Ok((withdrawals, withdrawal_index, processed_count)) +} + +/// `get_builders_sweep_withdrawals` (gloas `beacon-chain.md`): the builder +/// registry's own counterpart of +/// [`get_validators_sweep_withdrawals`](crate::beacon::stf::electra::get_validators_sweep_withdrawals), +/// swept +/// from [`gloas::BeaconState::next_withdrawal_builder_index`] the same +/// bounded, wrapping way. +pub fn get_builders_sweep_withdrawals( + state: &BeaconState, + mut withdrawal_index: WithdrawalIndex, + prior_withdrawals: &[capella::Withdrawal], +) -> Result<(Vec, WithdrawalIndex, u64)> { + let epoch = get_current_epoch(state); + let inner = gloas_state_ref(state, "get_builders_sweep_withdrawals")?; + let builders_len = inner.builders.len() as u64; + let builders_limit = builders_len.min(preset::MAX_BUILDERS_PER_WITHDRAWALS_SWEEP); + let withdrawals_limit = preset::MAX_WITHDRAWALS_PER_PAYLOAD - 1; + verify( + prior_withdrawals.len() <= withdrawals_limit, + "get_builders_sweep_withdrawals: len(prior_withdrawals) <= withdrawals_limit", + )?; + + let mut processed_count: u64 = 0; + let mut withdrawals = Vec::new(); + let mut builder_index = inner.next_withdrawal_builder_index; + for _ in 0..builders_limit { + if prior_withdrawals.len() + withdrawals.len() >= withdrawals_limit { + break; + } + + let builder = + inner + .builders + .get(builder_index as usize) + .ok_or(Error::IndexOutOfBounds { + index: builder_index as usize, + len: inner.builders.len(), + })?; + if builder.withdrawable_epoch <= epoch && builder.balance > 0 { + withdrawals.push(capella::Withdrawal { + index: withdrawal_index, + validator_index: convert_builder_index_to_validator_index(builder_index), + address: builder.execution_address, + amount: builder.balance, + }); + withdrawal_index = withdrawal_index + .checked_add(1) + .ok_or(Error::ArithmeticOverflow( + "get_builders_sweep_withdrawals: withdrawal_index + 1", + ))?; + } + + // `builders_limit <= builders_len`, and this loop only ever runs when + // `builders_limit > 0`, so `builders_len` is never zero here. + builder_index = builder_index + .checked_add(1) + .ok_or(Error::ArithmeticOverflow( + "get_builders_sweep_withdrawals: builder_index + 1", + ))? + % builders_len; + processed_count += 1; + } + + Ok((withdrawals, withdrawal_index, processed_count)) +} + +/// `get_expected_withdrawals` (modified, EIP-7732): builder withdrawals, then +/// pending partial withdrawals, then the builders sweep, then the validators +/// sweep, each sweep's own `prior_withdrawals` accumulating every withdrawal +/// an earlier sweep in this same call already produced, so none double-pays a +/// balance an earlier sweep already claimed. +/// +/// The last two sweeps are unmodified since electra (`beacon-chain.md` does +/// not redefine `get_pending_partial_withdrawals` or +/// `get_validators_sweep_withdrawals` for gloas), so they are shared calls +/// into [`crate::beacon::stf::electra`] rather than copies: the former +/// through [`crate::beacon::helpers::electra::PendingQueueFieldsRef`], which +/// abstracts over `pending_partial_withdrawals`'s type change (EIP-7688); the +/// latter touches no gloas-changed list at all, so it needs no widening +/// beyond [`BeaconState`]'s own fork-invariant accessors. +pub fn get_expected_withdrawals(state: &BeaconState) -> Result { + let (mut withdrawal_index, _) = state.withdrawal_cursor()?; + let mut withdrawals: Vec = Vec::new(); + + let (builder_withdrawals, next_index, processed_builder_withdrawals_count) = + get_builder_withdrawals(state, withdrawal_index, &withdrawals)?; + withdrawal_index = next_index; + withdrawals.extend(builder_withdrawals); + + let (partial_withdrawals, next_index, processed_partial_withdrawals_count) = + crate::beacon::stf::electra::get_pending_partial_withdrawals( + state, + withdrawal_index, + &withdrawals, + )?; + withdrawal_index = next_index; + withdrawals.extend(partial_withdrawals); + + let (builders_sweep_withdrawals, next_index, processed_builders_sweep_count) = + get_builders_sweep_withdrawals(state, withdrawal_index, &withdrawals)?; + withdrawal_index = next_index; + withdrawals.extend(builders_sweep_withdrawals); + + let (validators_sweep_withdrawals, _, processed_sweep_withdrawals_count) = + crate::beacon::stf::electra::get_validators_sweep_withdrawals( + state, + withdrawal_index, + &withdrawals, + )?; + withdrawals.extend(validators_sweep_withdrawals); + + Ok(ExpectedWithdrawals { + withdrawals, + processed_builder_withdrawals_count, + processed_partial_withdrawals_count, + processed_builders_sweep_count, + processed_sweep_withdrawals_count, + }) +} + +/// `apply_withdrawals` (modified, EIP-7732): a builder-indexed withdrawal +/// (`is_builder_index`) saturates the builder's own balance down rather than +/// going through [`decrease_balance`], the same "cannot go negative, no error +/// either" contract `saturating_sub` already gives ordinary validator +/// balances through that function. +pub fn apply_withdrawals( + state: &mut BeaconState, + withdrawals: &[capella::Withdrawal], +) -> Result<()> { + for withdrawal in withdrawals { + if is_builder_index(withdrawal.validator_index) { + let builder_index = + convert_validator_index_to_builder_index(withdrawal.validator_index); + let inner = gloas_state(state, "apply_withdrawals")?; + let len = inner.builders.len(); + let builder = + inner + .builders + .get_mut(builder_index as usize) + .ok_or(Error::IndexOutOfBounds { + index: builder_index as usize, + len, + })?; + builder.balance = builder.balance.saturating_sub(withdrawal.amount); + } else { + decrease_balance(state, withdrawal.validator_index, withdrawal.amount)?; + } + } + Ok(()) +} + +/// `update_payload_expected_withdrawals` (gloas `beacon-chain.md`): caches +/// this block's withdrawal sweep on the state itself, since there is no +/// payload for a block to carry it in for the execution layer to read back. +pub fn update_payload_expected_withdrawals( + state: &mut BeaconState, + withdrawals: &[capella::Withdrawal], +) -> Result<()> { + gloas_state(state, "update_payload_expected_withdrawals")?.payload_expected_withdrawals = + gloas::Withdrawals::from(withdrawals.to_vec()); + Ok(()) +} + +/// `update_builder_pending_withdrawals` (gloas `beacon-chain.md`): drops +/// however many entries [`get_builder_withdrawals`] actually consumed off the +/// front of the queue. +/// +/// No block queues a builder withdrawal at all most of the time, so the +/// common case returns before touching the queue; when there is work, +/// `mem::take` + `split_off` move the surviving entries into their +/// replacement rather than cloning the whole queue to build it, the same way +/// `crate::beacon::stf::electra::process_withdrawals` drops its own +/// `pending_partial_withdrawals` entries. +pub fn update_builder_pending_withdrawals( + state: &mut BeaconState, + processed_builder_withdrawals_count: u64, +) -> Result<()> { + if processed_builder_withdrawals_count == 0 { + return Ok(()); + } + let inner = gloas_state(state, "update_builder_pending_withdrawals")?; + let mut owned = core::mem::take(&mut inner.builder_pending_withdrawals).into_inner(); + let remaining = owned.split_off(processed_builder_withdrawals_count as usize); + inner.builder_pending_withdrawals = gloas::BuilderPendingWithdrawals::from(remaining); + Ok(()) +} + +/// `update_pending_partial_withdrawals` (unmodified since electra, whose +/// `process_withdrawals` drains the same queue inline). See +/// [`update_builder_pending_withdrawals`]'s own doc for why the common, +/// nothing-consumed case returns early and why the rest moves entries +/// instead of cloning them. +pub fn update_pending_partial_withdrawals( + state: &mut BeaconState, + processed_partial_withdrawals_count: u64, +) -> Result<()> { + if processed_partial_withdrawals_count == 0 { + return Ok(()); + } + let inner = gloas_state(state, "update_pending_partial_withdrawals")?; + let mut owned = core::mem::take(&mut inner.pending_partial_withdrawals).into_inner(); + let remaining = owned.split_off(processed_partial_withdrawals_count as usize); + inner.pending_partial_withdrawals = gloas::PendingPartialWithdrawals::from(remaining); + Ok(()) +} + +/// `update_next_withdrawal_builder_index` (gloas `beacon-chain.md`): the +/// builder-side counterpart of [`update_next_withdrawal_validator_index`], +/// advanced by however many builders [`get_builders_sweep_withdrawals`] +/// actually visited rather than by a fixed bound, since that sweep's own +/// bound is already `min(len(state.builders), MAX_BUILDERS_PER_WITHDRAWALS_SWEEP)`. +/// Left untouched on an empty builder registry, which has no valid index to +/// wrap into. +pub fn update_next_withdrawal_builder_index( + state: &mut BeaconState, + processed_builders_sweep_count: u64, +) -> Result<()> { + let inner = gloas_state(state, "update_next_withdrawal_builder_index")?; + let builders_len = inner.builders.len() as u64; + if builders_len > 0 { + let next_index = inner + .next_withdrawal_builder_index + .checked_add(processed_builders_sweep_count) + .ok_or(Error::ArithmeticOverflow( + "update_next_withdrawal_builder_index: next_withdrawal_builder_index + processed_builders_sweep_count", + ))?; + inner.next_withdrawal_builder_index = next_index % builders_len; + } + Ok(()) +} + +/// `update_next_withdrawal_index` (unchanged since capella; ported here +/// because gloas's own `process_withdrawals` still calls it by name). +fn update_next_withdrawal_index( + state: &mut BeaconState, + withdrawals: &[capella::Withdrawal], +) -> Result<()> { + if let Some(latest) = withdrawals.last() { + let (withdrawal_index, _) = state.withdrawal_cursor_mut()?; + *withdrawal_index = latest + .index + .checked_add(1) + .ok_or(Error::ArithmeticOverflow( + "update_next_withdrawal_index: latest_withdrawal.index + 1", + ))?; + } + Ok(()) +} + +/// `update_next_withdrawal_validator_index` (unchanged since capella; ported +/// here for the same reason as [`update_next_withdrawal_index`]). +fn update_next_withdrawal_validator_index( + state: &mut BeaconState, + withdrawals: &[capella::Withdrawal], +) -> Result<()> { + let validator_count = state.validator_count() as u64; + verify( + validator_count > 0, + "update_next_withdrawal_validator_index: len(state.validators) > 0", + )?; + let next_validator_index = if withdrawals.len() == preset::MAX_WITHDRAWALS_PER_PAYLOAD { + let latest = withdrawals + .last() + .expect("MAX_WITHDRAWALS_PER_PAYLOAD is never zero, so a full payload is non-empty"); + latest + .validator_index + .checked_add(1) + .ok_or(Error::ArithmeticOverflow( + "update_next_withdrawal_validator_index: latest_withdrawal.validator_index + 1", + ))? + % validator_count + } else { + let (_, current_cursor) = state.withdrawal_cursor()?; + current_cursor + .checked_add(preset::MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP) + .ok_or(Error::ArithmeticOverflow( + "update_next_withdrawal_validator_index: next_withdrawal_validator_index + MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP", + ))? + % validator_count + }; + let (_, validator_index) = state.withdrawal_cursor_mut()?; + *validator_index = next_validator_index; + Ok(()) +} + +/// `process_withdrawals` (modified, EIP-7732): takes only `state`. Returns +/// early, doing nothing at all, when the parent block was empty (its bid's +/// `block_hash` never landed in `state.latest_block_hash`): withdrawals are +/// deterministic from the state alone, and an empty parent leaves nothing new +/// to sweep beyond what the last block that actually revealed a payload +/// already swept. `payload_expected_withdrawals` is left exactly as that +/// last sweep set it rather than cleared: no execution payload has paid it +/// out yet either (there was none to reveal it), so the field still +/// correctly names what the *next* revealed payload owes. +pub fn process_withdrawals(state: &mut BeaconState) -> Result<()> { + let inner = gloas_state_ref(state, "process_withdrawals")?; + if inner.latest_block_hash != inner.latest_execution_payload_bid.block_hash { + return Ok(()); + } + + let expected = get_expected_withdrawals(state)?; + apply_withdrawals(state, &expected.withdrawals)?; + + update_next_withdrawal_index(state, &expected.withdrawals)?; + update_payload_expected_withdrawals(state, &expected.withdrawals)?; + update_builder_pending_withdrawals(state, expected.processed_builder_withdrawals_count)?; + update_pending_partial_withdrawals(state, expected.processed_partial_withdrawals_count)?; + update_next_withdrawal_builder_index(state, expected.processed_builders_sweep_count)?; + update_next_withdrawal_validator_index(state, &expected.withdrawals)?; + + Ok(()) +} + +// --------------------------------------------------------------------------- +// Execution payload bid +// --------------------------------------------------------------------------- + +/// `verify_execution_payload_bid_signature` (gloas `beacon-chain.md`). +/// +/// `Result` rather than a bare `bool`, unlike the specification's own +/// signature: [`process_execution_payload_bid`] only ever calls this after +/// `is_active_builder` has already proven `builder_index` is in range, but an +/// out-of-bounds index is still a real possibility for a caller that does +/// not, so this reports it rather than silently treating it as "signature +/// invalid". +pub fn verify_execution_payload_bid_signature( + state: &BeaconState, + signed_bid: &gloas::SignedExecutionPayloadBid, +) -> Result { + let builder_index = signed_bid.message.builder_index; + let inner = gloas_state_ref(state, "verify_execution_payload_bid_signature")?; + let builder = inner + .builders + .get(builder_index as usize) + .ok_or(Error::IndexOutOfBounds { + index: builder_index as usize, + len: inner.builders.len(), + })?; + let domain = get_domain(state, constants::DOMAIN_BEACON_BUILDER, None); + let signing_root = compute_signing_root(signed_bid.message.hash_tree_root(), domain); + Ok(bls::verify( + &builder.pubkey, + signing_root, + &signed_bid.signature, + )) +} + +/// `process_execution_payload_bid` (gloas `beacon-chain.md`). +pub fn process_execution_payload_bid( + state: &mut BeaconState, + signed_bid: &gloas::SignedExecutionPayloadBid, + config: &Config, +) -> Result<()> { + let bid = signed_bid.message.clone(); + let builder_index = bid.builder_index; + let amount = bid.value; + + if builder_index == constants::BUILDER_INDEX_SELF_BUILD { + // For self-builds, amount must be zero regardless of withdrawal + // credential prefix. + verify(amount == 0, "process_execution_payload_bid: amount == 0")?; + verify( + signed_bid.signature == g2_point_at_infinity(), + "process_execution_payload_bid: signature == G2_POINT_AT_INFINITY", + )?; + } else { + // Verify that the builder is active. + verify( + is_active_builder( + gloas_state_ref(state, "process_execution_payload_bid")?, + builder_index, + )?, + "process_execution_payload_bid: is_active_builder(state, builder_index)", + )?; + // Verify that the builder is a payload builder. + let version = { + let inner = gloas_state_ref(state, "process_execution_payload_bid")?; + inner + .builders + .get(builder_index as usize) + .ok_or(Error::IndexOutOfBounds { + index: builder_index as usize, + len: inner.builders.len(), + })? + .version + }; + verify( + version == constants::PAYLOAD_BUILDER_VERSION, + "process_execution_payload_bid: state.builders[builder_index].version == PAYLOAD_BUILDER_VERSION", + )?; + // Verify that the builder has funds to cover the bid. + verify( + can_builder_cover_bid( + gloas_state_ref(state, "process_execution_payload_bid")?, + builder_index, + amount, + )?, + "process_execution_payload_bid: can_builder_cover_bid(state, builder_index, amount)", + )?; + // Verify that the bid signature is valid. + verify( + verify_execution_payload_bid_signature(state, signed_bid)?, + "process_execution_payload_bid: verify_execution_payload_bid_signature(state, signed_bid)", + )?; + } + + // Verify commitments are under limit. + verify( + bid.blob_kzg_commitments.len() as u64 + <= config.max_blobs_per_block(get_current_epoch(state)), + "process_execution_payload_bid: len(bid.blob_kzg_commitments) <= get_blob_parameters(current_epoch).max_blobs_per_block", + )?; + + // Verify that the bid is for the current slot. + verify( + bid.slot == state.slot(), + "process_execution_payload_bid: bid.slot == state.slot", + )?; + verify( + state.slot() > constants::GENESIS_SLOT, + "process_execution_payload_bid: state.slot > GENESIS_SLOT", + )?; + // Verify that the bid is for the right parent block. + verify( + bid.parent_block_hash + == gloas_state_ref(state, "process_execution_payload_bid")?.latest_block_hash, + "process_execution_payload_bid: bid.parent_block_hash == state.latest_block_hash", + )?; + // Verify that the bid's block hash differs from its parent block hash. + verify( + bid.block_hash != bid.parent_block_hash, + "process_execution_payload_bid: bid.block_hash != bid.parent_block_hash", + )?; + verify( + bid.parent_block_root == get_block_root_at_slot(state, state.slot() - 1)?, + "process_execution_payload_bid: bid.parent_block_root == get_block_root_at_slot(state, state.slot - 1)", + )?; + verify( + bid.prev_randao == get_randao_mix(state, get_current_epoch(state)), + "process_execution_payload_bid: bid.prev_randao == get_randao_mix(state, get_current_epoch(state))", + )?; + + // Record the pending payment if there is some payment. + if amount > 0 { + let proposer_index = get_beacon_proposer_index(state)?; + let pending_payment = gloas::BuilderPendingPayment { + weight: 0, + withdrawal: gloas::BuilderPendingWithdrawal { + fee_recipient: bid.fee_recipient, + amount, + builder_index, + }, + proposer_index, + }; + let index = preset::SLOTS_PER_EPOCH + .checked_add(bid.slot % preset::SLOTS_PER_EPOCH) + .ok_or(Error::ArithmeticOverflow( + "process_execution_payload_bid: SLOTS_PER_EPOCH + bid.slot % SLOTS_PER_EPOCH", + ))? as usize; + let inner = gloas_state(state, "process_execution_payload_bid")?; + let len = inner.builder_pending_payments.len(); + if index >= len { + return Err(Error::IndexOutOfBounds { index, len }); + } + inner.builder_pending_payments[index] = pending_payment; + } + + // Cache the signed execution payload bid. + gloas_state(state, "process_execution_payload_bid")?.latest_execution_payload_bid = bid; + + Ok(()) +} + +// --------------------------------------------------------------------------- +// Parent execution payload +// --------------------------------------------------------------------------- + +/// `apply_parent_execution_payload` (gloas `beacon-chain.md`). +/// +/// Processes the parent's execution-layer-triggered requests, settles (or +/// directly queues) its builder payment, and marks its payload available. +/// Called by [`process_parent_execution_payload`] during block processing, +/// and (per the specification's own note) by the validator during block +/// production before computing withdrawals. +pub fn apply_parent_execution_payload( + state: &mut BeaconState, + requests: &gloas::ExecutionRequests, + config: &Config, +) -> Result<()> { + let (parent_bid, parent_slot, parent_epoch) = { + let inner = gloas_state_ref(state, "apply_parent_execution_payload")?; + let parent_slot = inner.latest_block_header.slot; + ( + inner.latest_execution_payload_bid.clone(), + parent_slot, + compute_epoch_at_slot(parent_slot), + ) + }; + + verify( + requests.withdrawals.len() <= preset::MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD, + "apply_parent_execution_payload: len(requests.withdrawals) <= MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD", + )?; + verify( + requests.consolidations.len() <= preset::MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD, + "apply_parent_execution_payload: len(requests.consolidations) <= MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD", + )?; + verify( + requests.builder_deposits.len() as u64 <= preset::MAX_BUILDER_DEPOSIT_REQUESTS_PER_PAYLOAD, + "apply_parent_execution_payload: len(requests.builder_deposits) <= MAX_BUILDER_DEPOSIT_REQUESTS_PER_PAYLOAD", + )?; + verify( + requests.builder_exits.len() as u64 <= preset::MAX_BUILDER_EXIT_REQUESTS_PER_PAYLOAD, + "apply_parent_execution_payload: len(requests.builder_exits) <= MAX_BUILDER_EXIT_REQUESTS_PER_PAYLOAD", + )?; + + // Process execution requests from the parent's payload. The execution + // requests are processed at state.slot (child's slot), not the parent's + // slot. + // + // Deposits, withdrawals, and consolidations are unmodified since electra + // (fulu, for deposits): see this module's own doc for why these three + // are shared calls into `stf::fulu`/`stf::electra` rather than copies. + for deposit in requests.deposits.iter() { + crate::beacon::stf::fulu::process_deposit_request(state, deposit)?; + } + for withdrawal in requests.withdrawals.iter() { + crate::beacon::stf::electra::process_withdrawal_request(state, withdrawal, config)?; + } + for consolidation in requests.consolidations.iter() { + crate::beacon::stf::electra::process_consolidation_request(state, consolidation, config)?; + } + for builder_deposit in requests.builder_deposits.iter() { + process_builder_deposit_request(state, builder_deposit, config)?; + } + for builder_exit in requests.builder_exits.iter() { + process_builder_exit_request(state, builder_exit, config)?; + } + + // Settle the builder payment. + let current_epoch = get_current_epoch(state); + if parent_epoch == current_epoch { + let payment_index = preset::SLOTS_PER_EPOCH + .checked_add(parent_slot % preset::SLOTS_PER_EPOCH) + .ok_or(Error::ArithmeticOverflow( + "apply_parent_execution_payload: SLOTS_PER_EPOCH + parent_slot % SLOTS_PER_EPOCH", + ))?; + settle_builder_payment( + gloas_state(state, "apply_parent_execution_payload")?, + payment_index, + )?; + } else if parent_epoch == get_previous_epoch(state) { + let payment_index = parent_slot % preset::SLOTS_PER_EPOCH; + settle_builder_payment( + gloas_state(state, "apply_parent_execution_payload")?, + payment_index, + )?; + } else if parent_bid.value > 0 { + // Parent is older than the previous epoch, its payment entry has + // been evicted from builder_pending_payments. Append the withdrawal + // directly. + gloas_state(state, "apply_parent_execution_payload")? + .builder_pending_withdrawals + .push(gloas::BuilderPendingWithdrawal { + fee_recipient: parent_bid.fee_recipient, + amount: parent_bid.value, + builder_index: parent_bid.builder_index, + }); + } + + // Update parent payload availability and latest block hash. + let slot_index = (parent_slot as usize) % preset::SLOTS_PER_HISTORICAL_ROOT; + let inner = gloas_state(state, "apply_parent_execution_payload")?; + inner + .execution_payload_availability + .set(slot_index, true) + .map_err(|err| Error::IndexOutOfBounds { + index: err.index, + len: err.len, + })?; + inner.latest_block_hash = parent_bid.block_hash; + + Ok(()) +} + +/// `process_parent_execution_payload` (gloas `beacon-chain.md`). +/// +/// *Note (from the specification):* must be called before +/// [`process_execution_payload_bid`] (which overwrites +/// `state.latest_execution_payload_bid`). +pub fn process_parent_execution_payload( + state: &mut BeaconState, + block: &gloas::BeaconBlock, + config: &Config, +) -> Result<()> { + let bid = &block.body.signed_execution_payload_bid.message; + let parent_bid = gloas_state_ref(state, "process_parent_execution_payload")? + .latest_execution_payload_bid + .clone(); + let requests = &block.body.parent_execution_requests; + + if bid.parent_block_hash != parent_bid.block_hash { + // Parent was EMPTY: no execution requests expected. + verify( + *requests == gloas::ExecutionRequests::default(), + "process_parent_execution_payload: requests == ExecutionRequests.empty()", + )?; + return Ok(()); + } + + // Parent was FULL: verify the bid commitment and apply the payload. + verify( + requests.hash_tree_root() == parent_bid.execution_requests_root, + "process_parent_execution_payload: hash_tree_root(requests) == parent_bid.execution_requests_root", + )?; + apply_parent_execution_payload(state, requests, config) +} + +// --------------------------------------------------------------------------- +// Builder deposit and exit requests (EIP-8282) +// --------------------------------------------------------------------------- + +/// `is_valid_builder_deposit_signature` (gloas `beacon-chain.md`). +/// +/// Signed under its own domain, `DOMAIN_BUILDER_DEPOSIT`, rather than +/// ordinary validator deposits' `DOMAIN_DEPOSIT`: the specification's own +/// note is that the dedicated domain is what keeps a validator deposit +/// signature and a builder deposit signature from being replayed against the +/// other deposit contract, since the two now sign over identical +/// `DepositMessage` bytes otherwise. +/// +/// Like [`crate::beacon::stf::electra::is_valid_deposit_signature`], it is +/// signed under `GENESIS_FORK_VERSION` and an all-zero genesis validators +/// root rather than [`get_domain`]'s state-dependent ones. The +/// specification's own `compute_domain(DOMAIN_BUILDER_DEPOSIT)`, passing +/// neither argument, defaults to exactly that; `config` is this file's way +/// of reaching `GENESIS_FORK_VERSION`, the same deviation +/// `is_valid_deposit_signature` already makes from the specification's +/// global-constant signature. +pub fn is_valid_builder_deposit_signature( + request: &gloas::BuilderDepositRequest, + config: &Config, +) -> bool { + let deposit_message = DepositMessage { + pubkey: request.pubkey, + withdrawal_credentials: request.withdrawal_credentials, + amount: request.amount, + }; + let domain = compute_domain( + constants::DOMAIN_BUILDER_DEPOSIT, + config.genesis_fork_version, + Root::ZERO, + ); + let signing_root = compute_signing_root(deposit_message.hash_tree_root(), domain); + bls::verify(&request.pubkey, signing_root, &request.signature) +} + +/// `process_builder_deposit_request` (gloas `beacon-chain.md`). +/// +/// *Note (from the specification):* builder indices are reusable. When a +/// builder exits, its index may later be reassigned to a different builder +/// with a new public key. +pub fn process_builder_deposit_request( + state: &mut BeaconState, + request: &gloas::BuilderDepositRequest, + config: &Config, +) -> Result<()> { + // Ignore deposits with unexpected withdrawal credential prefixes. + if !is_builder_withdrawal_credential(request.withdrawal_credentials) { + return Ok(()); + } + + let inner = gloas_state(state, "process_builder_deposit_request")?; + let existing = inner + .builders + .iter() + .position(|builder| builder.pubkey == request.pubkey); + + match existing { + None => { + if is_valid_builder_deposit_signature(request, config) { + let slot = inner.slot; + add_builder_to_registry( + inner, + request.pubkey, + constants::PAYLOAD_BUILDER_VERSION, + ExecutionAddress::from_slice(&request.withdrawal_credentials.0[12..]), + request.amount, + slot, + ); + } + } + Some(builder_index) => { + let epoch = compute_epoch_at_slot(inner.slot); + let builder = &mut inner.builders[builder_index]; + + // If exited and swept, reset the withdrawable epoch. + if builder.withdrawable_epoch != constants::FAR_FUTURE_EPOCH && builder.balance == 0 { + builder.withdrawable_epoch = + epoch + .checked_add(config.min_builder_withdrawability_delay) + .ok_or(Error::ArithmeticOverflow( + "process_builder_deposit_request: epoch + MIN_BUILDER_WITHDRAWABILITY_DELAY", + ))?; + } + + // Increase balance by deposit amount. + builder.balance = + builder + .balance + .checked_add(request.amount) + .ok_or(Error::ArithmeticOverflow( + "process_builder_deposit_request: builder.balance + request.amount", + ))?; + } + } + Ok(()) +} + +/// `process_builder_exit_request` (gloas `beacon-chain.md`). +pub fn process_builder_exit_request( + state: &mut BeaconState, + request: &gloas::BuilderExitRequest, + config: &Config, +) -> Result<()> { + let inner = gloas_state(state, "process_builder_exit_request")?; + let Some(builder_index) = inner + .builders + .iter() + .position(|builder| builder.pubkey == request.pubkey) + else { + return Ok(()); + }; + let builder_index = builder_index as gloas::BuilderIndex; + + if !is_active_builder(inner, builder_index)? { + return Ok(()); + } + if inner.builders[builder_index as usize].execution_address != request.source_address { + return Ok(()); + } + if crate::beacon::helpers::gloas::get_pending_balance_to_withdraw_for_builder( + inner, + builder_index, + )? != 0 + { + return Ok(()); + } + + initiate_builder_exit(inner, builder_index, config) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::beacon::fork::ForkName; + + fn gloas_state_with_validators(count: usize) -> BeaconState { + crate::beacon::helpers::test_state::with_validators_at(ForkName::Gloas, count) + } + + #[test] + fn a_self_build_bid_needs_no_signature_or_value() { + let mut state = gloas_state_with_validators(4); + let config = Config::mainnet(); + + let signed_bid = gloas::SignedExecutionPayloadBid { + message: gloas::ExecutionPayloadBid { + builder_index: constants::BUILDER_INDEX_SELF_BUILD, + slot: state.slot(), + value: 0, + parent_block_hash: gloas_state_ref(&state, "test").unwrap().latest_block_hash, + parent_block_root: get_block_root_at_slot(&state, state.slot() - 1).unwrap(), + prev_randao: get_randao_mix(&state, get_current_epoch(&state)), + block_hash: crate::beacon::primitives::ExecutionBlockHash::repeat_byte(1), + ..Default::default() + }, + signature: g2_point_at_infinity(), + }; + + process_execution_payload_bid(&mut state, &signed_bid, &config).unwrap(); + assert_eq!( + gloas_state_ref(&state, "test") + .unwrap() + .latest_execution_payload_bid + .block_hash, + signed_bid.message.block_hash + ); + } + + #[test] + fn a_self_build_bid_with_a_nonzero_value_is_rejected() { + let mut state = gloas_state_with_validators(4); + let config = Config::mainnet(); + + let signed_bid = gloas::SignedExecutionPayloadBid { + message: gloas::ExecutionPayloadBid { + builder_index: constants::BUILDER_INDEX_SELF_BUILD, + slot: state.slot(), + value: 1, + parent_block_hash: gloas_state_ref(&state, "test").unwrap().latest_block_hash, + parent_block_root: get_block_root_at_slot(&state, state.slot() - 1).unwrap(), + prev_randao: get_randao_mix(&state, get_current_epoch(&state)), + block_hash: crate::beacon::primitives::ExecutionBlockHash::repeat_byte(1), + ..Default::default() + }, + signature: g2_point_at_infinity(), + }; + + let result = process_execution_payload_bid(&mut state, &signed_bid, &config); + assert!( + matches!( + result, + Err(Error::SpecAssert( + "process_execution_payload_bid: amount == 0" + )) + ), + "expected the self-build amount == 0 assertion to fail, got {result:?}" + ); + } + + #[test] + fn apply_withdrawals_saturates_a_builders_balance_rather_than_erroring() { + let mut state = gloas_state_with_validators(1); + { + let inner = gloas_state(&mut state, "test").unwrap(); + inner.builders.push(gloas::Builder { + balance: 10, + ..Default::default() + }); + } + + let withdrawal = capella::Withdrawal { + index: 0, + validator_index: convert_builder_index_to_validator_index(0), + address: ExecutionAddress::ZERO, + amount: 20, + }; + apply_withdrawals(&mut state, std::slice::from_ref(&withdrawal)).unwrap(); + + let inner = gloas_state_ref(&state, "test").unwrap(); + assert_eq!(inner.builders.first().unwrap().balance, 0); + } +} diff --git a/crates/blockchain/state_transition/src/beacon/stf/mod.rs b/crates/blockchain/state_transition/src/beacon/stf/mod.rs index 5b17097cd..ea23506a1 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/mod.rs @@ -61,6 +61,7 @@ pub mod deneb; pub mod electra; pub mod epoch; pub mod fulu; +pub mod gloas; pub mod operations; use crate::beacon::containers; diff --git a/crates/blockchain/state_transition/src/beacon/upgrade.rs b/crates/blockchain/state_transition/src/beacon/upgrade.rs index a2aca6d2f..b0bf4880c 100644 --- a/crates/blockchain/state_transition/src/beacon/upgrade.rs +++ b/crates/blockchain/state_transition/src/beacon/upgrade.rs @@ -228,9 +228,14 @@ pub fn upgrade_to_altair( // here that mutates a post-state field-by-field rather than only building // one, and the fields it reaches for that [`BeaconState`]'s own accessors do // not cover (`exit_balance_to_consume`, `consolidation_balance_to_consume`, -// `pending_deposits`) already have a projection to reuse in -// `crate::beacon::helpers::electra` (`electra_state`, returning `ElectraOrFuluMut`), -// so this file does not need a second one of its own. +// `pending_deposits`) already have projections to reuse in +// `crate::beacon::helpers::electra`: the first two through `churn_cursors_mut` +// (returning `ChurnCursorsMut`), and `pending_deposits` indirectly, through +// [`crate::beacon::helpers::electra::queue_excess_active_balance`] and +// [`crate::beacon::helpers::electra::queue_entire_balance_and_reset_validator`] +// (both reaching it through `pending_queue_fields`, returning +// `PendingQueueFields`), so this file does not need a second projection of +// its own. /// The bellatrix state, or an error naming the function that needs one. fn bellatrix_state_ref<'a>( diff --git a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs index 0bc2946c7..6538bdc75 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs @@ -84,18 +84,42 @@ pub const PRESET: &str = if cfg!(feature = "preset-minimal") { /// is one line. pub const HIGHEST_IMPLEMENTED_FORK: ForkName = ForkName::Fulu; -/// Runners whose Gloas cases already run, while the fork is only partly -/// implemented. +/// Runners (or runner/handler pairs) whose Gloas cases already run, while the +/// fork is only partly implemented. /// -/// [`case_trial`] checks this alongside [`Case::in_scope`], so a runner named -/// here has its Gloas cases run (and counted toward pass/fail) even though -/// [`HIGHEST_IMPLEMENTED_FORK`] has not reached Gloas yet; every other -/// runner's Gloas cases stay ignored. Emptied, and removed along with this -/// check, when [`HIGHEST_IMPLEMENTED_FORK`] becomes `Gloas`. +/// An entry is either a bare runner name (`"ssz_static"`), which enables +/// every handler that runner ships, or `"runner/handler"` +/// (`"operations/withdrawals"`), which enables only that one handler under +/// that runner; `handler` is the directory right under the fork +/// (`gloas///...`), i.e. [`Case::handler`]. This lets a +/// runner whose Gloas support lands handler by handler (`operations`, whose +/// `execution_payload_bid`, `withdrawals`, and `parent_execution_payload` +/// went green well before `attestation` or `payload_attestation`) turn cases +/// on incrementally, without also running every other handler under it that +/// is still unimplemented. +/// +/// [`case_trial`] checks this alongside [`Case::in_scope`], so a case whose +/// runner or runner/handler is named here runs (and counts toward pass/fail) +/// even though [`HIGHEST_IMPLEMENTED_FORK`] has not reached Gloas yet; every +/// other runner's Gloas cases stay ignored. Emptied, and removed along with +/// this check, when [`HIGHEST_IMPLEMENTED_FORK`] becomes `Gloas`. /// /// `"transition"` is not listed: its Gloas cases also exercise Gloas block /// processing, which does not exist yet. -pub const GLOAS_RUNNERS: &[&str] = &["ssz_static", "fork", "epoch_processing", "rewards"]; +pub const GLOAS_RUNNERS: &[&str] = &[ + "ssz_static", + "fork", + "epoch_processing", + "rewards", + "operations/execution_payload_bid", + "operations/withdrawals", + "operations/parent_execution_payload", + "operations/deposit_request", + "operations/withdrawal_request", + "operations/consolidation_request", + "operations/builder_deposit_request", + "operations/builder_exit_request", +]; /// The root of the extracted fixture tree. /// @@ -161,6 +185,7 @@ pub fn collect_crypto(kind: &str, fork_of: impl Fn(&str) -> ForkName) -> Vec<(St Case { path: case_entry.path(), fork, + handler: handler.clone(), suite: handler.clone(), name: case_entry.file_name().to_string_lossy().into_owned(), }, @@ -177,6 +202,10 @@ pub struct Case { pub path: PathBuf, /// The fork whose rules apply. pub fork: ForkName, + /// The handler directory right under the fork's runner directory + /// (`gloas/operations//...`). [`GLOAS_RUNNERS`]'s + /// `"runner/handler"` form is matched against this. + pub handler: String, /// The suite directory name, which groups related cases. pub suite: String, /// The case directory name, which is what test output should identify. @@ -282,7 +311,7 @@ fn collect_in(root: &Path, config: &str, runner: &str, handler: &str) -> Vec Vec<(String, Case)> { // `collect` here would be wrong: `collect` walks every fork itself, // so nesting it inside this fork loop would yield each case once per // fork that happens to ship the runner. - collect_suites(&handler_entry.path(), fork, &mut |case| { + collect_suites(&handler_entry.path(), fork, &handler, &mut |case| { out.push((handler.clone(), case)) }); } @@ -323,8 +352,10 @@ pub fn collect_all_handlers(config: &str, runner: &str) -> Vec<(String, Case)> { /// Walks the suite and case directories under one fork's handler directory. /// /// Shared by both collectors so there is one definition of what a case directory -/// is, and so neither can drift from the other. -fn collect_suites(handler_dir: &Path, fork: ForkName, emit: &mut impl FnMut(Case)) { +/// is, and so neither can drift from the other. `handler` is recorded on each +/// emitted [`Case`] as [`Case::handler`], for [`GLOAS_RUNNERS`]'s +/// `"runner/handler"` matching. +fn collect_suites(handler_dir: &Path, fork: ForkName, handler: &str, emit: &mut impl FnMut(Case)) { if !handler_dir.is_dir() { return; } @@ -335,6 +366,7 @@ fn collect_suites(handler_dir: &Path, fork: ForkName, emit: &mut impl FnMut(Case emit(Case { path: case_entry.path(), fork, + handler: handler.to_string(), suite: suite.clone(), name: case_entry.file_name().to_string_lossy().into_owned(), }); @@ -487,7 +519,9 @@ pub fn case_trial( case: Case, run: impl FnOnce(&Case) -> Result<(), String> + Send + 'static, ) -> Trial { - let gloas_enabled = case.fork == ForkName::Gloas && GLOAS_RUNNERS.contains(&runner); + let runner_handler = format!("{runner}/{}", case.handler); + let gloas_enabled = case.fork == ForkName::Gloas + && (GLOAS_RUNNERS.contains(&runner) || GLOAS_RUNNERS.contains(&runner_handler.as_str())); let ignored = !(case.in_scope() || gloas_enabled); let name = format!("{runner}/{}", case.id()); Trial::test(name, move || run(&case).map_err(Failed::from)).with_ignored_flag(ignored) diff --git a/crates/blockchain/state_transition/tests/beacon_spec/operations.rs b/crates/blockchain/state_transition/tests/beacon_spec/operations.rs index 31c7e42cb..6cd078395 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/operations.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/operations.rs @@ -92,12 +92,21 @@ //! `deposit_requests_start_index`, since fulu retires the eth1-bridge //! deposit mechanism outright (`fulu_stf::process_deposit_request`'s own //! doc). +//! - `execution_payload_bid` and `parent_execution_payload` are new in gloas, +//! the block's binding commitment to a builder's payload and the next +//! block's processing of the previous one (EIP-7732); neither exists on any +//! earlier fork. `withdrawals` is modified again at gloas, and this time the +//! fixture's own operation file changes too: gloas's own +//! `process_withdrawals(state)` takes no payload at all (withdrawals are now +//! deterministic from the state alone), so unlike every earlier fork's arm +//! this one reads no `` file and calls `gloas_stf::process_withdrawals` +//! with only the state. use std::sync::Arc; use ethlambda_state_transition::beacon::ForkName; use ethlambda_state_transition::beacon::config::Config; use ethlambda_state_transition::beacon::containers::{ - BeaconState, altair, bellatrix, capella, deneb, electra, phase0, shared, + BeaconState, altair, bellatrix, capella, deneb, electra, gloas, phase0, shared, }; use ethlambda_state_transition::beacon::helpers::accessors::CommitteeCache; use ethlambda_state_transition::beacon::primitives::HashTreeRoot as _; @@ -107,6 +116,7 @@ use ethlambda_state_transition::beacon::stf::capella as capella_stf; use ethlambda_state_transition::beacon::stf::deneb as deneb_stf; use ethlambda_state_transition::beacon::stf::electra as electra_stf; use ethlambda_state_transition::beacon::stf::fulu as fulu_stf; +use ethlambda_state_transition::beacon::stf::gloas as gloas_stf; use ethlambda_state_transition::beacon::stf::{ExecutionEngine, block, operations}; use libtest_mimic::{Failed, Trial}; @@ -467,18 +477,35 @@ fn apply( fork @ (ForkName::Phase0 | ForkName::Altair | ForkName::Bellatrix) => { return Err(format!("withdrawals has no handler for fork `{fork}`")); } - ForkName::Gloas => Err( - ethlambda_state_transition::beacon::Error::UnsupportedForFork { - function: "withdrawals", - fork: ForkName::Gloas, - }, - ), + // Modified again at gloas (EIP-7732): `process_withdrawals` drops + // its `payload` parameter entirely, since withdrawals are now + // deterministic from the state alone. This is the one fork whose + // arm reads no operation file at all. + ForkName::Gloas => gloas_stf::process_withdrawals(state), ForkName::Lean => lean_is_not_a_fixture_fork("withdrawals"), }, + // New in gloas (EIP-7732): the block's binding commitment to a + // builder's payload. + "execution_payload_bid" => { + let signed_bid: gloas::SignedExecutionPayloadBid = case.ssz("execution_payload_bid"); + gloas_stf::process_execution_payload_bid(state, &signed_bid, config) + } + // New in gloas (EIP-7732): the *next* block's processing of the + // previous slot's payload, so the fixture's operation file is a whole + // `BeaconBlock`, named `block` like `block_header`'s. + "parent_execution_payload" => { + let block: gloas::BeaconBlock = case.ssz("block"); + gloas_stf::process_parent_execution_payload(state, &block, config) + } // `consolidation_request` and `withdrawal_request` are new in - // electra, alongside `withdrawals`; fulu's specification lists - // neither as modified, so electra's own functions serve fulu's cases - // too. + // electra, alongside `withdrawals`; neither fulu's nor gloas's own + // specification lists either as modified (gloas's own + // `apply_parent_execution_payload` is the only caller left once + // `process_operations` drops the call, but the function itself is + // unchanged), so electra's own functions serve every later fork's + // cases too. `ConsolidationRequest` and `WithdrawalRequest` are + // likewise electra's own containers, reused unchanged, so no fork + // match is needed for either the type or the call. "consolidation_request" => { let request: electra::ConsolidationRequest = case.ssz("consolidation_request"); electra_stf::process_consolidation_request(state, &request, config) @@ -488,12 +515,16 @@ fn apply( // `process_deposit_request`"): it drops the statement that sets // `deposit_requests_start_index` the first time a request is seen, // since fulu retires the eth1-bridge deposit mechanism outright (see - // `fulu_stf::process_deposit_request`'s own doc). Routing a fulu - // case to electra's version regardless would still pass every fixture - // shipped so far, since pyspec's own fulu genesis already sets that - // field, but it is not the function fulu's specification names. + // `fulu_stf::process_deposit_request`'s own doc). Gloas's own + // specification does not redefine the function again, so it inherits + // fulu's version rather than electra's, the same way gloas's own + // `apply_parent_execution_payload` calls straight into + // `fulu_stf::process_deposit_request`. Routing a fulu or gloas case to + // electra's version regardless would still pass every fixture shipped + // so far, since pyspec's own genesis for both already sets that + // field, but it is not the function either fork's specification names. "deposit_request" => match case.fork { - ForkName::Fulu => { + ForkName::Fulu | ForkName::Gloas => { let request: electra::DepositRequest = case.ssz("deposit_request"); fulu_stf::process_deposit_request(state, &request) } @@ -506,18 +537,24 @@ fn apply( let request: electra::DepositRequest = case.ssz("deposit_request"); electra_stf::process_deposit_request(state, &request) } - ForkName::Gloas => Err( - ethlambda_state_transition::beacon::Error::UnsupportedForFork { - function: "deposit_request", - fork: ForkName::Gloas, - }, - ), ForkName::Lean => lean_is_not_a_fixture_fork("deposit_request"), }, "withdrawal_request" => { let request: electra::WithdrawalRequest = case.ssz("withdrawal_request"); electra_stf::process_withdrawal_request(state, &request, config) } + // New in gloas (EIP-8282): the builder registry's own onboarding and + // exit requests, applied by `apply_parent_execution_payload` the same + // way the validator registry's own deposit/withdrawal/consolidation + // requests are. + "builder_deposit_request" => { + let request: gloas::BuilderDepositRequest = case.ssz("builder_deposit_request"); + gloas_stf::process_builder_deposit_request(state, &request, config) + } + "builder_exit_request" => { + let request: gloas::BuilderExitRequest = case.ssz("builder_exit_request"); + gloas_stf::process_builder_exit_request(state, &request, config) + } other => return Err(format!("unhandled operation `{other}`")), }; @@ -564,10 +601,14 @@ fn every_shipped_handler_is_dispatched() -> Result<(), Failed> { "attester_slashing", "block_header", "bls_to_execution_change", + "builder_deposit_request", + "builder_exit_request", "consolidation_request", "deposit", "deposit_request", "execution_payload", + "execution_payload_bid", + "parent_execution_payload", "proposer_slashing", "sync_aggregate", "voluntary_exit", From 0d9ff3725f1dce59f22c522a08fc359f2026d0b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 28 Sep 2026 12:26:03 -0300 Subject: [PATCH 019/132] feat(beacon): add the gloas fork choice nodes, weight and head ePBS (EIP-7732) gives every gloas block two branches that fork choice weighs separately: empty (no payload revealed, or not attested to as timely and available) and full (the builder's payload was revealed and attested to on time). Gloas's fork-choice.md models this with a payload dimension on ForkChoiceNode and LatestMessage, and with node derivation, weight and head descent that read it. A pre-gloas node stays a bare root, one per block. Nodes are not stored: a (root, PayloadStatus) node is derived on each call from the block tree, each block's bid and the set of verified payloads, the way the specification's Store derives it from blocks and payloads. The gloas get_weight walks every latest message per node, as the specification writes it, where pre-gloas compute_weights folds bottom-up; its doc lists what that costs and must change before live use. is_head_weak and is_parent_strong are not copied: the specification's gloas versions score a Pending node, which reduces to the pre-gloas, root-based attestation score, so both call sites use the pre-gloas functions. The store's block timeliness now holds one verdict per deadline (attestation and PTC), and it gains the verified payloads and the two PTC vote maps; pre-gloas code reads the attestation deadline's entry. The handlers that fill this state (gloas on_block, the payload envelope and payload attestation handlers, and the gloas anchor) need gloas block processing, so this commit is checked by unit tests on hand-built stores, each confirmed to fail on the defect it targets, and by every pre-gloas fork-choice fixture passing unchanged. --- .../src/beacon/fork_choice.rs | 1352 ++++++++++++++++- crates/common/types/src/beacon/constants.rs | 16 + crates/common/types/src/beacon/fork_choice.rs | 70 +- crates/common/types/src/beacon/preset.rs | 28 + crates/storage/src/store.rs | 148 +- 5 files changed, 1578 insertions(+), 36 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 96f0c6186..a8a6b39ce 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -167,7 +167,7 @@ use tracing::{error, warn}; use crate::beacon::config::Config; use crate::beacon::constants; use crate::beacon::containers::{AttestationData, BeaconState, Checkpoint, SignedBeaconBlock}; -use crate::beacon::containers::{bellatrix, deneb, electra, fulu, phase0}; +use crate::beacon::containers::{bellatrix, deneb, electra, fulu, gloas, phase0}; use crate::beacon::error::{Error, Result, verify}; use crate::beacon::fork::ForkName; use crate::beacon::helpers::accessors::{ @@ -181,7 +181,7 @@ use crate::beacon::helpers::misc::{ }; use crate::beacon::helpers::predicates::{is_active_validator, is_slashable_attestation_data}; use crate::beacon::kzg; -use crate::beacon::lean_boundary::lean_block_unreachable; +use crate::beacon::lean_boundary::{lean_block_unreachable, lean_fork_unreachable}; use crate::beacon::preset; use crate::beacon::primitives::{ Epoch, ExecutionBlockHash, Gwei, HashTreeRoot as _, KzgCommitment, KzgProof, Root, Slot, @@ -190,15 +190,19 @@ use crate::beacon::primitives::{ use crate::beacon::stf; // --------------------------------------------------------------------------- -// LatestMessage, PowBlock, PayloadStatusV1 +// LatestMessage, PowBlock, PayloadStatusV1, ForkChoiceNode, PayloadStatus // --------------------------------------------------------------------------- // All of these live in `ethlambda-types` rather than here, because // `ethlambda-storage` persists them and cannot depend on this crate. // Re-exported at the paths they had when they were defined here, so [`Store`] -// and its callers are unchanged. +// and its callers are unchanged. `ForkChoiceNode` and `PayloadStatus` are new +// with gloas, but join them here for the same reason: `Store::block_index`'s +// sibling accessors, this crate's own way of walking the block tree, are all +// keyed on plain `Root`s, and nothing about either type needs storage of its +// own beyond what `BeaconScratch` already keeps. pub use ethlambda_types::beacon::fork_choice::{ - LatestMessage, PayloadStatusEnum, PayloadStatusV1, PowBlock, + ForkChoiceNode, LatestMessage, PayloadStatus, PayloadStatusEnum, PayloadStatusV1, PowBlock, }; // --------------------------------------------------------------------------- @@ -1503,7 +1507,59 @@ fn compute_head( Ok(head) } -/// [`compute_head`], recorded as the store's own head. +/// [`compute_head`] pre-gloas, gloas's own [`gloas_get_head`] once the chain +/// has upgraded, as a full [`ForkChoiceNode`] rather than just a root. +/// +/// Dispatches on the justified checkpoint's own state fork, not the head's: +/// that is the state [`compute_weights`]/[`gloas_get_weight`] both score +/// every candidate against. Justification lags realized blocks by up to a +/// couple of epochs, so for a window after the fulu-to-gloas boundary the +/// block tree can already hold gloas blocks while the justified checkpoint +/// is still fulu-anchored; this dispatch takes the pre-gloas arm for that +/// whole window, weighing gloas blocks by [`compute_weights`] as if they had +/// no payload dimension. That is a real lag, not an oversight fixed here: +/// the specification's gloas `fork-choice.md` only ever describes a +/// gloas-anchored `Store` and says nothing about a mixed tree, and how this +/// crate should behave across the boundary is decided separately, before +/// gloas block import exists to reach it (see [`gloas_bid`]'s own doc for +/// the matching gap one level down, at a single block's own parent). A +/// pre-gloas head carries no payload dimension of its own, so it is +/// reported [`PayloadStatus::Pending`]: nothing on a pre-gloas chain ever +/// reads this field back. +/// +/// A fresh [`CommitteeCache`] on the gloas arm: gloas's own weight +/// (`gloas_get_weight` by way of `should_apply_proposer_boost`) now needs +/// committees a pre-gloas head computation never did, and this is the one +/// entry point both [`get_head`] and the fixture harness reach it through, +/// with no longer-lived cache to hand down instead. See [`gloas_get_weight`]'s +/// own performance note for what that costs repeated across a whole tree +/// walk. +pub fn get_head_node(store: &Store, config: &Config) -> Result { + let justified_checkpoint = store.beacon_justified_checkpoint(); + let justified_state = checkpoint_state(store, &justified_checkpoint, config)?; + match justified_state.fork_name() { + ForkName::Gloas => { + let committees = CommitteeCache::default(); + gloas_get_head(store, config, &committees) + } + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Fulu => { + let index = store.block_index(); + Ok(ForkChoiceNode { + root: compute_head(store, &index, config)?, + payload_status: PayloadStatus::Pending, + }) + } + ForkName::Lean => lean_fork_unreachable("fork_choice::get_head_node"), + } +} + +/// [`get_head_node`], recorded as the store's own head. /// /// Takes `&mut Store`, unlike most functions in this file: it records the head /// it just found through @@ -1519,12 +1575,11 @@ fn compute_head( /// that is empty whenever the head did not move, set against a whole weighted /// tree walk. pub fn get_head(store: &mut Store, config: &Config) -> Result { - let index = store.block_index(); - let head = compute_head(store, &index, config)?; + let head = get_head_node(store, config)?; store - .update_checkpoints(ForkCheckpoints::head_only(head)) + .update_checkpoints(ForkCheckpoints::head_only(head.root)) .expect("record beacon head"); - Ok(head) + Ok(head.root) } // --------------------------------------------------------------------------- @@ -1626,11 +1681,17 @@ pub fn get_aggregate_due_ms(_epoch: Epoch, config: &Config) -> u64 { /// Whether `head_root`'s block arrived after the attestation deadline of the /// slot it was imported in. +/// +/// Gloas widens `store.block_timeliness[root]` to two deadlines (see +/// [`Store::block_timeliness`](ethlambda_storage::Store::block_timeliness)); +/// this reads only [`constants::ATTESTATION_TIMELINESS_INDEX`], the one +/// deadline this function ever checked, so its own behaviour is unchanged by +/// that widening. pub fn is_head_late(store: &Store, head_root: Root) -> Result { let timely = store .block_timeliness(&head_root) .ok_or(Error::SpecAssert("head_root in store.block_timeliness"))?; - Ok(!timely) + Ok(!timely[constants::ATTESTATION_TIMELINESS_INDEX]) } /// Whether `slot` is not the first slot of its epoch, i.e. the proposer @@ -1872,6 +1933,746 @@ pub fn get_proposer_head( } } +// --------------------------------------------------------------------------- +// Gloas: payload-aware fork choice +// --------------------------------------------------------------------------- +// +// EIP-7732 (ePBS) splits a block into two branches fork choice must weigh +// separately: the block may end up *empty* (no payload revealed, or the +// payload was not attested to as timely and available) or *full* (its +// builder's payload was revealed and attested to on time). A gloas +// [`ForkChoiceNode`] names both the block and which branch, where a pre-gloas +// one is a bare [`Root`] (see [`ForkChoiceNode`]'s own doc for why). +// +// Nodes are not stored: a `(root, PayloadStatus)` node is derived, on every +// call, from the block tree ([`Store::block_index`]), each block's own +// execution payload bid, and the set of verified payloads +// ([`is_payload_verified`]), exactly as [`get_node_children`] does. That +// mirrors the specification's own `Store`, which keeps `blocks` and +// `payloads` as the source of truth and builds a `ForkChoiceNode` on demand +// wherever one is needed, never as a third, separately maintained index. +// +// Functions here are named exactly as the specification names them, with a +// `gloas_` prefix only where a pre-gloas function of the same name already +// exists in this file: `gloas_get_ancestor`, `gloas_get_checkpoint_block`, +// `gloas_get_weight`, `gloas_get_head`, and the four `*_due_ms` functions +// gloas narrows to its own basis-point fields (`gloas_get_attestation_due_ms`, +// `gloas_get_aggregate_due_ms`, `gloas_get_sync_message_due_ms`, +// `gloas_get_contribution_due_ms`; the last two rename an altair-shaped name +// no pre-gloas function of this file actually carries, since a bare +// `get_sync_message_due_ms` would otherwise misname a function that only +// ever reads gloas's own basis points). Two further functions +// (`gloas_bid`, `gloas_get_attestation_score`) are private and have no name +// of their own in the specification at all; they stay `gloas_`-prefixed for +// the same reason, since both exist only to bind gloas's own semantics +// (a block's bid, an ancestor check with a payload dimension) into either a +// projection this crate needs or spec-unmodified text (`get_attestation_score`) +// that the specification's own per-fork composition binds differently per +// fork. `is_head_weak` and `is_parent_strong` need no gloas copy at all: see +// their own call sites below for why the pre-gloas functions already answer +// gloas's question. Every other function below is new to this file, +// pre-gloas and gloas alike, and keeps the specification's bare name. + +/// `block`'s own execution payload bid, projected out of the fork-generic +/// [`SignedBeaconBlock`] enum. +/// +/// Every function in this section assumes `block` is itself gloas-shaped, +/// which does not hold at the fulu-to-gloas boundary: the first gloas +/// block's own parent is a fulu block with no bid at all, so +/// [`get_parent_payload_status`] on it fails here with +/// `Error::UnsupportedForFork` rather than answering a payload status for +/// it. The specification's gloas `fork-choice.md` only ever describes a +/// gloas-anchored `Store`; it says nothing about a pre-gloas parent (or, one +/// level up, a fulu-justified store scoring a mixed tree, see +/// [`get_head_node`]'s own doc), and how this crate should handle that +/// boundary is decided separately, before gloas block import exists to +/// reach it. A non-gloas block reaching here is a caller bug in every case +/// this crate does handle today, reported by name rather than assumed away, +/// the same projection `helpers::gloas::gloas_state_ref` gives a state. +fn gloas_bid(block: &SignedBeaconBlock) -> Result<&gloas::ExecutionPayloadBid> { + match block { + SignedBeaconBlock::Gloas(inner) => { + Ok(&inner.message.body.signed_execution_payload_bid.message) + } + other => Err(Error::UnsupportedForFork { + function: "gloas_bid", + fork: other.fork_name(), + }), + } +} + +/// `is_payload_verified` (gloas `fork-choice.md`): whether `root`'s execution +/// payload envelope has been locally delivered and verified via +/// `on_execution_payload_envelope`, which does not exist in this crate yet: +/// nothing calls [`Store::insert_verified_payload`](ethlambda_storage::Store::insert_verified_payload) +/// until the handler that reads a delivered envelope lands. +pub fn is_payload_verified(store: &Store, root: Root) -> bool { + store.has_verified_payload(&root) +} + +/// `payload_timeliness` (gloas `fork-choice.md`): whether `root`'s payload is +/// considered `timely` (or not, when `timely` is `false`), taking into +/// account both local availability and the payload timeliness committee's +/// votes. +pub fn payload_timeliness(store: &Store, root: Root, timely: bool) -> Result { + let vote = store + .payload_timeliness_vote(&root) + .ok_or(Error::SpecAssert("root in store.payload_timeliness_vote"))?; + + // If the payload is not locally available, it is not considered + // available regardless of the PTC vote. + if !is_payload_verified(store, root) { + return Ok(!timely); + } + + let votes: Vec = vote.into_iter().flatten().collect(); + let matching = votes.iter().filter(|&&v| v == timely).count() as u64; + Ok(matching > preset::PAYLOAD_TIMELY_THRESHOLD) +} + +/// `payload_data_availability` (gloas `fork-choice.md`): whether `root`'s +/// blob data is considered `available` (or not, when `available` is +/// `false`), the sibling check of [`payload_timeliness`]. +pub fn payload_data_availability(store: &Store, root: Root, available: bool) -> Result { + let vote = store + .payload_data_availability_vote(&root) + .ok_or(Error::SpecAssert( + "root in store.payload_data_availability_vote", + ))?; + + if !is_payload_verified(store, root) { + return Ok(!available); + } + + let votes: Vec = vote.into_iter().flatten().collect(); + let matching = votes.iter().filter(|&&v| v == available).count() as u64; + Ok(matching > preset::DATA_AVAILABILITY_TIMELY_THRESHOLD) +} + +/// `get_parent_payload_status` (gloas `fork-choice.md`): whether `block` +/// builds on its parent's full payload branch or its empty one, found by +/// comparing the two blocks' own bids. +/// +/// Takes `block: &SignedBeaconBlock`, not the specification's unsigned +/// `BeaconBlock`: see the module documentation's "Signed blocks" section for +/// why every function in this file does. +pub fn get_parent_payload_status( + store: &Store, + block: &SignedBeaconBlock, +) -> Result { + let parent = store + .get_signed_block(&block.parent_root()) + .expect("get") + .ok_or(Error::SpecAssert("block.parent_root in store.blocks"))?; + let parent_block_hash = gloas_bid(block)?.parent_block_hash; + let message_block_hash = gloas_bid(&parent)?.block_hash; + Ok(if parent_block_hash == message_block_hash { + PayloadStatus::Full + } else { + PayloadStatus::Empty + }) +} + +/// `is_parent_node_full` (gloas `fork-choice.md`). +pub fn is_parent_node_full(store: &Store, block: &SignedBeaconBlock) -> Result { + Ok(get_parent_payload_status(store, block)? == PayloadStatus::Full) +} + +/// `get_ancestor` (gloas `fork-choice.md`, modified): `node`'s ancestor at +/// `slot`, carrying the payload status of each parent step the descent +/// passes through. +/// +/// A loop rather than the specification's recursion, matching pre-gloas +/// [`get_ancestor`] for the same reason: a long unfinalized suffix must not +/// risk a stack overflow. +pub fn gloas_get_ancestor( + store: &Store, + node: ForkChoiceNode, + slot: Slot, +) -> Result { + let mut node = node; + loop { + let block = store + .get_signed_block(&node.root) + .expect("get") + .ok_or(Error::SpecAssert("node.root in store.blocks"))?; + if block.slot() > slot { + let payload_status = get_parent_payload_status(store, &block)?; + node = ForkChoiceNode { + root: block.parent_root(), + payload_status, + }; + } else { + return Ok(node); + } + } +} + +/// `is_ancestor` (gloas `fork-choice.md`, modified): whether `ancestor` is on +/// `node`'s chain, with `ancestor.payload_status == +/// PayloadStatus::Pending` acting as a wildcard that matches either branch. +/// +/// No pre-gloas counterpart exists in this file: a pre-gloas `ForkChoiceNode` +/// is a bare root (see that type's own doc), so every pre-gloas call this +/// question would need instead compares two roots directly. +/// +/// Against a `Pending` `ancestor`, the wildcard above makes this reduce to +/// `gloas_get_ancestor(node, ancestor_slot).root == ancestor.root`, and +/// `gloas_get_ancestor`'s own descent picks its next root by `block.slot` +/// alone (see [`gloas_get_checkpoint_block`]'s own doc), so that root is +/// exactly what the pre-gloas, index-only [`get_ancestor`] would answer for +/// the same walk. The specification's `is_head_weak` and `is_parent_strong` +/// always score a `Pending` node (a bare block root), so the attestation +/// score they need takes this reduction and equals the pre-gloas, +/// root-based one; that is what lets `should_apply_proposer_boost` and +/// `gloas_get_proposer_head` call the pre-gloas [`is_head_weak`]/ +/// [`is_parent_strong`] directly rather than keep a gloas copy of either. +/// `gloas_get_weight` also scores `Empty` and `Full` nodes, which do not +/// reduce this way, so it keeps its own attestation score. +pub fn is_ancestor(store: &Store, node: ForkChoiceNode, ancestor: ForkChoiceNode) -> Result { + let (ancestor_slot, _) = store + .block_entry(&ancestor.root) + .ok_or(Error::SpecAssert("ancestor.root in store.blocks"))?; + let node_ancestor = gloas_get_ancestor(store, node, ancestor_slot)?; + if node_ancestor.root != ancestor.root { + return Ok(false); + } + Ok(node_ancestor.payload_status == ancestor.payload_status + || ancestor.payload_status == PayloadStatus::Pending) +} + +/// `get_checkpoint_block` (gloas `fork-choice.md`, modified): the checkpoint +/// block for `epoch` on `root`'s chain. +/// +/// Its `.root` agrees with the pre-gloas, index-only [`get_checkpoint_block`] +/// on any chain both could answer: [`gloas_get_ancestor`]'s descent picks its +/// next step by `block.slot` alone, so the payload status threaded through +/// the walk never changes which root it lands on, only the label this +/// function then discards. [`filter_block_tree`] (the only caller of +/// [`get_checkpoint_block`] in this file; [`get_voting_source`] never calls +/// it) therefore keeps calling the pre-gloas version rather than growing a +/// gloas copy of its own, which is what lets [`get_filtered_block_tree`] +/// serve [`gloas_get_head`] with no per-block fetch. +pub fn gloas_get_checkpoint_block(store: &Store, root: Root, epoch: Epoch) -> Result { + let epoch_first_slot = compute_start_slot_at_epoch(epoch); + let node = ForkChoiceNode { + root, + payload_status: PayloadStatus::Pending, + }; + Ok(gloas_get_ancestor(store, node, epoch_first_slot)?.root) +} + +/// `get_supported_node` (gloas `fork-choice.md`): the node `message` +/// supports, full or empty once its slot has passed the block's own, pending +/// otherwise. +pub fn get_supported_node(store: &Store, message: LatestMessage) -> Result { + let (block_slot, _) = store + .block_entry(&message.root) + .ok_or(Error::SpecAssert("message.root in store.blocks"))?; + let payload_status = if block_slot < message.slot { + if message.payload_present { + PayloadStatus::Full + } else { + PayloadStatus::Empty + } + } else { + PayloadStatus::Pending + }; + Ok(ForkChoiceNode { + root: message.root, + payload_status, + }) +} + +/// `is_previous_slot_payload_decision` (gloas `fork-choice.md`): whether +/// `node` is the still-undecided empty/full choice for a block from the +/// previous slot, the one case [`gloas_get_weight`] cannot yet settle by +/// weight. +pub fn is_previous_slot_payload_decision( + store: &Store, + node: ForkChoiceNode, + config: &Config, +) -> Result { + let (block_slot, _) = store + .block_entry(&node.root) + .ok_or(Error::SpecAssert("node.root in store.blocks"))?; + let is_previous_slot = block_slot.checked_add(1) == Some(get_current_slot(store, config)); + let is_payload_decision = matches!( + node.payload_status, + PayloadStatus::Empty | PayloadStatus::Full + ); + Ok(is_previous_slot && is_payload_decision) +} + +/// `should_build_on_full` (gloas `fork-choice.md`): whether a proposer at +/// `slot` should build on `head`'s full payload branch rather than its empty +/// one. +pub fn should_build_on_full(store: &Store, head: ForkChoiceNode, slot: Slot) -> Result { + verify( + head.payload_status != PayloadStatus::Pending, + "head.payload_status != PAYLOAD_STATUS_PENDING", + )?; + let (head_slot, _) = store + .block_entry(&head.root) + .ok_or(Error::SpecAssert("head.root in store.blocks"))?; + if head_slot.checked_add(1) != Some(slot) { + return Ok(head.payload_status == PayloadStatus::Full); + } + if head.payload_status == PayloadStatus::Empty { + return Ok(false); + } + if payload_timeliness(store, head.root, false)? { + return Ok(false); + } + if payload_data_availability(store, head.root, false)? { + return Ok(false); + } + Ok(true) +} + +/// `should_extend_payload` (gloas `fork-choice.md`): whether a proposer +/// building on `root`'s (previous-slot) payload should extend it rather than +/// build empty, favoring extension unless the PTC view is against it and the +/// current proposer-boosted block itself chose empty. +pub fn should_extend_payload(store: &Store, root: Root, config: &Config) -> Result { + let (block_slot, _) = store + .block_entry(&root) + .ok_or(Error::SpecAssert("root in store.blocks"))?; + verify( + block_slot.checked_add(1) == Some(get_current_slot(store, config)), + "store.blocks[root].slot + 1 == get_current_slot(store)", + )?; + if !is_payload_verified(store, root) { + return Ok(false); + } + let proposer_root = store.proposer_boost_root(); + let payload_is_timely = payload_timeliness(store, root, true)?; + let payload_data_is_available = payload_data_availability(store, root, true)?; + if payload_is_timely && payload_data_is_available { + return Ok(true); + } + if proposer_root.is_zero() { + return Ok(true); + } + let proposer_block = + store + .get_signed_block(&proposer_root) + .expect("get") + .ok_or(Error::SpecAssert( + "store.proposer_boost_root in store.blocks", + ))?; + if proposer_block.parent_root() != root { + return Ok(true); + } + is_parent_node_full(store, &proposer_block) +} + +/// `get_payload_status_tiebreaker` (gloas `fork-choice.md`): [`gloas_get_head`]'s +/// tiebreaker between a block's full and empty branches, once their weights +/// alone cannot decide (their weight is `0` while +/// [`is_previous_slot_payload_decision`] holds; see [`gloas_get_weight`]). +pub fn get_payload_status_tiebreaker( + store: &Store, + node: ForkChoiceNode, + config: &Config, +) -> Result { + if is_previous_slot_payload_decision(store, node, config)? { + if node.payload_status == PayloadStatus::Empty { + return Ok(1); + } + if should_extend_payload(store, node.root, config)? { + return Ok(2); + } + Ok(0) + } else { + Ok(node.payload_status as u8) + } +} + +/// `should_apply_proposer_boost` (gloas `fork-choice.md`): whether the +/// current proposer-boosted block still deserves its boost. Applies +/// unconditionally unless the parent is exactly one slot behind it and +/// weak; even then, the boost is withheld only when a same-slot sibling of +/// the parent, sharing its proposer and itself timely by the PTC deadline +/// (an early equivocation), exists. With no such sibling, the boost still +/// applies. +pub fn should_apply_proposer_boost( + store: &Store, + config: &Config, + committees: &CommitteeCache, +) -> Result { + let proposer_boost_root = store.proposer_boost_root(); + if proposer_boost_root.is_zero() { + return Ok(false); + } + + let (slot, parent_root) = store + .block_entry(&proposer_boost_root) + .ok_or(Error::SpecAssert( + "store.proposer_boost_root in store.blocks", + ))?; + let (parent_slot, _) = store + .block_entry(&parent_root) + .ok_or(Error::SpecAssert("parent_root in store.blocks"))?; + + // Apply proposer boost if `parent` is not from the previous slot. + if parent_slot.checked_add(1).is_some_and(|next| next < slot) { + return Ok(true); + } + // Apply proposer boost if `parent` is not weak. Scored through the + // pre-gloas `is_head_weak`, not a gloas copy: see `is_ancestor`'s own + // doc for why a `Pending` node's score is the same either way. + if !is_head_weak(store, parent_root, config, committees)? { + return Ok(true); + } + + // If `parent` is weak and from the previous slot, apply proposer boost + // if there are no early equivocations. + let parent_block = store + .get_signed_block(&parent_root) + .expect("get") + .ok_or(Error::SpecAssert("parent_root in store.blocks"))?; + let index = store.block_index(); + let mut has_equivocation = false; + for (&candidate_root, &(candidate_slot, _)) in &index { + if candidate_root == parent_root { + continue; + } + if candidate_slot.checked_add(1) != Some(slot) { + continue; + } + let timely = store + .block_timeliness(&candidate_root) + .ok_or(Error::SpecAssert("root in store.block_timeliness"))?; + if !timely[constants::PTC_TIMELINESS_INDEX] { + continue; + } + let candidate_block = store + .get_signed_block(&candidate_root) + .expect("get") + .ok_or(Error::SpecAssert("root in store.blocks"))?; + if candidate_block.proposer_index() == parent_block.proposer_index() { + has_equivocation = true; + break; + } + } + + Ok(!has_equivocation) +} + +/// [`gloas_get_weight`]'s attestation half: unmodified spec text (phase0 +/// `fork-choice.md`'s own `get_attestation_score`), bound here to gloas's own +/// [`is_ancestor`] and [`get_supported_node`] the way the specification's own +/// per-fork composition binds it, rather than to the collapsed +/// root-comparison the pre-gloas [`get_attestation_score`] uses (sound only +/// because a pre-gloas `ForkChoiceNode` is a bare root; see that function's +/// own doc). +fn gloas_get_attestation_score( + store: &Store, + node: ForkChoiceNode, + state: &BeaconState, +) -> Result { + let current_epoch = get_current_epoch(state); + let mut attestation_score: Gwei = 0; + for validator_index in get_active_validator_indices(state, current_epoch) { + let validator = state.validator(validator_index)?; + if validator.slashed || store.is_equivocating(validator_index) { + continue; + } + let Some(message) = store.latest_message(validator_index) else { + continue; + }; + let supported = get_supported_node(store, message)?; + if is_ancestor(store, supported, node)? { + attestation_score = attestation_score.saturating_add(validator.effective_balance); + } + } + Ok(attestation_score) +} + +/// `get_weight` (gloas `fork-choice.md`, modified). +/// +/// **Performance note**: this is spec-literal, not the bottom-up form +/// pre-gloas [`get_weight`]/[`compute_weights`] moved to, and carries +/// several costs a later change must close before any live use, not only +/// the walk [`compute_weights`]'s own doc measured at 62% of a live mainnet +/// follower's CPU before that fold replaced repeated per-root calls: +/// - every latest message is walked afresh for every node [`gloas_get_head`] +/// visits, rather than folded bottom-up once per tree; +/// - each [`gloas_get_ancestor`] hop decodes two signed blocks (the node's +/// own, then its parent's inside [`get_parent_payload_status`]), where the +/// pre-gloas, index-only [`get_ancestor`] decodes none; +/// - [`should_apply_proposer_boost`] reruns a full [`is_head_weak`] score +/// and its own `store.block_index()` scan on every call, even though its +/// answer does not depend on `node` at all and so is identical for every +/// candidate one [`gloas_get_head`] call weighs; +/// - a vote for a block this store has pruned below its finalized anchor +/// aborts the whole head computation with `Error::SpecAssert`, exactly +/// the failure [`compute_weights`]'s own doc describes pre-gloas +/// `get_weight` having, which that fold was built to skip past instead. +/// +/// Fine for spec tests, where the tree is a handful of blocks and never +/// prunes; live use needs all four addressed, not just the first. +pub fn gloas_get_weight( + store: &Store, + node: ForkChoiceNode, + config: &Config, + committees: &CommitteeCache, +) -> Result { + if is_previous_slot_payload_decision(store, node, config)? { + return Ok(0); + } + + let justified_checkpoint = store.beacon_justified_checkpoint(); + let state = checkpoint_state(store, &justified_checkpoint, config)?; + let attestation_score = gloas_get_attestation_score(store, node, &state)?; + + if !should_apply_proposer_boost(store, config, committees)? { + // Return only attestation score if proposer boost should not apply. + return Ok(attestation_score); + } + + // Calculate proposer score if proposer boost should apply. + let proposer_boost_node = ForkChoiceNode { + root: store.proposer_boost_root(), + payload_status: PayloadStatus::Pending, + }; + // Boost is applied if `node` is an ancestor of `proposer_boost_node`. + let proposer_score = if is_ancestor(store, proposer_boost_node, node)? { + get_proposer_score(store, config)? + } else { + 0 + }; + + Ok(attestation_score.saturating_add(proposer_score)) +} + +/// `get_node_children` (gloas `fork-choice.md`, modified): the children of +/// `node` in the payload-aware tree, either its own empty/full branches (when +/// `node` is still `Pending`) or the next blocks that agree with `node`'s own +/// payload status about their shared parent. +/// +/// `blocks` is [`Store::block_index`]'s own shape, not the specification's +/// `Dict[Root, BeaconBlock]`: see the section documentation above for why +/// nodes, and the trees built from them, are derived rather than stored. +pub fn get_node_children( + store: &Store, + blocks: &HashMap, + node: ForkChoiceNode, +) -> Result> { + if node.payload_status == PayloadStatus::Pending { + let mut children = vec![ForkChoiceNode { + root: node.root, + payload_status: PayloadStatus::Empty, + }]; + if is_payload_verified(store, node.root) { + children.push(ForkChoiceNode { + root: node.root, + payload_status: PayloadStatus::Full, + }); + } + Ok(children) + } else { + let mut children = Vec::new(); + for (&root, &(_, parent_root)) in blocks { + if parent_root != node.root { + continue; + } + let block = store + .get_signed_block(&root) + .expect("get") + .ok_or(Error::SpecAssert("root in blocks"))?; + if node.payload_status == get_parent_payload_status(store, &block)? { + children.push(ForkChoiceNode { + root, + payload_status: PayloadStatus::Pending, + }); + } + } + Ok(children) + } +} + +/// `get_head` (gloas `fork-choice.md`, modified): the LMD GHOST head as a +/// full [`ForkChoiceNode`], breaking a weight tie between a block's full and +/// empty branches with [`get_payload_status_tiebreaker`] rather than weight +/// alone. +/// +/// Reuses the pre-gloas, index-only [`get_filtered_block_tree`] for the +/// candidate tree: see [`gloas_get_checkpoint_block`]'s own doc for why that +/// is sound rather than a shortcut. [`get_head_node`] is the dispatching +/// entry point this and pre-gloas's own [`compute_head`] share. +pub fn gloas_get_head( + store: &Store, + config: &Config, + committees: &CommitteeCache, +) -> Result { + let index = store.block_index(); + let blocks = get_filtered_block_tree(store, &index, config)?; + let mut head = ForkChoiceNode { + root: store.beacon_justified_checkpoint().root, + payload_status: PayloadStatus::Pending, + }; + loop { + let children = get_node_children(store, &blocks, head)?; + if children.is_empty() { + return Ok(head); + } + + // Sort by latest attesting balance with ties broken lexicographically + // by root, then by the payload-status tiebreaker: the same key the + // specification's own `max` sorts by. + let mut ranked = Vec::with_capacity(children.len()); + for child in children { + let weight = gloas_get_weight(store, child, config, committees)?; + let tiebreak = get_payload_status_tiebreaker(store, child, config)?; + ranked.push(((weight, child.root, tiebreak), child)); + } + head = ranked + .into_iter() + .max_by_key(|(key, _)| *key) + .expect("children is non-empty, checked above") + .1; + } +} + +/// `get_latest_message_epoch` (gloas `fork-choice.md`, modified): the epoch +/// `message`'s slot falls in, now that gloas keeps a [`LatestMessage`]'s slot +/// rather than its epoch directly. No pre-gloas counterpart exists in this +/// file: every pre-gloas reader of a [`LatestMessage`] reads its `epoch` +/// field directly instead. +pub fn get_latest_message_epoch(message: LatestMessage) -> Epoch { + compute_epoch_at_slot(message.slot) +} + +/// `get_attestation_due_ms` (gloas `fork-choice.md`, modified): gloas moves +/// the attestation deadline to its own `ATTESTATION_DUE_BPS_GLOAS`, earlier +/// than pre-gloas's [`get_attestation_due_ms`], to leave room in the same slot +/// for the payload and payload-attestation deadlines that follow it. +pub fn gloas_get_attestation_due_ms(config: &Config) -> u64 { + get_slot_component_duration_ms(config.attestation_due_bps_gloas, config) +} + +/// `get_aggregate_due_ms` (gloas `fork-choice.md`, modified). See +/// [`gloas_get_attestation_due_ms`]. +pub fn gloas_get_aggregate_due_ms(config: &Config) -> u64 { + get_slot_component_duration_ms(config.aggregate_due_bps_gloas, config) +} + +/// `get_sync_message_due_ms` (gloas `fork-choice.md`, modified): reads +/// `SYNC_MESSAGE_DUE_BPS_GLOAS`. `gloas_`-prefixed like its four siblings +/// above even though no altair-named `get_sync_message_due_ms` exists in +/// this file to collide with: the plain name takes altair's own, and this +/// function only ever reads gloas's basis points, never altair's, so the +/// bare name would misname what it does. +pub fn gloas_get_sync_message_due_ms(config: &Config) -> u64 { + get_slot_component_duration_ms(config.sync_message_due_bps_gloas, config) +} + +/// `get_contribution_due_ms` (gloas `fork-choice.md`, modified): reads +/// `CONTRIBUTION_DUE_BPS_GLOAS`. See [`gloas_get_sync_message_due_ms`]. +pub fn gloas_get_contribution_due_ms(config: &Config) -> u64 { + get_slot_component_duration_ms(config.contribution_due_bps_gloas, config) +} + +/// `get_payload_due_ms` (gloas `fork-choice.md`): how far into a slot, in +/// milliseconds, a builder's execution payload envelope is due. +pub fn get_payload_due_ms(config: &Config) -> u64 { + get_slot_component_duration_ms(config.payload_due_bps, config) +} + +/// `get_payload_attestation_due_ms` (gloas `fork-choice.md`): how far into a +/// slot, in milliseconds, the payload timeliness committee's votes are due. +pub fn get_payload_attestation_due_ms(config: &Config) -> u64 { + get_slot_component_duration_ms(config.payload_attestation_due_bps, config) +} + +/// `get_proposer_head` (gloas `fork-choice.md`, modified): the node a +/// proposer at `slot` should build on, as a full [`ForkChoiceNode`] so a +/// re-org preserves the parent's own payload status rather than collapsing +/// it back to `Pending`. +/// +/// Drops the pre-gloas `shuffling_stable` condition entirely, matching the +/// specification's own text: pre-gloas [`get_proposer_head`] still gates it +/// on `head_block.fork_name() >= ForkName::Fulu` for the forks before fulu, +/// but `head_block` here is always gloas's own, so that gate is +/// unconditionally true for every call this function ever gets and the +/// specification's own gloas text simply omits the condition rather than +/// spell out an always-true check. +pub fn gloas_get_proposer_head( + store: &Store, + head_node: ForkChoiceNode, + slot: Slot, + config: &Config, + committees: &CommitteeCache, +) -> Result { + let head_block = store + .get_signed_block(&head_node.root) + .expect("get") + .ok_or(Error::SpecAssert("head_node.root in store.blocks"))?; + let parent_root = head_block.parent_root(); + let (parent_slot, _) = store + .block_entry(&parent_root) + .ok_or(Error::SpecAssert("parent_root in store.blocks"))?; + let parent_payload_status = get_parent_payload_status(store, &head_block)?; + let parent_node = ForkChoiceNode { + root: parent_root, + payload_status: parent_payload_status, + }; + + // Only re-org the head block if it arrived later than the attestation + // deadline. + let head_late = is_head_late(store, head_node.root)?; + // Ensure that the FFG information of the new head will be competitive + // with the current head. + let ffg_competitive = is_ffg_competitive(store, head_node.root, parent_root)?; + // Do not re-org if the chain is not finalizing with acceptable frequency. + let finalization_ok = is_finalization_ok(store, slot, config); + // Only re-org if we are proposing on-time. + let proposing_on_time = is_proposing_on_time(store, config); + + let head_slot = head_block.slot(); + // Only re-org a single slot at most. + let parent_slot_ok = parent_slot.checked_add(1) == Some(head_slot); + let current_time_ok = head_slot.checked_add(1) == Some(slot); + let single_slot_reorg = parent_slot_ok && current_time_ok; + + // Check that the head has few enough votes to be overpowered by our + // proposer boost. + verify( + store.proposer_boost_root() != head_node.root, + "store.proposer_boost_root != head_node.root", + )?; + // Pre-gloas `is_head_weak`, not a gloas copy: see `is_ancestor`'s own + // doc for why a `Pending` node's score is the same either way. + let head_weak = is_head_weak(store, head_node.root, config, committees)?; + + // Check that the missing votes are assigned to the parent and not being + // hoarded. Pre-gloas `is_parent_strong`, for the same reason. + let parent_strong = is_parent_strong(store, head_node.root, config)?; + + // Re-org more aggressively if there is a proposer equivocation in the + // previous slot. + let index = store.block_index(); + let proposer_equivocation = is_proposer_equivocation(store, &index, head_node.root)?; + + if head_late + && ffg_competitive + && finalization_ok + && proposing_on_time + && single_slot_reorg + && head_weak + && parent_strong + { + // We can re-org the current head by building upon its parent node. + Ok(parent_node) + } else if head_weak && current_time_ok && proposer_equivocation { + Ok(parent_node) + } else { + Ok(head_node) + } +} + // --------------------------------------------------------------------------- // Merge transition helpers (bellatrix) // --------------------------------------------------------------------------- @@ -2174,6 +2975,14 @@ fn is_block_timely(store: &Store, block_slot: Slot, config: &Config) -> bool { /// `record_block_timeliness`, this one's own signature (`block_slot` as a /// parameter) is this crate's adaptation of it rather than a transcription, /// and [`on_block`] is its only caller. +/// +/// Writes the same verdict into both of `store.block_timeliness[root]`'s +/// entries: this function only ever runs on a pre-gloas chain (gloas's own +/// `on_block` records its two deadlines separately, per +/// `record_block_timeliness`'s gloas-modified text), so it has no PTC +/// deadline of its own to tell apart from the attestation one, and +/// [`is_head_late`] only ever reads [`constants::ATTESTATION_TIMELINESS_INDEX`] +/// back regardless. pub(crate) fn record_block_timeliness( store: &mut Store, root: Root, @@ -2181,7 +2990,7 @@ pub(crate) fn record_block_timeliness( config: &Config, ) { let is_timely = is_block_timely(store, block_slot, config); - store.set_block_timeliness(root, is_timely); + store.set_block_timeliness(root, [is_timely, is_timely]); } /// The first slot of the lookahead window `epoch`'s proposer shuffling opens @@ -2278,7 +3087,8 @@ pub(crate) fn update_proposer_boost_root( let is_first_block = store.proposer_boost_root().is_zero(); let is_timely = store .block_timeliness(&root) - .expect("record_block_timeliness always runs first, on the same root"); + .expect("record_block_timeliness always runs first, on the same root") + [constants::ATTESTATION_TIMELINESS_INDEX]; let epoch = get_current_store_epoch(store, config); let head_dependent_root = get_shuffling_dependent_root(index, head, epoch); let block_dependent_root = get_shuffling_dependent_root(index, root, epoch); @@ -2462,7 +3272,9 @@ pub fn update_latest_messages( index, LatestMessage { epoch: target.epoch, + slot: data.slot, root: beacon_block_root, + payload_present: false, }, ); } @@ -3060,6 +3872,38 @@ mod tests { SignedBeaconBlock::Phase0(inner) } + /// A gloas signed block with an empty body and a zero signature, and an + /// explicit bid `parent_block_hash`/`block_hash` pair: the gloas fork + /// choice tests below name their blocks' payload dimension entirely + /// through these two fields, per [`get_parent_payload_status`]. + fn gloas_block( + slot: Slot, + parent_root: Root, + parent_block_hash: ExecutionBlockHash, + block_hash: ExecutionBlockHash, + ) -> SignedBeaconBlock { + SignedBeaconBlock::Gloas(gloas::SignedBeaconBlock { + message: gloas::BeaconBlock { + slot, + proposer_index: 0, + parent_root, + state_root: Root::ZERO, + body: gloas::BeaconBlockBody { + signed_execution_payload_bid: gloas::SignedExecutionPayloadBid { + message: gloas::ExecutionPayloadBid { + parent_block_hash, + block_hash, + ..Default::default() + }, + ..Default::default() + }, + ..Default::default() + }, + }, + signature: Default::default(), + }) + } + /// An exact anchor pair: `state` is `block`'s own post-state. /// /// Built as a fixed point, the way the state transition produces one. The @@ -3162,20 +4006,32 @@ mod tests { // that counts for no candidate above it), and one equivocator whose // vote must not count at all. for (validator_index, root) in [(0, b_root), (1, b_root), (2, b_root), (3, c_root)] { - store.set_latest_message(validator_index, LatestMessage { epoch: 0, root }); + store.set_latest_message( + validator_index, + LatestMessage { + epoch: 0, + slot: 0, + root, + payload_present: false, + }, + ); } store.set_latest_message( 4, LatestMessage { epoch: 0, + slot: 0, root: anchor_root, + payload_present: false, }, ); store.set_latest_message( 5, LatestMessage { epoch: 0, + slot: 0, root: b_root, + payload_present: false, }, ); store.insert_equivocating_index(5); @@ -3216,15 +4072,19 @@ mod tests { 0, LatestMessage { epoch: 0, + slot: 0, root: a_root, + payload_present: false, }, ); store.set_latest_message( 1, LatestMessage { epoch: 0, + slot: 0, // Below the anchor, so no longer indexed. root: Root::repeat_byte(0xde), + payload_present: false, }, ); @@ -3472,7 +4332,7 @@ mod tests { // reorg branch (`head_late && ...`) from firing, so a `parent_root` // result below can only come from the new `head_weak && // current_time_ok && proposer_equivocation` branch. - store.set_block_timeliness(head_root, true); + store.set_block_timeliness(head_root, [true, true]); // `is_ffg_competitive` needs both roots to already have an // unrealized justification on record; its value is irrelevant to the @@ -4063,4 +4923,466 @@ mod tests { &store, 300, 90, parent, safe_slots )); } + + // ------------------------------------------------------------------ + // Gloas: payload-aware fork choice + // ------------------------------------------------------------------ + + #[test] + fn get_node_children_offers_the_full_branch_only_once_the_payload_is_verified() { + let mut store = empty_store(); + let a_root = Root::repeat_byte(0xa1); + let blocks = store.block_index(); + let node = ForkChoiceNode { + root: a_root, + payload_status: PayloadStatus::Pending, + }; + + assert_eq!( + get_node_children(&store, &blocks, node).unwrap(), + vec![ForkChoiceNode { + root: a_root, + payload_status: PayloadStatus::Empty + }], + "no verified payload yet: only the empty branch exists" + ); + + store.insert_verified_payload(a_root); + assert_eq!( + get_node_children(&store, &blocks, node).unwrap(), + vec![ + ForkChoiceNode { + root: a_root, + payload_status: PayloadStatus::Empty + }, + ForkChoiceNode { + root: a_root, + payload_status: PayloadStatus::Full + }, + ], + "a verified payload adds the full branch alongside the empty one" + ); + } + + #[test] + fn get_parent_payload_status_matches_the_childs_bid_against_the_parents_block_hash() { + let mut store = empty_store(); + let a_hash = ExecutionBlockHash::repeat_byte(0xaa); + let a_root = Root::repeat_byte(0xa1); + store + .insert_signed_block( + a_root, + gloas_block(1, Root::ZERO, ExecutionBlockHash::ZERO, a_hash), + ) + .unwrap(); + + let full_root = Root::repeat_byte(0xb1); + store + .insert_signed_block( + full_root, + // Its bid names `a_root`'s own revealed block hash. + gloas_block(2, a_root, a_hash, ExecutionBlockHash::repeat_byte(0xb2)), + ) + .unwrap(); + let empty_root = Root::repeat_byte(0xc1); + store + .insert_signed_block( + empty_root, + // Its bid names a block hash `a_root` never revealed. + gloas_block( + 2, + a_root, + ExecutionBlockHash::repeat_byte(0xff), + ExecutionBlockHash::repeat_byte(0xc2), + ), + ) + .unwrap(); + + let full_block = store.get_signed_block(&full_root).unwrap().unwrap(); + let empty_block = store.get_signed_block(&empty_root).unwrap().unwrap(); + + assert_eq!( + get_parent_payload_status(&store, &full_block).unwrap(), + PayloadStatus::Full, + "the child's bid names the parent's own revealed block hash" + ); + assert!(is_parent_node_full(&store, &full_block).unwrap()); + + assert_eq!( + get_parent_payload_status(&store, &empty_block).unwrap(), + PayloadStatus::Empty, + "the child's bid names a different block hash than the parent revealed" + ); + assert!(!is_parent_node_full(&store, &empty_block).unwrap()); + } + + /// A wrong ancestor walk that read the wrong child's bid at each step is + /// caught here: comparing `c_root`'s own bid against `a_root` directly + /// (skipping the step through `b_root`) would answer `Empty` (`c_root`'s + /// bid names a different hash than `a_root` revealed), not the correct + /// `Full` that `b_root`'s own bid (built on `a_root`'s revealed hash) + /// actually names. The second half below repeats the same shape with a + /// `b2_root` whose bid misses `a_hash`, so the walk must also come back + /// empty, not just full. + #[test] + fn get_ancestor_carries_the_childs_view_of_the_stepped_over_parents_payload_status() { + let mut store = empty_store(); + let a_hash = ExecutionBlockHash::repeat_byte(0xaa); + let a_root = Root::repeat_byte(0xa1); + store + .insert_signed_block( + a_root, + gloas_block(1, Root::ZERO, ExecutionBlockHash::ZERO, a_hash), + ) + .unwrap(); + + let b_root = Root::repeat_byte(0xb1); + let b_hash = ExecutionBlockHash::repeat_byte(0xbb); + store + .insert_signed_block(b_root, gloas_block(2, a_root, a_hash, b_hash)) + .unwrap(); + + let c_root = Root::repeat_byte(0xc1); + store + .insert_signed_block( + c_root, + gloas_block(3, b_root, b_hash, ExecutionBlockHash::repeat_byte(0xcc)), + ) + .unwrap(); + + let start = ForkChoiceNode { + root: c_root, + payload_status: PayloadStatus::Pending, + }; + let ancestor = gloas_get_ancestor(&store, start, 1).unwrap(); + + assert_eq!( + ancestor, + ForkChoiceNode { + root: a_root, + payload_status: PayloadStatus::Full, + }, + "b_root's bid names a_root's own revealed block hash, so stepping \ + past it labels a_root full" + ); + + // The same walk, but `b2_root`'s bid misses `a_hash`: stepping past + // it must label `a_root` empty instead. + let b2_root = Root::repeat_byte(0xb2); + let b2_hash = ExecutionBlockHash::repeat_byte(0xee); + store + .insert_signed_block( + b2_root, + gloas_block(2, a_root, ExecutionBlockHash::repeat_byte(0xff), b2_hash), + ) + .unwrap(); + let c2_root = Root::repeat_byte(0xc2); + store + .insert_signed_block( + c2_root, + gloas_block(3, b2_root, b2_hash, ExecutionBlockHash::repeat_byte(0xcc)), + ) + .unwrap(); + + let start2 = ForkChoiceNode { + root: c2_root, + payload_status: PayloadStatus::Pending, + }; + let ancestor2 = gloas_get_ancestor(&store, start2, 1).unwrap(); + + assert_eq!( + ancestor2, + ForkChoiceNode { + root: a_root, + payload_status: PayloadStatus::Empty, + }, + "b2_root's bid misses a_root's own revealed block hash, so \ + stepping past it labels a_root empty" + ); + } + + #[test] + fn get_payload_status_tiebreaker_prefers_empty_then_a_verified_extended_payload() { + let config = Config::active(); + let mut store = empty_store(); + + let a_root = Root::repeat_byte(0xa1); + let a_slot = 1; + store + .insert_signed_block( + a_root, + gloas_block( + a_slot, + Root::ZERO, + ExecutionBlockHash::ZERO, + ExecutionBlockHash::repeat_byte(0xaa), + ), + ) + .unwrap(); + // A previous-slot payload decision: `a_root`'s block is one slot + // behind the store clock, which is what makes the tiebreaker matter + // at all (`get_weight` scores it `0` otherwise). + store + .set_time_ms((a_slot + 1) * config.slot_duration_ms) + .unwrap(); + + let empty = ForkChoiceNode { + root: a_root, + payload_status: PayloadStatus::Empty, + }; + let full = ForkChoiceNode { + root: a_root, + payload_status: PayloadStatus::Full, + }; + + assert_eq!( + get_payload_status_tiebreaker(&store, empty, &config).unwrap(), + 1, + "the empty branch always wins its own tiebreaker value" + ); + assert_eq!( + get_payload_status_tiebreaker(&store, full, &config).unwrap(), + 0, + "a payload nobody verified must not be extended, so full loses the tiebreak" + ); + + store.insert_verified_payload(a_root); + let strong_votes = vec![Some(true); preset::PTC_SIZE]; + store.set_payload_timeliness_vote(a_root, strong_votes.clone()); + store.set_payload_data_availability_vote(a_root, strong_votes); + + assert_eq!( + get_payload_status_tiebreaker(&store, full, &config).unwrap(), + 2, + "a verified, PTC-approved payload must win the tiebreak by extending" + ); + } + + /// The spec compares with `>`, not `>=`: exactly the threshold's worth of + /// matching votes must not be enough, only one more than it. + #[test] + fn payload_timeliness_and_data_availability_require_strictly_more_than_the_threshold() { + let mut store = empty_store(); + let a_root = Root::repeat_byte(0xa1); + store.insert_verified_payload(a_root); + + let exactly_threshold = vec![Some(true); preset::PAYLOAD_TIMELY_THRESHOLD as usize]; + store.set_payload_timeliness_vote(a_root, exactly_threshold.clone()); + assert!( + !payload_timeliness(&store, a_root, true).unwrap(), + "exactly the threshold's worth of matching votes must not be enough" + ); + store.set_payload_timeliness_vote( + a_root, + vec![Some(true); preset::PAYLOAD_TIMELY_THRESHOLD as usize + 1], + ); + assert!( + payload_timeliness(&store, a_root, true).unwrap(), + "one more than the threshold must be enough" + ); + + store.set_payload_data_availability_vote(a_root, exactly_threshold); + assert!( + !payload_data_availability(&store, a_root, true).unwrap(), + "the same strict threshold applies to the data-availability vote" + ); + store.set_payload_data_availability_vote( + a_root, + vec![Some(true); preset::DATA_AVAILABILITY_TIMELY_THRESHOLD as usize + 1], + ); + assert!(payload_data_availability(&store, a_root, true).unwrap()); + } + + #[test] + fn should_extend_payload_follows_the_ptc_vote_once_verified() { + let config = Config::active(); + let mut store = empty_store(); + + let a_root = Root::repeat_byte(0xa1); + let a_slot = 1; + let a_hash = ExecutionBlockHash::repeat_byte(0xaa); + store + .insert_signed_block( + a_root, + gloas_block(a_slot, Root::ZERO, ExecutionBlockHash::ZERO, a_hash), + ) + .unwrap(); + + // The current proposer-boosted block builds on `a_root` and itself + // chose the empty branch, so neither of `should_extend_payload`'s two + // other conditions holds: only the PTC's own vote can make it extend. + let d_root = Root::repeat_byte(0xd1); + store + .insert_signed_block( + d_root, + gloas_block( + a_slot + 1, + a_root, + ExecutionBlockHash::repeat_byte(0xff), // does not match `a_hash` + ExecutionBlockHash::repeat_byte(0xdd), + ), + ) + .unwrap(); + store.set_proposer_boost_root(d_root); + store + .set_time_ms((a_slot + 1) * config.slot_duration_ms) + .unwrap(); + + assert!( + !should_extend_payload(&store, a_root, &config).unwrap(), + "a payload nobody verified must not be extended" + ); + + store.insert_verified_payload(a_root); + let weak_votes = vec![Some(false); preset::PTC_SIZE]; + store.set_payload_timeliness_vote(a_root, weak_votes.clone()); + store.set_payload_data_availability_vote(a_root, weak_votes); + assert!( + !should_extend_payload(&store, a_root, &config).unwrap(), + "verified but neither timely nor available, with the \ + proposer-boosted block itself choosing empty: must not extend" + ); + + // Timely but not available: both conditions are required (`&&`, not + // `||`), so this must not extend either. + store.set_payload_timeliness_vote(a_root, vec![Some(true); preset::PTC_SIZE]); + store.set_payload_data_availability_vote(a_root, vec![Some(false); preset::PTC_SIZE]); + assert!( + !should_extend_payload(&store, a_root, &config).unwrap(), + "timely alone, without availability, must not extend" + ); + + let strong_votes = vec![Some(true); preset::PTC_SIZE]; + store.set_payload_timeliness_vote(a_root, strong_votes.clone()); + store.set_payload_data_availability_vote(a_root, strong_votes); + assert!( + should_extend_payload(&store, a_root, &config).unwrap(), + "a verified payload the PTC saw as timely and available must extend" + ); + } + + #[test] + fn gloas_get_head_picks_the_more_heavily_voted_payload_branch() { + let config = Config::active(); + let anchor_root = Root::repeat_byte(1); + let mut store = store_anchored_at(anchor_root); + store + .insert_signed_block( + anchor_root, + gloas_block( + 0, + Root::ZERO, + ExecutionBlockHash::ZERO, + ExecutionBlockHash::ZERO, + ), + ) + .unwrap(); + + let a_root = Root::repeat_byte(2); + let a_slot = 1; + store + .insert_signed_block( + a_root, + gloas_block( + a_slot, + anchor_root, + // Does not match the anchor's own (zero) block hash, so + // `a_root` builds on the anchor's empty branch. + ExecutionBlockHash::repeat_byte(0xff), + ExecutionBlockHash::repeat_byte(0xaa), + ), + ) + .unwrap(); + store.insert_verified_payload(a_root); + + let state = test_state::with_validators_at(ForkName::Gloas, 4); + store.insert_state(anchor_root, state.clone()).unwrap(); + store.insert_state(a_root, state).unwrap(); + + // Past `a_root`'s own slot, but not by exactly one: + // `is_previous_slot_payload_decision` would otherwise zero both + // branches' weight and leave only the tiebreaker to decide, which is + // not what this test exercises. + store.set_time_ms(3 * config.slot_duration_ms).unwrap(); + + // Three of four votes go to the empty branch, whose own tiebreaker + // value (`PayloadStatus::Empty as u8 == 0`) is *lower* than full's + // (`PayloadStatus::Full as u8 == 1`): a descent that ignored weight + // and fell back on the tiebreaker alone would pick full here, so + // this only passes if weight actually decides. + for validator_index in 0..3 { + store.set_latest_message( + validator_index, + LatestMessage { + epoch: 0, + slot: a_slot + 1, + root: a_root, + payload_present: false, + }, + ); + } + store.set_latest_message( + 3, + LatestMessage { + epoch: 0, + slot: a_slot + 1, + root: a_root, + payload_present: true, + }, + ); + + let committees = CommitteeCache::default(); + let head = gloas_get_head(&store, &config, &committees).expect("known chain"); + assert_eq!( + head, + ForkChoiceNode { + root: a_root, + payload_status: PayloadStatus::Empty, + }, + "three of four votes for the empty branch must outweigh the lone \ + vote for full, despite full's higher tiebreak value" + ); + } + + /// `should_apply_proposer_boost`'s own blocks need no gloas bid at all + /// (it never calls `gloas_bid`/`get_parent_payload_status`), so the + /// plain phase0-shaped `block`/`block_with_proposer` helpers serve here. + #[test] + fn should_apply_proposer_boost_is_withheld_only_by_an_early_equivocation() { + let config = Config::active(); + let (mut store, anchor_root, anchor_slot) = anchored_store(1); + let committees = CommitteeCache::default(); + + let b_root = Root::repeat_byte(0xb0); + store + .insert_signed_block(b_root, block(anchor_slot + 1, anchor_root)) + .unwrap(); + store.set_proposer_boost_root(b_root); + + // No votes recorded at all, so the parent (the anchor) is trivially + // weak, per the same reading `is_head_weak_counts_an_equivocating_ + // validators_balance_from_the_head_slot_committees` gives "a head + // with no votes and no equivocators must be weak". The parent is + // also exactly one slot behind `b_root`. Neither of + // `should_apply_proposer_boost`'s two early-return conditions fires, + // so both checks below reach the equivocation scan. + assert!( + should_apply_proposer_boost(&store, &config, &committees).unwrap(), + "no same-slot, same-proposer, PTC-timely sibling of the parent \ + exists yet, so the boost still applies" + ); + + // A same-slot sibling of the parent, sharing its proposer (both + // default to index 0) and PTC-timely, is an early equivocation. + let twin_root = Root::repeat_byte(0xb1); + store + .insert_signed_block(twin_root, block(anchor_slot, Root::ZERO)) + .unwrap(); + store.set_block_timeliness(twin_root, [false, true]); + + assert!( + !should_apply_proposer_boost(&store, &config, &committees).unwrap(), + "an early equivocation against the parent must withhold the boost" + ); + } } diff --git a/crates/common/types/src/beacon/constants.rs b/crates/common/types/src/beacon/constants.rs index 2606be382..4bcd0d1f4 100644 --- a/crates/common/types/src/beacon/constants.rs +++ b/crates/common/types/src/beacon/constants.rs @@ -400,6 +400,22 @@ pub const BUILDER_DEPOSIT_REQUEST_TYPE: u8 = 0x03; /// request. pub const BUILDER_EXIT_REQUEST_TYPE: u8 = 0x04; +/// Index into `store.block_timeliness[root]` for whether the block itself +/// arrived before the (ordinary) attestation deadline: the one deadline every +/// fork before gloas checked, and still what `is_head_late` reads. See +/// [`PTC_TIMELINESS_INDEX`] for the other. +pub const ATTESTATION_TIMELINESS_INDEX: usize = 0; + +/// Index into `store.block_timeliness[root]` for whether the block arrived +/// before the payload timeliness committee's own, later deadline +/// (`get_payload_attestation_due_ms`). `should_apply_proposer_boost`'s +/// equivocation scan reads this index. +pub const PTC_TIMELINESS_INDEX: usize = 1; + +/// How many deadlines `store.block_timeliness[root]` records against: the +/// length every entry has, one slot per index above. +pub const NUM_BLOCK_TIMELINESS_DEADLINES: usize = 2; + #[cfg(test)] mod tests { use super::*; diff --git a/crates/common/types/src/beacon/fork_choice.rs b/crates/common/types/src/beacon/fork_choice.rs index 7002466d1..91890d2f1 100644 --- a/crates/common/types/src/beacon/fork_choice.rs +++ b/crates/common/types/src/beacon/fork_choice.rs @@ -1,28 +1,86 @@ //! Fork-choice-adjacent data that is neither a block nor a state. //! -//! Two kinds share this file. `LatestMessage` and `PowBlock` are SSZ +//! Three kinds share this file. `LatestMessage` and `PowBlock` are SSZ //! consensus containers moved out of the `beacon::fork_choice` module of //! `ethlambda-state-transition`, which re-exports both at their old paths so //! every use site inside it is unchanged. `PayloadStatusEnum` and //! `PayloadStatusV1` are plain Engine-API-shaped data with no such former -//! home. All four live here rather than there for the same reason: the -//! DB-backed `ethlambda_storage::Store` holds them, and `ethlambda-storage` -//! cannot depend on `ethlambda-state-transition`, which pulls in `blst` and -//! `c-kzg`. +//! home. `PayloadStatus` and `ForkChoiceNode` are gloas's own fork-choice +//! node types, new here rather than moved, since nothing named them before +//! gloas. All six live here for the same reason: the DB-backed +//! `ethlambda_storage::Store` holds `LatestMessage`, `PowBlock` and +//! `PayloadStatusV1`, and `ethlambda-storage` cannot depend on +//! `ethlambda-state-transition`, which pulls in `blst` and `c-kzg`; +//! `PayloadStatus` and `ForkChoiceNode` join them here rather than living +//! beside `ethlambda-state-transition`'s own fork choice, so that a +//! `LatestMessage`'s `payload_present` field and a stored node's own +//! `PayloadStatus` share one crate with no dependency to cross. use libssz_derive::{HashTreeRoot, SszDecode, SszEncode}; -use crate::beacon::primitives::{Epoch, ExecutionBlockHash, Root, Uint256}; +use crate::beacon::primitives::{Epoch, ExecutionBlockHash, Root, Slot, Uint256}; /// One validator's most recent attestation: the epoch it targeted, and the /// block it attested to (the LMD GHOST vote). /// /// `Copy`, matching the specification's `@dataclass(eq=True, frozen=True)`: /// there is nothing here worth borrowing rather than copying. +/// +/// `epoch` and `slot` both live here rather than one replacing the other: +/// every fork through fulu keeps `epoch`, the field gloas's own modified +/// `LatestMessage` drops in favour of `slot` (gloas compares messages by slot, +/// since a payload can be revealed a slot late and a slot-grained comparison +/// is what lets `update_latest_messages` tell such a re-vote apart from a +/// stale one). Splitting the two into per-fork types would mean every reader +/// of a [`LatestMessage`] picks a variant instead of a field, for a value that +/// is otherwise identical; carrying both instead lets every fork's own +/// constructor fill in the one it has and leave the other at its fork's own +/// neutral value (pre-gloas: `slot: data.slot, payload_present: false`; gloas +/// has no use for `epoch`, so nothing in this crate reads it off a gloas +/// message). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct LatestMessage { pub epoch: Epoch, + /// The attestation's slot. Gloas compares messages by slot; earlier + /// forks keep comparing `epoch`. + pub slot: Slot, + pub root: Root, + /// Gloas: whether the vote is for the block's full node (`data.index == + /// 1`). Always `false` before gloas, which has no payload dimension to + /// vote on. + pub payload_present: bool, +} + +/// A fork-choice node's payload dimension (gloas `fork-choice.md`'s new +/// `PayloadStatus`): whether a [`ForkChoiceNode`] stands for a block whose +/// payload is known to be empty, known to be full, or not yet decided either +/// way. +/// +/// Ordered `Empty < Full < Pending`, matching the specification's own integer +/// values (`PAYLOAD_STATUS_EMPTY = 0`, `PAYLOAD_STATUS_FULL = 1`, +/// `PAYLOAD_STATUS_PENDING = 2`): nothing in this crate compares two +/// `PayloadStatus` values by order today, but the derive is kept alongside +/// the discriminants it agrees with rather than left for a future caller to +/// get wrong. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum PayloadStatus { + Empty = 0, + Full = 1, + Pending = 2, +} + +/// A gloas fork-choice node (`fork-choice.md`'s modified `ForkChoiceNode`): a +/// block, and which of its payload branches. +/// +/// Every earlier fork's own `ForkChoiceNode` is a one-to-one mapping with a +/// `BeaconBlock` (see `specs/phase0/fork-choice.md`'s own note), so this +/// crate collapses it to a bare [`Root`] wherever a pre-gloas function reads +/// one; only gloas needs the pair, since ePBS (EIP-7732) splits a block into +/// two branches fork choice must weigh separately. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ForkChoiceNode { pub root: Root, + pub payload_status: PayloadStatus, } /// The execution chain's own block header, as far as bellatrix's merge diff --git a/crates/common/types/src/beacon/preset.rs b/crates/common/types/src/beacon/preset.rs index 7f8600125..afa81596b 100644 --- a/crates/common/types/src/beacon/preset.rs +++ b/crates/common/types/src/beacon/preset.rs @@ -514,6 +514,16 @@ pub mod mainnet { /// and each row of `BeaconState.ptc_window`. pub const PTC_SIZE: usize = 512; + /// `PTC_SIZE // 2`: how many PTC votes a payload needs, one way or the + /// other, before fork choice's `payload_timeliness` trusts the PTC's view + /// over "not verified, so not timely". + pub const PAYLOAD_TIMELY_THRESHOLD: u64 = (PTC_SIZE / 2) as u64; + + /// `PTC_SIZE // 2`, the `payload_data_availability` counterpart of + /// [`PAYLOAD_TIMELY_THRESHOLD`]. Equal to it in the specification's own + /// table; kept as its own name since the two answer different questions. + pub const DATA_AVAILABILITY_TIMELY_THRESHOLD: u64 = PAYLOAD_TIMELY_THRESHOLD; + // --- Max operations per block --- /// Bounds `BeaconBlockBody.payload_attestations` @@ -1017,6 +1027,12 @@ pub mod minimal { /// meaningful at minimal's validator counts. pub const PTC_SIZE: usize = 16; + /// `PTC_SIZE // 2`. See mainnet's own doc. + pub const PAYLOAD_TIMELY_THRESHOLD: u64 = (PTC_SIZE / 2) as u64; + + /// `PTC_SIZE // 2`. See mainnet's own doc. + pub const DATA_AVAILABILITY_TIMELY_THRESHOLD: u64 = PAYLOAD_TIMELY_THRESHOLD; + // --- Max operations per block --- /// Bounds `BeaconBlockBody.payload_attestations` @@ -1431,6 +1447,17 @@ mod tests { assert_eq!(mainnet::PTC_SIZE, 512); assert_eq!(minimal::PTC_SIZE, 16); + assert_eq!(mainnet::PAYLOAD_TIMELY_THRESHOLD, 256); + assert_eq!(minimal::PAYLOAD_TIMELY_THRESHOLD, 8); + assert_eq!( + mainnet::DATA_AVAILABILITY_TIMELY_THRESHOLD, + mainnet::PAYLOAD_TIMELY_THRESHOLD + ); + assert_eq!( + minimal::DATA_AVAILABILITY_TIMELY_THRESHOLD, + minimal::PAYLOAD_TIMELY_THRESHOLD + ); + assert_eq!(mainnet::MAX_BUILDERS_PER_WITHDRAWALS_SWEEP, 16_384); assert_eq!(minimal::MAX_BUILDERS_PER_WITHDRAWALS_SWEEP, 16); } @@ -1612,6 +1639,7 @@ mod tests { in_both!(u64: MAX_PAYLOAD_ATTESTATIONS, MAX_BUILDER_DEPOSIT_REQUESTS_PER_PAYLOAD, MAX_BUILDER_EXIT_REQUESTS_PER_PAYLOAD, MAX_BUILDERS_PER_WITHDRAWALS_SWEEP, + PAYLOAD_TIMELY_THRESHOLD, DATA_AVAILABILITY_TIMELY_THRESHOLD, ); } } diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 0bfefb7e6..a6d472bd8 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -17,6 +17,7 @@ use ethlambda_types::{ }, beacon::{ config::Config, + constants::NUM_BLOCK_TIMELINESS_DEADLINES, containers::{BeaconState, Checkpoint as BeaconCheckpoint, SignedBeaconBlock}, fork::ForkName, fork_choice::{LatestMessage, PayloadStatusV1, PowBlock}, @@ -656,23 +657,68 @@ impl GossipSignatureBuffer { /// for again, and `el_block_hashes` is a cache over data already decodable /// from the block itself. /// +/// Gloas's own four (`verified_payloads`, `payload_timeliness_vote`, +/// `payload_data_availability_vote`, and `block_timeliness`'s widened shape) +/// do not all fit the "a restarted node just gets it again" pattern above, +/// even though they live in this same scratch. `payload_timeliness_vote` and +/// `payload_data_availability_vote` do: they are the payload timeliness +/// committee's per-slot ballot, gone the moment the slot it was cast for is +/// no longer being decided, the same kind of throwaway `pow_blocks` is. +/// `verified_payloads` and `block_timeliness` do not, and both must be +/// persisted before this runs against a live, restarting node: losing +/// `verified_payloads` on restart does not error, it silently drops every +/// full payload branch from fork choice (`is_payload_verified` answers +/// `false` for a block this store genuinely verified before restarting), +/// and losing a `block_timeliness` entry the gloas fork choice's +/// `should_apply_proposer_boost` needs makes that call fail outright with +/// `Error::SpecAssert`. Unlike pre-gloas, where `block_timeliness` is read +/// only by the same-slot reorg helpers, gloas's own weight computation +/// reads it on the ordinary head-computation path, so that failure aborts +/// the whole `gloas_get_weight`/`gloas_get_head` walk, not just an optional +/// reorg decision. Nothing in this crate persists either yet. +/// /// Most of this is uncapped: the per-validator maps are bounded by the /// validator set, and the per-block ones (`block_timeliness`, -/// `unrealized_justifications`) grow with the blocks this process has -/// imported. Two are the exception, `el_block_hashes` and `optimistic_roots`, -/// each pruned to the unfinalized window by its own `prune_*` method. Both -/// fill on a path that runs for the whole life of the process and has no other -/// way of emptying them: `forkchoiceUpdated` reads the first once per head -/// move, and an execution client doing a long state sync answers -/// `NOT_VALIDATED` to every block, which writes the second once per import. +/// `unrealized_justifications`, and gloas's own three) grow with the blocks +/// this process has imported. Two are the exception, `el_block_hashes` and +/// `optimistic_roots`, each pruned to the unfinalized window by its own +/// `prune_*` method. Both fill on a path that runs for the whole life of the +/// process and has no other way of emptying them: `forkchoiceUpdated` reads +/// the first once per head move, and an execution client doing a long state +/// sync answers `NOT_VALIDATED` to every block, which writes the second once +/// per import. #[derive(Default)] pub(crate) struct BeaconScratch { pub(crate) proposer_boost_root: H256, - pub(crate) block_timeliness: HashMap, + /// Gloas widens this from one bool to two, `[ATTESTATION_TIMELINESS_INDEX, + /// PTC_TIMELINESS_INDEX]`: a block is now timed against both the + /// attestation deadline and the (later) payload-attestation-committee + /// deadline. Pre-gloas code reads `ATTESTATION_TIMELINESS_INDEX` only, + /// the same deadline it always checked. + pub(crate) block_timeliness: HashMap, pub(crate) equivocating_indices: HashSet, pub(crate) latest_messages: HashMap, pub(crate) pow_blocks: HashMap, pub(crate) unrealized_justifications: HashMap, + /// Gloas: beacon block roots whose execution payload envelope has been + /// locally delivered and verified (`on_execution_payload_envelope`'s own + /// `store.payloads[root] = envelope`), standing in for the specification's + /// `store.payloads` for exactly the question this crate asks of it today, + /// `is_payload_verified`'s membership test. The envelope itself is not + /// kept: nothing here reads one back out yet, and it is not clear this + /// scratch is even where a kept envelope should live, since (see this + /// struct's own doc) `verified_payloads` needs to survive a restart and + /// nothing else here does. + pub(crate) verified_payloads: HashSet, + /// Gloas: the payload timeliness committee's per-member vote on whether + /// the block's payload showed up on time, one slot per `PTC_SIZE` index, + /// `None` until that member votes. Keyed by beacon block root, like + /// `block_timeliness`. + pub(crate) payload_timeliness_vote: HashMap>>, + /// Gloas: the payload timeliness committee's per-member vote on whether + /// the block's blob data was available, the sibling of + /// `payload_timeliness_vote`. + pub(crate) payload_data_availability_vote: HashMap>>, /// Beacon roots imported on an execution client's `NOT_VALIDATED` answer, /// against the slot the unfinalized-window bound prunes them by. /// @@ -3181,9 +3227,17 @@ impl Store { self.beacon.lock().unwrap().proposer_boost_root = root; } - /// Whether `root` arrived within the same-slot reorg window. `None` when - /// no timeliness has been recorded for the block yet. - pub fn block_timeliness(&self, root: &H256) -> Option { + /// `root`'s two timeliness deadlines, + /// `[ATTESTATION_TIMELINESS_INDEX, PTC_TIMELINESS_INDEX]`. `None` when no + /// timeliness has been recorded for the block yet. + /// + /// A pre-gloas caller wants `ATTESTATION_TIMELINESS_INDEX`, the same-slot + /// reorg deadline this answered alone before gloas added the second; + /// `is_head_late` reads exactly that index rather than a narrower + /// accessor existing beside this one, since gloas and pre-gloas share + /// one record per block and pre-gloas code writes both entries of it + /// (see [`Store::set_block_timeliness`]). + pub fn block_timeliness(&self, root: &H256) -> Option<[bool; NUM_BLOCK_TIMELINESS_DEADLINES]> { self.beacon .lock() .unwrap() @@ -3192,8 +3246,12 @@ impl Store { .copied() } - /// Records whether `root` arrived within the same-slot reorg window. - pub fn set_block_timeliness(&mut self, root: H256, timely: bool) { + /// Records `root`'s two timeliness deadlines. See [`Store::block_timeliness`]. + pub fn set_block_timeliness( + &mut self, + root: H256, + timely: [bool; NUM_BLOCK_TIMELINESS_DEADLINES], + ) { self.beacon .lock() .unwrap() @@ -3220,6 +3278,18 @@ impl Store { .insert(index); } + /// Gloas: whether `root`'s execution payload envelope has been locally + /// delivered and verified, the specification's `is_payload_verified`. + pub fn has_verified_payload(&self, root: &H256) -> bool { + self.beacon.lock().unwrap().verified_payloads.contains(root) + } + + /// Gloas: records that `root`'s execution payload envelope has been + /// verified. + pub fn insert_verified_payload(&mut self, root: H256) { + self.beacon.lock().unwrap().verified_payloads.insert(root); + } + /// The latest attestation recorded for validator `index`, if any. pub fn latest_message(&self, index: u64) -> Option { self.beacon @@ -3239,6 +3309,49 @@ impl Store { .insert(index, message); } + /// Gloas: the payload timeliness committee's per-member votes on whether + /// `root`'s payload was timely, one entry per PTC member and `None` for a + /// member that has not voted yet. `None` when `root` has no vote vector at + /// all yet (`get_forkchoice_store`/`on_block` seed one per block). + pub fn payload_timeliness_vote(&self, root: &H256) -> Option>> { + self.beacon + .lock() + .unwrap() + .payload_timeliness_vote + .get(root) + .cloned() + } + + /// Replaces `root`'s payload-timeliness vote vector. + pub fn set_payload_timeliness_vote(&mut self, root: H256, votes: Vec>) { + self.beacon + .lock() + .unwrap() + .payload_timeliness_vote + .insert(root, votes); + } + + /// Gloas: the payload timeliness committee's per-member votes on whether + /// `root`'s blob data was available. The sibling of + /// [`Store::payload_timeliness_vote`]. + pub fn payload_data_availability_vote(&self, root: &H256) -> Option>> { + self.beacon + .lock() + .unwrap() + .payload_data_availability_vote + .get(root) + .cloned() + } + + /// Replaces `root`'s payload-data-availability vote vector. + pub fn set_payload_data_availability_vote(&mut self, root: H256, votes: Vec>) { + self.beacon + .lock() + .unwrap() + .payload_data_availability_vote + .insert(root, votes); + } + /// Calls `f` with `(validator_index, latest_message)` for every latest /// message whose validator has not been observed equivocating. /// @@ -6868,8 +6981,11 @@ mod tests { assert!(store.is_equivocating(42)); assert!(!store.is_equivocating(43)); - clone.set_block_timeliness(H256::from([1u8; 32]), true); - assert_eq!(store.block_timeliness(&H256::from([1u8; 32])), Some(true)); + clone.set_block_timeliness(H256::from([1u8; 32]), [true, false]); + assert_eq!( + store.block_timeliness(&H256::from([1u8; 32])), + Some([true, false]) + ); assert_eq!(store.block_timeliness(&H256::from([2u8; 32])), None); } @@ -6878,7 +6994,9 @@ mod tests { let mut store = Store::test_store(); let message = LatestMessage { epoch: 3, + slot: 24, root: H256::from([7u8; 32]), + payload_present: false, }; store.set_latest_message(1, message); From a66c0c6079b9e4b1ee43dd3c9fe966f9bb349fc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 28 Sep 2026 13:30:42 -0300 Subject: [PATCH 020/132] feat(beacon): add gloas block operations Ports gloas's modified process_operations and the operations it changes. An attestation's data.index is now the attester's vote on whether the slot's payload was present, and process_attestation takes the parent block's slot to tell a same-slot vote from a late one. A same-slot attester that newly earns a flag adds its effective balance to the builder's pending payment weight, which epoch processing checks against a quorum before paying the builder. The new process_payload_attestation checks the payload timeliness committee's vote on the previous slot's payload. process_proposer_slashing also clears the slashed proposal's pending builder payment. process_operations no longer calls the deposit, withdrawal and consolidation requests, which apply_parent_execution_payload applies, and adds the payload attestation loop. EIP-7688 makes every operations list progressive, so each one's length is checked against its preset maximum explicitly rather than by an SSZ bound. Operations the specification leaves unmodified are shared rather than copied: voluntary exits, BLS-to-execution changes and the sync aggregate run unchanged; the committee walk behind get_attesting_indices, the attester slashing tail and the proposer slashing checks are extracted and called from each fork, since only the list types behind them change. BeaconState::epoch_participation_mut gives process_attestation element writes to gloas's progressive participation lists. Every gloas operations handler now runs. The operations harness reads a case's own config.yaml when it ships one, and otherwise schedules every fork up to the case's own at genesis, through a helper it now shares with the gossip runner: get_ptc is the first operation to read a fork epoch from the config. --- .../src/beacon/helpers/electra.rs | 47 +- .../src/beacon/helpers/gloas.rs | 53 ++ .../src/beacon/stf/electra.rs | 42 +- .../state_transition/src/beacon/stf/gloas.rs | 734 +++++++++++++++++- .../src/beacon/stf/operations.rs | 117 ++- .../tests/beacon_spec/gossip.rs | 25 +- .../state_transition/tests/beacon_spec/mod.rs | 36 +- .../tests/beacon_spec/operations.rs | 196 +++-- .../common/types/src/beacon/containers/mod.rs | 27 + 9 files changed, 1113 insertions(+), 164 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/helpers/electra.rs b/crates/blockchain/state_transition/src/beacon/helpers/electra.rs index 4e14498a7..cca5d7bc9 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/electra.rs @@ -135,7 +135,7 @@ use crate::beacon::lean_state_unreachable; use crate::beacon::preset; use crate::beacon::primitives::{ BLS_SIGNATURE_SIZE, BlsSignature, Bytes32, CommitteeIndex, Epoch, Gwei, HashTreeRoot as _, - ValidatorIndex, + Slot, ValidatorIndex, }; use super::accessors::{ @@ -413,8 +413,8 @@ pub fn get_pending_balance_to_withdraw(state: &BeaconState, index: ValidatorInde Ok(total) } -/// The committee members whose bit is set in `attestation`, in ascending -/// order. +/// The attester indices a slot's named committees cover, filtered by +/// `is_set`, in ascending order. /// /// EIP-7549 moves the committee index out of `AttestationData` and lets one /// attestation cover every committee in a slot, so `aggregation_bits` is now @@ -448,22 +448,31 @@ pub fn get_pending_balance_to_withdraw(state: &BeaconState, index: ValidatorInde /// than `MAX_COMMITTEES_PER_SLOT` of each: every committee named by one /// attestation belongs to the same slot, and so to the same epoch's shuffling. /// See [`CommitteeCache`] for how far that sharing reaches beyond this call. -pub fn get_attesting_indices( +/// +/// `is_set` is this function's own share of what [`get_attesting_indices`] +/// otherwise is verbatim: electra's `aggregation_bits` is a bounded +/// `SszBitlist`, and [`crate::beacon::helpers::gloas::get_attesting_indices`]'s +/// is EIP-7688's unbounded `ProgressiveBitlist`, two different Rust types +/// whose only shared operation this walk needs is "is bit `n` set", which is +/// exactly what a closure abstracts over without either caller allocating or +/// duplicating the walk itself. +pub(crate) fn attesting_indices_from_committee_bits( state: &BeaconState, - attestation: &electra::Attestation, + slot: Slot, + committee_bits: &electra::CommitteeBits, + is_set: impl Fn(usize) -> bool, committees: &CommitteeCache, ) -> Result> { - let committee_indices = get_committee_indices(&attestation.committee_bits); - let epoch_committees = - committees.committees(state, compute_epoch_at_slot(attestation.data.slot)); + let committee_indices = get_committee_indices(committee_bits); + let epoch_committees = committees.committees(state, compute_epoch_at_slot(slot)); let mut indices = Vec::new(); let mut committee_offset = 0usize; for committee_index in committee_indices { - let committee = epoch_committees.committee(attestation.data.slot, committee_index)?; + let committee = epoch_committees.committee(slot, committee_index)?; for (position, attester_index) in committee.iter().enumerate() { let bit = committee_offset + position; - if attestation.aggregation_bits.get(bit).unwrap_or(false) { + if is_set(bit) { indices.push(*attester_index); } } @@ -474,6 +483,24 @@ pub fn get_attesting_indices( Ok(indices) } +/// The committee members whose bit is set in `attestation`, in ascending +/// order. See [`attesting_indices_from_committee_bits`] for the walk itself; +/// this is a thin wrapper over it, closing over `attestation`'s own +/// `aggregation_bits`. +pub fn get_attesting_indices( + state: &BeaconState, + attestation: &electra::Attestation, + committees: &CommitteeCache, +) -> Result> { + attesting_indices_from_committee_bits( + state, + attestation.data.slot, + &attestation.committee_bits, + |bit| attestation.aggregation_bits.get(bit).unwrap_or(false), + committees, + ) +} + /// The same attestation with its attesters named rather than bit-encoded. pub fn get_indexed_attestation( state: &BeaconState, diff --git a/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs b/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs index 36a3fe453..14816f09d 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs @@ -148,6 +148,59 @@ pub fn is_valid_indexed_attestation( bls::fast_aggregate_verify(&pubkeys, signing_root, &indexed_attestation.signature) } +// --------------------------------------------------------------------------- +// Attestations +// --------------------------------------------------------------------------- + +/// The committee members whose bit is set in `attestation`, in ascending +/// order. +/// +/// Not modified by gloas's own `beacon-chain.md` (which names no +/// `get_attesting_indices` of its own): [`gloas::Attestation`]'s +/// [`gloas::AggregationBits`] is a different Rust type from +/// [`super::electra::get_attesting_indices`]'s bounded bitlist (EIP-7688), but +/// that is the one thing +/// [`super::electra::attesting_indices_from_committee_bits`] already +/// abstracts over (see its own doc), so this is a thin wrapper over that +/// shared walk, the same shape electra's own `get_attesting_indices` is. +/// `committee_bits` is untouched by the EIP-7688 change ([`gloas::Attestation`] +/// reuses [`electra::CommitteeBits`] outright, see `containers::gloas`'s own +/// module doc), which is what lets both forks share one walk over one type. +pub fn get_attesting_indices( + state: &BeaconState, + attestation: &gloas::Attestation, + committees: &CommitteeCache, +) -> Result> { + super::electra::attesting_indices_from_committee_bits( + state, + attestation.data.slot, + &attestation.committee_bits, + |bit| attestation.aggregation_bits.get(bit).unwrap_or(false), + committees, + ) +} + +/// The same attestation with its attesters named rather than bit-encoded. +/// +/// See [`get_attesting_indices`] for why this is gloas's own copy rather than +/// a call into electra's: [`gloas::IndexedAttestation::attesting_indices`] is +/// the unbounded [`gloas::AttestingIndices`] (`ProgressiveList`), which is +/// built with `From>` rather than electra's bounds-checked +/// `TryFrom>`, since a progressive list has no maximum length to +/// exceed. +pub fn get_indexed_attestation( + state: &BeaconState, + attestation: &gloas::Attestation, + committees: &CommitteeCache, +) -> Result { + let indices = get_attesting_indices(state, attestation, committees)?; + Ok(gloas::IndexedAttestation { + attesting_indices: gloas::AttestingIndices::from(indices), + data: attestation.data, + signature: attestation.signature, + }) +} + /// Whether `validator_index` actually names a [`gloas::BuilderIndex`] /// (EIP-7732): builders and validators share one index space, distinguished /// by this one bit. diff --git a/crates/blockchain/state_transition/src/beacon/stf/electra.rs b/crates/blockchain/state_transition/src/beacon/stf/electra.rs index 0ec62ee86..104b2dd23 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/electra.rs @@ -42,8 +42,6 @@ //! ([`Config::max_blobs_per_block_electra`]) rather than deneb's fixed preset //! (see [`process_execution_payload`]). -use std::collections::HashSet; - use libssz::SszEncode as _; use crate::beacon::bls; @@ -75,10 +73,8 @@ use crate::beacon::helpers::misc::{ compute_deposit_domain, compute_domain, compute_epoch_at_slot, compute_signing_root, is_valid_merkle_branch, }; -use crate::beacon::helpers::mutators::{decrease_balance, increase_balance, slash_validator}; -use crate::beacon::helpers::predicates::{ - is_active_validator, is_slashable_attestation_data, is_slashable_validator, -}; +use crate::beacon::helpers::mutators::{decrease_balance, increase_balance}; +use crate::beacon::helpers::predicates::{is_active_validator, is_slashable_attestation_data}; use crate::beacon::preset; use crate::beacon::primitives::{ BlsPubkey, BlsSignature, Bytes32, Gwei, HashTreeRoot as _, ParticipationFlags, ValidatorIndex, @@ -582,7 +578,9 @@ pub fn process_bls_to_execution_change( /// [`electra::AttesterSlashing`] and [`crate::beacon::helpers::electra::is_valid_indexed_attestation`] /// instead, the same reason that helper exists as its own copy (see its own /// documentation): a different concrete `IndexedAttestation` type, not -/// different logic. +/// different logic. See +/// `crate::beacon::stf::operations::slash_attesting_index_intersection` for the +/// shared walk this delegates to once both indexed attestations check out. pub fn process_attester_slashing( state: &mut BeaconState, attester_slashing: &electra::AttesterSlashing, @@ -604,30 +602,12 @@ pub fn process_attester_slashing( "is_valid_indexed_attestation(state, attestation_2)", )?; - let current_epoch = get_current_epoch(state); - // `is_valid_indexed_attestation` already required both index lists to be - // sorted and unique; see `crate::beacon::stf::operations::process_attester_slashing` - // for why walking `attestation_1`'s list in order while filtering by - // membership in `attestation_2`'s set yields the intersection already - // sorted. - let indices_2: HashSet = - attestation_2.attesting_indices.iter().copied().collect(); - - let mut slashed_any = false; - for &index in attestation_1.attesting_indices.iter() { - if !indices_2.contains(&index) { - continue; - } - if is_slashable_validator(state.validator(index)?, current_epoch) { - slash_validator(state, index, None, config)?; - slashed_any = true; - } - } - verify( - slashed_any, - "at least one validator in the intersection of the two attesting index sets was slashed", - )?; - Ok(()) + super::operations::slash_attesting_index_intersection( + state, + &attestation_1.attesting_indices, + &attestation_2.attesting_indices, + config, + ) } // --------------------------------------------------------------------------- diff --git a/crates/blockchain/state_transition/src/beacon/stf/gloas.rs b/crates/blockchain/state_transition/src/beacon/stf/gloas.rs index 84c220c73..0a97c9a6d 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/gloas.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/gloas.rs @@ -44,26 +44,36 @@ //! read-only counterpart), which abstracts over exactly that type change, //! rather than through a gloas-only copy. +use libssz::SszEncode as _; + use crate::beacon::bls; use crate::beacon::config::Config; use crate::beacon::constants; -use crate::beacon::containers::shared::DepositMessage; +use crate::beacon::containers::shared::{DepositMessage, ProposerSlashing}; use crate::beacon::containers::{BeaconState, capella, gloas}; use crate::beacon::error::{Error, Result, verify}; use crate::beacon::helpers::accessors::{ - get_beacon_proposer_index, get_block_root_at_slot, get_current_epoch, get_domain, - get_previous_epoch, get_randao_mix, + CommitteeCache, CommitteeCacheExt, get_beacon_proposer_index, get_block_root_at_slot, + get_current_epoch, get_domain, get_previous_epoch, get_randao_mix, }; -use crate::beacon::helpers::electra::g2_point_at_infinity; +use crate::beacon::helpers::altair::{add_flag, get_base_reward_per_increment, has_flag}; +use crate::beacon::helpers::electra::{g2_point_at_infinity, get_committee_indices}; use crate::beacon::helpers::gloas::{ add_builder_to_registry, can_builder_cover_bid, convert_builder_index_to_validator_index, - convert_validator_index_to_builder_index, gloas_state, gloas_state_ref, initiate_builder_exit, - is_active_builder, is_builder_index, is_builder_withdrawal_credential, settle_builder_payment, + convert_validator_index_to_builder_index, get_attestation_participation_flag_indices, + get_indexed_attestation, get_indexed_payload_attestation, gloas_state, gloas_state_ref, + initiate_builder_exit, is_active_builder, is_attestation_same_slot, is_builder_index, + is_builder_withdrawal_credential, is_valid_indexed_attestation, + is_valid_indexed_payload_attestation, settle_builder_payment, }; use crate::beacon::helpers::misc::{compute_domain, compute_epoch_at_slot, compute_signing_root}; -use crate::beacon::helpers::mutators::decrease_balance; +use crate::beacon::helpers::mutators::{decrease_balance, increase_balance, slash_validator}; +use crate::beacon::helpers::predicates::is_slashable_attestation_data; use crate::beacon::preset; -use crate::beacon::primitives::{ExecutionAddress, HashTreeRoot as _, Root, WithdrawalIndex}; +use crate::beacon::primitives::{ + ExecutionAddress, Gwei, HashTreeRoot as _, ParticipationFlags, Root, Slot, ValidatorIndex, + WithdrawalIndex, +}; // --------------------------------------------------------------------------- // Withdrawals @@ -434,6 +444,71 @@ pub fn process_withdrawals(state: &mut BeaconState) -> Result<()> { Ok(()) } +// --------------------------------------------------------------------------- +// Execution payload +// --------------------------------------------------------------------------- + +/// `get_execution_requests_list` (modified, EIP-8282): electra's three +/// request-type prefixes, plus [`gloas::ExecutionRequests::builder_deposits`] +/// and [`gloas::ExecutionRequests::builder_exits`] under their own new +/// prefixes. +/// +/// The specification files this under "Execution payload", not "Operations": +/// it is read by `execution_engine.verify_and_notify_new_payload`'s own +/// `NewPayloadRequest.execution_requests`, in `verify_execution_payload_envelope` +/// (gloas `fork-choice.md`), built from the revealed envelope's own +/// `execution_requests`. Not from `state.latest_execution_payload_bid`: the +/// bid commits to that list only by its hash +/// ([`gloas::ExecutionPayloadBid::execution_requests_root`]), it never carries +/// the list itself. Envelope verification is not transcribed yet, so nothing +/// calls this today, the same place electra's own +/// [`crate::beacon::stf::electra::get_execution_requests_list`] already is +/// (see [`crate::beacon::stf::deneb::process_execution_payload`]'s own +/// documentation for why: [`crate::beacon::stf::ExecutionEngine`] collapses the whole +/// `verify_and_notify_new_payload` interface to one boolean and never +/// inspects the list either implementation builds). +pub fn get_execution_requests_list(requests: &gloas::ExecutionRequests) -> Vec> { + let mut list = Vec::new(); + + let mut push = |request_type: u8, is_empty: bool, encoded: Vec| { + if is_empty { + return; + } + let mut element = Vec::with_capacity(1 + encoded.len()); + element.push(request_type); + element.extend_from_slice(&encoded); + list.push(element); + }; + + push( + constants::DEPOSIT_REQUEST_TYPE, + requests.deposits.is_empty(), + requests.deposits.to_ssz(), + ); + push( + constants::WITHDRAWAL_REQUEST_TYPE, + requests.withdrawals.is_empty(), + requests.withdrawals.to_ssz(), + ); + push( + constants::CONSOLIDATION_REQUEST_TYPE, + requests.consolidations.is_empty(), + requests.consolidations.to_ssz(), + ); + push( + constants::BUILDER_DEPOSIT_REQUEST_TYPE, + requests.builder_deposits.is_empty(), + requests.builder_deposits.to_ssz(), + ); + push( + constants::BUILDER_EXIT_REQUEST_TYPE, + requests.builder_exits.is_empty(), + requests.builder_exits.to_ssz(), + ); + + list +} + // --------------------------------------------------------------------------- // Execution payload bid // --------------------------------------------------------------------------- @@ -873,6 +948,460 @@ pub fn process_builder_exit_request( initiate_builder_exit(inner, builder_index, config) } +// --------------------------------------------------------------------------- +// Operations +// --------------------------------------------------------------------------- + +/// `process_proposer_slashing` (modified, EIP-7732): the shared +/// `crate::beacon::stf::operations::verify_proposer_slashing` prologue, plus +/// clearing the [`gloas::BeaconState::builder_pending_payments`] entry tied +/// to the slashed proposal, if the slashing lands the slashed validator +/// itself as the payment's own proposer and the payment is still inside the +/// live two-epoch window. An unrelated same-slot equivocation (a different +/// validator's evidence about the same proposal) must not grief an honest +/// proposer's payment, which is exactly what comparing `payment.proposer_index` +/// against the slashed proposer's own index, rather than clearing +/// unconditionally, prevents. +pub fn process_proposer_slashing( + state: &mut BeaconState, + proposer_slashing: &ProposerSlashing, + config: &Config, +) -> Result<()> { + let (proposer_index, current_epoch) = + crate::beacon::stf::operations::verify_proposer_slashing(state, proposer_slashing)?; + + // [New in Gloas:EIP7732] Remove the `BuilderPendingPayment` corresponding + // to this proposal if it is still in the 2-epoch window. + let slot = proposer_slashing.signed_header_1.message.slot; + let proposal_epoch = compute_epoch_at_slot(slot); + if proposal_epoch == current_epoch { + let payment_index = preset::SLOTS_PER_EPOCH + .checked_add(slot % preset::SLOTS_PER_EPOCH) + .ok_or(Error::ArithmeticOverflow( + "process_proposer_slashing: SLOTS_PER_EPOCH + slot % SLOTS_PER_EPOCH", + ))? as usize; + clear_builder_pending_payment_if_owned_by(state, payment_index, proposer_index)?; + } else if proposal_epoch == get_previous_epoch(state) { + let payment_index = (slot % preset::SLOTS_PER_EPOCH) as usize; + clear_builder_pending_payment_if_owned_by(state, payment_index, proposer_index)?; + } + + slash_validator(state, proposer_index, None, config)?; + Ok(()) +} + +/// Clears `state.builder_pending_payments[payment_index]` if it is recorded +/// against `proposer_index`, [`process_proposer_slashing`]'s own helper for +/// its two (current-epoch, previous-epoch) payment-window branches. +fn clear_builder_pending_payment_if_owned_by( + state: &mut BeaconState, + payment_index: usize, + proposer_index: ValidatorIndex, +) -> Result<()> { + let inner = gloas_state(state, "process_proposer_slashing")?; + let len = inner.builder_pending_payments.len(); + let payment = inner + .builder_pending_payments + .get_mut(payment_index) + .ok_or(Error::IndexOutOfBounds { + index: payment_index, + len, + })?; + if payment.proposer_index == proposer_index { + *payment = gloas::BuilderPendingPayment::default(); + } + Ok(()) +} + +/// `process_attester_slashing` (unmodified since electra). Transcribed here, +/// not called through [`crate::beacon::stf::electra::process_attester_slashing`], +/// only because [`gloas::AttesterSlashing`] is its own Rust type: EIP-7688 +/// makes [`gloas::IndexedAttestation::attesting_indices`] the unbounded +/// [`gloas::AttestingIndices`] rather than electra's bounded one, the same +/// reason [`crate::beacon::helpers::gloas::get_attesting_indices`] cannot reuse +/// electra's copy either. See that function's own doc. The prologue below is +/// the one part that genuinely cannot be shared (it is checked against +/// gloas's own [`gloas::IndexedAttestation`]); once both attestations check +/// out, the rest is +/// `crate::beacon::stf::operations::slash_attesting_index_intersection`, the same +/// shared walk phase0's and electra's own versions delegate to. +pub fn process_attester_slashing( + state: &mut BeaconState, + attester_slashing: &gloas::AttesterSlashing, + config: &Config, +) -> Result<()> { + let attestation_1 = &attester_slashing.attestation_1; + let attestation_2 = &attester_slashing.attestation_2; + + verify( + is_slashable_attestation_data(&attestation_1.data, &attestation_2.data), + "process_attester_slashing: is_slashable_attestation_data(attestation_1.data, attestation_2.data)", + )?; + verify( + is_valid_indexed_attestation(state, attestation_1), + "process_attester_slashing: is_valid_indexed_attestation(state, attestation_1)", + )?; + verify( + is_valid_indexed_attestation(state, attestation_2), + "process_attester_slashing: is_valid_indexed_attestation(state, attestation_2)", + )?; + + crate::beacon::stf::operations::slash_attesting_index_intersection( + state, + &attestation_1.attesting_indices, + &attestation_2.attesting_indices, + config, + ) +} + +/// `process_attestation` (modified, EIP-7732): the new `parent_slot` +/// parameter (the parent block's slot) is threaded straight into +/// [`get_attestation_participation_flag_indices`], which is what lets an +/// attester's payload vote (`data.index`, now 0 or 1 rather than always 0) +/// be checked against the payload the attested block actually revealed. The +/// other addition is builder-payment weight accounting: each attester whose +/// participation for that epoch was still empty (`had_no_participation`, the +/// specification's own name for it) who newly satisfies a flag via a +/// same-slot attestation, while a nonzero builder payment for that slot is +/// still pending, adds their effective balance to that payment's `weight`, +/// which is what +/// `crate::beacon::stf::epoch::gloas::process_builder_pending_payments` later +/// checks against [`crate::beacon::helpers::gloas::get_builder_payment_quorum_threshold`] +/// to decide whether the builder gets paid in full or not at all. +/// +/// Structured as the same read-then-write split +/// [`crate::beacon::stf::altair::process_attestation`] uses, and for the identical +/// borrow-checker reason: the read phase borrows `state` immutably (through +/// [`BeaconState::altair_validator_lists`], which covers gloas), and the +/// write phase borrows it mutably (through +/// [`BeaconState::epoch_participation_mut`], gloas's own widened element-only +/// view of the same list; see that accessor's own doc for why it, and not +/// [`BeaconState::altair_validator_lists_mut`], is the one gloas can use). +pub fn process_attestation( + state: &mut BeaconState, + attestation: &gloas::Attestation, + parent_slot: Slot, + committees: &CommitteeCache, +) -> Result<()> { + let data = attestation.data; + let current_epoch = get_current_epoch(state); + let previous_epoch = get_previous_epoch(state); + + verify( + data.target.epoch == previous_epoch || data.target.epoch == current_epoch, + "process_attestation: data.target.epoch in (get_previous_epoch(state), get_current_epoch(state))", + )?; + verify( + data.target.epoch == compute_epoch_at_slot(data.slot), + "process_attestation: data.target.epoch == compute_epoch_at_slot(data.slot)", + )?; + let min_slot = data + .slot + .checked_add(preset::MIN_ATTESTATION_INCLUSION_DELAY) + .ok_or(Error::ArithmeticOverflow( + "process_attestation: data.slot + MIN_ATTESTATION_INCLUSION_DELAY", + ))?; + verify( + min_slot <= state.slot(), + "process_attestation: data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot", + )?; + + // [Modified in Gloas:EIP7732] `data.index` is now a payload-availability + // bit (0 or 1), not always zero. + verify(data.index < 2, "process_attestation: data.index < 2")?; + let committee_indices = get_committee_indices(&attestation.committee_bits); + let epoch_committees = committees.committees(state, data.target.epoch); + let mut committee_offset = 0usize; + for committee_index in committee_indices { + verify( + committee_index < epoch_committees.committees_per_slot(), + "process_attestation: committee_index < get_committee_count_per_slot(state, data.target.epoch)", + )?; + let committee = epoch_committees.committee(data.slot, committee_index)?; + let committee_has_an_attester = (0..committee.len()).any(|position| { + attestation + .aggregation_bits + .get(committee_offset + position) + .unwrap_or(false) + }); + verify( + committee_has_an_attester, + "process_attestation: len(committee_attesters) > 0", + )?; + committee_offset += committee.len(); + } + verify( + attestation.aggregation_bits.len() == committee_offset, + "process_attestation: len(attestation.aggregation_bits) == committee_offset", + )?; + + // Safe: `min_slot <= state.slot()` above and `min_slot >= data.slot` (the + // inclusion delay is non-negative), so `data.slot <= state.slot()`. + let inclusion_delay = state.slot() - data.slot; + let participation_flag_indices = + get_attestation_participation_flag_indices(state, &data, inclusion_delay, parent_slot)?; + + let indexed_attestation = get_indexed_attestation(state, attestation, committees)?; + verify( + is_valid_indexed_attestation(state, &indexed_attestation), + "process_attestation: is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation))", + )?; + + let current_epoch_target = data.target.epoch == current_epoch; + let payment_index = if current_epoch_target { + preset::SLOTS_PER_EPOCH + .checked_add(data.slot % preset::SLOTS_PER_EPOCH) + .ok_or(Error::ArithmeticOverflow( + "process_attestation: SLOTS_PER_EPOCH + data.slot % SLOTS_PER_EPOCH", + ))? as usize + } else { + (data.slot % preset::SLOTS_PER_EPOCH) as usize + }; + let mut payment = { + let inner = gloas_state_ref(state, "process_attestation")?; + let len = inner.builder_pending_payments.len(); + inner + .builder_pending_payments + .get(payment_index) + .cloned() + .ok_or(Error::IndexOutOfBounds { + index: payment_index, + len, + })? + }; + + let attesting_indices: Vec = indexed_attestation.attesting_indices.to_vec(); + // The specification calls `is_attestation_same_slot(state, data)` inside + // the per-attester short-circuit below, so hoisting it out to a single + // call here changes nothing: `get_attestation_participation_flag_indices` + // above already called it once against this same, still-unmutated + // `state`, so every attester's own check would read the identical + // result regardless of where it is evaluated. + let is_same_slot = is_attestation_same_slot(state, &data)?; + // Hoisted for the same reason `crate::beacon::stf::electra::process_attestation` + // hoists it: `get_base_reward(state, index)` is `increments * + // get_base_reward_per_increment(state)`, and the second factor is + // constant across this whole read phase (nothing here mutates `state` + // yet), so computing it once outside the loop below avoids one + // `get_total_active_balance` scan of the registry per attester per flag. + let base_reward_per_increment = get_base_reward_per_increment(state)?; + + // Read phase: for every attester, decide which flags this attestation + // newly satisfies, add up the proposer's reward for granting them, and + // credit the pending builder payment's weight where the specification + // says to. Nothing here mutates `state`. + let mut proposer_reward_numerator: Gwei = 0; + let mut flag_updates: Vec<(ValidatorIndex, ParticipationFlags)> = Vec::new(); + { + let (previous_participation, current_participation, _) = state.altair_validator_lists()?; + let epoch_participation = if current_epoch_target { + current_participation + } else { + previous_participation + }; + for index in attesting_indices { + let current_flags = epoch_participation.get(index as usize).copied().ok_or( + Error::IndexOutOfBounds { + index: index as usize, + len: epoch_participation.len(), + }, + )?; + let had_no_participation = current_flags == 0; + + let mut new_flags: ParticipationFlags = 0; + for &flag_index in &participation_flag_indices { + if has_flag(current_flags, flag_index) { + continue; + } + new_flags = add_flag(new_flags, flag_index); + let weight = constants::PARTICIPATION_FLAG_WEIGHTS[flag_index]; + let increments = + state.validator(index)?.effective_balance / preset::EFFECTIVE_BALANCE_INCREMENT; + let base_reward = increments.checked_mul(base_reward_per_increment).ok_or( + Error::ArithmeticOverflow("process_attestation: get_base_reward(state, index)"), + )?; + let reward = base_reward + .checked_mul(weight) + .ok_or(Error::ArithmeticOverflow( + "process_attestation: get_base_reward(state, index) * weight", + ))?; + proposer_reward_numerator = proposer_reward_numerator.checked_add(reward).ok_or( + Error::ArithmeticOverflow("process_attestation: proposer_reward_numerator"), + )?; + } + let will_set_new_flag = new_flags != 0; + + if will_set_new_flag + && had_no_participation + && is_same_slot + && payment.withdrawal.amount > 0 + { + let effective_balance = state.validator(index)?.effective_balance; + payment.weight = payment.weight.checked_add(effective_balance).ok_or( + Error::ArithmeticOverflow( + "process_attestation: payment.weight + validator.effective_balance", + ), + )?; + } + if will_set_new_flag { + flag_updates.push((index, new_flags)); + } + } + } + + // Write phase: apply exactly the flags the read phase decided on. + { + let participation_mut = state.epoch_participation_mut(current_epoch_target)?; + let participation_len = participation_mut.len(); + for (index, new_flags) in flag_updates { + let flags = + participation_mut + .get_mut(index as usize) + .ok_or(Error::IndexOutOfBounds { + index: index as usize, + len: participation_len, + })?; + *flags |= new_flags; + } + } + + const NON_PROPOSER_WEIGHT: u64 = constants::WEIGHT_DENOMINATOR - constants::PROPOSER_WEIGHT; + const PROPOSER_REWARD_DENOMINATOR: u64 = + NON_PROPOSER_WEIGHT * constants::WEIGHT_DENOMINATOR / constants::PROPOSER_WEIGHT; + let proposer_reward = proposer_reward_numerator / PROPOSER_REWARD_DENOMINATOR; + let proposer_index = get_beacon_proposer_index(state)?; + increase_balance(state, proposer_index, proposer_reward)?; + + // Update builder payment weight. + { + let inner = gloas_state(state, "process_attestation")?; + let len = inner.builder_pending_payments.len(); + let entry = inner + .builder_pending_payments + .get_mut(payment_index) + .ok_or(Error::IndexOutOfBounds { + index: payment_index, + len, + })?; + *entry = payment; + } + + Ok(()) +} + +/// `process_payload_attestation` (gloas `beacon-chain.md`): checks a payload +/// timeliness committee vote against the *parent* block (the previous slot's +/// proposal, whose payload it is attesting to) rather than against the head +/// [`process_attestation`] checks attestations against. +pub fn process_payload_attestation( + state: &mut BeaconState, + payload_attestation: &gloas::PayloadAttestation, + config: &Config, +) -> Result<()> { + let data = payload_attestation.data; + + // Check that the attestation is for the parent beacon block. + verify( + data.beacon_block_root == state.latest_block_header().parent_root, + "process_payload_attestation: data.beacon_block_root == state.latest_block_header.parent_root", + )?; + // Check that the attestation is for the previous slot. + let expected_slot = data.slot.checked_add(1).ok_or(Error::ArithmeticOverflow( + "process_payload_attestation: data.slot + 1", + ))?; + verify( + expected_slot == state.slot(), + "process_payload_attestation: data.slot + 1 == state.slot", + )?; + + // Verify signature. + let indexed_payload_attestation = + get_indexed_payload_attestation(state, payload_attestation, config)?; + verify( + is_valid_indexed_payload_attestation(state, &indexed_payload_attestation), + "process_payload_attestation: is_valid_indexed_payload_attestation(state, indexed_payload_attestation)", + )?; + Ok(()) +} + +/// `process_operations` (modified, EIP-7732 and EIP-7688): removes the calls +/// to `process_deposit_request`, `process_withdrawal_request`, and +/// `process_consolidation_request` (see this module's own doc for where the +/// five execution-layer-triggered requests go instead), adds the new +/// `parent_slot` argument [`process_attestation`] needs and the payload +/// timeliness committee's own operation list, and, since every list here is +/// now an EIP-7688 progressive one with no SSZ-enforced bound of its own, +/// checks each list's length against its preset maximum explicitly (bounds +/// every earlier fork's container shape enforced by construction). +/// +/// `assert len(body.deposits) == 0` unconditionally, with no `process_deposit` +/// call at all: like fulu, gloas has no eth1-bridge deposit path left to +/// drain (`crate::beacon::stf::fulu::process_operations`'s own doc explains why). +pub fn process_operations( + state: &mut BeaconState, + body: &gloas::BeaconBlockBody, + parent_slot: Slot, + config: &Config, + committees: &CommitteeCache, +) -> Result<()> { + verify( + body.deposits.is_empty(), + "process_operations: len(body.deposits) == 0", + )?; + + // [New in Gloas:EIP7688] + verify( + body.proposer_slashings.len() <= preset::MAX_PROPOSER_SLASHINGS, + "process_operations: len(body.proposer_slashings) <= MAX_PROPOSER_SLASHINGS", + )?; + verify( + body.attester_slashings.len() <= preset::MAX_ATTESTER_SLASHINGS_ELECTRA, + "process_operations: len(body.attester_slashings) <= MAX_ATTESTER_SLASHINGS_ELECTRA", + )?; + verify( + body.attestations.len() <= preset::MAX_ATTESTATIONS_ELECTRA, + "process_operations: len(body.attestations) <= MAX_ATTESTATIONS_ELECTRA", + )?; + verify( + body.voluntary_exits.len() <= preset::MAX_VOLUNTARY_EXITS, + "process_operations: len(body.voluntary_exits) <= MAX_VOLUNTARY_EXITS", + )?; + verify( + body.bls_to_execution_changes.len() <= preset::MAX_BLS_TO_EXECUTION_CHANGES, + "process_operations: len(body.bls_to_execution_changes) <= MAX_BLS_TO_EXECUTION_CHANGES", + )?; + verify( + body.payload_attestations.len() as u64 <= preset::MAX_PAYLOAD_ATTESTATIONS, + "process_operations: len(body.payload_attestations) <= MAX_PAYLOAD_ATTESTATIONS", + )?; + + // [Modified in Gloas:EIP7732] + for proposer_slashing in body.proposer_slashings.iter() { + process_proposer_slashing(state, proposer_slashing, config)?; + } + for attester_slashing in body.attester_slashings.iter() { + process_attester_slashing(state, attester_slashing, config)?; + } + // [Modified in Gloas:EIP7732] + for attestation in body.attestations.iter() { + process_attestation(state, attestation, parent_slot, committees)?; + } + for voluntary_exit in body.voluntary_exits.iter() { + crate::beacon::stf::electra::process_voluntary_exit(state, voluntary_exit, config)?; + } + for signed_change in body.bls_to_execution_changes.iter() { + crate::beacon::stf::capella::process_bls_to_execution_change(state, signed_change, config)?; + } + // [Modified in Gloas:EIP7732] Removed `process_deposit_request`, + // `process_withdrawal_request`, and `process_consolidation_request`. + // [New in Gloas:EIP7732] + for payload_attestation in body.payload_attestations.iter() { + process_payload_attestation(state, payload_attestation, config)?; + } + + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -964,4 +1493,193 @@ mod tests { let inner = gloas_state_ref(&state, "test").unwrap(); assert_eq!(inner.builders.first().unwrap().balance, 0); } + + #[test] + fn get_execution_requests_list_orders_every_kind_by_its_own_prefix() { + use crate::beacon::containers::electra; + use crate::beacon::primitives::Bytes32; + + let requests = gloas::ExecutionRequests { + deposits: gloas::DepositRequests::from(vec![electra::DepositRequest { + pubkey: Default::default(), + withdrawal_credentials: Bytes32::ZERO, + amount: 0, + signature: Default::default(), + index: 0, + }]), + withdrawals: gloas::WithdrawalRequests::from(vec![electra::WithdrawalRequest { + source_address: ExecutionAddress::ZERO, + validator_pubkey: Default::default(), + amount: 0, + }]), + consolidations: gloas::ConsolidationRequests::from(vec![ + electra::ConsolidationRequest { + source_address: ExecutionAddress::ZERO, + source_pubkey: Default::default(), + target_pubkey: Default::default(), + }, + ]), + builder_deposits: gloas::BuilderDepositRequests::from(vec![ + gloas::BuilderDepositRequest::default(), + ]), + builder_exits: gloas::BuilderExitRequests::from(vec![ + gloas::BuilderExitRequest::default(), + ]), + }; + + let list = get_execution_requests_list(&requests); + + let prefixes: Vec = list.iter().map(|element| element[0]).collect(); + assert_eq!( + prefixes, + vec![ + constants::DEPOSIT_REQUEST_TYPE, + constants::WITHDRAWAL_REQUEST_TYPE, + constants::CONSOLIDATION_REQUEST_TYPE, + constants::BUILDER_DEPOSIT_REQUEST_TYPE, + constants::BUILDER_EXIT_REQUEST_TYPE, + ], + "every non-empty list appears, in its own fixed prefix order: \ + DEPOSIT_REQUEST_TYPE, WITHDRAWAL_REQUEST_TYPE, CONSOLIDATION_REQUEST_TYPE, \ + BUILDER_DEPOSIT_REQUEST_TYPE, BUILDER_EXIT_REQUEST_TYPE" + ); + assert_eq!( + &list[0][1..], + requests.deposits.to_ssz(), + "the element after the prefix byte is the request list's own SSZ encoding" + ); + } + + #[test] + fn get_execution_requests_list_skips_empty_lists() { + let requests = gloas::ExecutionRequests { + builder_exits: gloas::BuilderExitRequests::from(vec![ + gloas::BuilderExitRequest::default(), + ]), + ..Default::default() + }; + + // Every list but `builder_exits` is empty, so only its own element + // appears; an all-empty `ExecutionRequests` produces an empty list. + assert_eq!( + get_execution_requests_list(&requests) + .iter() + .map(|element| element[0]) + .collect::>(), + vec![constants::BUILDER_EXIT_REQUEST_TYPE] + ); + assert!(get_execution_requests_list(&gloas::ExecutionRequests::default()).is_empty()); + } + + #[test] + fn process_operations_rejects_each_length_checked_list_past_its_max() { + use crate::beacon::containers::shared; + + // Sanity does not run for gloas yet, so nothing else pins + // these EIP-7688 length checks: every list here lost the SSZ bound a + // bounded `SszList` used to enforce, so `process_operations` itself + // is now the only thing standing between an oversized list and + // `for_ops` processing it anyway. + let state = gloas_state_with_validators(4); + let config = Config::mainnet(); + + let empty_indexed_attestation = || gloas::IndexedAttestation { + attesting_indices: Default::default(), + data: Default::default(), + signature: Default::default(), + }; + + let cases: Vec<(&str, gloas::BeaconBlockBody)> = vec![ + ( + "process_operations: len(body.proposer_slashings) <= MAX_PROPOSER_SLASHINGS", + gloas::BeaconBlockBody { + proposer_slashings: gloas::ProposerSlashings::from(vec![ + shared::ProposerSlashing::default(); + preset::MAX_PROPOSER_SLASHINGS + 1 + ]), + ..gloas::BeaconBlockBody::empty() + }, + ), + ( + "process_operations: len(body.attester_slashings) <= MAX_ATTESTER_SLASHINGS_ELECTRA", + gloas::BeaconBlockBody { + attester_slashings: gloas::AttesterSlashings::from(vec![ + gloas::AttesterSlashing { + attestation_1: empty_indexed_attestation(), + attestation_2: empty_indexed_attestation(), + }; + preset::MAX_ATTESTER_SLASHINGS_ELECTRA + 1 + ]), + ..gloas::BeaconBlockBody::empty() + }, + ), + ( + "process_operations: len(body.attestations) <= MAX_ATTESTATIONS_ELECTRA", + gloas::BeaconBlockBody { + attestations: gloas::Attestations::from(vec![ + gloas::Attestation { + aggregation_bits: Default::default(), + data: Default::default(), + signature: Default::default(), + committee_bits: Default::default(), + }; + preset::MAX_ATTESTATIONS_ELECTRA + + 1 + ]), + ..gloas::BeaconBlockBody::empty() + }, + ), + ( + "process_operations: len(body.voluntary_exits) <= MAX_VOLUNTARY_EXITS", + gloas::BeaconBlockBody { + voluntary_exits: gloas::VoluntaryExits::from(vec![ + shared::SignedVoluntaryExit::default(); + preset::MAX_VOLUNTARY_EXITS + 1 + ]), + ..gloas::BeaconBlockBody::empty() + }, + ), + ( + "process_operations: len(body.bls_to_execution_changes) <= MAX_BLS_TO_EXECUTION_CHANGES", + gloas::BeaconBlockBody { + bls_to_execution_changes: gloas::BlsToExecutionChanges::from(vec![ + capella::SignedBLSToExecutionChange { + message: capella::BLSToExecutionChange { + validator_index: 0, + from_bls_pubkey: Default::default(), + to_execution_address: Default::default(), + }, + signature: Default::default(), + }; + preset::MAX_BLS_TO_EXECUTION_CHANGES + 1 + ]), + ..gloas::BeaconBlockBody::empty() + }, + ), + ( + "process_operations: len(body.payload_attestations) <= MAX_PAYLOAD_ATTESTATIONS", + gloas::BeaconBlockBody { + payload_attestations: gloas::PayloadAttestations::from(vec![ + gloas::PayloadAttestation { + aggregation_bits: Default::default(), + data: Default::default(), + signature: Default::default(), + }; + (preset::MAX_PAYLOAD_ATTESTATIONS + 1) as usize + ]), + ..gloas::BeaconBlockBody::empty() + }, + ), + ]; + + for (expected_message, body) in cases { + let mut state = state.clone(); + let committees = CommitteeCache::default(); + let result = process_operations(&mut state, &body, 0, &config, &committees); + assert!( + matches!(result, Err(Error::SpecAssert(message)) if message == expected_message), + "expected {expected_message:?} to fail, got {result:?}" + ); + } + } } diff --git a/crates/blockchain/state_transition/src/beacon/stf/operations.rs b/crates/blockchain/state_transition/src/beacon/stf/operations.rs index 43b0a2394..10f2d8689 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/operations.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/operations.rs @@ -38,7 +38,7 @@ use crate::beacon::helpers::predicates::{ }; use crate::beacon::lean_state_unreachable; use crate::beacon::preset; -use crate::beacon::primitives::{Gwei, HashTreeRoot as _, ValidatorIndex}; +use crate::beacon::primitives::{Epoch, Gwei, HashTreeRoot as _, ValidatorIndex}; /// Runs every operation in a block, in the specification's order. /// @@ -152,17 +152,26 @@ pub fn process_operations( // Proposer slashings // --------------------------------------------------------------------------- -/// Slashes a proposer caught signing two different headers for the same slot. +/// Checks a proposer slashing's evidence without slashing anyone: the two +/// headers agree on slot and proposer, differ from each other, name a +/// still-slashable proposer, and both signatures check out. +/// +/// Shared between phase0's own [`process_proposer_slashing`] below and +/// `crate::beacon::stf::gloas::process_proposer_slashing`: gloas's own +/// specification modifies the function (EIP-7732: it also clears a +/// `BuilderPendingPayment`), but not this prologue, which the two share line +/// for line. Returns the slashed proposer's index and the state's current +/// epoch, both of which gloas's own caller also needs for its own +/// payment-window check, so neither has to be recomputed. /// /// The two headers must actually differ: a proposer can be asked to co-sign /// the same header twice (by different requesters, or the same one twice), /// and that is not evidence of anything. Only a genuine equivocation, two /// distinct headers for the one slot, is slashable. -pub fn process_proposer_slashing( - state: &mut BeaconState, +pub(crate) fn verify_proposer_slashing( + state: &BeaconState, proposer_slashing: &ProposerSlashing, - config: &Config, -) -> Result<()> { +) -> Result<(ValidatorIndex, Epoch)> { let header_1 = &proposer_slashing.signed_header_1.message; let header_2 = &proposer_slashing.signed_header_2.message; @@ -201,6 +210,18 @@ pub fn process_proposer_slashing( )?; } + Ok((proposer_index, current_epoch)) +} + +/// Slashes a proposer caught signing two different headers for the same +/// slot. See [`verify_proposer_slashing`] for the shared prologue this +/// delegates to. +pub fn process_proposer_slashing( + state: &mut BeaconState, + proposer_slashing: &ProposerSlashing, + config: &Config, +) -> Result<()> { + let (proposer_index, _current_epoch) = verify_proposer_slashing(state, proposer_slashing)?; slash_validator(state, proposer_index, None, config)?; Ok(()) } @@ -209,17 +230,61 @@ pub fn process_proposer_slashing( // Attester slashings // --------------------------------------------------------------------------- -/// Slashes every slashable validator in the overlap of two conflicting -/// attestations' attesting sets. +/// Slashes every slashable validator in the overlap of two attesting index +/// sets, given each attestation's own already-verified indices. +/// +/// The shared tail of `process_attester_slashing`, unchanged across every +/// fork that has its own copy (phase0's below, electra's and gloas's own): +/// what differs between them is only the prologue above this, validating an +/// indexed attestation against that fork's own container type +/// (`is_valid_indexed_attestation`). Once both are valid, this walk needs +/// nothing fork-specific, only the two already-sorted, already-deduplicated +/// index slices. /// /// The two indexed attestations must each be independently valid (sorted, /// unique, unslashed-signature-correct) before their overlap means anything: -/// evidence built from a forged or malformed attestation proves nothing. -/// It is not enough for the overlap to be non-empty either. If every -/// validator in it has already been slashed (and so is past +/// evidence built from a forged or malformed attestation proves nothing. It +/// is not enough for the overlap to be non-empty either. If every validator +/// in it has already been slashed (and so is past /// [`is_slashable_validator`]'s reach) or has already withdrawn, the /// operation has no effect and including it would let a block waste space /// (or, worse, let a proposer replay old evidence) for free. +/// +/// `attesting_indices_1` walked in order while filtering by membership in +/// `attesting_indices_2`'s set yields the intersection already sorted, +/// matching the specification's `sorted(indices)` without a separate sort: +/// `is_valid_indexed_attestation` already required both to be sorted and +/// unique before either reaches here. +pub(crate) fn slash_attesting_index_intersection( + state: &mut BeaconState, + attesting_indices_1: &[ValidatorIndex], + attesting_indices_2: &[ValidatorIndex], + config: &Config, +) -> Result<()> { + let current_epoch = get_current_epoch(state); + let indices_2: HashSet = attesting_indices_2.iter().copied().collect(); + + let mut slashed_any = false; + for &index in attesting_indices_1 { + if !indices_2.contains(&index) { + continue; + } + if is_slashable_validator(state.validator(index)?, current_epoch) { + slash_validator(state, index, None, config)?; + slashed_any = true; + } + } + verify( + slashed_any, + "at least one validator in the intersection of the two attesting index sets was slashed", + )?; + Ok(()) +} + +/// Slashes every slashable validator in the overlap of two conflicting +/// attestations' attesting sets. See [`slash_attesting_index_intersection`] +/// for the shared walk this delegates to once both indexed attestations +/// check out against phase0's own container type. pub fn process_attester_slashing( state: &mut BeaconState, attester_slashing: &phase0::AttesterSlashing, @@ -241,30 +306,12 @@ pub fn process_attester_slashing( "is_valid_indexed_attestation(state, attestation_2)", )?; - let current_epoch = get_current_epoch(state); - // `is_valid_indexed_attestation` already required both index lists to be - // sorted and unique, so walking `attestation_1`'s list in order while - // filtering by membership in `attestation_2`'s set yields the - // intersection already sorted, matching the specification's - // `sorted(indices)` without a separate sort. - let indices_2: HashSet = - attestation_2.attesting_indices.iter().copied().collect(); - - let mut slashed_any = false; - for &index in attestation_1.attesting_indices.iter() { - if !indices_2.contains(&index) { - continue; - } - if is_slashable_validator(state.validator(index)?, current_epoch) { - slash_validator(state, index, None, config)?; - slashed_any = true; - } - } - verify( - slashed_any, - "at least one validator in the intersection of the two attesting index sets was slashed", - )?; - Ok(()) + slash_attesting_index_intersection( + state, + &attestation_1.attesting_indices, + &attestation_2.attesting_indices, + config, + ) } // --------------------------------------------------------------------------- diff --git a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs index aae025f99..45188d54f 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs @@ -30,7 +30,7 @@ use ethlambda_storage::ForkCheckpoints; use libssz::SszDecode; use libtest_mimic::{Failed, Trial}; -use super::{Case, PRESET, collect, collect_all_handlers}; +use super::{Case, PRESET, collect, collect_all_handlers, fork_active_from_genesis}; /// The handlers this runner covers, and runs. const HANDLERS: &[&str] = &[ @@ -177,25 +177,12 @@ fn decode_signed_aggregate(case: &Case, name: &str) -> Result Config { - let mut config: Config = case.yaml_opt("config").unwrap_or_else(|| { - ForkName::ALL - .into_iter() - .take_while(|fork| *fork <= case.fork) - .filter(|fork| *fork != ForkName::Phase0) - .fold(Config::active(), |config, fork| { - config.with_fork_epoch(fork, 0) - }) - }); + let mut config: Config = case + .yaml_opt("config") + .unwrap_or_else(|| fork_active_from_genesis(case)); config.genesis_time = state.genesis_time(); // Newer configs name only `SLOT_DURATION_MS`; without `SECONDS_PER_SLOT` // the field would keep mainnet's default. diff --git a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs index 6538bdc75..79ac88e9f 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs @@ -60,6 +60,7 @@ use std::fs; use std::path::{Path, PathBuf}; use ethlambda_state_transition::beacon::ForkName; +use ethlambda_state_transition::beacon::config::Config; use ethlambda_state_transition::beacon::containers::BeaconState; use libssz::SszDecode; use libtest_mimic::{Failed, Trial}; @@ -72,6 +73,29 @@ pub const PRESET: &str = if cfg!(feature = "preset-minimal") { "mainnet" }; +/// The compiled-in preset with every fork up to and including `case.fork` +/// pulled back to genesis, never a later one. +/// +/// The fallback the gossip and operations runners use when a case carries no +/// `config.yaml` of its own to read a fork schedule from (see [`gossip`]'s +/// own `case_config`). Those vectors are generated as if their own fork, and +/// every one before it, had activated at genesis: the ones that ship a +/// `config.yaml` set exactly that, apart from deliberate pre-fork cases such +/// as `payload_attestation`'s `process_payload_attestation_pre_fork_epoch`, +/// which schedules the fork later. Never a later fork: pulling forward one this +/// case does not itself reach would let `Config::fork_at_epoch` resolve a +/// state the case never claims to be, on any config a later runner builds +/// this way. +pub fn fork_active_from_genesis(case: &Case) -> Config { + ForkName::ALL + .into_iter() + .take_while(|fork| *fork <= case.fork) + .filter(|fork| *fork != ForkName::Phase0) + .fold(Config::active(), |config, fork| { + config.with_fork_epoch(fork, 0) + }) +} + /// The newest fork whose state transition this crate implements. /// /// Every runner gates on this, and a case past it becomes an ignored test @@ -111,14 +135,10 @@ pub const GLOAS_RUNNERS: &[&str] = &[ "fork", "epoch_processing", "rewards", - "operations/execution_payload_bid", - "operations/withdrawals", - "operations/parent_execution_payload", - "operations/deposit_request", - "operations/withdrawal_request", - "operations/consolidation_request", - "operations/builder_deposit_request", - "operations/builder_exit_request", + // Every handler `operations` ships for gloas now runs, so the bare + // runner name serves all of it, the same way `"ssz_static"` already + // does for its own runner. + "operations", ]; /// The root of the extracted fixture tree. diff --git a/crates/blockchain/state_transition/tests/beacon_spec/operations.rs b/crates/blockchain/state_transition/tests/beacon_spec/operations.rs index 6cd078395..9162c2340 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/operations.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/operations.rs @@ -25,9 +25,11 @@ //! given fork keeps calling whichever earlier fork's function last introduced //! or changed it. //! -//! - `proposer_slashing` needs no routing at all: no fork's specification ever -//! lists a modified `process_proposer_slashing`, so phase0's function serves -//! every fork this crate implements. +//! - `proposer_slashing` needs no routing through fulu: no fork's specification +//! before gloas ever lists a modified `process_proposer_slashing`, so +//! phase0's function serves every earlier fork. Gloas's own specification +//! does modify it (EIP-7732: clearing the `BuilderPendingPayment` tied to a +//! slashed proposal), so it gets its own routed arm and its own function. //! - `attester_slashing` and `deposit` need no *function* routing through //! deneb (neither fork's specification lists either as modified before //! electra), but `attester_slashing`'s container does change shape at @@ -35,7 +37,9 @@ //! `process_attester_slashing` (transcribed against the new container, not //! behaviorally different) and `process_deposit` (behaviorally different: //! a deposit is queued rather than credited) as its own functions from -//! there on. +//! there on. `attester_slashing`'s container changes shape again at gloas +//! (EIP-7688's unbounded `AttestingIndices`), so gloas gets its own +//! transcription too, not behaviorally different either. //! - `attestation` needs both. The container //! ([`phase0::Attestation`]) is the one phase0, altair, bellatrix, capella, //! and deneb all share, and [`electra::Attestation`] (EIP-7549's committee @@ -50,21 +54,33 @@ //! on a later case would not even fail loudly in every instance, since it //! would return [`ethlambda_state_transition::beacon::Error::UnsupportedForFork`] only where //! the two forks' state shapes actually differ, which is still a rejection, -//! just not the one the fixture is testing for. +//! just not the one the fixture is testing for. Gloas reshapes the +//! container a third time (EIP-7688) and modifies the function again +//! (EIP-7732's `parent_slot` argument, read from the case's own `meta.yaml` +//! since it is not part of the operation file itself, and the +//! builder-payment weight accounting), so it gets its own arm and its own +//! function too, called [`gloas_stf::process_attestation`] rather than +//! [`electra_stf::process_attestation`]. //! - `voluntary_exit` changes twice after phase0: deneb pins the signature to //! a fixed fork version for EIP-7044, and electra adds a //! pending-partial-withdrawal check and swaps in electra's own -//! exit-queue accounting for EIP-7251. +//! exit-queue accounting for EIP-7251. Neither fulu's nor gloas's own +//! specification lists a further change, so electra's function serves both; +//! `voluntary_exit_churn` is a second fixture handler gloas ships with the +//! same operation file and the same processing call, exercising exit-queue +//! churn scenarios specifically, so it routes identically. //! - `block_header`'s file is a whole `BeaconBlock`, and that type is //! different per fork (altair's carries a `sync_aggregate` its body root -//! folds in, bellatrix's an execution payload on top of that, and so on). +//! folds in, bellatrix's an execution payload on top of that, and so on; +//! gloas's drops the payload entirely in favor of a builder's bid). //! [`block::process_block_header`] itself takes only the four //! fork-invariant fields a header needs, not a block, precisely so this //! runner (and every per-fork `process_block_*` driver) can decode with the //! fork's own concrete type and still call one shared function. //! - `sync_aggregate` is new in altair, with no phase0 case, and no later -//! fork's specification ever lists a modified version of it, so it needs -//! no fork match beyond that. +//! fork's specification, gloas included, ever lists a modified version of +//! it, so it needs no fork match at all: `sync_aggregate` is byte-for-byte +//! the same field on every state and body from altair on. //! - `execution_payload` is new in bellatrix and its specification lists a //! modified version at every fork from capella on: capella and later drop //! the still-mid-merge-transition check, deneb folds in the blob @@ -73,12 +89,14 @@ //! Five forks, five distinct functions, none reusable for another fork's //! case. //! - `bls_to_execution_change` and `withdrawals` are both new in capella. -//! `bls_to_execution_change` is never listed as modified again: its -//! function reads and writes only fork-invariant fields, so capella's own -//! serves every later fork too. `withdrawals` is listed as modified once -//! more, at electra (EIP-7251's partial-withdrawal queue); deneb's own -//! specification lists no change to it at all, so deneb's copy exists only -//! to hold a differently-typed `payload` parameter, not different logic. +//! `bls_to_execution_change` is never listed as modified again, gloas +//! included: its function reads and writes only fork-invariant fields +//! (the validator registry, genesis validators root), so capella's own +//! serves every later fork too, with no fork match needed. `withdrawals` is +//! listed as modified once more, at electra (EIP-7251's partial-withdrawal +//! queue); deneb's own specification lists no change to it at all, so +//! deneb's copy exists only to hold a differently-typed `payload` parameter, +//! not different logic. //! - `consolidation_request`, `deposit_request`, and `withdrawal_request` are //! all new in electra, alongside `withdrawals`, `process_attestation`, //! `process_deposit`, and `process_voluntary_exit`. Fulu's specification @@ -101,15 +119,20 @@ //! deterministic from the state alone), so unlike every earlier fork's arm //! this one reads no `` file and calls `gloas_stf::process_withdrawals` //! with only the state. -use std::sync::Arc; - +//! - `payload_attestation` is new in gloas too, the payload timeliness +//! committee's vote on whether the *previous* slot's payload showed up on +//! time; no earlier fork has anything to route away from. `execution_payload` +//! must not appear for gloas at all: a block no longer carries an embedded +//! payload for this handler's own `process_execution_payload` family to +//! check, so its Gloas arm returns an error naming the handler rather than +//! silently falling through to `unhandled operation`. use ethlambda_state_transition::beacon::ForkName; use ethlambda_state_transition::beacon::config::Config; use ethlambda_state_transition::beacon::containers::{ BeaconState, altair, bellatrix, capella, deneb, electra, gloas, phase0, shared, }; use ethlambda_state_transition::beacon::helpers::accessors::CommitteeCache; -use ethlambda_state_transition::beacon::primitives::HashTreeRoot as _; +use ethlambda_state_transition::beacon::primitives::{HashTreeRoot as _, Slot}; use ethlambda_state_transition::beacon::stf::altair as altair_stf; use ethlambda_state_transition::beacon::stf::bellatrix as bellatrix_stf; use ethlambda_state_transition::beacon::stf::capella as capella_stf; @@ -120,7 +143,9 @@ use ethlambda_state_transition::beacon::stf::gloas as gloas_stf; use ethlambda_state_transition::beacon::stf::{ExecutionEngine, block, operations}; use libtest_mimic::{Failed, Trial}; -use super::{Case, PRESET, collect_all_handlers, lean_is_not_a_fixture_fork}; +use super::{ + Case, PRESET, collect_all_handlers, fork_active_from_genesis, lean_is_not_a_fixture_fork, +}; /// The `{execution_valid: bool}` an `execution_payload` case ships alongside /// its block, standing in for whatever a real execution client would have @@ -131,6 +156,15 @@ struct ExecutionYaml { execution_valid: bool, } +/// The `{parent_slot: int}` a gloas `attestation` case's own `meta.yaml` +/// carries: the parent block's slot, an extra input +/// [`gloas_stf::process_attestation`] needs that no earlier fork's version +/// does (EIP-7732's payload-availability bit). +#[derive(serde::Deserialize)] +struct AttestationMeta { + parent_slot: Slot, +} + /// Applies one operation to the case's pre-state. /// /// Returns the post-state on success. Anything the crate rejects comes back as an @@ -217,12 +251,20 @@ fn apply( block.body.hash_tree_root(), ) } - ForkName::Gloas => Err( - ethlambda_state_transition::beacon::Error::UnsupportedForFork { - function: "block_header", - fork: ForkName::Gloas, - }, - ), + // Gloas drops the embedded execution payload (EIP-7732) but keeps + // `process_block_header` unmodified: a `BeaconBlockHeader` never + // carried a payload to begin with, so the four fields this shared + // function reads are unaffected. + ForkName::Gloas => { + let block: gloas::BeaconBlock = case.ssz("block"); + block::process_block_header( + state, + block.slot, + block.proposer_index, + block.parent_root, + block.body.hash_tree_root(), + ) + } // No `other` arm: the patterns above already cover every `ForkName` // there is, so a catch-all here would be dead code rather than a // safety net. `Lean` is covered by name for the same reason, rather @@ -255,12 +297,17 @@ fn apply( let attestation: electra::Attestation = case.ssz("attestation"); electra_stf::process_attestation(state, &attestation, &committees) } - ForkName::Gloas => Err( - ethlambda_state_transition::beacon::Error::UnsupportedForFork { - function: "attestation", - fork: ForkName::Gloas, - }, - ), + // Gloas reshapes the container a third time (EIP-7688's unbounded + // `AttestingIndices`) and modifies the function again (the new + // `parent_slot` argument and the builder-payment weight + // accounting, EIP-7732), so this reads the case's own `meta.yaml` + // for `parent_slot` rather than only the operation file every + // other fork's arm reads. + ForkName::Gloas => { + let attestation: gloas::Attestation = case.ssz("attestation"); + let meta: AttestationMeta = case.yaml("meta"); + gloas_stf::process_attestation(state, &attestation, meta.parent_slot, &committees) + } ForkName::Lean => lean_is_not_a_fixture_fork("attestation"), }, "attester_slashing" => match case.fork { @@ -281,20 +328,34 @@ fn apply( let slashing: electra::AttesterSlashing = case.ssz("attester_slashing"); electra_stf::process_attester_slashing(state, &slashing, config) } - ForkName::Gloas => Err( - ethlambda_state_transition::beacon::Error::UnsupportedForFork { - function: "attester_slashing", - fork: ForkName::Gloas, - }, - ), + // Gloas's container widens again (EIP-7688's unbounded + // `AttestingIndices`); the function is still not behaviorally + // modified, only transcribed against gloas's own type. + ForkName::Gloas => { + let slashing: gloas::AttesterSlashing = case.ssz("attester_slashing"); + gloas_stf::process_attester_slashing(state, &slashing, config) + } ForkName::Lean => lean_is_not_a_fixture_fork("attester_slashing"), }, - // `ProposerSlashing` never changes shape, and no fork's specification - // ever lists a modified `process_proposer_slashing`, so this needs no - // per-fork routing at all. + // `ProposerSlashing` never changes shape, container included, gloas's + // own not excepted. What changes at gloas is the function: it clears + // the `BuilderPendingPayment` tied to a slashed proposal (EIP-7732), + // which every earlier fork's `process_proposer_slashing` has no + // notion of, so this alone needs the routing every other fork's + // `proposer_slashing` handler above already has. "proposer_slashing" => { let slashing: shared::ProposerSlashing = case.ssz("proposer_slashing"); - operations::process_proposer_slashing(state, &slashing, config) + match case.fork { + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Fulu => operations::process_proposer_slashing(state, &slashing, config), + ForkName::Gloas => gloas_stf::process_proposer_slashing(state, &slashing, config), + ForkName::Lean => lean_is_not_a_fixture_fork("proposer_slashing"), + } } "deposit" => match case.fork { // The `Deposit` container never changes shape; through deneb a @@ -330,7 +391,12 @@ fn apply( ), ForkName::Lean => lean_is_not_a_fixture_fork("deposit"), }, - "voluntary_exit" => match case.fork { + // `voluntary_exit_churn` is a second gloas-only fixture handler + // exercising exit-queue churn scenarios specifically; its own + // operation file is still named `voluntary_exit.ssz_snappy`, and it + // routes through the identical processing call, so it shares this + // whole arm rather than getting one of its own. + "voluntary_exit" | "voluntary_exit_churn" => match case.fork { // Unchanged through capella. ForkName::Phase0 | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => { let exit: shared::SignedVoluntaryExit = case.ssz("voluntary_exit"); @@ -344,18 +410,15 @@ fn apply( deneb_stf::process_voluntary_exit(state, &exit, config) } // Electra adds a pending-partial-withdrawal check and its own - // balance-churn exit-queue accounting (EIP-7251); fulu's - // specification changes neither. - ForkName::Electra | ForkName::Fulu => { + // balance-churn exit-queue accounting (EIP-7251); neither fulu's + // nor gloas's specification changes it further (gloas's own churn + // limits are read by `crate::beacon::helpers::electra::compute_exit_epoch_and_update_churn` + // dispatching on the state's own fork, not by a gloas copy of + // `process_voluntary_exit`; see that function's own doc). + ForkName::Electra | ForkName::Fulu | ForkName::Gloas => { let exit: shared::SignedVoluntaryExit = case.ssz("voluntary_exit"); electra_stf::process_voluntary_exit(state, &exit, config) } - ForkName::Gloas => Err( - ethlambda_state_transition::beacon::Error::UnsupportedForFork { - function: "voluntary_exit", - fork: ForkName::Gloas, - }, - ), ForkName::Lean => lean_is_not_a_fixture_fork("voluntary_exit"), }, // New in altair, and never listed as modified again, so this needs no @@ -555,23 +618,48 @@ fn apply( let request: gloas::BuilderExitRequest = case.ssz("builder_exit_request"); gloas_stf::process_builder_exit_request(state, &request, config) } + // New in gloas (EIP-7732): the payload timeliness committee's vote on + // whether the *previous* slot's payload showed up on time. + "payload_attestation" => { + let payload_attestation: gloas::PayloadAttestation = case.ssz("payload_attestation"); + gloas_stf::process_payload_attestation(state, &payload_attestation, config) + } other => return Err(format!("unhandled operation `{other}`")), }; outcome.map_err(|err| format!("{err:?}")) } +/// The config to process one case's operation against. +/// +/// Most cases carry no `config.yaml` of their own, and +/// [`fork_active_from_genesis`]'s all-forks-at-genesis default serves +/// them; see its own doc for why that is the right fallback. Only +/// `gloas_stf::get_ptc` (reached through `process_payload_attestation`) reads +/// a fork-epoch config field directly in this whole suite +/// (`config.gloas_fork_epoch`, for the spec's own `assert epoch >= +/// GLOAS_FORK_EPOCH`), which is why gloas's own `payload_attestation` cases +/// are the ones that need a real answer here rather than the default: every +/// one of them ships a full `config.yaml`, and +/// `process_payload_attestation_pre_fork_epoch` specifically sets +/// `GLOAS_FORK_EPOCH: 1` to exercise that assertion's rejection path, so +/// reading the case's own file first (rather than always overriding) is what +/// keeps that case's expected rejection intact. +fn case_config(case: &Case) -> Config { + case.yaml_opt("config") + .unwrap_or_else(|| fork_active_from_genesis(case)) +} + pub fn trials() -> Vec { - let config = Arc::new(Config::active()); let cases = collect_all_handlers(PRESET, "operations"); let mut trials = vec![super::discovery_trial("operations", cases.len())]; for (handler, case) in cases { - let config = Arc::clone(&config); trials.push(super::case_trial("operations", case, move |case| { let mut state = BeaconState::from_ssz(case.fork, &case.ssz_bytes("pre")) .map_err(|err| format!("the fixture's pre-state does not decode: {err:?}"))?; + let config = case_config(case); let outcome = apply(&handler, case, &mut state, &config); super::check_transition(case, outcome, &state) })); @@ -609,9 +697,11 @@ fn every_shipped_handler_is_dispatched() -> Result<(), Failed> { "execution_payload", "execution_payload_bid", "parent_execution_payload", + "payload_attestation", "proposer_slashing", "sync_aggregate", "voluntary_exit", + "voluntary_exit_churn", "withdrawal_request", "withdrawals", ]; diff --git a/crates/common/types/src/beacon/containers/mod.rs b/crates/common/types/src/beacon/containers/mod.rs index fe6139668..882b59698 100644 --- a/crates/common/types/src/beacon/containers/mod.rs +++ b/crates/common/types/src/beacon/containers/mod.rs @@ -989,6 +989,33 @@ impl BeaconState { ) } + /// `previous_epoch_participation` or `current_epoch_participation`, + /// mutably and as a plain slice, whichever `current` selects: element + /// writes only, the same contract [`Self::inactivity_scores_mut`]'s own + /// doc gives for why that is enough to include gloas where + /// [`Self::altair_validator_lists_mut`] cannot. + /// + /// `crate::beacon::stf::gloas::process_attestation` is the one caller: like + /// every earlier fork's own version, it only ever ORs a newly-satisfied + /// flag into an existing validator's entry, never grows or replaces + /// either list (that only ever happens through + /// [`Self::altair_validator_lists_mut`], on a fork that carries it, or + /// through a gloas caller's own per-fork projection, the way + /// [`Self::altair_validator_lists_mut`]'s own doc describes). + pub fn epoch_participation_mut(&mut self, current: bool) -> Result<&mut [ParticipationFlags]> { + dispatch_state_from!( + self, + "BeaconState::epoch_participation_mut", + |state| if current { + &mut state.current_epoch_participation[..] + } else { + &mut state.previous_epoch_participation[..] + }, + carried_by: [Altair, Bellatrix, Capella, Deneb, Electra, Fulu, Gloas], + absent_from: [Phase0], + ) + } + /// The current and next sync committee, by reference. /// /// Both exist from altair on, byte-for-byte the same field in every later From 8200261ea8921ee3cf99fff5d411c6cfbb445bf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:00:26 -0300 Subject: [PATCH 021/132] feat(beacon): carry the gloas fork choice across the fulu boundary The gloas fork choice in consensus-specs v1.7.0-beta.2 only describes a store anchored in gloas. get_parent_payload_status reads the parent block's execution payload bid, which a fulu block does not have, and the node walks step onto such parents. The gap is known upstream: issue ethereum/consensus-specs#5096 was closed as not planned, and PR ethereum/consensus-specs#5125 (open) would answer the existing PAYLOAD_STATUS_PENDING for a pre-gloas parent (it describes that as Lighthouse's behavior; Lighthouse v8.2.2 answers EMPTY). Lodestar and Prysm treat the pre-gloas block as full. This follows Lodestar and Prysm. A pre-gloas block is a single fork-choice node whose payload status is FULL, since its payload was executed inside the block: it is the parent payload status of the first gloas block, its only child node, the node every vote for it supports, and its payload counts as verified, timely and available. The last point is what lets the first gloas block and payload-present votes for the last fulu block pass the gloas checks that require a verified parent payload. get_head_node switches on the current slot's fork rather than the justified state's: from GLOAS_FORK_EPOCH on, the gloas head runs over the whole tree, pre-gloas blocks included, the way clients run one walk across the boundary. Because that makes the gloas head run on live networks from the fork epoch, it tolerates what the pre-gloas compute_weights already tolerates: a vote whose ancestor walk leaves the pruned index (its own block, or a parent below the finalized slot) weighs nothing, in the root-based attestation score the proposer boost check shares as well, and a block timeliness entry lost on restart reads as not timely. The first cannot change the head, since a walk that leaves the index cannot descend from a scored root; the second can move it for at most one slot right after a restart. Both are marked as implementation choices beside the specification text. --- .../src/beacon/fork_choice.rs | 1072 ++++++++++++++++- 1 file changed, 1009 insertions(+), 63 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index a8a6b39ce..91a5a3f7c 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -1151,6 +1151,22 @@ pub fn get_proposer_score(store: &Store, config: &Config) -> Result { /// can score a root against the justified state without also asking whether /// the proposer boost applies to it; the specification makes the same split, /// since neither of those checks the boost. +/// +/// **Implementation choice, not spec text**: a voter's last message counts +/// only when the ancestor walk from its root succeeds and lands on `root`; +/// a walk that meets a root missing from `index` (its own, or an ancestor +/// pruned below the finalized slot) contributes nothing, where +/// [`get_ancestor`] would raise `Error::SpecAssert`. [`compute_weights`]'s +/// own doc gives the reasoning this mirrors: pruning removes only index +/// entries below the finalized slot, and every root scored here is indexed +/// at or above it, so a walk that leaves the index cannot descend from, or +/// equal, the scored root, and the specification's own contribution for +/// such a vote is `0` too. This function is not only +/// [`get_weight`]'s own attestation half; [`is_head_weak`] and +/// [`is_parent_strong`] call it too, and both run underneath +/// [`should_apply_proposer_boost`], which the gloas head visits on every +/// call whose boosted block's parent is from the previous slot, so a pruned +/// vote reaching this function is not only a pre-gloas concern. pub fn get_attestation_score( store: &Store, index: &HashMap, @@ -1172,7 +1188,8 @@ pub fn get_attestation_score( let Some(message) = store.latest_message(validator_index) else { continue; }; - if get_ancestor(index, message.root, block_slot)? == root { + if matches!(get_ancestor(index, message.root, block_slot), Ok(ancestor) if ancestor == root) + { attestation_score = attestation_score.saturating_add(validator.effective_balance); } } @@ -1510,22 +1527,30 @@ fn compute_head( /// [`compute_head`] pre-gloas, gloas's own [`gloas_get_head`] once the chain /// has upgraded, as a full [`ForkChoiceNode`] rather than just a root. /// -/// Dispatches on the justified checkpoint's own state fork, not the head's: -/// that is the state [`compute_weights`]/[`gloas_get_weight`] both score -/// every candidate against. Justification lags realized blocks by up to a -/// couple of epochs, so for a window after the fulu-to-gloas boundary the -/// block tree can already hold gloas blocks while the justified checkpoint -/// is still fulu-anchored; this dispatch takes the pre-gloas arm for that -/// whole window, weighing gloas blocks by [`compute_weights`] as if they had -/// no payload dimension. That is a real lag, not an oversight fixed here: -/// the specification's gloas `fork-choice.md` only ever describes a -/// gloas-anchored `Store` and says nothing about a mixed tree, and how this -/// crate should behave across the boundary is decided separately, before -/// gloas block import exists to reach it (see [`gloas_bid`]'s own doc for -/// the matching gap one level down, at a single block's own parent). A -/// pre-gloas head carries no payload dimension of its own, so it is -/// reported [`PayloadStatus::Pending`]: nothing on a pre-gloas chain ever -/// reads this field back. +/// Dispatches on the current slot's own fork, via [`Config::fork_at_epoch`], +/// not on the head's fork or the justified checkpoint's: from +/// `GLOAS_FORK_EPOCH` on, every call runs [`gloas_get_head`] over the whole +/// tree, pre-gloas blocks included, so the payload-aware algorithm takes +/// over as soon as the chain could hold a gloas block at all, rather than +/// waiting for a block, or justification, to actually reach one. Before that +/// epoch, the pre-gloas [`compute_head`]/[`compute_weights`] path runs +/// exactly as it did before gloas existed. +/// +/// A pre-gloas block has no payload dimension of its own: this crate treats +/// it as a single-variant node whose status is always +/// [`PayloadStatus::Full`] (its payload ran inside the block, with no +/// separate reveal to attest to), so a pre-gloas head is reported as +/// `(root, Full)` rather than `Pending`. That is a decided rule, not +/// something the specification states: gloas's own `fork-choice.md` only +/// ever describes a gloas-anchored `Store` and says nothing about a +/// pre-gloas parent, or a mixed tree that still has one in it (see +/// [`gloas_bid`]'s own doc for the matching gap one level down, at a single +/// block's own parent, and the module documentation's "Gloas: payload-aware +/// fork choice" section for the rule in full). consensus-specs PR +/// ethereum/consensus-specs#5125 (open, unmerged as of this writing) answers +/// a pre-gloas parent with the existing `PENDING` status instead of a new +/// one, and says Lighthouse does the same; our FULL rule differs from both. +/// Adopt PENDING here if that PR lands. /// /// A fresh [`CommitteeCache`] on the gloas arm: gloas's own weight /// (`gloas_get_weight` by way of `should_apply_proposer_boost`) now needs @@ -1535,9 +1560,8 @@ fn compute_head( /// own performance note for what that costs repeated across a whole tree /// walk. pub fn get_head_node(store: &Store, config: &Config) -> Result { - let justified_checkpoint = store.beacon_justified_checkpoint(); - let justified_state = checkpoint_state(store, &justified_checkpoint, config)?; - match justified_state.fork_name() { + let current_slot = get_current_slot(store, config); + match config.fork_at_epoch(compute_epoch_at_slot(current_slot)) { ForkName::Gloas => { let committees = CommitteeCache::default(); gloas_get_head(store, config, &committees) @@ -1552,7 +1576,7 @@ pub fn get_head_node(store: &Store, config: &Config) -> Result { let index = store.block_index(); Ok(ForkChoiceNode { root: compute_head(store, &index, config)?, - payload_status: PayloadStatus::Pending, + payload_status: PayloadStatus::Full, }) } ForkName::Lean => lean_fork_unreachable("fork_choice::get_head_node"), @@ -1952,6 +1976,29 @@ pub fn get_proposer_head( // `payloads` as the source of truth and builds a `ForkChoiceNode` on demand // wherever one is needed, never as a third, separately maintained index. // +// The specification's own gloas `fork-choice.md` only ever describes a +// gloas-anchored `Store`: `get_parent_payload_status` reads a parent's own +// execution payload bid, which a fulu (or earlier) parent does not have, and +// says nothing about the fulu-to-gloas boundary a real chain has to cross to +// ever reach a gloas-anchored store in the first place. This crate resolves +// that gap with a decided rule: a pre-gloas block is a single-variant node +// whose payload status is unconditionally FULL, since its payload ran inside +// the block itself and never had a separate empty branch to weigh against. +// Concretely: [`get_parent_payload_status`] of a gloas block with a +// pre-gloas parent answers FULL; a pre-gloas block's only child +// ([`get_node_children`]) is its FULL node; a vote for a pre-gloas block +// ([`get_supported_node`]) supports that FULL node whatever its own +// `payload_present` carries; [`gloas_get_ancestor`] stepping onto a +// pre-gloas block lands on FULL; a pre-gloas payload reads as already +// verified, timely and available ([`is_payload_verified`], +// [`payload_timeliness`], [`payload_data_availability`]), since it ran +// inside the block with no separate reveal to judge; and a pre-gloas head +// ([`get_head_node`]) is reported as `(root, Full)`. consensus-specs PR +// ethereum/consensus-specs#5125 (open, unmerged as of this writing) answers +// a pre-gloas parent with the existing `PENDING` status instead of a new +// one, and says Lighthouse does the same; our FULL rule differs from both. +// Adopt PENDING here if that PR lands. +// // Functions here are named exactly as the specification names them, with a // `gloas_` prefix only where a pre-gloas function of the same name already // exists in this file: `gloas_get_ancestor`, `gloas_get_checkpoint_block`, @@ -1977,18 +2024,15 @@ pub fn get_proposer_head( /// [`SignedBeaconBlock`] enum. /// /// Every function in this section assumes `block` is itself gloas-shaped, -/// which does not hold at the fulu-to-gloas boundary: the first gloas -/// block's own parent is a fulu block with no bid at all, so -/// [`get_parent_payload_status`] on it fails here with -/// `Error::UnsupportedForFork` rather than answering a payload status for -/// it. The specification's gloas `fork-choice.md` only ever describes a -/// gloas-anchored `Store`; it says nothing about a pre-gloas parent (or, one -/// level up, a fulu-justified store scoring a mixed tree, see -/// [`get_head_node`]'s own doc), and how this crate should handle that -/// boundary is decided separately, before gloas block import exists to -/// reach it. A non-gloas block reaching here is a caller bug in every case -/// this crate does handle today, reported by name rather than assumed away, -/// the same projection `helpers::gloas::gloas_state_ref` gives a state. +/// which does not hold at the fulu-to-gloas boundary on its own: the first +/// gloas block's own parent is a fulu block with no bid at all. +/// [`get_parent_payload_status`], this function's only caller, never reaches +/// here for a pre-gloas block though: it answers [`PayloadStatus::Full`] for +/// one directly, per this crate's decided rule for that boundary (see the +/// module documentation's "Gloas: payload-aware fork choice" section). A +/// non-gloas block still reaching here is therefore a caller bug, reported +/// by name rather than assumed away, the same projection +/// `helpers::gloas::gloas_state_ref` gives a state. fn gloas_bid(block: &SignedBeaconBlock) -> Result<&gloas::ExecutionPayloadBid> { match block { SignedBeaconBlock::Gloas(inner) => { @@ -2001,27 +2045,76 @@ fn gloas_bid(block: &SignedBeaconBlock) -> Result<&gloas::ExecutionPayloadBid> { } } +/// Whether `root` names a block this store holds and that block is +/// pre-gloas: the one bit every function in this section needs to tell a +/// real payload-bearing block from a pre-gloas one whose payload ran inside +/// the block itself (this crate's decided rule for the fulu-to-gloas +/// boundary; see the module documentation's "Gloas: payload-aware fork +/// choice" section). +/// +/// `false` for a root this store has never heard of, not `true`: a `None` +/// from [`Store::get_signed_block`](ethlambda_storage::Store::get_signed_block) +/// answers `Some(block) if ...` the same as a gloas one would, so callers +/// fall through to their own gloas-shaped behavior rather than assuming an +/// unknown root is pre-gloas, which would answer, say, +/// [`is_payload_verified`] `true` for a root the specification would never +/// call verified. +/// +/// Decodes `root`'s full signed block to answer, which every caller here +/// otherwise avoids doing more than once: [`is_payload_verified`], +/// [`payload_timeliness`] and [`payload_data_availability`] call this first +/// and then read [`Store::has_verified_payload`](ethlambda_storage::Store::has_verified_payload) +/// directly rather than back through [`is_payload_verified`], and +/// [`get_node_children`]'s own decode of a `Pending` node's block answers +/// this same question without a second call here. +fn is_known_pre_gloas_block(store: &Store, root: Root) -> bool { + matches!( + store.get_signed_block(&root).expect("get"), + Some(block) if !matches!(block, SignedBeaconBlock::Gloas(_)) + ) +} + /// `is_payload_verified` (gloas `fork-choice.md`): whether `root`'s execution /// payload envelope has been locally delivered and verified via /// `on_execution_payload_envelope`, which does not exist in this crate yet: /// nothing calls [`Store::insert_verified_payload`](ethlambda_storage::Store::insert_verified_payload) /// until the handler that reads a delivered envelope lands. +/// +/// A pre-gloas block has no envelope to deliver at all: its payload ran +/// inside the block itself, so this answers `true` for one unconditionally, +/// per this crate's decided rule for the fulu-to-gloas boundary (see the +/// module documentation's "Gloas: payload-aware fork choice" section). pub fn is_payload_verified(store: &Store, root: Root) -> bool { - store.has_verified_payload(&root) + is_known_pre_gloas_block(store, root) || store.has_verified_payload(&root) } /// `payload_timeliness` (gloas `fork-choice.md`): whether `root`'s payload is /// considered `timely` (or not, when `timely` is `false`), taking into /// account both local availability and the payload timeliness committee's /// votes. +/// +/// A pre-gloas block has no payload timeliness committee vote to read at +/// all, since the committee did not exist yet, and no separate reveal to +/// judge the timeliness of: it answers `timely` back unconditionally +/// (matching [`payload_data_availability`]'s own `available`), part of this +/// crate's decided rule for the fulu-to-gloas boundary (see the module +/// documentation's "Gloas: payload-aware fork choice" section). Checked +/// before the vote lookup below, not after: a pre-gloas root was never +/// registered in [`Store::payload_timeliness_vote`](ethlambda_storage::Store::payload_timeliness_vote) +/// in the first place, so reading it first would raise +/// `Error::SpecAssert` rather than ever reach the fallback that answers it. pub fn payload_timeliness(store: &Store, root: Root, timely: bool) -> Result { + if is_known_pre_gloas_block(store, root) { + return Ok(timely); + } + let vote = store .payload_timeliness_vote(&root) .ok_or(Error::SpecAssert("root in store.payload_timeliness_vote"))?; // If the payload is not locally available, it is not considered // available regardless of the PTC vote. - if !is_payload_verified(store, root) { + if !store.has_verified_payload(&root) { return Ok(!timely); } @@ -2033,14 +2126,22 @@ pub fn payload_timeliness(store: &Store, root: Root, timely: bool) -> Result Result { + if is_known_pre_gloas_block(store, root) { + return Ok(available); + } + let vote = store .payload_data_availability_vote(&root) .ok_or(Error::SpecAssert( "root in store.payload_data_availability_vote", ))?; - if !is_payload_verified(store, root) { + if !store.has_verified_payload(&root) { return Ok(!available); } @@ -2053,6 +2154,11 @@ pub fn payload_data_availability(store: &Store, root: Root, available: bool) -> /// builds on its parent's full payload branch or its empty one, found by /// comparing the two blocks' own bids. /// +/// A pre-gloas parent has no bid to compare against, and no empty branch of +/// its own to have built one against: it answers FULL unconditionally, per +/// this crate's decided rule for the fulu-to-gloas boundary (see the module +/// documentation's "Gloas: payload-aware fork choice" section). +/// /// Takes `block: &SignedBeaconBlock`, not the specification's unsigned /// `BeaconBlock`: see the module documentation's "Signed blocks" section for /// why every function in this file does. @@ -2064,6 +2170,9 @@ pub fn get_parent_payload_status( .get_signed_block(&block.parent_root()) .expect("get") .ok_or(Error::SpecAssert("block.parent_root in store.blocks"))?; + if !matches!(parent, SignedBeaconBlock::Gloas(_)) { + return Ok(PayloadStatus::Full); + } let parent_block_hash = gloas_bid(block)?.parent_block_hash; let message_block_hash = gloas_bid(&parent)?.block_hash; Ok(if parent_block_hash == message_block_hash { @@ -2165,12 +2274,28 @@ pub fn gloas_get_checkpoint_block(store: &Store, root: Root, epoch: Epoch) -> Re /// `get_supported_node` (gloas `fork-choice.md`): the node `message` /// supports, full or empty once its slot has passed the block's own, pending /// otherwise. +/// +/// A vote for a pre-gloas block always supports its FULL node, whatever +/// `message.payload_present` carries: every pre-gloas attestation leaves +/// that field `false` (see [`LatestMessage::payload_present`]'s own doc), +/// which would otherwise read as a vote for an empty branch a pre-gloas +/// block never had. Part of this crate's decided rule for the +/// fulu-to-gloas boundary; see the module documentation's "Gloas: +/// payload-aware fork choice" section. +/// +/// One [`Store::get_signed_block`](ethlambda_storage::Store::get_signed_block) +/// read answers both `block_slot` and the fork-shape check above, rather +/// than the cheaper [`Store::block_entry`](ethlambda_storage::Store::block_entry) +/// for the slot and a second decode for the shape: this runs once per voter +/// in [`gloas_get_attestation_score`]'s own per-candidate walk, so a second +/// decode there is not free. pub fn get_supported_node(store: &Store, message: LatestMessage) -> Result { - let (block_slot, _) = store - .block_entry(&message.root) + let block = store + .get_signed_block(&message.root) + .expect("get") .ok_or(Error::SpecAssert("message.root in store.blocks"))?; - let payload_status = if block_slot < message.slot { - if message.payload_present { + let payload_status = if block.slot() < message.slot { + if !matches!(block, SignedBeaconBlock::Gloas(_)) || message.payload_present { PayloadStatus::Full } else { PayloadStatus::Empty @@ -2296,6 +2421,16 @@ pub fn get_payload_status_tiebreaker( /// the parent, sharing its proposer and itself timely by the PTC deadline /// (an early equivocation), exists. With no such sibling, the boost still /// applies. +/// +/// **Implementation choice, not spec text**: a candidate with no recorded +/// [`Store::block_timeliness`](ethlambda_storage::Store::block_timeliness) +/// entry reads as not timely by either deadline, rather than raising +/// `Error::SpecAssert`. That scratch is in-memory only, so every entry is +/// gone after a restart; reading a gap as "not an early equivocation" +/// rather than aborting the whole weight computation over it is the +/// conservative answer (it can only ever miss withholding a boost, never +/// wrongly withhold one), the same shape of tolerance +/// [`gloas_get_attestation_score`]'s own doc gives for a pruned vote. pub fn should_apply_proposer_boost( store: &Store, config: &Config, @@ -2343,7 +2478,7 @@ pub fn should_apply_proposer_boost( } let timely = store .block_timeliness(&candidate_root) - .ok_or(Error::SpecAssert("root in store.block_timeliness"))?; + .unwrap_or([false; constants::NUM_BLOCK_TIMELINESS_DEADLINES]); if !timely[constants::PTC_TIMELINESS_INDEX] { continue; } @@ -2367,6 +2502,20 @@ pub fn should_apply_proposer_boost( /// root-comparison the pre-gloas [`get_attestation_score`] uses (sound only /// because a pre-gloas `ForkChoiceNode` is a bare root; see that function's /// own doc). +/// +/// **Implementation choice, not spec text**: a voter's last message is +/// skipped when its own root is not in the store at all, rather than +/// raising `Error::SpecAssert` the way [`get_supported_node`] otherwise +/// would, so one stale voter cannot abort the whole head computation. A +/// block pruned from the fork-choice index is still in the store, so a +/// vote for it passes this check and walks the block table, which is safe +/// and contributes nothing (it cannot descend from a scored node, the +/// reasoning [`compute_weights`]'s own doc gives); the skip fires only for +/// a root the store never held. Checked with `store.has_block`, not +/// [`Store::block_entry`](ethlambda_storage::Store::block_entry): the latter +/// decodes the whole block on a beacon store to answer two fields this +/// check does not even need, which would spend a decode on every voter just +/// to maybe skip [`get_supported_node`]'s own. fn gloas_get_attestation_score( store: &Store, node: ForkChoiceNode, @@ -2382,6 +2531,9 @@ fn gloas_get_attestation_score( let Some(message) = store.latest_message(validator_index) else { continue; }; + if !store.has_block(&message.root) { + continue; + } let supported = get_supported_node(store, message)?; if is_ancestor(store, supported, node)? { attestation_score = attestation_score.saturating_add(validator.effective_balance); @@ -2400,19 +2552,32 @@ fn gloas_get_attestation_score( /// - every latest message is walked afresh for every node [`gloas_get_head`] /// visits, rather than folded bottom-up once per tree; /// - each [`gloas_get_ancestor`] hop decodes two signed blocks (the node's -/// own, then its parent's inside [`get_parent_payload_status`]), where the +/// own, then its parent's inside [`get_parent_payload_status`]), and +/// [`get_supported_node`] decodes a third, once per voter, where the /// pre-gloas, index-only [`get_ancestor`] decodes none; +/// - telling a pre-gloas root from a gloas one is itself a decode a +/// pre-gloas tree never needed at all, paid by [`is_payload_verified`], +/// [`payload_timeliness`], [`payload_data_availability`], and +/// [`get_node_children`]'s own check on every `Pending` node it visits; /// - [`should_apply_proposer_boost`] reruns a full [`is_head_weak`] score /// and its own `store.block_index()` scan on every call, even though its /// answer does not depend on `node` at all and so is identical for every -/// candidate one [`gloas_get_head`] call weighs; -/// - a vote for a block this store has pruned below its finalized anchor -/// aborts the whole head computation with `Error::SpecAssert`, exactly -/// the failure [`compute_weights`]'s own doc describes pre-gloas -/// `get_weight` having, which that fold was built to skip past instead. +/// candidate one [`gloas_get_head`] call weighs. +/// +/// One cost this list used to carry is closed: a vote for a block this +/// store has pruned below its finalized anchor no longer aborts the whole +/// head computation with `Error::SpecAssert`, the failure +/// [`compute_weights`]'s own doc describes pre-gloas `get_weight` having. +/// Closing it took two fixes, not one: [`gloas_get_attestation_score`]'s +/// own doc has the direct one; the other is reached indirectly, through +/// [`should_apply_proposer_boost`] (called above regardless of `node`) into +/// pre-gloas [`is_head_weak`]'s own [`get_attestation_score`], which fires +/// whenever the boosted block's parent is from the previous slot, the +/// ordinary case rather than a corner one; [`get_attestation_score`]'s own +/// doc has that fix. /// /// Fine for spec tests, where the tree is a handful of blocks and never -/// prunes; live use needs all four addressed, not just the first. +/// prunes; live use needs the remaining costs addressed. pub fn gloas_get_weight( store: &Store, node: ForkChoiceNode, @@ -2452,6 +2617,18 @@ pub fn gloas_get_weight( /// `node` is still `Pending`) or the next blocks that agree with `node`'s own /// payload status about their shared parent. /// +/// A pre-gloas `node` has no empty branch of its own: its only child is its +/// FULL node, per this crate's decided rule for the fulu-to-gloas boundary +/// (see the module documentation's "Gloas: payload-aware fork choice" +/// section). The specification's own gloas `fork-choice.md` never describes +/// this case, since it only ever anchors a gloas `Store`. +/// +/// **Performance note**: telling a pre-gloas `node` apart from a gloas one +/// costs a full block decode, on every `Pending` node this walks, that a +/// pre-gloas tree never needed. See [`gloas_get_weight`]'s own performance +/// note for the other costs this section's spec-literal walk carries above +/// pre-gloas [`compute_weights`]'s bottom-up fold. +/// /// `blocks` is [`Store::block_index`]'s own shape, not the specification's /// `Dict[Root, BeaconBlock]`: see the section documentation above for why /// nodes, and the trees built from them, are derived rather than stored. @@ -2461,11 +2638,25 @@ pub fn get_node_children( node: ForkChoiceNode, ) -> Result> { if node.payload_status == PayloadStatus::Pending { + let block = store + .get_signed_block(&node.root) + .expect("get") + .ok_or(Error::SpecAssert("node.root in store.blocks"))?; + if !matches!(block, SignedBeaconBlock::Gloas(_)) { + return Ok(vec![ForkChoiceNode { + root: node.root, + payload_status: PayloadStatus::Full, + }]); + } let mut children = vec![ForkChoiceNode { root: node.root, payload_status: PayloadStatus::Empty, }]; - if is_payload_verified(store, node.root) { + // `store.has_verified_payload` directly, not `is_payload_verified`: + // `block` above already confirmed this is a gloas node, so + // `is_payload_verified`'s own fork-shape check would only redecode + // the same block to reach the answer this already has in hand. + if store.has_verified_payload(&node.root) { children.push(ForkChoiceNode { root: node.root, payload_status: PayloadStatus::Full, @@ -2594,11 +2785,14 @@ pub fn get_payload_attestation_due_ms(config: &Config) -> u64 { /// /// Drops the pre-gloas `shuffling_stable` condition entirely, matching the /// specification's own text: pre-gloas [`get_proposer_head`] still gates it -/// on `head_block.fork_name() >= ForkName::Fulu` for the forks before fulu, -/// but `head_block` here is always gloas's own, so that gate is -/// unconditionally true for every call this function ever gets and the -/// specification's own gloas text simply omits the condition rather than -/// spell out an always-true check. +/// on `head_block.fork_name() >= ForkName::Fulu` for the forks before fulu. +/// [`get_head_node`] only ever reaches this function's gloas arm once the +/// chain has reached `GLOAS_FORK_EPOCH`, so `head_block` here is gloas, or +/// fulu right at the fulu-to-gloas boundary (this crate's decided rule for +/// a pre-gloas head; see the module documentation's "Gloas: payload-aware +/// fork choice" section), so `>= ForkName::Fulu` holds unconditionally for +/// every call this function gets, and the specification's own gloas text +/// simply omits the condition rather than spell out an always-true check. pub fn gloas_get_proposer_head( store: &Store, head_node: ForkChoiceNode, @@ -3904,6 +4098,24 @@ mod tests { }) } + /// A fulu signed block with an empty body and a zero signature, for the + /// fulu-to-gloas boundary tests below, which need a pre-gloas block + /// genuinely of fulu's own fork rather than the phase0-shaped [`block`] + /// stand-in. Wraps [`electra::SignedBeaconBlock`], the same shape + /// [`SignedBeaconBlock::Fulu`] itself wraps (see that variant's own doc). + fn fulu_block(slot: Slot, parent_root: Root) -> SignedBeaconBlock { + SignedBeaconBlock::Fulu(electra::SignedBeaconBlock { + message: electra::BeaconBlock { + slot, + proposer_index: 0, + parent_root, + state_root: Root::ZERO, + body: electra::BeaconBlockBody::empty(), + }, + signature: Default::default(), + }) + } + /// An exact anchor pair: `state` is `block`'s own post-state. /// /// Built as a fixed point, the way the state transition produces one. The @@ -4054,10 +4266,12 @@ mod tests { /// The failure a live mainnet follower hit: `promote_beacon_anchor` prunes /// the block index below the oldest kept anchor, and any validator whose - /// freshest vote was for a block down there kept pointing at it. The - /// specification's `get_weight` raises on that vote and takes the whole - /// head computation with it; the head froze for as long as one stale voter - /// stayed stale. + /// freshest vote was for a block down there kept pointing at it. Both + /// [`get_weight`] (through [`get_attestation_score`]'s own decided rule; + /// see its doc) and [`compute_weights`] (its own independent bottom-up + /// fold, which never called `get_attestation_score` and so never shared + /// this bug) skip such a vote rather than raising, and agree on the + /// weight it leaves behind. #[test] fn a_vote_for_a_pruned_block_weighs_nothing_instead_of_failing() { let config = Config::active(); @@ -4088,16 +4302,18 @@ mod tests { }, ); - assert!( - get_weight(&store, &index, a_root, &config).is_err(), - "the specification's own version is what raises here; this test \ - exists because that took the whole head computation with it" - ); + let per_root_weight = get_weight(&store, &index, a_root, &config) + .expect("a pruned vote is skipped, not fatal, in the per-root version either"); + assert!(per_root_weight > 0, "the live vote still counts"); let weights = compute_weights(&store, &index, &config).expect("a pruned vote is not fatal"); let one_validator_balance = weights[&a_root]; assert!(one_validator_balance > 0, "the live vote still counts"); + assert_eq!( + one_validator_balance, per_root_weight, + "the per-root and bottom-up versions must agree" + ); assert_eq!( weights[&anchor_root], one_validator_balance, "and it counts exactly once, for a_root and everything it descends from" @@ -4932,6 +5148,17 @@ mod tests { fn get_node_children_offers_the_full_branch_only_once_the_payload_is_verified() { let mut store = empty_store(); let a_root = Root::repeat_byte(0xa1); + store + .insert_signed_block( + a_root, + gloas_block( + 1, + Root::ZERO, + ExecutionBlockHash::ZERO, + ExecutionBlockHash::repeat_byte(0xaa), + ), + ) + .unwrap(); let blocks = store.block_index(); let node = ForkChoiceNode { root: a_root, @@ -5164,6 +5391,17 @@ mod tests { fn payload_timeliness_and_data_availability_require_strictly_more_than_the_threshold() { let mut store = empty_store(); let a_root = Root::repeat_byte(0xa1); + store + .insert_signed_block( + a_root, + gloas_block( + 1, + Root::ZERO, + ExecutionBlockHash::ZERO, + ExecutionBlockHash::repeat_byte(0xaa), + ), + ) + .unwrap(); store.insert_verified_payload(a_root); let exactly_threshold = vec![Some(true); preset::PAYLOAD_TIMELY_THRESHOLD as usize]; @@ -5385,4 +5623,712 @@ mod tests { "an early equivocation against the parent must withhold the boost" ); } + + // ------------------------------------------------------------------ + // Gloas: the fulu-to-gloas boundary + // ------------------------------------------------------------------ + // + // A pre-gloas block is a single-variant node whose payload status is + // always FULL: its payload ran inside the block, with no separate empty + // branch to weigh against. The tests below exercise that rule at each of + // its entry points (`get_parent_payload_status`, `get_node_children`, + // `gloas_get_ancestor`, `get_supported_node`) and at the two places that + // compose them (`gloas_get_head`, `get_head_node`'s own dispatch). + + #[test] + fn mixed_tree_treats_the_fulu_prefix_as_a_single_full_node_at_every_gloas_entry_point() { + let mut store = empty_store(); + + let anchor_root = Root::repeat_byte(0xa0); + store + .insert_signed_block(anchor_root, fulu_block(0, Root::ZERO)) + .unwrap(); + + let fulu_root = Root::repeat_byte(0xf1); + store + .insert_signed_block(fulu_root, fulu_block(1, anchor_root)) + .unwrap(); + + // Its own bid never matters to whether it counts as fulu_root's + // child: a pre-gloas parent is FULL regardless of what a gloas + // child's bid claims about it. + let gloas_root = Root::repeat_byte(0x90); + store + .insert_signed_block( + gloas_root, + gloas_block( + 2, + fulu_root, + ExecutionBlockHash::repeat_byte(0xff), + ExecutionBlockHash::repeat_byte(0x99), + ), + ) + .unwrap(); + let gloas_block_value = store.get_signed_block(&gloas_root).unwrap().unwrap(); + + assert_eq!( + get_parent_payload_status(&store, &gloas_block_value).unwrap(), + PayloadStatus::Full, + "the first gloas block's fulu parent is a single-variant FULL node" + ); + + let blocks = store.block_index(); + let pending_fulu = ForkChoiceNode { + root: fulu_root, + payload_status: PayloadStatus::Pending, + }; + assert_eq!( + get_node_children(&store, &blocks, pending_fulu).unwrap(), + vec![ForkChoiceNode { + root: fulu_root, + payload_status: PayloadStatus::Full, + }], + "a fulu block's only child node is its own FULL node, with no \ + empty branch and no dependency on a verified payload" + ); + + let start = ForkChoiceNode { + root: gloas_root, + payload_status: PayloadStatus::Pending, + }; + assert_eq!( + gloas_get_ancestor(&store, start, 1).unwrap(), + ForkChoiceNode { + root: fulu_root, + payload_status: PayloadStatus::Full, + }, + "stepping from a gloas node onto a pre-gloas block lands on its \ + FULL node rather than erroring" + ); + } + + #[test] + fn get_supported_node_counts_a_pre_gloas_vote_toward_its_full_node() { + let mut store = empty_store(); + let a_root = Root::repeat_byte(0xa1); + store + .insert_signed_block(a_root, fulu_block(1, Root::ZERO)) + .unwrap(); + + let message = LatestMessage { + epoch: 0, + slot: 2, + root: a_root, + // Every real pre-gloas attestation leaves this `false`: there is + // no payload dimension to vote on before gloas. + payload_present: false, + }; + + assert_eq!( + get_supported_node(&store, message).unwrap(), + ForkChoiceNode { + root: a_root, + payload_status: PayloadStatus::Full, + }, + "a vote for a pre-gloas block supports its single FULL node, \ + whatever payload_present carries" + ); + } + + #[test] + fn gloas_get_head_descends_from_a_fulu_justified_root_through_the_boundary_to_the_heaviest_gloas_branch() + { + let config = Config::active(); + let anchor_root = Root::repeat_byte(0xa0); + let mut store = store_anchored_at(anchor_root); + store + .insert_signed_block(anchor_root, fulu_block(0, Root::ZERO)) + .unwrap(); + let state = test_state::with_validators_at(ForkName::Fulu, 4); + store.insert_state(anchor_root, state.clone()).unwrap(); + + // Two gloas children of the fulu anchor: both are its FULL children + // regardless of their own bid, since a pre-gloas parent is always + // FULL. + let heavy_root = Root::repeat_byte(0x91); + store + .insert_signed_block( + heavy_root, + gloas_block( + 1, + anchor_root, + ExecutionBlockHash::repeat_byte(0xff), + ExecutionBlockHash::repeat_byte(0xaa), + ), + ) + .unwrap(); + store.insert_state(heavy_root, state.clone()).unwrap(); + + let light_root = Root::repeat_byte(0x92); + store + .insert_signed_block( + light_root, + gloas_block( + 1, + anchor_root, + ExecutionBlockHash::repeat_byte(0xff), + ExecutionBlockHash::repeat_byte(0xbb), + ), + ) + .unwrap(); + store.insert_state(light_root, state).unwrap(); + store.set_time_ms(2 * config.slot_duration_ms).unwrap(); + + // Three votes for the heavier branch, one for the lighter. Neither + // payload is verified, so once a branch is picked it offers only its + // empty node next, and only the vote weight decides which branch + // wins. + for validator_index in 0..3 { + store.set_latest_message( + validator_index, + LatestMessage { + epoch: 0, + slot: 2, + root: heavy_root, + payload_present: false, + }, + ); + } + store.set_latest_message( + 3, + LatestMessage { + epoch: 0, + slot: 2, + root: light_root, + payload_present: false, + }, + ); + + let committees = CommitteeCache::default(); + let head = gloas_get_head(&store, &config, &committees).expect("known chain"); + assert_eq!( + head, + ForkChoiceNode { + root: heavy_root, + payload_status: PayloadStatus::Empty, + }, + "the walk must cross the fulu anchor and pick the heavier gloas \ + branch" + ); + } + + #[test] + fn get_head_node_switches_from_the_pre_gloas_algorithm_to_gloas_at_the_fork_epoch() { + let gloas_fork_epoch = 1; + let config = Config::active().with_fork_epoch(ForkName::Gloas, gloas_fork_epoch); + let first_gloas_slot = compute_start_slot_at_epoch(gloas_fork_epoch); + + let anchor_root = Root::repeat_byte(0xa0); + let mut store = store_anchored_at(anchor_root); + store + .insert_signed_block(anchor_root, fulu_block(0, Root::ZERO)) + .unwrap(); + let state = test_state::with_validators_at(ForkName::Fulu, 1); + store.insert_state(anchor_root, state.clone()).unwrap(); + + // An unverified gloas block: gloas's own algorithm only ever offers + // its EMPTY node as a candidate for one, while the pre-gloas + // algorithm has no payload dimension at all and treats it as a + // single, opaque, always-full root. + let b_root = Root::repeat_byte(0xb0); + store + .insert_signed_block( + b_root, + gloas_block( + first_gloas_slot, + anchor_root, + ExecutionBlockHash::ZERO, + ExecutionBlockHash::repeat_byte(0xbb), + ), + ) + .unwrap(); + store.insert_state(b_root, state).unwrap(); + store.set_unrealized_justification(b_root, Checkpoint::default()); + + store + .set_time_ms((first_gloas_slot - 1) * config.slot_duration_ms) + .unwrap(); + assert_eq!( + get_head_node(&store, &config).unwrap(), + ForkChoiceNode { + root: b_root, + payload_status: PayloadStatus::Full, + }, + "one slot before the fork epoch, the pre-gloas algorithm still \ + runs and reports the gloas block as a single, full root" + ); + + store + .set_time_ms(first_gloas_slot * config.slot_duration_ms) + .unwrap(); + assert_eq!( + get_head_node(&store, &config).unwrap(), + ForkChoiceNode { + root: b_root, + payload_status: PayloadStatus::Empty, + }, + "at its first slot, gloas's own algorithm takes over, and the \ + unverified payload leaves only the empty branch" + ); + } + + /// The complement of `gloas_get_head_picks_the_more_heavily_voted_ + /// payload_branch`'s 3-empty/1-full case: a `get_supported_node` that + /// ignored `payload_present` entirely (reading every vote below the + /// block's own slot as empty) would send all four votes here to empty + /// too, and empty would win instead of full. + #[test] + fn gloas_get_head_picks_full_when_it_has_the_greater_vote_weight() { + let config = Config::active(); + let anchor_root = Root::repeat_byte(1); + let mut store = store_anchored_at(anchor_root); + store + .insert_signed_block( + anchor_root, + gloas_block( + 0, + Root::ZERO, + ExecutionBlockHash::ZERO, + ExecutionBlockHash::ZERO, + ), + ) + .unwrap(); + + let a_root = Root::repeat_byte(2); + let a_slot = 1; + store + .insert_signed_block( + a_root, + gloas_block( + a_slot, + anchor_root, + ExecutionBlockHash::repeat_byte(0xff), + ExecutionBlockHash::repeat_byte(0xaa), + ), + ) + .unwrap(); + store.insert_verified_payload(a_root); + + let state = test_state::with_validators_at(ForkName::Gloas, 4); + store.insert_state(anchor_root, state.clone()).unwrap(); + store.insert_state(a_root, state).unwrap(); + + // Past `a_root`'s own slot, but not by exactly one, so weight (not + // the tiebreaker) decides. + store.set_time_ms(3 * config.slot_duration_ms).unwrap(); + + // Three of four votes go to the full branch, one to the empty + // branch. + for validator_index in 0..3 { + store.set_latest_message( + validator_index, + LatestMessage { + epoch: 0, + slot: a_slot + 1, + root: a_root, + payload_present: true, + }, + ); + } + store.set_latest_message( + 3, + LatestMessage { + epoch: 0, + slot: a_slot + 1, + root: a_root, + payload_present: false, + }, + ); + + let committees = CommitteeCache::default(); + let head = gloas_get_head(&store, &config, &committees).expect("known chain"); + assert_eq!( + head, + ForkChoiceNode { + root: a_root, + payload_status: PayloadStatus::Full, + }, + "three of four votes for the full branch must outweigh the lone \ + vote for empty" + ); + } + + #[test] + fn should_apply_proposer_boost_applies_when_the_parent_is_not_from_the_previous_slot() { + let config = Config::active(); + let (mut store, anchor_root, anchor_slot) = anchored_store(1); + let committees = CommitteeCache::default(); + + // Two slots ahead of the parent, not one: the first early return + // fires before `is_head_weak` is ever consulted. + let b_root = Root::repeat_byte(0xb0); + store + .insert_signed_block(b_root, block(anchor_slot + 2, anchor_root)) + .unwrap(); + store.set_proposer_boost_root(b_root); + + // Plant an equivocation the equivocation scan would find, were the + // first early return missing: with no votes at all, the parent is + // otherwise trivially weak (falling through to that scan), and a + // same-proposer, PTC-timely block at `anchor_slot + 1` (one slot + // behind `b_root`) matches its own condition. + let poison_root = Root::repeat_byte(0xb2); + store + .insert_signed_block(poison_root, block(anchor_slot + 1, Root::ZERO)) + .unwrap(); + store.set_block_timeliness(poison_root, [false, true]); + + assert!( + should_apply_proposer_boost(&store, &config, &committees).unwrap(), + "a parent more than one slot behind the boosted block always \ + keeps the boost, even with an equivocation in place that would \ + otherwise withhold it" + ); + } + + #[test] + fn should_apply_proposer_boost_applies_when_the_parent_is_not_weak() { + let config = Config::active(); + let (mut store, anchor_root, anchor_slot) = + anchored_store(preset::SLOTS_PER_EPOCH as usize); + let committees = CommitteeCache::default(); + + let b_root = Root::repeat_byte(0xb0); + store + .insert_signed_block(b_root, block(anchor_slot + 1, anchor_root)) + .unwrap(); + store.set_proposer_boost_root(b_root); + + // A single committee member's vote for the parent (the anchor) is + // already enough to clear the reorg threshold, the same reading + // `is_head_weak_counts_an_equivocating_validators_balance_from_the_ + // head_slot_committees` relies on for its own equivocator. + store.set_latest_message( + 0, + LatestMessage { + epoch: 0, + slot: anchor_slot, + root: anchor_root, + payload_present: false, + }, + ); + + // Also plant a same-slot, same-proposer, PTC-timely sibling of the + // parent: were the "parent is not weak" early return missing, the + // equivocation scan below would find this sibling and withhold the + // boost, flipping the assertion below. + let twin_root = Root::repeat_byte(0xb1); + store + .insert_signed_block(twin_root, block(anchor_slot, Root::ZERO)) + .unwrap(); + store.set_block_timeliness(twin_root, [false, true]); + + assert!( + should_apply_proposer_boost(&store, &config, &committees).unwrap(), + "a parent with enough votes to not be weak keeps the boost \ + without ever reaching the equivocation scan" + ); + } + + #[test] + fn should_apply_proposer_boost_still_applies_against_a_same_slot_sibling_from_a_different_proposer() + { + let config = Config::active(); + let (mut store, anchor_root, anchor_slot) = anchored_store(1); + let committees = CommitteeCache::default(); + + let b_root = Root::repeat_byte(0xb0); + store + .insert_signed_block(b_root, block(anchor_slot + 1, anchor_root)) + .unwrap(); + store.set_proposer_boost_root(b_root); + + // A same-slot sibling of the parent, PTC-timely, but proposed by a + // different validator than the parent's own (default) proposer: not + // an equivocation, since that requires the same proposer, so the + // boost still applies. + let twin_root = Root::repeat_byte(0xb1); + store + .insert_signed_block(twin_root, block_with_proposer(anchor_slot, Root::ZERO, 1)) + .unwrap(); + store.set_block_timeliness(twin_root, [false, true]); + + assert!( + should_apply_proposer_boost(&store, &config, &committees).unwrap(), + "a same-slot sibling from a different proposer is not an \ + equivocation, so the boost still applies" + ); + } + + #[test] + fn is_payload_verified_is_true_for_a_fulu_root() { + let mut store = empty_store(); + let a_root = Root::repeat_byte(0xa1); + store + .insert_signed_block(a_root, fulu_block(1, Root::ZERO)) + .unwrap(); + + assert!( + is_payload_verified(&store, a_root), + "a pre-gloas block's payload ran inside the block itself, with \ + no separate envelope left to verify" + ); + } + + #[test] + fn is_payload_verified_is_false_for_an_unknown_root() { + let store = empty_store(); + assert!( + !is_payload_verified(&store, Root::repeat_byte(0xff)), + "a root this store has never heard of must not be assumed \ + pre-gloas and so already verified" + ); + } + + #[test] + fn should_build_on_full_and_should_extend_payload_answer_without_error_for_a_previous_slot_pre_gloas_head() + { + let config = Config::active(); + let mut store = empty_store(); + let a_root = Root::repeat_byte(0xa1); + let a_slot = 1; + store + .insert_signed_block(a_root, fulu_block(a_slot, Root::ZERO)) + .unwrap(); + store + .set_time_ms((a_slot + 1) * config.slot_duration_ms) + .unwrap(); + + // `head` carries this crate's own decided status for a pre-gloas + // block (Full), per `get_head_node`'s pre-gloas arm. + let head = ForkChoiceNode { + root: a_root, + payload_status: PayloadStatus::Full, + }; + assert!( + should_build_on_full(&store, head, a_slot + 1).unwrap(), + "a pre-gloas head is always FULL and its payload always \ + verified, timely and available, so a proposer one slot later \ + must build on full" + ); + assert!( + should_extend_payload(&store, a_root, &config).unwrap(), + "a pre-gloas payload is always considered verified, timely and \ + available, so it must always be extended" + ); + } + + /// The gloas-side counterpart of `a_vote_for_a_pruned_block_weighs_ + /// nothing_instead_of_failing`: a stale voter's last message can name a + /// root pruned below the retained window on a live gloas chain too, and + /// `gloas_get_attestation_score` must skip it rather than abort the + /// whole weight computation. + #[test] + fn a_gloas_vote_for_a_root_not_in_the_store_weighs_nothing_instead_of_failing() { + let config = Config::active(); + let anchor_root = Root::repeat_byte(1); + let mut store = store_anchored_at(anchor_root); + store + .insert_signed_block( + anchor_root, + gloas_block( + 0, + Root::ZERO, + ExecutionBlockHash::ZERO, + ExecutionBlockHash::ZERO, + ), + ) + .unwrap(); + + let a_root = Root::repeat_byte(2); + let a_slot = 1; + store + .insert_signed_block( + a_root, + gloas_block( + a_slot, + anchor_root, + ExecutionBlockHash::repeat_byte(0xff), + ExecutionBlockHash::repeat_byte(0xaa), + ), + ) + .unwrap(); + + let state = test_state::with_validators_at(ForkName::Gloas, 2); + store.insert_state(anchor_root, state.clone()).unwrap(); + store.insert_state(a_root, state).unwrap(); + store.set_time_ms(3 * config.slot_duration_ms).unwrap(); + + // A live vote for `a_root`. + store.set_latest_message( + 0, + LatestMessage { + epoch: 0, + slot: a_slot + 1, + root: a_root, + payload_present: false, + }, + ); + // A vote for a root the store never indexed at all (pruned below + // the retained window). + store.set_latest_message( + 1, + LatestMessage { + epoch: 0, + slot: a_slot + 1, + root: Root::repeat_byte(0xde), + payload_present: false, + }, + ); + + let committees = CommitteeCache::default(); + let node = ForkChoiceNode { + root: a_root, + payload_status: PayloadStatus::Empty, + }; + let weight = gloas_get_weight(&store, node, &config, &committees) + .expect("a vote for a root not in the store must not be fatal"); + assert!(weight > 0, "the live vote still counts"); + } + + #[test] + fn should_apply_proposer_boost_reads_a_missing_block_timeliness_entry_as_not_timely() { + let config = Config::active(); + let (mut store, anchor_root, anchor_slot) = anchored_store(1); + let committees = CommitteeCache::default(); + + let b_root = Root::repeat_byte(0xb0); + store + .insert_signed_block(b_root, block(anchor_slot + 1, anchor_root)) + .unwrap(); + store.set_proposer_boost_root(b_root); + + // A same-slot, same-proposer sibling of the parent that would be an + // early equivocation if its timeliness were known, but whose + // `block_timeliness` entry was never recorded: the scratch is + // in-memory only and does not survive a restart. Deliberately no + // `store.set_block_timeliness(twin_root, ...)` call. + let twin_root = Root::repeat_byte(0xb1); + store + .insert_signed_block(twin_root, block(anchor_slot, Root::ZERO)) + .unwrap(); + + assert!( + should_apply_proposer_boost(&store, &config, &committees).unwrap(), + "a same-slot, same-proposer sibling with no recorded \ + timeliness must read as not timely, so it is not treated as \ + an equivocation and the boost still applies" + ); + } + + /// The direct case `get_attestation_score`'s own doc describes: a pruned + /// vote must not abort `is_head_weak`, which shares this function with + /// [`get_weight`] and, through `should_apply_proposer_boost`, with + /// [`gloas_get_weight`] too. + #[test] + fn is_head_weak_succeeds_with_a_pruned_vote() { + let config = Config::active(); + let (mut store, anchor_root, _anchor_slot) = + anchored_store(preset::SLOTS_PER_EPOCH as usize); + let committees = CommitteeCache::default(); + + // A vote for a root the store never indexed at all (pruned below + // the retained window). + store.set_latest_message( + 0, + LatestMessage { + epoch: 0, + slot: 0, + root: Root::repeat_byte(0xde), + payload_present: false, + }, + ); + + assert!( + is_head_weak(&store, anchor_root, &config, &committees).is_ok(), + "a pruned vote must not abort is_head_weak" + ); + } + + /// A vote for a block that is still indexed, on a branch whose parent the + /// index no longer holds: the ancestor walk leaves the index part-way + /// down, which must contribute nothing rather than abort `is_head_weak`. + #[test] + fn is_head_weak_succeeds_when_a_vote_walks_onto_a_pruned_parent() { + let config = Config::active(); + let (mut store, anchor_root, anchor_slot) = + anchored_store(preset::SLOTS_PER_EPOCH as usize); + let committees = CommitteeCache::default(); + + let orphan_root = Root::repeat_byte(0xd1); + store + .insert_signed_block(orphan_root, block(anchor_slot + 1, Root::repeat_byte(0xd0))) + .unwrap(); + store.set_latest_message( + 0, + LatestMessage { + epoch: 0, + slot: anchor_slot + 1, + root: orphan_root, + payload_present: false, + }, + ); + + assert!( + is_head_weak(&store, anchor_root, &config, &committees).is_ok(), + "a vote whose ancestor walk meets a pruned parent must not abort is_head_weak" + ); + } + + /// `gloas_get_weight` reaches pre-gloas `get_attestation_score` too, not + /// only its own `gloas_get_attestation_score`: it calls + /// `should_apply_proposer_boost` on every node it weighs, which calls + /// `is_head_weak` whenever the boosted block's parent is from the + /// previous slot, the ordinary case rather than a corner one. A pruned + /// vote reached that way must not abort `gloas_get_weight` or + /// `gloas_get_head` either. + #[test] + fn gloas_get_weight_tolerates_a_pruned_vote_reached_through_should_apply_proposer_boost() { + let config = Config::active(); + let (mut store, anchor_root, anchor_slot) = + anchored_store(preset::SLOTS_PER_EPOCH as usize); + let committees = CommitteeCache::default(); + + let b_root = Root::repeat_byte(0xb0); + store + .insert_signed_block(b_root, block(anchor_slot + 1, anchor_root)) + .unwrap(); + store + .insert_state( + b_root, + test_state::with_validators(preset::SLOTS_PER_EPOCH as usize), + ) + .unwrap(); + store.set_unrealized_justification(b_root, Checkpoint::default()); + store.set_proposer_boost_root(b_root); + + // No votes for the anchor (the boosted block's parent, one slot + // behind it) except a pruned one, so it is trivially weak, the same + // reading `is_head_weak_counts_an_equivocating_validators_balance_ + // from_the_head_slot_committees` gives, and `should_apply_proposer_ + // boost` reaches `is_head_weak(anchor_root)`. + store.set_latest_message( + 0, + LatestMessage { + epoch: 0, + slot: 0, + root: Root::repeat_byte(0xde), + payload_present: false, + }, + ); + + let node = ForkChoiceNode { + root: b_root, + payload_status: PayloadStatus::Full, + }; + gloas_get_weight(&store, node, &config, &committees) + .expect("a pruned vote reached through should_apply_proposer_boost must not be fatal"); + gloas_get_head(&store, &config, &committees) + .expect("the same tolerance must hold through the whole head walk"); + } } From 435480e2139c4e9705b8764b1373382332b18901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:27:38 -0300 Subject: [PATCH 022/132] feat(beacon): add the gloas state transition Ports gloas's process_block in the specification's order and wires it into the block dispatch, so the full state transition runs on gloas blocks. A gloas block carries no payload: it commits to a builder's bid, and the previous slot's payload is applied by process_parent_execution_payload, which runs before process_execution_payload_bid replaces the bid it reads. parent_slot is read from the latest block header first, since process_block_header overwrites it. process_block needs no execution engine on this path. A revealed payload is checked by verify_execution_payload_envelope (fork-choice.md): the envelope's signature, its consistency with the block it belongs to and the bid it fulfils, then the engine's verdict. It mutates nothing. The block header it rebuilds takes the state's cached state root when the state carries one, since this repository stores post-block states with that root filled in and the specification's own states leave it zero; hashing a stored state again would reject every valid envelope. The signature check reads the proposer's key for a self-build and the builder's key otherwise. The sanity, finality, random and transition suites now run for gloas. The sanity, operations and gossip runners share one case_config: a case's own config.yaml, with seconds_per_slot derived from SLOT_DURATION_MS, or else every fork up to the case's own scheduled at genesis. get_ptc reads the gloas fork epoch from the config, so a runner using the compiled-in default would reject a gloas case's attestations. Fork upgrades read only the next fork's epoch, which the fallback never changes, so pre-gloas cases run as before. Fork choice still refuses gloas blocks: gloas on_block, the payload envelope and the payload timeliness committee have no fork-choice handling yet. --- .../src/beacon/fork_choice.rs | 12 +- .../state_transition/src/beacon/stf/block.rs | 12 +- .../state_transition/src/beacon/stf/gloas.rs | 571 +++++++++++++++++- .../tests/beacon_spec/gossip.rs | 26 +- .../state_transition/tests/beacon_spec/mod.rs | 59 +- .../tests/beacon_spec/operations.rs | 26 +- .../tests/beacon_spec/sanity.rs | 5 +- 7 files changed, 632 insertions(+), 79 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 91a5a3f7c..82da78183 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -3642,11 +3642,13 @@ pub fn on_block( // ePBS (EIP-7732) moves the availability question from the block's // blob/column commitments to the separately gossiped execution // payload envelope; the check above the match on `signed_block` does - // not apply to a gloas block at all. `state_transition` below is the - // gloas gap this repository has not closed yet (Part B), so this - // returns the same `Error::UnsupportedForFork` any other - // fork-mismatched call in this crate does, rather than skipping the - // availability question with a silent `{}` arm. + // not apply to a gloas block at all. This arm refuses because gloas's + // own `on_block` is not ported yet, not because `state_transition` + // below is missing anything: envelope processing and the payload + // timeliness committee's vote still need their own fork-choice + // wiring, so this returns the same `Error::UnsupportedForFork` any + // other fork-mismatched call in this crate does, rather than + // skipping the availability question with a silent `{}` arm. SignedBeaconBlock::Gloas(_) => { return Err(Error::UnsupportedForFork { function: "fork_choice::on_block", diff --git a/crates/blockchain/state_transition/src/beacon/stf/block.rs b/crates/blockchain/state_transition/src/beacon/stf/block.rs index d7500ca75..bacb60dfa 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/block.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/block.rs @@ -31,7 +31,7 @@ use crate::beacon::helpers::misc::compute_signing_root; use crate::beacon::preset; use crate::beacon::primitives::{BlsSignature, HashTreeRoot as _, Root, Slot, ValidatorIndex}; -use super::{ExecutionEngine, bellatrix, capella, deneb, electra, fulu}; +use super::{ExecutionEngine, bellatrix, capella, deneb, electra, fulu, gloas}; /// Dispatches on the block's fork and runs that fork's own block processing. /// @@ -70,12 +70,10 @@ pub fn process_block( fulu::process_block(state, &signed.message, config, engine, committees) } // ePBS (EIP-7732) restructures block processing around a builder's - // bid rather than an embedded payload; a later task ports it. - containers::SignedBeaconBlock::Gloas(_) => { - Err(crate::beacon::error::Error::UnsupportedForFork { - function: "process_block", - fork: crate::beacon::fork::ForkName::Gloas, - }) + // bid rather than an embedded payload: a gloas block carries no + // payload for `engine` to answer for, so this arm never reaches it. + containers::SignedBeaconBlock::Gloas(signed) => { + gloas::process_block(state, &signed.message, config, committees) } containers::SignedBeaconBlock::Lean(_) => { crate::beacon::lean_block_unreachable("process_block") diff --git a/crates/blockchain/state_transition/src/beacon/stf/gloas.rs b/crates/blockchain/state_transition/src/beacon/stf/gloas.rs index 0a97c9a6d..65615b528 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/gloas.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/gloas.rs @@ -1,5 +1,6 @@ -//! Gloas's withdrawals, execution payload bid, and parent execution payload -//! processing (EIP-7732). +//! Gloas's block processing, withdrawals, execution payload bid, parent +//! execution payload processing, and execution payload envelope verification +//! (EIP-7732). //! //! Withdrawals gain two new sweeps of their own: builders can be paid out //! ([`get_builder_withdrawals`], draining @@ -71,10 +72,62 @@ use crate::beacon::helpers::mutators::{decrease_balance, increase_balance, slash use crate::beacon::helpers::predicates::is_slashable_attestation_data; use crate::beacon::preset; use crate::beacon::primitives::{ - ExecutionAddress, Gwei, HashTreeRoot as _, ParticipationFlags, Root, Slot, ValidatorIndex, - WithdrawalIndex, + Bytes32, ExecutionAddress, Gwei, HashTreeRoot as _, ParticipationFlags, Root, Slot, + ValidatorIndex, WithdrawalIndex, }; +use super::ExecutionEngine; + +// --------------------------------------------------------------------------- +// Block processing +// --------------------------------------------------------------------------- + +/// `process_block` (gloas `beacon-chain.md`). +/// +/// No execution payload is embedded in a gloas block, unlike every earlier +/// fork's own `process_block`: the body only commits to a builder's bid +/// ([`process_execution_payload_bid`]), and the payload itself is verified +/// separately, once revealed, by [`verify_execution_payload_envelope`]. So, +/// unlike [`crate::beacon::stf::fulu::process_block`] and its own siblings, +/// this function takes no [`ExecutionEngine`]. +/// +/// `parent_slot` is read from `state.latest_block_header` before anything +/// else runs: [`process_parent_execution_payload`], the very next step, +/// processes the payload that header's own slot names, and +/// [`super::block::process_block_header`], two steps after that, overwrites +/// the header with this block's own. Reading it any later would read this +/// block's slot instead of its parent's. +pub fn process_block( + state: &mut BeaconState, + block: &gloas::BeaconBlock, + config: &Config, + committees: &CommitteeCache, +) -> Result<()> { + // [New in Gloas:EIP7732] + let parent_slot = state.latest_block_header().slot; + + // [New in Gloas:EIP7732] + process_parent_execution_payload(state, block, config)?; + super::block::process_block_header( + state, + block.slot, + block.proposer_index, + block.parent_root, + block.body.hash_tree_root(), + )?; + // [Modified in Gloas:EIP7732] + process_withdrawals(state)?; + // [Modified in Gloas:EIP7732] Removed `process_execution_payload`. + // [New in Gloas:EIP7732] + process_execution_payload_bid(state, &block.body.signed_execution_payload_bid, config)?; + super::block::process_randao(state, &block.body.randao_reveal)?; + super::block::process_eth1_data(state, &block.body.eth1_data)?; + // [Modified in Gloas:EIP7732] + process_operations(state, &block.body, parent_slot, config, committees)?; + super::altair::process_sync_aggregate(state, &block.body.sync_aggregate)?; + Ok(()) +} + // --------------------------------------------------------------------------- // Withdrawals // --------------------------------------------------------------------------- @@ -455,16 +508,15 @@ pub fn process_withdrawals(state: &mut BeaconState) -> Result<()> { /// /// The specification files this under "Execution payload", not "Operations": /// it is read by `execution_engine.verify_and_notify_new_payload`'s own -/// `NewPayloadRequest.execution_requests`, in `verify_execution_payload_envelope` -/// (gloas `fork-choice.md`), built from the revealed envelope's own -/// `execution_requests`. Not from `state.latest_execution_payload_bid`: the -/// bid commits to that list only by its hash -/// ([`gloas::ExecutionPayloadBid::execution_requests_root`]), it never carries -/// the list itself. Envelope verification is not transcribed yet, so nothing -/// calls this today, the same place electra's own +/// `NewPayloadRequest.execution_requests`, in [`verify_execution_payload_envelope`], +/// built from the revealed envelope's own `execution_requests`. Not from +/// `state.latest_execution_payload_bid`: the bid commits to that list only +/// by its hash ([`gloas::ExecutionPayloadBid::execution_requests_root`]), it +/// never carries the list itself. Nothing calls this even now that envelope +/// verification is transcribed, the same place electra's own /// [`crate::beacon::stf::electra::get_execution_requests_list`] already is /// (see [`crate::beacon::stf::deneb::process_execution_payload`]'s own -/// documentation for why: [`crate::beacon::stf::ExecutionEngine`] collapses the whole +/// documentation for why: [`ExecutionEngine`] collapses the whole /// `verify_and_notify_new_payload` interface to one boolean and never /// inspects the list either implementation builds). pub fn get_execution_requests_list(requests: &gloas::ExecutionRequests) -> Vec> { @@ -509,6 +561,163 @@ pub fn get_execution_requests_list(requests: &gloas::ExecutionRequests) -> Vec` rather than a bare `bool`, the same deviation +/// [`verify_execution_payload_bid_signature`]'s own doc explains: a +/// self-build reads a real validator index +/// (`state.latest_block_header.proposer_index`) into `state.validators`, +/// and a non-self-build one reads `builder_index` into `state.builders`; +/// either can be out of range for a caller that has not already checked it, +/// which this reports rather than folding into "signature invalid". +pub fn verify_execution_payload_envelope_signature( + state: &BeaconState, + signed_envelope: &gloas::SignedExecutionPayloadEnvelope, +) -> Result { + let builder_index = signed_envelope.message.builder_index; + let pubkey = if builder_index == constants::BUILDER_INDEX_SELF_BUILD { + let validator_index = state.latest_block_header().proposer_index; + state.validator(validator_index)?.pubkey + } else { + let inner = gloas_state_ref(state, "verify_execution_payload_envelope_signature")?; + inner + .builders + .get(builder_index as usize) + .ok_or(Error::IndexOutOfBounds { + index: builder_index as usize, + len: inner.builders.len(), + })? + .pubkey + }; + + let domain = get_domain(state, constants::DOMAIN_BEACON_BUILDER, None); + let signing_root = compute_signing_root(signed_envelope.message.hash_tree_root(), domain); + Ok(bls::verify( + &pubkey, + signing_root, + &signed_envelope.signature, + )) +} + +/// `verify_execution_payload_envelope` (gloas `fork-choice.md`). +/// +/// A pure verification helper: it mutates nothing, matching the +/// specification's own note that `process_execution_payload` has been +/// replaced by this function plus the deferred [`apply_parent_execution_payload`] +/// (see this module's own doc). Called once a builder's envelope for the +/// slot's committed bid has been received, in addition to the checks +/// [`process_execution_payload_bid`] already made against the bid itself: +/// this checks the *envelope* is the one thing that bid actually promised. +/// +/// The engine check has the same collapsed shape +/// [`crate::beacon::stf::fulu::process_execution_payload`]'s own: see +/// [`ExecutionEngine`]'s own doc for why the versioned hashes below are +/// computed but never themselves checked against anything beyond the +/// engine's own opaque verdict. +pub fn verify_execution_payload_envelope( + state: &BeaconState, + signed_envelope: &gloas::SignedExecutionPayloadEnvelope, + config: &Config, + engine: &ExecutionEngine, +) -> Result<()> { + let envelope = &signed_envelope.message; + let payload = &envelope.payload; + + // Verify signature. + verify( + verify_execution_payload_envelope_signature(state, signed_envelope)?, + "verify_execution_payload_envelope: verify_execution_payload_envelope_signature(state, signed_envelope)", + )?; + + // Verify consistency with the beacon block. + // + // `BeaconState::compute_state_root`, not a bare `hash_tree_root`. The + // specification's own `header.state_root = hash_tree_root(state)` is + // exactly right for a state fresh out of block processing, whose own + // `latest_block_header.state_root` is still left zero (see + // `super::process_slot`'s own doc for why). But this function's + // real caller, `on_execution_payload_envelope`, hands in a *stored* + // state, and this repository caches the real root into that same field + // right after `state_transition` returns (`fork_choice::on_block`, the + // checkpoint anchor), not one slot later the way a raw spec state would. + // Hashing such a state unconditionally would hash a header whose own + // `state_root` is already set, a different value from the one the + // header actually committed to when it was still zero, so every stored + // state's envelope would be rejected. `compute_state_root` returns the + // cached value directly once it is set and only falls back to a fresh + // hash while it is still zero, so both shapes resolve to the same root. + let mut header = state.latest_block_header().clone(); + header.state_root = state.compute_state_root(); + verify( + envelope.beacon_block_root == header.hash_tree_root(), + "verify_execution_payload_envelope: envelope.beacon_block_root == hash_tree_root(header)", + )?; + verify( + envelope.parent_beacon_block_root == state.latest_block_header().parent_root, + "verify_execution_payload_envelope: envelope.parent_beacon_block_root == state.latest_block_header.parent_root", + )?; + + // Verify consistency with the committed bid. + let inner = gloas_state_ref(state, "verify_execution_payload_envelope")?; + let bid = &inner.latest_execution_payload_bid; + verify( + envelope.builder_index == bid.builder_index, + "verify_execution_payload_envelope: envelope.builder_index == bid.builder_index", + )?; + verify( + payload.prev_randao == bid.prev_randao, + "verify_execution_payload_envelope: payload.prev_randao == bid.prev_randao", + )?; + verify( + payload.gas_limit == bid.gas_limit, + "verify_execution_payload_envelope: payload.gas_limit == bid.gas_limit", + )?; + verify( + payload.block_hash == bid.block_hash, + "verify_execution_payload_envelope: payload.block_hash == bid.block_hash", + )?; + verify( + envelope.execution_requests.hash_tree_root() == bid.execution_requests_root, + "verify_execution_payload_envelope: hash_tree_root(envelope.execution_requests) == bid.execution_requests_root", + )?; + + // Verify the execution payload is valid. + verify( + payload.slot_number == state.slot(), + "verify_execution_payload_envelope: payload.slot_number == state.slot", + )?; + verify( + payload.parent_hash == inner.latest_block_hash, + "verify_execution_payload_envelope: payload.parent_hash == state.latest_block_hash", + )?; + verify( + payload.timestamp + == super::bellatrix::compute_timestamp_at_slot(state, state.slot(), config), + "verify_execution_payload_envelope: payload.timestamp == compute_time_at_slot(state, state.slot)", + )?; + verify( + payload.withdrawals.hash_tree_root() == inner.payload_expected_withdrawals.hash_tree_root(), + "verify_execution_payload_envelope: hash_tree_root(payload.withdrawals) == hash_tree_root(state.payload_expected_withdrawals)", + )?; + + // Compute versioned hashes. + let _versioned_hashes: Vec = bid + .blob_kzg_commitments + .iter() + .map(super::deneb::kzg_commitment_to_versioned_hash) + .collect(); + + verify( + engine.execution_valid, + "verify_execution_payload_envelope: execution_engine.verify_and_notify_new_payload(\ + NewPayloadRequest(execution_payload=payload, versioned_hashes=versioned_hashes, \ + parent_beacon_block_root=envelope.parent_beacon_block_root, \ + execution_requests=envelope.execution_requests))", + )?; + + Ok(()) +} + // --------------------------------------------------------------------------- // Execution payload bid // --------------------------------------------------------------------------- @@ -1405,7 +1614,9 @@ pub fn process_operations( #[cfg(test)] mod tests { use super::*; + use crate::beacon::containers::BeaconBlockHeader; use crate::beacon::fork::ForkName; + use crate::beacon::primitives::{BlsPubkey, BlsSignature, ExecutionBlockHash, Uint256}; fn gloas_state_with_validators(count: usize) -> BeaconState { crate::beacon::helpers::test_state::with_validators_at(ForkName::Gloas, count) @@ -1682,4 +1893,340 @@ mod tests { ); } } + + // ----------------------------------------------------------------------- + // verify_execution_payload_envelope + // ----------------------------------------------------------------------- + + /// A syntactically valid but otherwise arbitrary [`gloas::ExecutionPayload`], + /// for a test that only cares about the envelope wrapping it, not the + /// payload's own contents. [`gloas::ExecutionPayload`] has no + /// `#[derive(Default)]` (see its own doc: `logs_bloom` has no meaningful + /// empty value), so every field is filled by hand instead. + fn arbitrary_execution_payload() -> gloas::ExecutionPayload { + gloas::ExecutionPayload { + parent_hash: ExecutionBlockHash::ZERO, + fee_recipient: ExecutionAddress::ZERO, + state_root: Bytes32::ZERO, + receipts_root: Bytes32::ZERO, + logs_bloom: crate::beacon::containers::bellatrix::LogsBloom::try_from(vec![ + 0u8; + preset::BYTES_PER_LOGS_BLOOM + ]) + .expect("built at exactly BYTES_PER_LOGS_BLOOM"), + prev_randao: Bytes32::ZERO, + block_number: 0, + gas_limit: 0, + gas_used: 0, + timestamp: 0, + extra_data: Default::default(), + base_fee_per_gas: Uint256::ZERO, + block_hash: ExecutionBlockHash::ZERO, + transactions: Default::default(), + withdrawals: Default::default(), + blob_gas_used: 0, + excess_blob_gas: 0, + block_access_list: Default::default(), + slot_number: 0, + } + } + + /// A self-build gloas state and the unsigned message an envelope for its + /// current slot must match field for field to pass every check in + /// [`verify_execution_payload_envelope`] but the engine's own. + /// + /// Returned unsigned and paired with the proposer's real secret key, so + /// a test can tamper with exactly one field before signing: signing + /// *after* the tamper keeps the signature real, over the tampered + /// message, rather than merely malformed, which is what lets a test + /// isolate the one assertion downstream of the signature check that the + /// tamper is meant to trip. `tweak_bid` runs before the state's own + /// fields (and so its `hash_tree_root`) are finalized, for a tamper that + /// must land in the committed bid itself; `tweak_message` runs after, + /// against the otherwise-consistent message, for one that must not. + fn consistent_envelope_message( + tweak_bid: impl FnOnce(&mut gloas::ExecutionPayloadBid), + tweak_message: impl FnOnce(&mut gloas::ExecutionPayloadEnvelope), + ) -> ( + BeaconState, + Config, + blst::min_pk::SecretKey, + gloas::ExecutionPayloadEnvelope, + ) { + let mut state = gloas_state_with_validators(1); + let secret = crate::beacon::helpers::test_state::secret_key_for(0); + state.validator_mut(0).unwrap().pubkey = BlsPubkey(secret.sk_to_pk().to_bytes()); + + let slot = state.slot(); + *state.latest_block_header_mut() = BeaconBlockHeader { + slot, + proposer_index: 0, + parent_root: Root::repeat_byte(0x11), + state_root: Root::ZERO, + body_root: Root::repeat_byte(0x22), + }; + + let config = Config::mainnet(); + let parent_hash = ExecutionBlockHash::repeat_byte(0x33); + let execution_requests = gloas::ExecutionRequests::default(); + let mut bid = gloas::ExecutionPayloadBid { + parent_block_hash: ExecutionBlockHash::ZERO, + parent_block_root: Root::ZERO, + block_hash: ExecutionBlockHash::repeat_byte(0x44), + prev_randao: Bytes32::repeat_byte(0x55), + fee_recipient: ExecutionAddress::ZERO, + gas_limit: 30_000_000, + builder_index: constants::BUILDER_INDEX_SELF_BUILD, + slot, + value: 0, + execution_payment: 0, + blob_kzg_commitments: Default::default(), + execution_requests_root: execution_requests.hash_tree_root(), + }; + tweak_bid(&mut bid); + + { + let inner = gloas_state(&mut state, "test").unwrap(); + inner.latest_block_hash = parent_hash; + inner.latest_execution_payload_bid = bid.clone(); + inner.payload_expected_withdrawals = gloas::Withdrawals::default(); + } + // `validators` is the tree-backed field `validator_mut` above + // buffered a write against; flush it before `state.compute_state_root()` + // below, the same way `gossip::test_support::fulu_parent` does. + state.apply_pending_mutations(); + + // The same `compute_state_root` call `verify_execution_payload_envelope` + // itself now makes; see that function's own doc for why. The header's + // own `state_root` is still zero at this point, so this falls to a + // fresh `hash_tree_root`, matching a state fresh out of block + // processing. + let mut header = state.latest_block_header().clone(); + header.state_root = state.compute_state_root(); + let beacon_block_root = header.hash_tree_root(); + + let payload = gloas::ExecutionPayload { + parent_hash, + fee_recipient: ExecutionAddress::ZERO, + state_root: Bytes32::ZERO, + receipts_root: Bytes32::ZERO, + logs_bloom: crate::beacon::containers::bellatrix::LogsBloom::try_from(vec![ + 0u8; + preset::BYTES_PER_LOGS_BLOOM + ]) + .expect("built at exactly BYTES_PER_LOGS_BLOOM"), + prev_randao: bid.prev_randao, + block_number: 0, + gas_limit: bid.gas_limit, + gas_used: 0, + timestamp: crate::beacon::stf::bellatrix::compute_timestamp_at_slot( + &state, slot, &config, + ), + extra_data: Default::default(), + base_fee_per_gas: Uint256::ZERO, + block_hash: bid.block_hash, + transactions: Default::default(), + withdrawals: gloas::Withdrawals::default(), + blob_gas_used: 0, + excess_blob_gas: 0, + block_access_list: Default::default(), + slot_number: slot, + }; + + let mut message = gloas::ExecutionPayloadEnvelope { + payload, + execution_requests, + builder_index: constants::BUILDER_INDEX_SELF_BUILD, + beacon_block_root, + parent_beacon_block_root: state.latest_block_header().parent_root, + }; + tweak_message(&mut message); + + (state, config, secret, message) + } + + /// Signs `message` the way a builder (or, for a self-build, the + /// proposer) would: over `hash_tree_root(message)` under + /// `DOMAIN_BEACON_BUILDER`, matching + /// [`verify_execution_payload_envelope_signature`] exactly. + fn sign_envelope( + state: &BeaconState, + secret: &blst::min_pk::SecretKey, + message: gloas::ExecutionPayloadEnvelope, + ) -> gloas::SignedExecutionPayloadEnvelope { + let domain = get_domain(state, constants::DOMAIN_BEACON_BUILDER, None); + let signing_root = compute_signing_root(message.hash_tree_root(), domain); + let signature = BlsSignature( + secret + .sign(signing_root.as_slice(), bls::DST, &[]) + .to_bytes(), + ); + gloas::SignedExecutionPayloadEnvelope { message, signature } + } + + #[test] + fn a_consistent_envelope_only_fails_the_engine_check() { + let (state, config, secret, message) = consistent_envelope_message(|_| {}, |_| {}); + let signed = sign_envelope(&state, &secret, message); + + assert!( + matches!( + verify_execution_payload_envelope( + &state, + &signed, + &config, + &ExecutionEngine::invalid() + ), + Err(Error::SpecAssert( + "verify_execution_payload_envelope: execution_engine.verify_and_notify_new_payload(\ + NewPayloadRequest(execution_payload=payload, versioned_hashes=versioned_hashes, \ + parent_beacon_block_root=envelope.parent_beacon_block_root, \ + execution_requests=envelope.execution_requests))" + )) + ), + "every check but the engine's should have already passed" + ); + verify_execution_payload_envelope(&state, &signed, &config, &ExecutionEngine::valid()) + .expect("a consistent envelope passes every check, including a valid engine's"); + } + + #[test] + fn a_stored_states_cached_state_root_still_matches_the_envelope() { + // `consistent_envelope_message` builds the state the way a state + // fresh out of block processing looks: `latest_block_header.state_root` + // still zero. This repository's stored states do not stay that way: + // `fork_choice::on_block` (and the checkpoint anchor) cache the real + // root into that same field right after `state_transition` returns, + // which is what `verify_execution_payload_envelope`'s only real + // caller, `on_execution_payload_envelope`, actually hands in. + let (mut state, config, secret, message) = consistent_envelope_message(|_| {}, |_| {}); + let signed = sign_envelope(&state, &secret, message); + + // Computed, not asserted: `compute_state_root` on this still-zero + // state falls to a fresh hash, the same value + // `consistent_envelope_message` already built `envelope.beacon_block_root` + // from. Caching it into the header mimics the stored shape without + // changing what that value actually is. + let cached_root = state.compute_state_root(); + state.latest_block_header_mut().state_root = cached_root; + + verify_execution_payload_envelope(&state, &signed, &config, &ExecutionEngine::valid()) + .expect("a stored state's cached header.state_root must still match the envelope"); + } + + #[test] + fn a_wrong_beacon_block_root_is_rejected() { + let (state, config, secret, message) = consistent_envelope_message( + |_| {}, + |message| message.beacon_block_root = Root::repeat_byte(0xee), + ); + // Signed *after* the tamper, so the signature itself still checks + // out: what should fail is the block-root consistency check, not + // the signature. + let signed = sign_envelope(&state, &secret, message); + + let result = + verify_execution_payload_envelope(&state, &signed, &config, &ExecutionEngine::valid()); + assert!( + matches!( + result, + Err(Error::SpecAssert( + "verify_execution_payload_envelope: envelope.beacon_block_root == hash_tree_root(header)" + )) + ), + "expected the beacon_block_root check to fail, got {result:?}" + ); + } + + #[test] + fn a_wrong_builder_index_is_rejected() { + // Only the committed bid's own `builder_index` disagrees; the + // envelope stays self-build-consistent (signed by the proposer, + // `builder_index == BUILDER_INDEX_SELF_BUILD`), so the signature + // check still passes and the mismatch is isolated to the one + // assertion it is meant to trip. `bid.builder_index` is otherwise + // unread by `verify_execution_payload_envelope`. + let (state, config, secret, message) = + consistent_envelope_message(|bid| bid.builder_index = 0, |_| {}); + let signed = sign_envelope(&state, &secret, message); + + let result = + verify_execution_payload_envelope(&state, &signed, &config, &ExecutionEngine::valid()); + assert!( + matches!( + result, + Err(Error::SpecAssert( + "verify_execution_payload_envelope: envelope.builder_index == bid.builder_index" + )) + ), + "expected the builder_index check to fail, got {result:?}" + ); + } + + // ----------------------------------------------------------------------- + // verify_execution_payload_envelope_signature + // ----------------------------------------------------------------------- + + #[test] + fn a_builder_signed_envelope_passes_the_signature_check() { + let mut state = gloas_state_with_validators(1); + let secret = crate::beacon::helpers::test_state::secret_key_for(1); + let builder_index: gloas::BuilderIndex = { + let inner = gloas_state(&mut state, "test").unwrap(); + inner.builders.push(gloas::Builder { + pubkey: BlsPubkey(secret.sk_to_pk().to_bytes()), + ..Default::default() + }); + (inner.builders.len() - 1) as gloas::BuilderIndex + }; + state.apply_pending_mutations(); + + let message = gloas::ExecutionPayloadEnvelope { + payload: arbitrary_execution_payload(), + execution_requests: gloas::ExecutionRequests::default(), + builder_index, + beacon_block_root: Root::ZERO, + parent_beacon_block_root: Root::ZERO, + }; + let signed = sign_envelope(&state, &secret, message); + + assert!( + verify_execution_payload_envelope_signature(&state, &signed).unwrap(), + "a builder's own signature over an envelope naming its own index must verify" + ); + } + + #[test] + fn a_self_build_envelope_signed_with_a_builders_key_is_rejected() { + let mut state = gloas_state_with_validators(1); + // The proposer's own key, distinct from the builder's below, so a + // self-build envelope genuinely needs it rather than accepting any + // signature. + let proposer_secret = crate::beacon::helpers::test_state::secret_key_for(0); + state.validator_mut(0).unwrap().pubkey = BlsPubkey(proposer_secret.sk_to_pk().to_bytes()); + *state.latest_block_header_mut() = BeaconBlockHeader { + proposer_index: 0, + ..Default::default() + }; + state.apply_pending_mutations(); + + let builder_secret = crate::beacon::helpers::test_state::secret_key_for(1); + let message = gloas::ExecutionPayloadEnvelope { + payload: arbitrary_execution_payload(), + execution_requests: gloas::ExecutionRequests::default(), + builder_index: constants::BUILDER_INDEX_SELF_BUILD, + beacon_block_root: Root::ZERO, + parent_beacon_block_root: Root::ZERO, + }; + // Signed with the builder's key, not the proposer's: a self-build + // envelope must verify against the proposer named in + // `state.latest_block_header.proposer_index`, so this signature must + // not check out. + let signed = sign_envelope(&state, &builder_secret, message); + + assert!( + !verify_execution_payload_envelope_signature(&state, &signed).unwrap(), + "a self-build envelope signed by a builder, not the proposer, must not verify" + ); + } } diff --git a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs index 45188d54f..0ef599b16 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs @@ -30,7 +30,7 @@ use ethlambda_storage::ForkCheckpoints; use libssz::SszDecode; use libtest_mimic::{Failed, Trial}; -use super::{Case, PRESET, collect, collect_all_handlers, fork_active_from_genesis}; +use super::{Case, PRESET, collect, collect_all_handlers}; /// The handlers this runner covers, and runs. const HANDLERS: &[&str] = &[ @@ -171,22 +171,20 @@ fn decode_signed_aggregate(case: &Case, name: &str) -> Result Config { - let mut config: Config = case - .yaml_opt("config") - .unwrap_or_else(|| fork_active_from_genesis(case)); + let mut config = super::case_config(case); config.genesis_time = state.genesis_time(); - // Newer configs name only `SLOT_DURATION_MS`; without `SECONDS_PER_SLOT` - // the field would keep mainnet's default. - config.seconds_per_slot = config.slot_duration_ms / 1000; config } diff --git a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs index 79ac88e9f..ce7d310be 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs @@ -76,16 +76,15 @@ pub const PRESET: &str = if cfg!(feature = "preset-minimal") { /// The compiled-in preset with every fork up to and including `case.fork` /// pulled back to genesis, never a later one. /// -/// The fallback the gossip and operations runners use when a case carries no -/// `config.yaml` of its own to read a fork schedule from (see [`gossip`]'s -/// own `case_config`). Those vectors are generated as if their own fork, and -/// every one before it, had activated at genesis: the ones that ship a -/// `config.yaml` set exactly that, apart from deliberate pre-fork cases such -/// as `payload_attestation`'s `process_payload_attestation_pre_fork_epoch`, -/// which schedules the fork later. Never a later fork: pulling forward one this -/// case does not itself reach would let `Config::fork_at_epoch` resolve a -/// state the case never claims to be, on any config a later runner builds -/// this way. +/// The fallback [`case_config`] uses when a case carries no `config.yaml` of +/// its own to read a fork schedule from. Those vectors are generated as if +/// their own fork, and every one before it, had activated at genesis: the +/// ones that ship a `config.yaml` set exactly that, apart from deliberate +/// pre-fork cases such as `payload_attestation`'s +/// `process_payload_attestation_pre_fork_epoch`, which schedules the fork +/// later. Never a later fork: pulling forward one this case does not itself +/// reach would let `Config::fork_at_epoch` resolve a state the case never +/// claims to be, on any config a later runner builds this way. pub fn fork_active_from_genesis(case: &Case) -> Config { ForkName::ALL .into_iter() @@ -96,6 +95,39 @@ pub fn fork_active_from_genesis(case: &Case) -> Config { }) } +/// The config to process one case against: its own `config.yaml` when it +/// ships one, [`fork_active_from_genesis`]'s all-forks-at-genesis default +/// otherwise. Shared by [`gossip`], [`operations`] and [`sanity`], the three +/// runners whose cases can carry a fork schedule (or, for gossip, a blob +/// schedule) of their own. +/// +/// The fork schedule matters in these suites because gloas's `get_ptc`, +/// reached through attestation and payload attestation processing, reads +/// `config.gloas_fork_epoch` for the specification's `assert epoch >= +/// GLOAS_FORK_EPOCH`. [`Config::active`] alone leaves that epoch at +/// `FAR_FUTURE_EPOCH`, which would reject every gloas case's attestations. +/// The case's own file is read first, rather than always overridden, because +/// deliberate pre-fork cases depend on it: `payload_attestation`'s +/// `process_payload_attestation_pre_fork_epoch` sets `GLOAS_FORK_EPOCH: 1` to +/// exercise that assertion's rejection path. +/// +/// `seconds_per_slot` is corrected from `slot_duration_ms` unconditionally, +/// whichever branch supplied `config`: per the consensus-specs tests format +/// README, a case's `config.yaml` may name only `SLOT_DURATION_MS`, the field +/// newer configs use, and leaving `seconds_per_slot` alone would keep +/// whatever [`Config`]'s own `#[serde(default)]` gives it +/// ([`Config::mainnet`]'s `12`, via `Config::default`) rather than the value +/// that config's own slot duration actually implies. [`fork_active_from_genesis`]'s +/// own default already keeps the two fields in agreement, so the correction +/// is a no-op on that branch. +pub fn case_config(case: &Case) -> Config { + let mut config: Config = case + .yaml_opt("config") + .unwrap_or_else(|| fork_active_from_genesis(case)); + config.seconds_per_slot = config.slot_duration_ms / 1000; + config +} + /// The newest fork whose state transition this crate implements. /// /// Every runner gates on this, and a case past it becomes an ignored test @@ -127,9 +159,6 @@ pub const HIGHEST_IMPLEMENTED_FORK: ForkName = ForkName::Fulu; /// even though [`HIGHEST_IMPLEMENTED_FORK`] has not reached Gloas yet; every /// other runner's Gloas cases stay ignored. Emptied, and removed along with /// this check, when [`HIGHEST_IMPLEMENTED_FORK`] becomes `Gloas`. -/// -/// `"transition"` is not listed: its Gloas cases also exercise Gloas block -/// processing, which does not exist yet. pub const GLOAS_RUNNERS: &[&str] = &[ "ssz_static", "fork", @@ -139,6 +168,10 @@ pub const GLOAS_RUNNERS: &[&str] = &[ // runner name serves all of it, the same way `"ssz_static"` already // does for its own runner. "operations", + "sanity", + "finality", + "random", + "transition", ]; /// The root of the extracted fixture tree. diff --git a/crates/blockchain/state_transition/tests/beacon_spec/operations.rs b/crates/blockchain/state_transition/tests/beacon_spec/operations.rs index 9162c2340..927d0a4ff 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/operations.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/operations.rs @@ -143,9 +143,7 @@ use ethlambda_state_transition::beacon::stf::gloas as gloas_stf; use ethlambda_state_transition::beacon::stf::{ExecutionEngine, block, operations}; use libtest_mimic::{Failed, Trial}; -use super::{ - Case, PRESET, collect_all_handlers, fork_active_from_genesis, lean_is_not_a_fixture_fork, -}; +use super::{Case, PRESET, collect_all_handlers, lean_is_not_a_fixture_fork}; /// The `{execution_valid: bool}` an `execution_payload` case ships alongside /// its block, standing in for whatever a real execution client would have @@ -630,26 +628,6 @@ fn apply( outcome.map_err(|err| format!("{err:?}")) } -/// The config to process one case's operation against. -/// -/// Most cases carry no `config.yaml` of their own, and -/// [`fork_active_from_genesis`]'s all-forks-at-genesis default serves -/// them; see its own doc for why that is the right fallback. Only -/// `gloas_stf::get_ptc` (reached through `process_payload_attestation`) reads -/// a fork-epoch config field directly in this whole suite -/// (`config.gloas_fork_epoch`, for the spec's own `assert epoch >= -/// GLOAS_FORK_EPOCH`), which is why gloas's own `payload_attestation` cases -/// are the ones that need a real answer here rather than the default: every -/// one of them ships a full `config.yaml`, and -/// `process_payload_attestation_pre_fork_epoch` specifically sets -/// `GLOAS_FORK_EPOCH: 1` to exercise that assertion's rejection path, so -/// reading the case's own file first (rather than always overriding) is what -/// keeps that case's expected rejection intact. -fn case_config(case: &Case) -> Config { - case.yaml_opt("config") - .unwrap_or_else(|| fork_active_from_genesis(case)) -} - pub fn trials() -> Vec { let cases = collect_all_handlers(PRESET, "operations"); let mut trials = vec![super::discovery_trial("operations", cases.len())]; @@ -659,7 +637,7 @@ pub fn trials() -> Vec { let mut state = BeaconState::from_ssz(case.fork, &case.ssz_bytes("pre")) .map_err(|err| format!("the fixture's pre-state does not decode: {err:?}"))?; - let config = case_config(case); + let config = super::case_config(case); let outcome = apply(&handler, case, &mut state, &config); super::check_transition(case, outcome, &state) })); diff --git a/crates/blockchain/state_transition/tests/beacon_spec/sanity.rs b/crates/blockchain/state_transition/tests/beacon_spec/sanity.rs index eb18e67a8..224a9f5ef 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/sanity.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/sanity.rs @@ -13,8 +13,6 @@ //! Unlike the `operations` suite, these run the full [`state_transition`], so //! they check the proposer signature and the committed `state_root` too. -use std::sync::Arc; - use ethlambda_state_transition::beacon::config::Config; use ethlambda_state_transition::beacon::containers::{BeaconState, SignedBeaconBlock}; use ethlambda_state_transition::beacon::helpers::accessors::CommitteeCache; @@ -80,15 +78,14 @@ fn apply_slots(case: &Case, state: &mut BeaconState, config: &Config) -> Result< /// Builds one runner and handler pair's trials, since all four share this shape. fn run(runner: &str, handler: &str, slots: bool) -> Vec { - let config = Arc::new(Config::active()); let cases = collect(PRESET, runner, handler); let mut trials = vec![super::discovery_trial(runner, cases.len())]; for case in cases { - let config = Arc::clone(&config); trials.push(super::case_trial(runner, case, move |case| { let mut state = BeaconState::from_ssz(case.fork, &case.ssz_bytes("pre")) .map_err(|err| format!("the fixture's pre-state does not decode: {err:?}"))?; + let config = super::case_config(case); let outcome = if slots { apply_slots(case, &mut state, &config) From bf7cf122f3c1549c798fce268c21eae064ada48c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:45:32 -0300 Subject: [PATCH 023/132] feat(beacon): add the gloas fork-choice handlers Fork choice had gloas's node model, head and proposer-head functions but no handler that could put a gloas block, envelope or vote into the store, so nothing could exercise them end to end. This adds the handlers. - get_forkchoice_store accepts a gloas anchor and seeds block timeliness, the two payload-attestation vote vectors and an empty verified-payload set for it. The live follower still cannot follow gloas (nothing delivers envelopes or payload attestations), so the refusal moves to startup in main.rs, ahead of both store constructions and the import benchmark's, and keeps its distinct error. - on_block handles a gloas block: a block on its parent's full branch needs that payload verified, availability is not checked at the block (it gates the envelope), the vote vectors are initialized, the block's payload attestations reach the store, and the two timeliness deadlines are recorded. A failing head computation fails the import, as it does before gloas. - on_execution_payload_envelope checks data availability against the bid's commitments, verifies the envelope, and records the payload as verified. It takes the sampled gloas sidecars directly rather than a DataAvailability value, so block evidence and envelope evidence cannot be confused; the deneb and fulu availability checks now reject the other fork's evidence instead of reading it as available. - on_payload_attestation_message and notify_ptc_messages write a member's vote into every committee seat it holds. - validate_on_attestation and update_latest_messages apply gloas's index rules and slot ordering to a gloas attestation and leave earlier forks as they were. Attestation and AttesterSlashing gain gloas variants, and block_operations reads a gloas block's own attestations and slashings. - Gloas column sidecars get their own structural and KZG checks, sharing the cell batch verification with fulu's. Which rules apply is decided in one place, ForkRules::of, by an exhaustive match over forks. The earlier "is this block pre-gloas" tests were negative matches on the gloas variant, which would have sent any later fork down the pre-gloas path; they now go through the same decision. Comments and messages that said the state transition or on_block refuse gloas blocks are corrected. --- bin/ethlambda/src/beacon.rs | 7 +- bin/ethlambda/src/benchmark/import/replay.rs | 3 + bin/ethlambda/src/checkpoint_sync.rs | 6 +- bin/ethlambda/src/main.rs | 146 +- bin/ethlambda/src/network/mod.rs | 18 +- crates/blockchain/src/lib.rs | 25 +- .../src/beacon/fork_choice.rs | 1351 ++++++++++++++--- .../src/beacon/gossip/block.rs | 22 +- docs/cli.md | 7 +- 9 files changed, 1290 insertions(+), 295 deletions(-) diff --git a/bin/ethlambda/src/beacon.rs b/bin/ethlambda/src/beacon.rs index 3b15d23a4..0528e5724 100644 --- a/bin/ethlambda/src/beacon.rs +++ b/bin/ethlambda/src/beacon.rs @@ -194,9 +194,10 @@ pub fn wire_params( // restart there recomputes the digest and resubscribes to the right // topics, the same as any other boundary, but it does not make this // build any more able to import a gloas block or cross a gloas epoch - // boundary (`state_transition` refuses both). Suggesting a restart - // "crosses" it would read as a stall a restart clears, which this is - // not. + // boundary (the chain actor refuses every gloas block, since nothing + // delivers payload envelopes or payload attestations to it yet). + // Suggesting a restart "crosses" it would read as a stall a restart + // clears, which this is not. match next_fork_boundary(&chain, epoch) { Some(boundary) if boundary == chain.gloas_fork_epoch => warn!( boundary_epoch = boundary, diff --git a/bin/ethlambda/src/benchmark/import/replay.rs b/bin/ethlambda/src/benchmark/import/replay.rs index 104e8dea7..a6a267675 100644 --- a/bin/ethlambda/src/benchmark/import/replay.rs +++ b/bin/ethlambda/src/benchmark/import/replay.rs @@ -107,6 +107,9 @@ pub(crate) async fn replay_corpus(dir: &Path, options: &ReplayOptions) -> eyre:: options.data_dir.display() ) })?); + // The same refusal startup makes, so a gloas corpus fails with its real + // reason rather than partway through replay. + crate::refuse_unfollowable_fork(anchor_state.fork_name())?; let store = fork_choice::get_forkchoice_store(backend, anchor_state, anchor_block, &config)?; // A clone, not a borrow: `Store::set_time_ms` writes to the shared // backend's metadata, so this clone's write is the write every other diff --git a/bin/ethlambda/src/checkpoint_sync.rs b/bin/ethlambda/src/checkpoint_sync.rs index 6eb50c154..882d49be3 100644 --- a/bin/ethlambda/src/checkpoint_sync.rs +++ b/bin/ethlambda/src/checkpoint_sync.rs @@ -180,9 +180,9 @@ pub enum CheckpointSyncError { BlockHeaderJustifiedRootMismatch, #[error("anchor block does not match anchor state")] AnchorPairingMismatch, - /// `get_forkchoice_store` refused the anchor outright because this build - /// cannot process its fork yet (currently only gloas): a real anchor, not - /// a peer serving a mismatched pair, so it is named separately from + /// Startup refused the anchor outright because this build cannot follow + /// its fork yet (currently only gloas): a real anchor, not a peer serving + /// a mismatched pair, so it is named separately from /// [`Self::AnchorPairingMismatch`] rather than folded into it. #[error( "this build cannot follow {fork} yet; the anchor is at {fork} and nothing past this \ diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index 1cab37343..2ce1be2b9 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -1691,21 +1691,35 @@ fn first_config_difference(persisted: &Config, supplied: &Config) -> Option checkpoint_sync::CheckpointSyncError { - match err { - ethlambda_state_transition::beacon::Error::UnsupportedForFork { fork, .. } => { - checkpoint_sync::CheckpointSyncError::UnsupportedFork { fork } - } - _ => checkpoint_sync::CheckpointSyncError::AnchorPairingMismatch, +/// Refuses an anchor state in `fork` if this node cannot follow that fork yet, +/// currently gloas. +/// +/// `fork_choice::get_forkchoice_store` accepts a gloas anchor, since fork choice +/// itself handles the fork. This node's wiring does not: nothing delivers +/// payload envelopes or payload attestations to the chain actor, and +/// `process_or_pend_block` refuses every gloas block, so a follower anchored +/// here would sit at its anchor forever, looking alive while importing +/// nothing. Refusing at startup reports the real reason instead. Checked ahead +/// of any store construction, on both anchor sources (a loaded network's +/// genesis state can schedule `GLOAS_FORK_EPOCH: 0`, and a checkpoint provider +/// can serve a gloas finalized state), so a rejected anchor writes nothing to +/// the data directory. +/// +/// Keeps the refusal distinguishable from a peer serving a mismatched anchor +/// pair: reporting both as `AnchorPairingMismatch` would tell an operator to +/// look for a bad peer when the real answer is "wait for this build to support +/// the fork". +fn refuse_unfollowable_fork(fork: ForkName) -> Result<(), checkpoint_sync::CheckpointSyncError> { + match fork { + ForkName::Gloas => Err(checkpoint_sync::CheckpointSyncError::UnsupportedFork { fork }), + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Fulu => Ok(()), + ForkName::Lean => unreachable!("a beacon anchor is never ForkName::Lean"), } } @@ -1866,6 +1880,8 @@ async fn fetch_initial_beacon_state( }; let state = loaded.genesis_state.as_ref().clone(); + refuse_unfollowable_fork(state.fork_name()) + .inspect_err(|err| error!(%err, "Cannot anchor at this network's genesis state"))?; let block = genesis_anchor_block(&state); info!( genesis_time = genesis.genesis_time, @@ -1874,7 +1890,7 @@ async fn fetch_initial_beacon_state( ); return fork_choice::get_forkchoice_store(backend, state, block, &config) .inspect_err(|err| error!(%err, "Failed to initialize store from the genesis state")) - .map_err(anchor_store_error); + .map_err(|_| checkpoint_sync::CheckpointSyncError::AnchorPairingMismatch); } info!(?checkpoint_urls, "Starting beacon checkpoint sync"); @@ -1896,9 +1912,11 @@ async fn fetch_initial_beacon_state( "Beacon checkpoint sync complete" ); + refuse_unfollowable_fork(state.fork_name()) + .inspect_err(|err| error!(%err, "Cannot anchor at the checkpoint provider's state"))?; fork_choice::get_forkchoice_store(backend, state, block, &config) .inspect_err(|err| error!(%err, "Failed to initialize store from anchor state and block")) - .map_err(anchor_store_error) + .map_err(|_| checkpoint_sync::CheckpointSyncError::AnchorPairingMismatch) } /// The block the specification pairs with a genesis anchor state. @@ -2150,6 +2168,27 @@ mod tests { assert!(validate_aggregate_subnet_ids(Some(&[]), 4).is_ok()); } + /// Fork choice accepts a gloas anchor, so the node has to be the one to + /// refuse it: nothing here delivers the payload envelopes a gloas chain + /// needs. Every fork this node does follow must still be let through. + #[test] + fn startup_refuses_a_gloas_anchor_and_only_a_gloas_anchor() { + assert!(matches!( + refuse_unfollowable_fork(ForkName::Gloas), + Err(checkpoint_sync::CheckpointSyncError::UnsupportedFork { + fork: ForkName::Gloas + }) + )); + for fork in ForkName::ALL { + if fork != ForkName::Gloas { + assert!( + refuse_unfollowable_fork(fork).is_ok(), + "{fork} must stay followable" + ); + } + } + } + /// Validator-config snippet matching `lean-quickstart`'s ansible-devnet /// where networks share a non-default committee count. const VC_WITH_COMMITTEE_COUNT: &str = r#" @@ -2661,6 +2700,79 @@ validators: assert_eq!(head_slot, 0, "a genesis anchor is at slot 0"); } + /// A loaded network can schedule gloas at epoch 0, which makes its own + /// genesis state a gloas one. Fork choice accepts that anchor, so startup + /// has to be what refuses it, with the reason that names the fork, and + /// before anything is written to the data directory. + #[tokio::test] + async fn a_gloas_genesis_is_refused_before_the_store_is_built() { + use ethlambda_state_transition::beacon::config::Config; + use ethlambda_state_transition::beacon::upgrade::upgrade_state; + + // Every fork from altair on scheduled at epoch 0, in the file and in + // the config the genesis state is upgraded with. + let scheduled = [ + ForkName::Altair, + ForkName::Bellatrix, + ForkName::Capella, + ForkName::Deneb, + ForkName::Electra, + ForkName::Fulu, + ForkName::Gloas, + ]; + let fixture = std::fs::read_to_string( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/networks/devnet/config.yaml"), + ) + .unwrap(); + let config_text: String = fixture + .lines() + .map(|line| { + match scheduled.iter().find(|fork| { + let key = format!("{}_FORK_EPOCH:", fork.as_str().to_uppercase()); + line.starts_with(&key) + }) { + Some(fork) => format!("{}_FORK_EPOCH: 0", fork.as_str().to_uppercase()), + None => line.to_string(), + } + }) + .collect::>() + .join("\n"); + + let mut config = Config::mainnet(); + let mut state = beacon::mainnet_genesis_state().unwrap(); + for fork in scheduled { + config = config.with_fork_epoch(fork, 0); + state = upgrade_state(&state, fork, &config).unwrap(); + } + assert_eq!(state.fork_name(), ForkName::Gloas); + + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("config.yaml"), config_text).unwrap(); + std::fs::write(dir.path().join("genesis.ssz"), state.to_ssz()).unwrap(); + let loaded = network::dir::NetworkDir::load(dir.path()).unwrap(); + let source = network::NetworkSource::Loaded(Box::new(loaded)); + let backend: Arc = Arc::new(InMemoryBackend::new()); + + // `Store` is not `Debug`, so take the error by pattern. + let Err(err) = fetch_initial_beacon_state(&[], backend.clone(), &source).await else { + panic!("a gloas genesis must not become an anchor"); + }; + assert!( + matches!( + err, + checkpoint_sync::CheckpointSyncError::UnsupportedFork { + fork: ForkName::Gloas + } + ), + "the refusal must name the fork: {err}" + ); + assert!( + Store::from_db_state(backend).unwrap().is_none(), + "a refused anchor leaves the directory empty" + ); + } + /// A changed fork epoch leaves genesis time and the validators root /// untouched, so `verify_state_genesis` cannot see it, while putting this /// node on a different chain from its peers from that epoch on. Resuming diff --git a/bin/ethlambda/src/network/mod.rs b/bin/ethlambda/src/network/mod.rs index 9f6495eae..29f2cbf49 100644 --- a/bin/ethlambda/src/network/mod.rs +++ b/bin/ethlambda/src/network/mod.rs @@ -240,22 +240,24 @@ fn derive_genesis_fields(config: &mut Config, genesis_time: u64) { /// (`ConfigFile::warn_about_ignored_keys`) no longer implies this the way it /// used to, now that `GLOAS_*` keys are claimed rather than reported as /// unknown, so this says it explicitly instead of leaving it to be -/// discovered as a stall: this build's state transition and fork choice -/// refuse every gloas block and epoch boundary, so a follower stops making -/// progress there regardless of how cleanly its config parsed. A loaded +/// discovered as a stall: the chain actor refuses every gloas block +/// (`process_or_pend_block`), because nothing delivers payload envelopes or +/// payload attestations to it yet, so a follower stops making progress there +/// regardless of how cleanly its config parsed. A loaded /// network reaches this the same as a built-in one, and can even schedule /// gloas at epoch 0, in which case its own genesis state already decodes as -/// gloas and `fork_choice::get_forkchoice_store` refuses it outright at -/// startup; this warning fires first either way. +/// gloas and startup refuses it outright (`refuse_unfollowable_fork` in +/// `main.rs`, since fork choice itself accepts a gloas anchor); this warning +/// fires first either way. fn warn_if_gloas_scheduled(network: &str, config: &Config) { if config.gloas_fork_epoch != ethlambda_types::beacon::constants::FAR_FUTURE_EPOCH { tracing::warn!( network, gloas_fork_epoch = config.gloas_fork_epoch, "This build stops following this chain at its gloas fork epoch; \ - crossing it is not a stall a restart clears, since state_transition \ - refuses every gloas block and epoch boundary until a later release \ - implements it" + crossing it is not a stall a restart clears, since the chain actor \ + refuses every gloas block until a later release delivers payload \ + envelopes and payload attestations to it" ); } } diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 5a075f1ee..35e4c3529 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -845,13 +845,12 @@ fn data_availability_for( // `blob_kzg_commitments`, so this gate has nothing gloas-shaped to // check yet. Unreachable from network input today: // `process_or_pend_block` refuses every gloas block before this - // function is ever called, precisely so a gloas block never reaches - // this gate, `fork_choice::block_operations`, or `on_block`'s own - // `Error::UnsupportedForFork`, none of which it can pay for - // productively. `None` here regardless, defensively: were that gate - // ever bypassed, holding the block rather than answering - // `NotRequired` keeps it from being treated as available and handed - // to `on_block` under a false pretense. + // function is ever called, because this node cannot yet deliver the + // payload envelopes and payload attestations a gloas chain needs. + // `on_block` itself accepts a gloas block and reads no availability + // evidence for it. `None` here regardless, defensively: were that + // gate ever bypassed, `None` keeps a block this node cannot follow + // out of `on_block` entirely. SignedBeaconBlock::Gloas(_) => None, // `process_block` dispatches a lean block to `store::on_block` before // this function is ever reached, so this arm is never observed for @@ -2595,10 +2594,11 @@ impl BlockChainServer { timings.guards_start = Some(Instant::now()); // Refused before anything else, including the columns/finalized-slot - // checks below: `state_transition` refuses every gloas block anyway - // (`Error::UnsupportedForFork`), so persisting one as pending and - // discovering that on the other end of a clone-parent-state-then-fail - // round trip is pure waste. Worse than waste were this not here + // checks below: this node cannot follow a gloas chain yet, since + // nothing delivers the payload envelopes a block's full branch needs + // (`fork_choice::on_execution_payload_envelope`) or the payload + // attestations that vote on them, so persisting a gloas block as + // pending is pure waste. Worse than waste were this not here // first: a gloas block whose parent is missing would otherwise be // `insert_pending_block`-ed and tracked in `pending_block_parents`, // and every later block naming it as an ancestor would walk back to @@ -2612,8 +2612,7 @@ impl BlockChainServer { warn!( %slot, block_root = %ShortRoot(&block_root.0), - "Refusing a gloas block: this build does not implement the gloas state \ - transition yet" + "Refusing a gloas block: this build cannot follow a gloas chain yet" ); self.discard_pending_subtree(block_root); return None; diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 82da78183..13384c1c4 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -6,16 +6,19 @@ //! `Store` therefore accepts a block from any fork this module implements, //! even though the algorithm applied to it is unconditionally phase0's. //! `Store` tracks the block tree, attester votes, and the checkpoints fork -//! choice reasons about; the four handlers at the bottom of this file -//! ([`on_tick`], [`on_block`], [`on_attestation`], [`on_attester_slashing`]) -//! are the only ones the specification lists as sole ways to change it, -//! matching its own framing: "Invalid calls to handlers must not modify -//! `store`." Most other functions in this file take `&Store`; the exceptions -//! are [`get_head`] (records the head it just computed; see its own -//! documentation for why that write belongs there rather than in a caller) -//! and `record_block_timeliness`/`update_proposer_boost_root`, which write -//! `store.block_timeliness`/`store.proposer_boost_root` on [`on_block`]'s -//! behalf: [`on_block`] is their only caller. +//! choice reasons about; the handlers at the bottom of this file +//! ([`on_tick`], [`on_block`], [`on_attestation`], [`on_attester_slashing`], +//! and, from gloas on, [`on_execution_payload_envelope`] and +//! [`on_payload_attestation_message`]) are the ones the specification lists as +//! sole ways to change it, matching its own framing: "Invalid calls to +//! handlers must not modify `store`." Most other functions in this file take +//! `&Store`; the exceptions are [`get_head`] (records the head it just +//! computed; see its own documentation for why that write belongs there +//! rather than in a caller), [`notify_ptc_messages`] (a helper of +//! [`on_block`] that applies payload attestations through +//! [`on_payload_attestation_message`]), and `update_proposer_boost_root`, +//! which writes `store.proposer_boost_root` on [`on_block`]'s behalf: +//! [`on_block`] is its only caller. //! //! # Units: one seconds-granularity clock, read out in milliseconds at the edges //! @@ -111,22 +114,26 @@ //! different concrete type, not just a wider bound on the same one. //! //! [`Attestation`] and [`AttesterSlashing`] mirror [`SignedBeaconBlock`]'s own -//! answer to that problem: an enum over the two shapes, with two variants -//! rather than one per fork for the same reason `SignedBeaconBlock` has only -//! seven, not one per fork through fulu. But fork choice reads far less out -//! of an attestation than a block: `data.slot`, `data.target`, -//! `data.beacon_block_root`, and the attesting indices, per the module -//! documentation above. `AttestationData` (`crate::beacon::containers::shared`) is -//! already fork-invariant, so every function below except the two enums' -//! own methods reads it directly rather than matching on a fork tag it does -//! not need: [`validate_on_attestation`] and [`update_latest_messages`] take -//! `AttestationData` and a resolved `&[ValidatorIndex]`, not an -//! [`Attestation`]. The one place a fork's own shape actually matters is -//! resolving an [`Attestation`] into the attesters it names and checking -//! their aggregate signature, which needs the fork-specific -//! `get_indexed_attestation`/`is_valid_indexed_attestation` pair -//! ([`crate::beacon::helpers::attestation`] for phase0, [`crate::beacon::helpers::electra`] -//! for electra); [`Attestation::verified_attesting_indices`] and +//! answer to that problem: an enum over the shapes, with one variant per +//! shape rather than one per fork: a fork that leaves a container unchanged +//! reuses the earlier one, as [`SignedBeaconBlock::Fulu`] wraps electra's. +//! But fork choice reads far less out of an attestation than a block: +//! `data.slot`, `data.target`, `data.beacon_block_root`, and the attesting +//! indices, per the module documentation above. `AttestationData` +//! (`crate::beacon::containers::shared`) is already fork-invariant, so every +//! function below except the two enums' own methods reads it directly rather +//! than matching on a fork tag it does not need: [`validate_on_attestation`] +//! and [`update_latest_messages`] take `AttestationData` and a resolved +//! `&[ValidatorIndex]`, not an [`Attestation`]. Gloas gives `data.index` a +//! meaning and orders votes by slot, so those two also take a [`ForkRules`] +//! value, which [`Attestation::rules`] derives from the variant. The one place +//! a fork's own shape otherwise matters is resolving an [`Attestation`] into +//! the attesters it names and checking their aggregate signature, which needs +//! the fork-specific `get_indexed_attestation`/`is_valid_indexed_attestation` +//! pair ([`crate::beacon::helpers::attestation`] for phase0, +//! [`crate::beacon::helpers::electra`] for electra, +//! [`crate::beacon::helpers::gloas`] for gloas); +//! [`Attestation::verified_attesting_indices`] and //! [`AttesterSlashing::verified_attesting_indices`] are where that dispatch //! happens, once, so [`on_attestation`] and [`on_attester_slashing`] //! themselves never match on a fork at all. @@ -176,6 +183,7 @@ use crate::beacon::helpers::accessors::{ }; use crate::beacon::helpers::attestation as phase0_attestation; use crate::beacon::helpers::electra as electra_helpers; +use crate::beacon::helpers::gloas as gloas_helpers; use crate::beacon::helpers::misc::{ compute_epoch_at_slot, compute_start_slot_at_epoch, is_valid_merkle_branch, }; @@ -213,14 +221,60 @@ pub use ethlambda_types::beacon::fork_choice::{ /// documentation for why this exists and what it lets the rest of this file /// stay generic over. /// -/// Two variants, not one per fork: every fork through deneb shares +/// Three variants, not one per fork: every fork through deneb shares /// [`phase0::Attestation`] outright, and fulu shares [`electra::Attestation`] /// the same way [`SignedBeaconBlock::Fulu`] shares -/// [`electra::SignedBeaconBlock`]. +/// [`electra::SignedBeaconBlock`]. Gloas has its own, since EIP-7688 makes its +/// `aggregation_bits` unbounded, which is a different Rust type. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Attestation { Phase0(phase0::Attestation), Electra(electra::Attestation), + Gloas(gloas::Attestation), +} + +/// Which fork's handler rules apply: to an attestation's `data.index` and to +/// the order of an attester's votes, and to a block or anchor's gloas-only +/// state. +/// +/// Before gloas, `data.index` is a committee index (zero from electra on) and +/// votes are compared by target epoch. Gloas repurposes `index` as the payload +/// flag (`0` for the empty branch or a same-slot vote, `1` for the full +/// branch), and its `LatestMessage` is compared by slot. For attesters who do +/// not equivocate, the slot order and the epoch order agree. +/// +/// Every value comes from an exhaustive match on a fork or a container +/// ([`ForkRules::of`], [`Attestation::rules`]), so a fork added after gloas has +/// to be given its rules there rather than falling into the pre-gloas ones. An +/// attestation's rules follow its own container, not the store's clock: an +/// attestation carried by a gloas block is a [`gloas::Attestation`] even when +/// its slot is in the last pre-gloas epoch. Such a vote has `index == 0`, which +/// both rule sets accept, and is then ordered the way gloas's own +/// `update_latest_messages` orders it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ForkRules { + /// Phase0 through fulu. + PreGloas, + /// Gloas (EIP-7732). + Gloas, +} + +impl ForkRules { + /// The rules a value of `fork` (a block, a state, or an anchor) is handled + /// under. Names every fork, so the next one has to be placed here. + pub fn of(fork: ForkName) -> Self { + match fork { + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Fulu => ForkRules::PreGloas, + ForkName::Gloas => ForkRules::Gloas, + ForkName::Lean => lean_fork_unreachable("ForkRules::of"), + } + } } impl Attestation { @@ -231,13 +285,23 @@ impl Attestation { match self { Attestation::Phase0(attestation) => attestation.data, Attestation::Electra(attestation) => attestation.data, + Attestation::Gloas(attestation) => attestation.data, + } + } + + /// The rules [`validate_on_attestation`] and [`update_latest_messages`] + /// apply to this attestation; see [`ForkRules`]. + pub fn rules(&self) -> ForkRules { + match self { + Attestation::Phase0(_) | Attestation::Electra(_) => ForkRules::PreGloas, + Attestation::Gloas(_) => ForkRules::Gloas, } } /// The attesters this attestation names, once its aggregate signature and /// index ordering have both been checked against `state`. /// - /// The one place this enum's two shapes actually matter: building the + /// The one place this enum's shapes actually matter: building the /// indexed form and checking it needs the fork-specific /// `get_indexed_attestation`/`is_valid_indexed_attestation` pair, so this /// dispatches once here rather than leaving that match to every caller. @@ -266,7 +330,7 @@ impl Attestation { self.indices(state, false, committees) } - /// The body both accessors above share: the one place this enum's two + /// The body both accessors above share: the one place this enum's /// shapes actually matter, since building the indexed form and checking it /// needs the fork-specific /// `get_indexed_attestation`/`is_valid_indexed_attestation` pair. Kept as @@ -301,17 +365,29 @@ impl Attestation { } Ok(indexed.attesting_indices.into_inner()) } + Attestation::Gloas(attestation) => { + let indexed = + gloas_helpers::get_indexed_attestation(state, attestation, committees)?; + if verify_signature { + verify( + gloas_helpers::is_valid_indexed_attestation(state, &indexed), + "is_valid_indexed_attestation(target_state, indexed_attestation)", + )?; + } + Ok(indexed.attesting_indices.iter().copied().collect()) + } } } } /// Evidence that a set of validators made two conflicting attestations, in /// whichever fork's shape it currently has. See [`Attestation`] for why this -/// has the same two variants and no more. +/// has the same variants and no more. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AttesterSlashing { Phase0(phase0::AttesterSlashing), Electra(electra::AttesterSlashing), + Gloas(gloas::AttesterSlashing), } impl AttesterSlashing { @@ -327,6 +403,9 @@ impl AttesterSlashing { AttesterSlashing::Electra(slashing) => { (slashing.attestation_1.data, slashing.attestation_2.data) } + AttesterSlashing::Gloas(slashing) => { + (slashing.attestation_1.data, slashing.attestation_2.data) + } } } @@ -393,6 +472,30 @@ impl AttesterSlashing { .collect(), )) } + AttesterSlashing::Gloas(slashing) => { + verify( + gloas_helpers::is_valid_indexed_attestation(state, &slashing.attestation_1), + "is_valid_indexed_attestation(state, attestation_1)", + )?; + verify( + gloas_helpers::is_valid_indexed_attestation(state, &slashing.attestation_2), + "is_valid_indexed_attestation(state, attestation_2)", + )?; + Ok(( + slashing + .attestation_1 + .attesting_indices + .iter() + .copied() + .collect(), + slashing + .attestation_2 + .attesting_indices + .iter() + .copied() + .collect(), + )) + } } } } @@ -404,97 +507,79 @@ impl AttesterSlashing { /// Lives here, beside the two enums it builds, because the fork-to-shape /// mapping is theirs: phase0 through deneb share /// [`phase0::Attestation`]/[`phase0::AttesterSlashing`], electra and fulu the -/// `electra` pair. Both consumers of a block's own operations, the chain actor -/// and the `fork_choice` fixture runner, read it from here, so a new fork -/// reshaping `body.attestations` cannot be handled in one and forgotten in the -/// other. +/// `electra` pair, and gloas its own. Both consumers of a block's own +/// operations, the chain actor and the `fork_choice` fixture runner, read it +/// from here, so a new fork reshaping `body.attestations` cannot be handled in +/// one and forgotten in the other. pub fn block_operations(block: &SignedBeaconBlock) -> (Vec, Vec) { match block { - SignedBeaconBlock::Electra(block) => ( - block - .message - .body - .attestations - .iter() - .cloned() - .map(Attestation::Electra) - .collect(), - block - .message - .body - .attester_slashings - .iter() - .cloned() - .map(AttesterSlashing::Electra) - .collect(), + SignedBeaconBlock::Phase0(block) => wrap_operations( + block.message.body.attestations.iter(), + block.message.body.attester_slashings.iter(), + Attestation::Phase0, + AttesterSlashing::Phase0, + ), + SignedBeaconBlock::Altair(block) => wrap_operations( + block.message.body.attestations.iter(), + block.message.body.attester_slashings.iter(), + Attestation::Phase0, + AttesterSlashing::Phase0, ), - SignedBeaconBlock::Fulu(block) => ( - block - .message - .body - .attestations - .iter() - .cloned() - .map(Attestation::Electra) - .collect(), - block - .message - .body - .attester_slashings - .iter() - .cloned() - .map(AttesterSlashing::Electra) - .collect(), + SignedBeaconBlock::Bellatrix(block) => wrap_operations( + block.message.body.attestations.iter(), + block.message.body.attester_slashings.iter(), + Attestation::Phase0, + AttesterSlashing::Phase0, ), - SignedBeaconBlock::Phase0(block) => phase0_operations( + SignedBeaconBlock::Capella(block) => wrap_operations( block.message.body.attestations.iter(), block.message.body.attester_slashings.iter(), + Attestation::Phase0, + AttesterSlashing::Phase0, ), - SignedBeaconBlock::Altair(block) => phase0_operations( + SignedBeaconBlock::Deneb(block) => wrap_operations( block.message.body.attestations.iter(), block.message.body.attester_slashings.iter(), + Attestation::Phase0, + AttesterSlashing::Phase0, ), - SignedBeaconBlock::Bellatrix(block) => phase0_operations( + SignedBeaconBlock::Electra(block) => wrap_operations( block.message.body.attestations.iter(), block.message.body.attester_slashings.iter(), + Attestation::Electra, + AttesterSlashing::Electra, ), - SignedBeaconBlock::Capella(block) => phase0_operations( + SignedBeaconBlock::Fulu(block) => wrap_operations( block.message.body.attestations.iter(), block.message.body.attester_slashings.iter(), + Attestation::Electra, + AttesterSlashing::Electra, ), - SignedBeaconBlock::Deneb(block) => phase0_operations( + SignedBeaconBlock::Gloas(block) => wrap_operations( block.message.body.attestations.iter(), block.message.body.attester_slashings.iter(), + Attestation::Gloas, + AttesterSlashing::Gloas, ), - // EIP-7688 makes a gloas block's own `attestations`/`attester_slashings` - // a `ProgressiveList`, which `on_block_attestation` and - // `on_attester_slashing` are not yet shaped to take, so this cannot - // decode them the way every other arm does. Unreachable from network - // input today: `crates/blockchain`'s `process_or_pend_block` refuses - // every gloas block before this is ever called, precisely so this - // arm, `data_availability_for`, and `on_block`'s own - // `Error::UnsupportedForFork` are never paid for on a block that - // cannot be imported regardless. Empty rather than a panic - // regardless, defensively: were that gate ever bypassed, this must - // not crash the actor on the way to `on_block`'s own refusal. - SignedBeaconBlock::Gloas(_) => (Vec::new(), Vec::new()), SignedBeaconBlock::Lean(_) => lean_block_unreachable("fork_choice::block_operations"), } } -/// Phase0 through deneb share one attestation and slashing shape, so their -/// five arms above share one body. +/// Wraps one fork's attestations and slashings in the fork-generic enums, given +/// the two variant constructors for that fork's shape. /// /// Takes iterators rather than the lists themselves: each fork's body names -/// its own `SszList` bound, so a parameter typed on the list would need one +/// its own list type and bound, so a parameter typed on the list would need one /// generic per bound, and `.iter()` erases exactly that difference. -fn phase0_operations<'a>( - attestations: impl Iterator, - slashings: impl Iterator, +fn wrap_operations<'a, A: Clone + 'a, S: Clone + 'a>( + attestations: impl Iterator, + slashings: impl Iterator, + wrap_attestation: fn(A) -> Attestation, + wrap_slashing: fn(S) -> AttesterSlashing, ) -> (Vec, Vec) { ( - attestations.cloned().map(Attestation::Phase0).collect(), - slashings.cloned().map(AttesterSlashing::Phase0).collect(), + attestations.cloned().map(wrap_attestation).collect(), + slashings.cloned().map(wrap_slashing).collect(), ) } @@ -905,28 +990,20 @@ pub fn checkpoint_state( /// "Trusted" means fork choice will never roll back past this point: a full /// client anchors at genesis, and a checkpoint-syncing client anchors at /// whatever finalized state and block it fetched instead. +/// +/// A gloas anchor is accepted and seeds the three gloas-only tables the way +/// gloas's own `get_forkchoice_store` does: both block timeliness deadlines +/// met, an empty payload-attestation vote vector for each of the two vote +/// kinds, and no verified payload. Whether a node can *follow* a gloas chain +/// from there is the caller's question, not this function's: the live follower +/// refuses a gloas anchor at startup, since its node wiring cannot deliver +/// payload envelopes yet. pub fn get_forkchoice_store( backend: Arc, mut anchor_state: BeaconState, anchor_block: SignedBeaconBlock, config: &Config, ) -> Result { - // Refused before any other check: a gloas anchor (checkpoint sync - // decodes one structurally today, and a loaded network scheduling - // `GLOAS_FORK_EPOCH: 0` reaches this from `genesis_anchor_block` too) - // would otherwise boot a store this build can never advance past: - // `state_transition` already refuses every gloas block and epoch - // boundary, so a node anchored here would sit at genesis forever, - // looking alive while importing nothing. Checked ahead of the - // fork-matching `verify` below so a gloas anchor reports its real reason - // rather than a spurious "fork mismatch". - if anchor_state.fork_name() == ForkName::Gloas { - return Err(Error::UnsupportedForFork { - function: "get_forkchoice_store", - fork: ForkName::Gloas, - }); - } - // The specification's `BeaconState` and `BeaconBlock` are already one // fork's own types, so a mismatch between them cannot even be expressed // there; here both are enums, so this module has to enforce the invariant @@ -938,6 +1015,7 @@ pub fn get_forkchoice_store( )?; let anchor_root = anchor_block.message_hash_tree_root(); + let anchor_rules = ForkRules::of(anchor_state.fork_name()); // The specification asserts `anchor_block.state_root == // hash_tree_root(anchor_state)`, which holds only when the anchor state is @@ -1029,6 +1107,20 @@ pub fn get_forkchoice_store( .expect("insert"); store.set_unrealized_justification(anchor_root, justified_checkpoint); + // [New in Gloas:EIP7732] `block_timeliness={anchor_root: [True, True]}`, + // both vote vectors seeded with `[None] * PTC_SIZE`, and `payloads={}` + // (nothing to insert: the anchor's own payload is not verified until an + // envelope arrives for it). Earlier forks record none of these for the + // anchor, and nothing reads them there. + match anchor_rules { + ForkRules::Gloas => { + store.set_block_timeliness(anchor_root, [true, true]); + store.set_payload_timeliness_vote(anchor_root, vec![None; preset::PTC_SIZE]); + store.set_payload_data_availability_vote(anchor_root, vec![None; preset::PTC_SIZE]); + } + ForkRules::PreGloas => {} + } + Ok(store) } @@ -2026,13 +2118,13 @@ pub fn get_proposer_head( /// Every function in this section assumes `block` is itself gloas-shaped, /// which does not hold at the fulu-to-gloas boundary on its own: the first /// gloas block's own parent is a fulu block with no bid at all. -/// [`get_parent_payload_status`], this function's only caller, never reaches -/// here for a pre-gloas block though: it answers [`PayloadStatus::Full`] for -/// one directly, per this crate's decided rule for that boundary (see the -/// module documentation's "Gloas: payload-aware fork choice" section). A -/// non-gloas block still reaching here is therefore a caller bug, reported -/// by name rather than assumed away, the same projection -/// `helpers::gloas::gloas_state_ref` gives a state. +/// [`get_parent_payload_status`] and [`on_execution_payload_envelope`] call +/// this. The first never reaches here for a pre-gloas block though: it +/// answers [`PayloadStatus::Full`] for one directly, per this crate's decided +/// rule for that boundary (see the module documentation's "Gloas: +/// payload-aware fork choice" section). A non-gloas block still reaching here +/// is therefore a caller bug, reported by name rather than assumed away, the +/// same projection `helpers::gloas::gloas_state_ref` gives a state. fn gloas_bid(block: &SignedBeaconBlock) -> Result<&gloas::ExecutionPayloadBid> { match block { SignedBeaconBlock::Gloas(inner) => { @@ -2070,22 +2162,31 @@ fn gloas_bid(block: &SignedBeaconBlock) -> Result<&gloas::ExecutionPayloadBid> { fn is_known_pre_gloas_block(store: &Store, root: Root) -> bool { matches!( store.get_signed_block(&root).expect("get"), - Some(block) if !matches!(block, SignedBeaconBlock::Gloas(_)) + Some(block) if is_pre_gloas(&block) ) } +/// Whether `block` is handled under the pre-gloas rules, from an exhaustive +/// match on [`ForkRules`], so a fork added after gloas has to be placed there +/// rather than being read as "not gloas, therefore pre-gloas" by each caller. +fn is_pre_gloas(block: &SignedBeaconBlock) -> bool { + match ForkRules::of(block.fork_name()) { + ForkRules::PreGloas => true, + ForkRules::Gloas => false, + } +} + /// `is_payload_verified` (gloas `fork-choice.md`): whether `root`'s execution /// payload envelope has been locally delivered and verified via -/// `on_execution_payload_envelope`, which does not exist in this crate yet: -/// nothing calls [`Store::insert_verified_payload`](ethlambda_storage::Store::insert_verified_payload) -/// until the handler that reads a delivered envelope lands. +/// [`on_execution_payload_envelope`], the only caller of +/// [`Store::insert_verified_payload`](ethlambda_storage::Store::insert_verified_payload). /// /// A pre-gloas block has no envelope to deliver at all: its payload ran /// inside the block itself, so this answers `true` for one unconditionally, /// per this crate's decided rule for the fulu-to-gloas boundary (see the /// module documentation's "Gloas: payload-aware fork choice" section). pub fn is_payload_verified(store: &Store, root: Root) -> bool { - is_known_pre_gloas_block(store, root) || store.has_verified_payload(&root) + store.has_verified_payload(&root) || is_known_pre_gloas_block(store, root) } /// `payload_timeliness` (gloas `fork-choice.md`): whether `root`'s payload is @@ -2170,7 +2271,7 @@ pub fn get_parent_payload_status( .get_signed_block(&block.parent_root()) .expect("get") .ok_or(Error::SpecAssert("block.parent_root in store.blocks"))?; - if !matches!(parent, SignedBeaconBlock::Gloas(_)) { + if is_pre_gloas(&parent) { return Ok(PayloadStatus::Full); } let parent_block_hash = gloas_bid(block)?.parent_block_hash; @@ -2295,7 +2396,7 @@ pub fn get_supported_node(store: &Store, message: LatestMessage) -> Result Option { } /// Records `pow_block` so later [`get_pow_block`] lookups by its own hash can -/// find it. Not one of the four handlers at the bottom of this file: there is +/// find it. Not one of the handlers at the bottom of this file: there is /// no validity condition to check first, since this only ever adds data a /// fixture suite's `on_merge_block` step already trusts. pub fn insert_pow_block(store: &mut Store, pow_block: PowBlock) { @@ -2960,7 +3061,12 @@ pub fn is_data_available_blobs( ) -> Result { let (blobs, proofs) = match evidence { DataAvailability::Blobs { blobs, proofs } => (blobs.as_slice(), proofs.as_slice()), - _ => (&[][..], &[][..]), + DataAvailability::NotRequired => (&[][..], &[][..]), + DataAvailability::Columns(_) => { + return Err(Error::SpecAssert( + "a deneb or electra block's data availability is judged on blobs", + )); + } }; let blob_slices: Vec<&[u8]> = blobs.iter().map(|blob| &blob[..]).collect(); kzg::verify_blob_kzg_proof_batch(&blob_slices, commitments, proofs) @@ -2994,18 +3100,73 @@ pub fn verify_data_column_sidecar(sidecar: &fulu::DataColumnSidecar, config: &Co /// `sidecar.index` as its cell index, since a column names one cell position /// across every blob in the block. pub fn verify_data_column_sidecar_kzg_proofs(sidecar: &fulu::DataColumnSidecar) -> Result { - let cell_indices = vec![sidecar.index; sidecar.column.len()]; - let mut cells = Vec::with_capacity(sidecar.column.len()); - for cell in sidecar.column.iter() { + verify_column_cells( + sidecar.index, + &sidecar.column, + &sidecar.kzg_commitments, + &sidecar.kzg_proofs, + ) +} + +/// The batch check both forks' `verify_data_column_sidecar_kzg_proofs` run. +/// +/// The one thing that differs between them is where `commitments` comes from: +/// fulu's sidecar carries them, and gloas's takes them from the bid of the +/// block it names. The cell check itself is the same, so it takes the four +/// slices and nothing about either container. +fn verify_column_cells( + index: u64, + column: &[fulu::Cell], + commitments: &[KzgCommitment], + proofs: &[KzgProof], +) -> Result { + let cell_indices = vec![index; column.len()]; + let mut cells = Vec::with_capacity(column.len()); + for cell in column { cells.push( c_kzg::Cell::from_bytes(&cell[..]) .map_err(|_| Error::SpecAssert("len(cell) == BYTES_PER_CELL"))?, ); } - kzg::verify_cell_kzg_proof_batch( - &sidecar.kzg_commitments, - &cell_indices, - &cells, + kzg::verify_cell_kzg_proof_batch(commitments, &cell_indices, &cells, proofs) +} + +/// `verify_data_column_sidecar` (gloas `p2p-interface.md`, modified): the +/// structural checks a gloas column sidecar must pass before its KZG proofs +/// are worth checking, against `kzg_commitments` from the bid of the block the +/// sidecar names. +/// +/// Fulu's version reads the commitments off the sidecar and bounds their count +/// by the blob schedule. Gloas's does neither: the commitments are the +/// caller's, so the length checks compare the column against them directly. +pub fn gloas_verify_data_column_sidecar( + sidecar: &gloas::DataColumnSidecar, + kzg_commitments: &[KzgCommitment], +) -> bool { + // The sidecar index must be within the valid range. + if sidecar.index as usize >= preset::NUMBER_OF_COLUMNS { + return false; + } + // A sidecar for zero blobs is invalid. + if sidecar.column.is_empty() { + return false; + } + // The column length must be equal to the number of commitments and proofs. + sidecar.column.len() == kzg_commitments.len() + && sidecar.column.len() == sidecar.kzg_proofs.len() +} + +/// `verify_data_column_sidecar_kzg_proofs` (gloas `p2p-interface.md`, +/// modified): [`verify_data_column_sidecar_kzg_proofs`] with `kzg_commitments` +/// supplied by the caller rather than carried by the sidecar. +pub fn gloas_verify_data_column_sidecar_kzg_proofs( + sidecar: &gloas::DataColumnSidecar, + kzg_commitments: &[KzgCommitment], +) -> Result { + verify_column_cells( + sidecar.index, + &sidecar.column, + kzg_commitments, &sidecar.kzg_proofs, ) } @@ -3066,8 +3227,14 @@ pub fn verify_data_column_sidecar_inclusion_proof(sidecar: &fulu::DataColumnSide /// than relying on this to do it, since nothing about an empty list is /// distinguishable here from "this block needed no sampling at all". pub fn is_data_available_columns(evidence: &DataAvailability, config: &Config) -> Result { - let DataAvailability::Columns(sidecars) = evidence else { - return Ok(true); + let sidecars = match evidence { + DataAvailability::Columns(sidecars) => sidecars, + DataAvailability::NotRequired => return Ok(true), + DataAvailability::Blobs { .. } => { + return Err(Error::SpecAssert( + "a fulu block's data availability is judged on column sidecars", + )); + } }; for sidecar in sidecars { if !(verify_data_column_sidecar(sidecar, config) @@ -3079,6 +3246,33 @@ pub fn is_data_available_columns(evidence: &DataAvailability, config: &Config) - Ok(true) } +/// `is_data_available` (gloas `fork-choice.md`, modified): every column +/// sidecar sampled for the payload must be individually valid against +/// `kzg_commitments`, the commitments on the bid of the block the payload +/// belongs to. +/// +/// `sidecars` and `kzg_commitments` are the two values the specification's +/// `retrieve_column_sidecars_and_kzg_commitments` returns. The caller reads +/// the commitments off the stored block instead, since the store already holds +/// that bid. An empty `sidecars` is that retrieval returning none, and +/// `all()` over an empty list holds, so a live caller must not pass an empty +/// slice for a payload it has not sampled. That is the only encoding of "no +/// sidecars": gloas evidence has no variant of [`DataAvailability`], so +/// neither an older fork's check nor this one can read the other's evidence. +pub fn is_data_available_gloas_columns( + sidecars: &[gloas::DataColumnSidecar], + kzg_commitments: &[KzgCommitment], +) -> Result { + for sidecar in sidecars { + if !(gloas_verify_data_column_sidecar(sidecar, kzg_commitments) + && gloas_verify_data_column_sidecar_kzg_proofs(sidecar, kzg_commitments)?) + { + return Ok(false); + } + } + Ok(true) +} + // --------------------------------------------------------------------------- // Pull-up tip helper // --------------------------------------------------------------------------- @@ -3136,55 +3330,50 @@ pub fn compute_pulled_up_tip( // on_block helpers // --------------------------------------------------------------------------- -/// Whether a block at `block_slot` arrived before its slot's attestation -/// deadline: [`record_block_timeliness`]'s own definition of timely, minus -/// the write. +/// `record_block_timeliness`'s verdict for a block at `block_slot`: whether it +/// arrived before each of its slot's two deadlines, +/// `[ATTESTATION_TIMELINESS_INDEX, PTC_TIMELINESS_INDEX]`, which is the value +/// [`on_block`] then stores as +/// [`Store::block_timeliness`](ethlambda_storage::Store::block_timeliness) for +/// [`update_proposer_boost_root`] and [`is_head_late`] to read back. /// -/// Split out so [`on_block`] can gate the (otherwise unconditional) -/// proposer-boost head computation on the same answer *before* the block -/// joins the store, without either recomputing the formula by hand or paying -/// to look the block back up once it has, the way [`record_block_timeliness`] -/// does. -fn is_block_timely(store: &Store, block_slot: Slot, config: &Config) -> bool { - let time_into_slot_ms = store.ms_since_genesis() % config.slot_duration_ms; - let epoch = get_current_store_epoch(store, config); - let attestation_threshold_ms = get_attestation_due_ms(epoch, config); - let is_before_attesting_interval = time_into_slot_ms < attestation_threshold_ms; - get_current_slot(store, config) == block_slot && is_before_attesting_interval -} - -/// Records whether `root`'s block, at `block_slot`, arrived before its slot's -/// attestation deadline: the timeliness [`update_proposer_boost_root`] and -/// [`is_head_late`] both read back out of -/// [`Store::block_timeliness`](ethlambda_storage::Store::block_timeliness). -/// -/// Takes `block_slot` directly rather than reading it back out of the block -/// [`on_block`] just inserted: `Store::block_entry` decodes the whole signed -/// block to answer it on a beacon directory (see its own documentation), and -/// the one caller here always already has the slot, from the block it just -/// inserted. -/// -/// `pub(crate)` rather than `pub`, unlike most functions in this file: unlike -/// [`is_block_timely`], which mirrors a piece of the specification's own -/// `record_block_timeliness`, this one's own signature (`block_slot` as a -/// parameter) is this crate's adaptation of it rather than a transcription, -/// and [`on_block`] is its only caller. -/// -/// Writes the same verdict into both of `store.block_timeliness[root]`'s -/// entries: this function only ever runs on a pre-gloas chain (gloas's own -/// `on_block` records its two deadlines separately, per -/// `record_block_timeliness`'s gloas-modified text), so it has no PTC -/// deadline of its own to tell apart from the attestation one, and -/// [`is_head_late`] only ever reads [`constants::ATTESTATION_TIMELINESS_INDEX`] -/// back regardless. -pub(crate) fn record_block_timeliness( - store: &mut Store, - root: Root, +/// Split from the write so [`on_block`] can gate the (otherwise unconditional) +/// proposer-boost head computation on the same answer *before* the block joins +/// the store, and take the slot from the block it already holds rather than +/// decoding it back out of the store (`Store::block_entry` decodes the whole +/// signed block on a beacon directory). +/// +/// Before gloas there is one deadline, the attestation one, so both entries +/// repeat its verdict: nothing before gloas tells a PTC deadline apart, and +/// [`is_head_late`] only reads the attestation entry. Gloas modifies the +/// function: a block is timely for a deadline only if it arrived in its own +/// slot and before that deadline, and the two deadlines are gloas's own +/// attestation deadline ([`gloas_get_attestation_due_ms`]) and the payload +/// timeliness committee's ([`get_payload_attestation_due_ms`]). +fn block_timeliness( + store: &Store, block_slot: Slot, + rules: ForkRules, config: &Config, -) { - let is_timely = is_block_timely(store, block_slot, config); - store.set_block_timeliness(root, [is_timely, is_timely]); +) -> [bool; 2] { + let time_into_slot_ms = store.ms_since_genesis() % config.slot_duration_ms; + let is_current_slot = get_current_slot(store, config) == block_slot; + match rules { + ForkRules::PreGloas => { + let epoch = get_current_store_epoch(store, config); + let attestation_threshold_ms = get_attestation_due_ms(epoch, config); + let is_timely = is_current_slot && time_into_slot_ms < attestation_threshold_ms; + [is_timely, is_timely] + } + ForkRules::Gloas => { + let attestation_threshold_ms = gloas_get_attestation_due_ms(config); + let ptc_threshold_ms = get_payload_attestation_due_ms(config); + [ + is_current_slot && time_into_slot_ms < attestation_threshold_ms, + is_current_slot && time_into_slot_ms < ptc_threshold_ms, + ] + } + } } /// The first slot of the lookahead window `epoch`'s proposer shuffling opens @@ -3244,6 +3433,13 @@ fn get_ancestor_or_lowest_indexed( /// ancestor of `root` at [`compute_shuffling_dependent_slot`], or the lowest /// indexed ancestor on `root`'s chain if that slot is not covered; see /// [`get_ancestor_or_lowest_indexed`] for why the second case is sound. +/// +/// Serves gloas unchanged. Gloas's `get_shuffling_dependent_root` walks from a +/// `PENDING` node with its payload-aware `get_ancestor` and returns the root of +/// the node it lands on; that walk picks each step by `block.slot` alone (see +/// [`gloas_get_checkpoint_block`]), so the root is the one this index walk +/// answers, and the payload status the gloas walk threads through is +/// discarded. pub fn get_shuffling_dependent_root( index: &HashMap, root: Root, @@ -3261,16 +3457,16 @@ pub fn get_shuffling_dependent_root( /// the boost cannot itself be the thing that drags the head onto a branch with /// a different, no-longer-relevant view of who was supposed to propose it. /// -/// `head` must be the head [`compute_head`] found *before* `root` joined the -/// store: [`on_block`] is this function's only caller, and it passes exactly -/// that. `index` should include `root`'s own entry (`on_block` extends its -/// own pre-insertion index with it rather than re-scanning `LiveChain`), or -/// the walk below falls back to whatever the closest indexed ancestor -/// answers, per [`get_ancestor_or_lowest_indexed`]. +/// `head` must be the head [`compute_head`] (pre-gloas) or [`gloas_get_head`] +/// (gloas) found *before* `root` joined the store: [`on_block`] is this +/// function's only caller, and it passes exactly that. `index` should include +/// `root`'s own entry (`on_block` extends its own pre-insertion index with it +/// rather than re-scanning `LiveChain`), or the walk below falls back to +/// whatever the closest indexed ancestor answers, per +/// [`get_ancestor_or_lowest_indexed`]. /// -/// `pub(crate)`, not `pub`: like [`record_block_timeliness`], this one's -/// `index` parameter is this crate's own adaptation, not a transcription, and -/// [`on_block`] is its only caller. +/// `pub(crate)`, not `pub`: this one's `index` parameter is this crate's own +/// adaptation, not a transcription, and [`on_block`] is its only caller. pub(crate) fn update_proposer_boost_root( store: &mut Store, index: &HashMap, @@ -3281,7 +3477,7 @@ pub(crate) fn update_proposer_boost_root( let is_first_block = store.proposer_boost_root().is_zero(); let is_timely = store .block_timeliness(&root) - .expect("record_block_timeliness always runs first, on the same root") + .expect("on_block records the block's timeliness before this runs, on the same root") [constants::ATTESTATION_TIMELINESS_INDEX]; let epoch = get_current_store_epoch(store, config); let head_dependent_root = get_shuffling_dependent_root(index, head, epoch); @@ -3341,7 +3537,7 @@ pub fn on_tick_per_slot(store: &mut Store, time: u64, config: &Config) { /// Takes `data` directly rather than an [`Attestation`]: this and /// [`validate_on_attestation`] read nothing from an attestation besides its /// fork-invariant `data`, so neither needs to know which of -/// [`Attestation`]'s two shapes the caller actually has. See the module +/// [`Attestation`]'s shapes the caller actually has. See the module /// documentation. pub fn validate_target_epoch_against_current_time( store: &Store, @@ -3365,13 +3561,24 @@ pub fn validate_target_epoch_against_current_time( /// Every check `on_attestation` requires before it may look up or update /// anything in `store`. See [`validate_target_epoch_against_current_time`] /// for why this takes `data` rather than an [`Attestation`]. +/// +/// `rules` selects gloas's `data.index` rules where the attestation is a gloas +/// one; see [`ForkRules`]. pub fn validate_on_attestation( store: &Store, data: AttestationData, + rules: ForkRules, is_from_block: bool, config: &Config, ) -> Result<()> { - validate_on_attestation_indexed(store, data, is_from_block, config, &store.block_index()) + validate_on_attestation_indexed( + store, + data, + rules, + is_from_block, + config, + &store.block_index(), + ) } /// [`validate_on_attestation`] against an already-built block index. @@ -3384,6 +3591,7 @@ pub fn validate_on_attestation( fn validate_on_attestation_indexed( store: &Store, data: AttestationData, + rules: ForkRules, is_from_block: bool, config: &Config, index: &HashMap, @@ -3418,7 +3626,28 @@ fn validate_on_attestation_indexed( "store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot", )?; - // LMD vote must be consistent with FFG vote target. + // [New in Gloas:EIP7732] `index` is the payload flag, not a committee index. + if rules == ForkRules::Gloas { + verify(data.index <= 1, "attestation.data.index in [0, 1]")?; + if head_block_slot == data.slot { + verify( + data.index == 0, + "attestation.data.index == 0 for a same-slot vote", + )?; + } + // If attesting for a full node, the payload must be known. + if data.index == 1 { + verify( + is_payload_verified(store, data.beacon_block_root), + "is_payload_verified(store, attestation.data.beacon_block_root)", + )?; + } + } + + // LMD vote must be consistent with FFG vote target. Gloas's own + // `get_checkpoint_block` answers the same root through a payload-aware walk + // (see [`gloas_get_checkpoint_block`]), so the index-only walk serves both + // forks and spares a gloas attestation one block decode per hop. let checkpoint_block = get_checkpoint_block(index, data.beacon_block_root, target.epoch)?; verify( target.root == checkpoint_block, @@ -3438,9 +3667,12 @@ fn validate_on_attestation_indexed( /// Records an attestation as each attester's latest message, for every /// attesting index that is not a known equivocator. /// -/// An attester's latest message only ever moves to a later target epoch: an -/// attestation for an epoch already superseded by that attester's own later -/// vote is simply not the freshest thing known about them anymore. +/// Before gloas, an attester's latest message only ever moves to a later +/// target epoch: an attestation for an epoch already superseded by that +/// attester's own later vote is simply not the freshest thing known about +/// them anymore. Gloas orders by the attestation's slot instead, and records +/// whether the vote is for the block's full payload (`data.index == 1`); see +/// [`ForkRules`]. /// /// Takes `attesting_indices` and `data` rather than an [`Attestation`]: by /// the time [`on_attestation`] calls this, [`Attestation::verified_attesting_indices`] @@ -3449,17 +3681,20 @@ pub fn update_latest_messages( store: &mut Store, attesting_indices: &[ValidatorIndex], data: AttestationData, + rules: ForkRules, ) { let target = data.target; let beacon_block_root = data.beacon_block_root; + let payload_present = rules == ForkRules::Gloas && data.index == 1; for &index in attesting_indices { if store.is_equivocating(index) { continue; } - let should_update = match store.latest_message(index) { - None => true, - Some(existing) => target.epoch > existing.epoch, + let should_update = match (store.latest_message(index), rules) { + (None, _) => true, + (Some(existing), ForkRules::PreGloas) => target.epoch > existing.epoch, + (Some(existing), ForkRules::Gloas) => data.slot > existing.slot, }; if should_update { store.set_latest_message( @@ -3468,7 +3703,7 @@ pub fn update_latest_messages( epoch: target.epoch, slot: data.slot, root: beacon_block_root, - payload_present: false, + payload_present, }, ); } @@ -3479,14 +3714,16 @@ pub fn update_latest_messages( // Handlers // --------------------------------------------------------------------------- // -// These four are the only functions in this file the specification itself +// These six are the only functions in this file the specification itself // lists as the sole ways to change `store`; each validates before it mutates // anything, so a rejected call leaves `store` exactly as it found it, matching // its requirement that "invalid calls to handlers must not modify store". -// [`get_head`], above, is the one non-handler that also takes `&mut Store`: -// it records the head it just computed, which is not a validity-gated -// mutation a rejected call would need rolled back, just a derived value kept -// in sync with every call. +// Two non-handlers also take `&mut Store`. [`get_head`], above, records the +// head it just computed, which is not a validity-gated mutation a rejected +// call would need rolled back, just a derived value kept in sync with every +// call. [`notify_ptc_messages`] applies a block's payload attestations through +// the same checks as [`on_payload_attestation_message`]; a failure partway +// leaves the votes of the messages before it, which no valid block can cause. /// Advances `store` to `time` (Unix seconds), running [`on_tick_per_slot`] /// once per slot boundary crossed so that none of them are skipped even if @@ -3539,7 +3776,7 @@ pub fn on_block( let parent_root = signed_block.parent_root(); // Return early if the block already has a post-state: a re-delivery must - // not re-run `record_block_timeliness`/`update_proposer_boost_root` below + // not re-run `set_block_timeliness`/`update_proposer_boost_root` below // on a block already fully imported. The chain actor's own import cascade // already deduplicates a known root before ever reaching here // (`Store::has_state`, which also skips re-running `state_transition`; @@ -3577,6 +3814,23 @@ pub fn on_block( .ok_or(Error::SpecAssert("block.parent_root in store.block_states"))?; let mut state = (*parent_state).clone(); + // [New in Gloas:EIP7732] If this block builds on its parent's full payload, + // that payload must have been verified by `on_execution_payload_envelope`. + // A pre-gloas parent is a full node whose payload is verified by + // definition (see `is_payload_verified`), so the first gloas block passes. + let rules = ForkRules::of(signed_block.fork_name()); + match rules { + ForkRules::Gloas => { + if is_parent_node_full(store, &signed_block)? { + verify( + is_payload_verified(store, parent_root), + "is_payload_verified(store, block.parent_root)", + )?; + } + } + ForkRules::PreGloas => {} + } + // Blocks cannot be in the future. If they are, their consideration must // be delayed until they are in the past. verify( @@ -3616,47 +3870,45 @@ pub fn on_block( // The same check, over column sidecars instead of blobs. Both run before // `state_transition`, matching the specification's own ordering: an // unavailable block is not even worth transitioning. - match &signed_block { + // + // The same match hands back the block's payload attestations, which only a + // gloas body carries, so the one place that names every fork also decides + // whether there is anything to notify the store about below. + let payload_attestations: Option<&[gloas::PayloadAttestation]> = match &signed_block { SignedBeaconBlock::Deneb(block) => { verify( is_data_available_blobs(&block.message.body.blob_kzg_commitments, blob_evidence)?, "is_data_available(hash_tree_root(block), block.body.blob_kzg_commitments)", )?; + None } SignedBeaconBlock::Electra(block) => { verify( is_data_available_blobs(&block.message.body.blob_kzg_commitments, blob_evidence)?, "is_data_available(hash_tree_root(block), block.body.blob_kzg_commitments)", )?; + None } SignedBeaconBlock::Fulu(_) => { verify( is_data_available_columns(blob_evidence, config)?, "is_data_available(hash_tree_root(block))", )?; + None } SignedBeaconBlock::Phase0(_) | SignedBeaconBlock::Altair(_) | SignedBeaconBlock::Bellatrix(_) - | SignedBeaconBlock::Capella(_) => {} - // ePBS (EIP-7732) moves the availability question from the block's - // blob/column commitments to the separately gossiped execution - // payload envelope; the check above the match on `signed_block` does - // not apply to a gloas block at all. This arm refuses because gloas's - // own `on_block` is not ported yet, not because `state_transition` - // below is missing anything: envelope processing and the payload - // timeliness committee's vote still need their own fork-choice - // wiring, so this returns the same `Error::UnsupportedForFork` any - // other fork-mismatched call in this crate does, rather than - // skipping the availability question with a silent `{}` arm. - SignedBeaconBlock::Gloas(_) => { - return Err(Error::UnsupportedForFork { - function: "fork_choice::on_block", - fork: ForkName::Gloas, - }); - } + | SignedBeaconBlock::Capella(_) => None, + // [Modified in Gloas:EIP7732] The block itself is not gated on data + // availability: the blob data arrives with the payload, so the check + // moves to `on_execution_payload_envelope`, which is where a gloas + // payload becomes usable. A block whose data never arrives is still + // imported, and its full payload branch stays unreachable, since + // building on it requires `is_payload_verified` above. + SignedBeaconBlock::Gloas(block) => Some(&block.message.body.payload_attestations), SignedBeaconBlock::Lean(_) => lean_block_unreachable("fork_choice::on_block"), - } + }; // Check the block is valid and compute the post-state. The engine's answer // is read the way the specification reads it: an `INVALIDATED` verdict makes @@ -3760,15 +4012,41 @@ pub fn on_block( // `compute_weights` for why the last of those matters at scale) is not // worth paying for on a block the shuffling check could not change the // answer for anyway. + // + // Gloas gates on its own attestation deadline, and computes the head with + // its payload-aware algorithm. When it is computed, its `get_head` failing + // fails the import, as the pre-gloas head computation does. The head is + // only computed for a block that is timely, which puts it in the current + // slot, so `get_head_node`'s dispatch on the current slot's fork would + // pick gloas's algorithm here anyway; calling it directly reuses + // `committees` instead of building a fresh cache. let block_slot = signed_block.slot(); - let is_timely = is_block_timely(store, block_slot, config); + let timeliness = block_timeliness(store, block_slot, rules, config); + let is_timely = timeliness[constants::ATTESTATION_TIMELINESS_INDEX]; let is_first_block = store.proposer_boost_root().is_zero(); let pre_block_head = if is_timely && is_first_block { - Some(compute_head(store, &index, config)?) + Some(match rules { + ForkRules::Gloas => gloas_get_head(store, config, committees)?.root, + ForkRules::PreGloas => compute_head(store, &index, config)?, + }) } else { None }; + // [New in Gloas:EIP7732] Notify the store about the payload attestations + // the block carries. Runs before the block joins the store, unlike the + // specification, so a failure leaves no block behind. The votes it writes + // are for the parent (`process_payload_attestation` pins + // `data.beacon_block_root` to `parent_root`), so nothing here reads the + // block being imported. No failure is expected after a valid transition: + // `process_payload_attestation` already ran the same `get_ptc` and + // membership checks, and the parent's vote vectors always exist. Were one + // to fail midway anyway, the parent would keep the votes of the messages + // before it. + if let Some(payload_attestations) = payload_attestations { + notify_ptc_messages(store, &state, payload_attestations, config)?; + } + // Add new block to the store, and the new state for this block to the // store. let signed_block_el_hash = signed_block.execution_block_hash(); @@ -3776,6 +4054,15 @@ pub fn on_block( .insert_signed_block(block_root, signed_block) .expect("insert"); store.insert_state(block_root, state).expect("insert"); + // [New in Gloas:EIP7732] A new payload timeliness committee vote for this + // block, for each of the two questions it votes on. + match rules { + ForkRules::Gloas => { + store.set_payload_timeliness_vote(block_root, vec![None; preset::PTC_SIZE]); + store.set_payload_data_availability_vote(block_root, vec![None; preset::PTC_SIZE]); + } + ForkRules::PreGloas => {} + } // Keep `index` in step with the one block that changed, rather than // re-scanning `LiveChain` for it: see this function's earlier comment on // `index` for who below still needs it. @@ -3812,7 +4099,7 @@ pub fn on_block( // first, and shares the pre-import head's proposer shuffling. Both calls // are infallible: nothing from here to the end of this function can turn // into an `Err`, and the store has already been mutated above. - record_block_timeliness(store, block_root, block_slot, config); + store.set_block_timeliness(block_root, timeliness); if let Some(pre_block_head) = pre_block_head { update_proposer_boost_root(store, &index, pre_block_head, block_root, config); } @@ -3826,6 +4113,222 @@ pub fn on_block( Ok(()) } +/// `notify_ptc_messages` (gloas `fork-choice.md`): feeds the payload +/// attestations a block carries to [`on_payload_attestation_message`], one +/// message per attester, so the store's votes reflect them. +/// +/// `state` is the block's own post-state, which names the attesters through +/// `get_indexed_payload_attestation`, and the attestations are taken as +/// already verified by `process_block`, which is why each message is applied +/// with `is_from_block` set and a default signature. Each message is judged +/// against the state of the block its attestation names (the specification's +/// `store.block_states[data.beacon_block_root]`), read once per payload +/// attestation rather than once per attester. A genesis-slot state has no +/// payload attestation to read. +pub fn notify_ptc_messages( + store: &mut Store, + state: &BeaconState, + payload_attestations: &[gloas::PayloadAttestation], + config: &Config, +) -> Result<()> { + if state.slot() == 0 { + return Ok(()); + } + for payload_attestation in payload_attestations { + let indexed = + gloas_helpers::get_indexed_payload_attestation(state, payload_attestation, config)?; + if indexed.attesting_indices.is_empty() { + continue; + } + let attested_state = store + .get_state(&payload_attestation.data.beacon_block_root) + .expect("get") + .ok_or(Error::SpecAssert( + "data.beacon_block_root in store.block_states", + ))?; + for &validator_index in indexed.attesting_indices.iter() { + let message = gloas::PayloadAttestationMessage { + validator_index, + data: payload_attestation.data, + signature: Default::default(), + }; + apply_payload_attestation_message(store, &attested_state, &message, true, config)?; + } + } + Ok(()) +} + +/// `on_payload_attestation_message` (gloas `fork-choice.md`): records a +/// payload timeliness committee member's vote on whether a block's payload was +/// revealed on time and its blob data is available. +/// +/// `is_from_block` marks a vote carried inside a block, which +/// [`notify_ptc_messages`] has already had verified; a vote received directly +/// must be for the current slot and carry a valid signature. +/// +/// A member can hold more than one seat in the committee (it is drawn with +/// replacement), and the vote lands in every one of them. Every check runs +/// before either vote vector is written, so a rejected message changes +/// nothing. +pub fn on_payload_attestation_message( + store: &mut Store, + ptc_message: &gloas::PayloadAttestationMessage, + is_from_block: bool, + config: &Config, +) -> Result<()> { + // PTC attestation must be for a known block. If block is unknown, delay + // consideration until the block is found. + let state = store + .get_state(&ptc_message.data.beacon_block_root) + .expect("get") + .ok_or(Error::SpecAssert( + "data.beacon_block_root in store.block_states", + ))?; + apply_payload_attestation_message(store, &state, ptc_message, is_from_block, config) +} + +/// The body of [`on_payload_attestation_message`] after its state read, given +/// `state`, the state of the block `ptc_message` names, so +/// [`notify_ptc_messages`] can read that state once for every attester of a +/// payload attestation. +fn apply_payload_attestation_message( + store: &mut Store, + state: &BeaconState, + ptc_message: &gloas::PayloadAttestationMessage, + is_from_block: bool, + config: &Config, +) -> Result<()> { + let data = ptc_message.data; + + // PTC votes can only change the vote for their assigned beacon block, + // return early otherwise. + if data.slot != state.slot() { + return Ok(()); + } + + // Get all positions of the attester in the PTC. + let ptc = gloas_helpers::get_ptc(state, data.slot, config)?; + let ptc_indices: Vec = ptc + .iter() + .enumerate() + .filter(|&(_, &validator_index)| validator_index == ptc_message.validator_index) + .map(|(ptc_index, _)| ptc_index) + .collect(); + + // Check that the attester is from the PTC. + verify(!ptc_indices.is_empty(), "len(ptc_indices) > 0")?; + + // Verify the signature and check that it is for the current slot if it is + // coming from the wire. + if !is_from_block { + verify( + data.slot == get_current_slot(store, config), + "data.slot == get_current_slot(store)", + )?; + let indexed = gloas::IndexedPayloadAttestation { + attesting_indices: gloas::PayloadTimelinessCommitteeIndices::try_from(vec![ + ptc_message.validator_index, + ])?, + data, + signature: ptc_message.signature, + }; + verify( + gloas_helpers::is_valid_indexed_payload_attestation(state, &indexed), + "is_valid_indexed_payload_attestation(state, indexed_payload_attestation)", + )?; + } + + // Update the votes for the block. + let mut payload_timeliness_vote = store + .payload_timeliness_vote(&data.beacon_block_root) + .ok_or(Error::SpecAssert( + "data.beacon_block_root in store.payload_timeliness_vote", + ))?; + let mut payload_data_availability_vote = store + .payload_data_availability_vote(&data.beacon_block_root) + .ok_or(Error::SpecAssert( + "data.beacon_block_root in store.payload_data_availability_vote", + ))?; + for ptc_index in ptc_indices { + let len = payload_timeliness_vote.len(); + *payload_timeliness_vote + .get_mut(ptc_index) + .ok_or(Error::IndexOutOfBounds { + index: ptc_index, + len, + })? = Some(data.payload_present); + let len = payload_data_availability_vote.len(); + *payload_data_availability_vote + .get_mut(ptc_index) + .ok_or(Error::IndexOutOfBounds { + index: ptc_index, + len, + })? = Some(data.blob_data_available); + } + store.set_payload_timeliness_vote(data.beacon_block_root, payload_timeliness_vote); + store + .set_payload_data_availability_vote(data.beacon_block_root, payload_data_availability_vote); + + Ok(()) +} + +/// `on_execution_payload_envelope` (gloas `fork-choice.md`): verifies a +/// builder's payload envelope against the block it belongs to and, if it +/// holds, records the payload as verified, which is what lets a block build +/// on its full branch (see [`on_block`]) and a vote name it. +/// +/// `sidecars` is what the specification's implementation-dependent +/// `retrieve_column_sidecars_and_kzg_commitments` returns for the block: the +/// sampled column sidecars. The commitments they are checked against are the +/// block's bid's own, read from the store. An empty slice reads as available; +/// see [`is_data_available_gloas_columns`]. `engine` answers the execution +/// layer's part of `verify_execution_payload_envelope`. +/// +/// Every check runs before the payload is recorded, so a rejected envelope +/// changes nothing. An envelope delivered again for a payload already +/// verified is checked again and recorded again, which is idempotent. +pub fn on_execution_payload_envelope( + store: &mut Store, + signed_envelope: &gloas::SignedExecutionPayloadEnvelope, + config: &Config, + sidecars: &[gloas::DataColumnSidecar], + engine: &stf::ExecutionEngine, +) -> Result<()> { + let envelope = &signed_envelope.message; + let block_root = envelope.beacon_block_root; + + // The corresponding beacon block root needs to be known. + let state = store + .get_state(&block_root) + .expect("get") + .ok_or(Error::SpecAssert( + "envelope.beacon_block_root in store.block_states", + ))?; + + // Check if blob data is available. If not, this payload MAY be queued and + // subsequently considered when blob data becomes available. + let block = store + .get_signed_block(&block_root) + .expect("get") + .ok_or(Error::SpecAssert( + "envelope.beacon_block_root in store.blocks", + ))?; + let kzg_commitments = &gloas_bid(&block)?.blob_kzg_commitments; + verify( + is_data_available_gloas_columns(sidecars, kzg_commitments)?, + "is_data_available(envelope.beacon_block_root)", + )?; + + // Verify the execution payload envelope. + stf::gloas::verify_execution_payload_envelope(&state, signed_envelope, config, engine)?; + + // Add execution payload envelope to the store. Only the fact that it is + // verified is kept: nothing in fork choice reads the payload itself. + store.insert_verified_payload(block_root); + + Ok(()) +} + /// Validates `attestation` and, if valid, records it as each attester's /// latest message. /// @@ -3841,7 +4344,8 @@ pub fn on_attestation( committees: &CommitteeCache, ) -> Result<()> { let data = attestation.data(); - validate_on_attestation(store, data, is_from_block, config)?; + let rules = attestation.rules(); + validate_on_attestation(store, data, rules, is_from_block, config)?; // The state at the `target` to fully validate attestation against. // `checkpoint_state` hands back an owned value now, so there is no borrow @@ -3852,7 +4356,7 @@ pub fn on_attestation( let attesting_indices = attestation.verified_attesting_indices(&target_state, committees)?; // Update latest messages for attesting indices. - update_latest_messages(store, &attesting_indices, data); + update_latest_messages(store, &attesting_indices, data, rules); Ok(()) } @@ -3900,10 +4404,11 @@ pub fn on_block_attestation( committees: &CommitteeCache, ) -> Result<()> { let data = attestation.data(); - validate_on_attestation_indexed(store, data, true, config, index)?; + let rules = attestation.rules(); + validate_on_attestation_indexed(store, data, rules, true, config, index)?; let attesting_indices = attestation.attesting_indices(block_state, committees)?; - update_latest_messages(store, &attesting_indices, data); + update_latest_messages(store, &attesting_indices, data, rules); Ok(()) } @@ -3928,6 +4433,11 @@ pub fn on_block_attestation( /// records it against `attesting_indices`, resolved by the caller's gossip /// validation rather than recomputed here. /// +/// The rules are [`ForkRules::PreGloas`]: an aggregate reaches this function +/// only as a pre-gloas (phase0- or electra-shaped) one, since gloas's own +/// aggregate is refused when the gossip payload is decoded and never gets as +/// far as gossip validation. +/// /// `is_from_block` is fixed at `false`, matching [`on_attestation`]'s call for /// this topic: an aggregate here is by definition not carried in a block, so /// the current-or-previous-epoch target check applies. @@ -3943,8 +4453,8 @@ pub fn apply_verified_aggregate( config: &Config, index: &HashMap, ) -> Result<()> { - validate_on_attestation_indexed(store, data, false, config, index)?; - update_latest_messages(store, attesting_indices, data); + validate_on_attestation_indexed(store, data, ForkRules::PreGloas, false, config, index)?; + update_latest_messages(store, attesting_indices, data, ForkRules::PreGloas); Ok(()) } @@ -4807,7 +5317,7 @@ mod tests { // `on_block` refuses a block from the future (`get_current_slot(store) // >= block.slot`), so the store's clock has to have reached the // child's own slot before this test's `on_block` call below. Set to - // the slot's own start, its first block, so `is_block_timely` is + // the slot's own start, its first block, so `block_timeliness` is // true and `on_block` runs its pre-import `compute_head` call too // (see this test's own doc for why that matters). store @@ -6333,4 +6843,371 @@ mod tests { gloas_get_head(&store, &config, &committees) .expect("the same tolerance must hold through the whole head walk"); } + + // ----------------------------------------------------------------------- + // Gloas handlers + // ----------------------------------------------------------------------- + + const UNVERIFIED_PAYLOAD_PARENT: &str = "is_payload_verified(store, block.parent_root)"; + + /// The message of the specification assertion `result` failed, or `None` + /// if it succeeded or failed some other way. `Error` has no `PartialEq`, + /// and what these tests pin is which assertion fired. + fn failed_assertion(result: Result) -> Option<&'static str> { + match result { + Err(Error::SpecAssert(what)) => Some(what), + _ => None, + } + } + + #[test] + fn on_block_rejects_a_block_built_on_a_full_parent_whose_payload_is_unverified() { + let config = Config::active().with_fork_epoch(ForkName::Gloas, 0); + let anchor_root = Root::repeat_byte(0xa1); + let anchor_hash = ExecutionBlockHash::repeat_byte(0x11); + let mut store = store_anchored_at(anchor_root); + store + .insert_signed_block( + anchor_root, + gloas_block(0, Root::ZERO, ExecutionBlockHash::ZERO, anchor_hash), + ) + .unwrap(); + store + .insert_state( + anchor_root, + test_state::with_validators_at(ForkName::Gloas, 1), + ) + .unwrap(); + store.set_time_ms(2 * config.slot_duration_ms).unwrap(); + + let import = |store: &mut Store, block: SignedBeaconBlock| { + on_block( + store, + block, + &config, + &DataAvailability::NotRequired, + &PayloadValidity::NotRequired, + &CommitteeCache::default(), + ) + }; + + // Names the anchor's own block hash as its parent's, so it builds on + // the anchor's full branch. + let full_child = gloas_block( + 1, + anchor_root, + anchor_hash, + ExecutionBlockHash::repeat_byte(0x22), + ); + let full_child_root = full_child.message_hash_tree_root(); + assert_eq!( + failed_assertion(import(&mut store, full_child.clone())), + Some(UNVERIFIED_PAYLOAD_PARENT), + "a full parent whose envelope was never verified must fail the import" + ); + assert!( + !store.has_state(&full_child_root).unwrap(), + "a rejected block leaves no post-state behind" + ); + + // A block on the empty branch needs no envelope, so it gets past this + // check (and fails later, on the empty body it carries). + let empty_child = gloas_block( + 1, + anchor_root, + ExecutionBlockHash::repeat_byte(0xee), + ExecutionBlockHash::repeat_byte(0x33), + ); + let rejected = import(&mut store, empty_child); + assert!(rejected.is_err()); + assert_ne!(failed_assertion(rejected), Some(UNVERIFIED_PAYLOAD_PARENT)); + + // Once the envelope is verified the full child clears the check too. + store.insert_verified_payload(anchor_root); + let rejected = import(&mut store, full_child); + assert!(rejected.is_err()); + assert_ne!(failed_assertion(rejected), Some(UNVERIFIED_PAYLOAD_PARENT)); + } + + fn gloas_attestation_data(slot: Slot, index: u64, root: Root, target: Root) -> AttestationData { + AttestationData { + slot, + index, + beacon_block_root: root, + source: Checkpoint::default(), + target: Checkpoint { + epoch: 0, + root: target, + }, + } + } + + #[test] + fn validate_on_attestation_applies_the_gloas_index_rules_only_to_a_gloas_attestation() { + let config = Config::active().with_fork_epoch(ForkName::Gloas, 0); + let anchor_root = Root::repeat_byte(0xa1); + let b_root = Root::repeat_byte(0xb1); + let mut store = store_anchored_at(anchor_root); + store + .insert_signed_block( + anchor_root, + gloas_block( + 0, + Root::ZERO, + ExecutionBlockHash::ZERO, + ExecutionBlockHash::repeat_byte(0x11), + ), + ) + .unwrap(); + store + .insert_signed_block( + b_root, + gloas_block( + 1, + anchor_root, + ExecutionBlockHash::repeat_byte(0x11), + ExecutionBlockHash::repeat_byte(0x22), + ), + ) + .unwrap(); + store.set_time_ms(4 * config.slot_duration_ms).unwrap(); + + let vote = + |slot: Slot, index: u64| gloas_attestation_data(slot, index, b_root, anchor_root); + + // `index` is the payload flag: 0 and 1 only. + let validate = |store: &Store, data: AttestationData, rules: ForkRules| { + validate_on_attestation(store, data, rules, true, &config) + }; + assert!(validate(&store, vote(2, 0), ForkRules::Gloas).is_ok()); + assert_eq!( + failed_assertion(validate(&store, vote(2, 2), ForkRules::Gloas)), + Some("attestation.data.index in [0, 1]") + ); + // A vote in the block's own slot cannot name the full branch: the + // payload is revealed after the block. + assert_eq!( + failed_assertion(validate(&store, vote(1, 1), ForkRules::Gloas)), + Some("attestation.data.index == 0 for a same-slot vote") + ); + // A later vote for the full branch needs the payload to be verified. + assert_eq!( + failed_assertion(validate(&store, vote(2, 1), ForkRules::Gloas)), + Some("is_payload_verified(store, attestation.data.beacon_block_root)") + ); + // The same attestations before gloas have no such rules. + assert!(validate(&store, vote(2, 1), ForkRules::PreGloas).is_ok()); + assert!(validate(&store, vote(2, 2), ForkRules::PreGloas).is_ok()); + + store.insert_verified_payload(b_root); + assert!(validate(&store, vote(2, 1), ForkRules::Gloas).is_ok()); + } + + #[test] + fn update_latest_messages_orders_gloas_votes_by_slot_and_records_the_payload_flag() { + let root = Root::repeat_byte(0xb1); + let mut store = empty_store(); + let vote = |slot: Slot, index: u64| gloas_attestation_data(slot, index, root, root); + + update_latest_messages(&mut store, &[0], vote(5, 0), ForkRules::Gloas); + // Same target epoch, later slot: gloas replaces it, since it orders by + // slot, and records that the vote is for the full branch. + update_latest_messages(&mut store, &[0], vote(6, 1), ForkRules::Gloas); + let message = store.latest_message(0).unwrap(); + assert_eq!((message.slot, message.payload_present), (6, true)); + // An earlier slot never replaces a later one. + update_latest_messages(&mut store, &[0], vote(4, 0), ForkRules::Gloas); + assert_eq!(store.latest_message(0).unwrap().slot, 6); + + // Before gloas the same pair of votes shares a target epoch, so the + // second does not replace the first, and no vote is for a full branch. + update_latest_messages(&mut store, &[1], vote(5, 0), ForkRules::PreGloas); + update_latest_messages(&mut store, &[1], vote(6, 1), ForkRules::PreGloas); + let message = store.latest_message(1).unwrap(); + assert_eq!((message.slot, message.payload_present), (5, false)); + + // An equivocator's vote is dropped under either set of rules. + store.insert_equivocating_index(2); + for rules in [ForkRules::Gloas, ForkRules::PreGloas] { + update_latest_messages(&mut store, &[2], vote(5, 0), rules); + assert_eq!(store.latest_message(2), None); + } + } + + #[test] + fn on_payload_attestation_message_writes_every_seat_the_validator_holds() { + let config = Config::active().with_fork_epoch(ForkName::Gloas, 0); + let root = Root::repeat_byte(0xb1); + let mut store = store_anchored_at(root); + store + .insert_signed_block( + root, + gloas_block( + 1, + Root::ZERO, + ExecutionBlockHash::ZERO, + ExecutionBlockHash::repeat_byte(0x22), + ), + ) + .unwrap(); + + // The committee for slot 1 in a state at slot 1 is the window entry + // `get_ptc` reads one epoch of slots in. Validator 3 is seated twice + // and every other seat is validator 0. + let mut state = test_state::with_validators_at(ForkName::Gloas, 4); + let BeaconState::Gloas(inner) = &mut state else { + unreachable!("with_validators_at(Gloas) builds a gloas state"); + }; + inner.slot = 1; + let mut committee = vec![0; preset::PTC_SIZE]; + committee[2] = 3; + committee[7] = 3; + let window_index = (preset::SLOTS_PER_EPOCH + 1) as usize; + inner.ptc_window[window_index] = committee.try_into().unwrap(); + store.insert_state(root, state).unwrap(); + store.set_payload_timeliness_vote(root, vec![None; preset::PTC_SIZE]); + store.set_payload_data_availability_vote(root, vec![None; preset::PTC_SIZE]); + + let message = |validator_index: u64, slot: Slot| gloas::PayloadAttestationMessage { + validator_index, + data: gloas::PayloadAttestationData { + beacon_block_root: root, + slot, + payload_present: true, + blob_data_available: false, + }, + signature: Default::default(), + }; + let seats = |votes: Vec>| -> Vec<(usize, bool)> { + votes + .into_iter() + .enumerate() + .filter_map(|(seat, vote)| vote.map(|vote| (seat, vote))) + .collect() + }; + + // A vote for a slot other than the block's own changes nothing. + on_payload_attestation_message(&mut store, &message(3, 2), true, &config).unwrap(); + assert!(seats(store.payload_timeliness_vote(&root).unwrap()).is_empty()); + + // A validator outside the committee is rejected, and writes nothing. + assert_eq!( + failed_assertion(on_payload_attestation_message( + &mut store, + &message(2, 1), + true, + &config + )), + Some("len(ptc_indices) > 0") + ); + assert!(seats(store.payload_timeliness_vote(&root).unwrap()).is_empty()); + + // Both seats of validator 3 receive both answers. + on_payload_attestation_message(&mut store, &message(3, 1), true, &config).unwrap(); + assert_eq!( + seats(store.payload_timeliness_vote(&root).unwrap()), + vec![(2, true), (7, true)] + ); + assert_eq!( + seats(store.payload_data_availability_vote(&root).unwrap()), + vec![(2, false), (7, false)] + ); + } + + #[test] + fn get_forkchoice_store_seeds_a_gloas_anchor_and_leaves_an_earlier_one_alone() { + // An exact anchor pair, built as `anchor_pair_with` builds a phase0 one. + let mut state = test_state::with_validators_at(ForkName::Gloas, 1); + let mut signed = gloas_block( + state.slot(), + Root::ZERO, + ExecutionBlockHash::ZERO, + ExecutionBlockHash::ZERO, + ); + let SignedBeaconBlock::Gloas(inner) = &mut signed else { + unreachable!("`gloas_block` builds a gloas signed block"); + }; + *state.latest_block_header_mut() = BeaconBlockHeader { + slot: inner.message.slot, + proposer_index: 0, + parent_root: Root::ZERO, + state_root: Root::ZERO, + body_root: inner.message.body.hash_tree_root(), + }; + inner.message.state_root = state.hash_tree_root(); + let anchor_root = signed.message_hash_tree_root(); + + let store = get_forkchoice_store( + Arc::new(InMemoryBackend::new()), + state, + signed, + &Config::active().with_fork_epoch(ForkName::Gloas, 0), + ) + .expect("a gloas anchor is accepted"); + assert_eq!(store.block_timeliness(&anchor_root), Some([true, true])); + assert_eq!( + store.payload_timeliness_vote(&anchor_root), + Some(vec![None; preset::PTC_SIZE]) + ); + assert_eq!( + store.payload_data_availability_vote(&anchor_root), + Some(vec![None; preset::PTC_SIZE]) + ); + assert!(!store.has_verified_payload(&anchor_root)); + + // Earlier forks record none of the three for their anchor. + let (state, block) = anchor_pair(); + let anchor_root = block.message_hash_tree_root(); + let store = get_forkchoice_store( + Arc::new(InMemoryBackend::new()), + state, + block, + &Config::active(), + ) + .unwrap(); + assert_eq!(store.block_timeliness(&anchor_root), None); + assert_eq!(store.payload_timeliness_vote(&anchor_root), None); + assert_eq!(store.payload_data_availability_vote(&anchor_root), None); + } + + #[test] + fn gloas_data_availability_is_judged_on_gloas_sidecars_only() { + // No sidecars sampled: `all()` over an empty list holds. + assert!(is_data_available_gloas_columns(&[], &[]).unwrap()); + + // A sidecar for zero blobs is invalid. + let empty = gloas::DataColumnSidecar::default(); + assert!(!is_data_available_gloas_columns(&[empty], &[]).unwrap()); + + // One cell and one proof against no commitments: past the zero-blob + // rule, and stopped by the length check against the bid. + let cell = fulu::Cell::try_from(vec![0u8; preset::BYTES_PER_CELL]).unwrap(); + let one_blob = gloas::DataColumnSidecar { + column: vec![cell].into(), + kzg_proofs: vec![KzgProof::default()].into(), + ..Default::default() + }; + assert!(!gloas_verify_data_column_sidecar(&one_blob, &[])); + assert!(gloas_verify_data_column_sidecar( + &one_blob, + &[KzgCommitment::default()] + )); + assert!(!is_data_available_gloas_columns(&[one_blob], &[]).unwrap()); + + // Gloas evidence has no `DataAvailability` variant, so an older fork's + // check cannot read it as its own, and each of theirs refuses the + // other's evidence rather than reading it as available. + let config = Config::active(); + assert!( + is_data_available_columns( + &DataAvailability::Blobs { + blobs: Vec::new(), + proofs: Vec::new(), + }, + &config, + ) + .is_err() + ); + assert!(is_data_available_blobs(&[], &DataAvailability::Columns(Vec::new())).is_err()); + } } diff --git a/crates/blockchain/state_transition/src/beacon/gossip/block.rs b/crates/blockchain/state_transition/src/beacon/gossip/block.rs index 177b2b637..163125ee4 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/block.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/block.rs @@ -8,8 +8,7 @@ use super::{ finalized_start_slot, is_future_slot, }; use crate::beacon::containers::SignedBeaconBlock; -use crate::beacon::fork::ForkName; -use crate::beacon::fork_choice::Store; +use crate::beacon::fork_choice::{ForkRules, Store}; use crate::beacon::helpers::misc::compute_epoch_at_slot; use crate::beacon::precheck::{self, PrecheckError, Reference, precheck_block}; use crate::beacon::primitives::Root; @@ -29,15 +28,16 @@ pub fn cheap_checks( // [IGNORE] This build does not validate a gloas-shaped block against // these (fulu) rules: EIP-7732 changes what a block commits to enough // that accepting one here would propagate something this node cannot - // itself import (`fork_choice::on_block` refuses every gloas block with - // `Error::UnsupportedForFork`). Checked first, so a gloas block never - // reaches a rule below whose fields it happens to still answer (its - // `blob_kzg_commitment_count` reads the builder's bid) but whose verdict - // would misdescribe it. `Ignore`, not `Reject`: the sender did nothing - // wrong, and every honest peer sends exactly this once this node's own - // clock reaches gloas. - if block.fork_name() == ForkName::Gloas { - return Err(Outcome::Ignore(IgnoreReason::UnsupportedFork)); + // itself follow (its node wiring cannot deliver a gloas block's payload + // envelope yet, though `fork_choice::on_block` accepts the block). Checked + // first, so a gloas block never reaches a rule below whose fields it + // happens to still answer (its `blob_kzg_commitment_count` reads the + // builder's bid) but whose verdict would misdescribe it. `Ignore`, not + // `Reject`: the sender did nothing wrong, and every honest peer sends + // exactly this once this node's own clock reaches gloas. + match ForkRules::of(block.fork_name()) { + ForkRules::Gloas => return Err(Outcome::Ignore(IgnoreReason::UnsupportedFork)), + ForkRules::PreGloas => {} } // [IGNORE] The block is not from a future slot. if is_future_slot(&config, slot, now_ms) { diff --git a/docs/cli.md b/docs/cli.md index d9086f3b2..48cf91180 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -167,9 +167,10 @@ config, the heze schedule this build does not claim) are dropped with one warning line naming each. Its `PRESET_BASE` is checked against the compiled preset; a mismatch is a hard startup error naming the cargo feature that would fix it. Sepolia's config schedules gloas, whose keys this build -does claim but whose state transition it does not implement yet: a separate, -explicit warning at startup names that boundary, since the chain follower -stops tracking the chain there regardless of what parsed cleanly. The keys this +does claim but whose blocks the chain actor refuses, since nothing delivers +payload envelopes or payload attestations to it yet: a separate, explicit +warning at startup names that boundary, since the chain follower stops +tracking the chain there regardless of what parsed cleanly. The keys this build runs on compile-time constants for (the custody and subnet counts, the `MAX_REQUEST_*` limits, `MAX_PAYLOAD_SIZE`, the snappy message domains and `MAXIMUM_GOSSIP_CLOCK_DISPARITY`) must equal those From 69d9bd72ee5faaed2ec8c82e8f22fc3a4d1e3312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:38:44 -0300 Subject: [PATCH 024/132] test(beacon): run the gloas fork-choice suite The gloas fork-choice handlers had never run against a fixture. The fork_choice runner now drives them: each case is processed under its own fork schedule (a shared config that schedules gloas at the far future epoch would have run the pre-gloas head), and it understands gloas's execution_payload and payload_attestation_message steps, the head's payload status, and the payload timeliness and data availability vote checks. Anchor blocks, attestations and attester slashings decode into their gloas containers. The should_apply_proposer_boost cases also name viable_for_head_roots_and_weights. The runner walks the payload-aware tree from the justified root the way the reference generator does and compares each leaf's root, weight and payload status as an unordered set: the store's children come from a hash map, the generator's from insertion-ordered dicts. Without it, a wrong weight for the previous slot's payload decision passed every case. Every step and check struct now rejects unknown keys, so a check the runner does not model fails its case instead of being dropped silently. The harness does not expand a block's payload attestations itself: on_block already does, and writing the same vote again changes nothing. --- .../tests/beacon_spec/fork_choice.rs | 329 ++++++++++++++---- .../state_transition/tests/beacon_spec/mod.rs | 7 +- 2 files changed, 269 insertions(+), 67 deletions(-) diff --git a/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs b/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs index 8fb610278..c0835edad 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs @@ -4,11 +4,15 @@ //! fork's modifications to it. Each case builds a [`Store`] from an anchor //! state and block via [`fork_choice::get_forkchoice_store`], then replays //! `steps.yaml` in order. A step is one of `tick`, `block`, `attestation`, -//! `attester_slashing`, or `on_merge_block` (bellatrix's own `pow_block` -//! step), driving the correspondingly named handler, or `checks`, which -//! asserts on the store's own fields and on [`fork_choice::get_head`] and -//! [`fork_choice::get_proposer_head`]. See `tests/formats/fork_choice/README.md` -//! in the pinned specification checkout for the format in full. +//! `attester_slashing`, `on_merge_block` (bellatrix's own `pow_block` step), +//! or gloas's `execution_payload` and `payload_attestation_message`, driving +//! the correspondingly named handler, or `checks`, which asserts on the +//! store's own fields and on [`fork_choice::get_head`] and +//! [`fork_choice::get_proposer_head`] (gloas adds the head's payload status, +//! the two payload committee vote checks and the viable leaf weights). Each +//! case runs against [`super::case_config`]. See +//! `tests/formats/fork_choice/README.md` in the pinned specification checkout +//! for the format in full. //! //! No released fixture targets phase0 directly: the earliest suite is //! altair's, built from altair-shaped states because the generator needs a @@ -21,13 +25,35 @@ //! # A step's `valid: false` means the call must be rejected //! //! Matching [`super::check_transition`]'s rule for a missing `post` state, -//! `valid: false` on a `block`, `attestation`, or `attester_slashing` step -//! means the handler must return an error, and the store must be left exactly -//! as it was; treating a should-fail call that happens to succeed as a pass -//! would let this suite go green while checking nothing. `on_tick` cannot -//! fail in this crate, nor in the specification (it has no assertion to -//! fail), so a `tick` step asking for rejection is reported as a failure of -//! this suite's own assumptions rather than silently accepted. +//! `valid: false` on a `block`, `attestation`, `attester_slashing`, +//! `execution_payload`, or `payload_attestation_message` step means the +//! handler must return an error, and the store must be left exactly as it +//! was; treating a should-fail call that happens to succeed as a pass would +//! let this suite go green while checking nothing. `on_tick` cannot fail in +//! this crate, nor in the specification (it has no assertion to fail), so a +//! `tick` step asking for rejection is reported as a failure of this suite's +//! own assumptions rather than silently accepted. +//! +//! # Gloas payload attestations are not replayed here +//! +//! The README has the harness expand a gloas block's `payload_attestations` +//! into messages and apply them with `is_from_block` set. +//! [`fork_choice::on_block`] already does that through `notify_ptc_messages`, +//! so expanding them again in [`apply_block`] is redundant: writing the same +//! vote a second time changes nothing. +//! +//! # `viable_for_head_roots_and_weights` is compared as a set +//! +//! The reference generator (`get_viable_for_head_checks` in +//! `tests/core/pyspec/eth_consensus_specs/test/helpers/fork_choice.py`) walks +//! from the justified root's PENDING node with a stack, through +//! `get_node_children` over `get_filtered_block_tree`, and reports every leaf +//! with its `get_weight` and payload status. [`check_viable_for_head`] does the +//! same walk but compares `{root, weight, payload_status}` as an unordered set, +//! since the order the fixture lists the leaves in is not reproducible here: +//! `get_node_children` iterates a `HashMap`, while the generator walks dicts +//! in insertion order. +//! Only gloas cases name it; the walk needs gloas's payload-aware nodes. //! //! # An `on_block` step implies more than the README documents //! @@ -58,15 +84,9 @@ //! //! # Checks this runner does not model //! -//! `viable_for_head_roots_and_weights` is part of the format, but no case at -//! any implemented fork's `checks` step names it, on either preset, so it is -//! not modeled here rather than guessed at. `should_override_forkchoice_update` -//! is in the format's README too, but no `fork-choice.md` at `v1.7.0-beta.2` -//! defines it any more (not even `fast-confirmation.md`, which absorbed most -//! of what bellatrix's `fork-choice.md` dropped): it is gone from the -//! specification outright, not merely relocated, so this crate drops it with -//! it rather than keeping an implementation of a function the pinned release -//! no longer has. +//! Every key of a `checks` step this runner models is a field of [`Checks`], +//! which rejects unknown keys, so a check the runner does not model fails its +//! case instead of being dropped without a word. //! //! # `columns: []` means "simulate unavailable", not "vacuously available" //! @@ -104,12 +124,15 @@ use ethlambda_state_transition::beacon::ForkName; use ethlambda_state_transition::beacon::config::Config; use ethlambda_state_transition::beacon::containers::{ BeaconState, Checkpoint, SignedBeaconBlock, altair, bellatrix, capella, deneb, electra, fulu, - phase0, + gloas, phase0, +}; +use ethlambda_state_transition::beacon::fork_choice::{ + self, DataAvailability, ForkChoiceNode, PayloadStatus, Store, }; -use ethlambda_state_transition::beacon::fork_choice::{self, DataAvailability, Store}; use ethlambda_state_transition::beacon::helpers::accessors::CommitteeCache; use ethlambda_state_transition::beacon::preset; use ethlambda_state_transition::beacon::primitives::{KzgProof, Root}; +use ethlambda_state_transition::beacon::stf::ExecutionEngine; use libssz::SszDecode; use libssz_types::SszList; use libtest_mimic::Trial; @@ -122,15 +145,17 @@ use super::{Case, PRESET, collect_all_handlers, lean_is_not_a_fixture_fork}; /// One entry of a case's `steps.yaml`. /// -/// The format overlays six step kinds into one YAML sequence item: exactly +/// The format overlays eight step kinds into one YAML sequence item: exactly /// one of [`Step::tick`], [`Step::block`], [`Step::attestation`], -/// [`Step::attester_slashing`], and [`Step::pow_block`] is set for an -/// execution step, or none of them for a [`Step::checks`] step. Modeled as -/// one struct with every field optional, rather than a `#[serde(untagged)]` -/// enum over five variants, because the four execution kinds that carry a -/// validity outcome also share [`Step::valid`], which an enum would have to -/// repeat on every variant instead of naming once. +/// [`Step::attester_slashing`], [`Step::pow_block`], +/// [`Step::execution_payload`], and [`Step::payload_attestation_message`] is +/// set for an execution step, or none of them for a [`Step::checks`] step. +/// Modeled as one struct with every field optional, rather than a +/// `#[serde(untagged)]` enum over seven variants, because the execution kinds +/// that carry a validity outcome also share [`Step::valid`], which an enum +/// would have to repeat on every variant instead of naming once. #[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] struct Step { /// The Unix-second time to advance the store to, for an `on_tick` step. tick: Option, @@ -158,9 +183,16 @@ struct Step { /// an `on_merge_block` step adds to the store for later `get_pow_block` /// lookups. pow_block: Option, + /// `[New in Gloas]` the `execution_payload_envelope_` file naming the + /// envelope an `on_execution_payload_envelope` step delivers. + execution_payload: Option, + /// `[New in Gloas]` the `payload_attestation_message_` file naming + /// the message an `on_payload_attestation_message` step delivers. + payload_attestation_message: Option, /// Whether this step's call is expected to succeed. Only `block`, - /// `attestation`, and `attester_slashing` steps carry `false` in any - /// released fixture, but the format allows it on any execution step. + /// `attestation`, `attester_slashing`, `execution_payload`, and + /// `payload_attestation_message` steps carry `false` in any released + /// fixture, but the format allows it on any execution step. #[serde(default = "default_valid")] valid: bool, /// The assertions to check against the current store. @@ -175,9 +207,11 @@ fn default_valid() -> bool { /// A `checks` step's assertions against the store. /// -/// See the module documentation for the two fields of the format this leaves -/// out, and why. +/// Every field of the format's `checks` step is modeled, and unknown keys are +/// rejected, so a key added to the format fails its case instead of being +/// dropped. #[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] pub(super) struct Checks { time: Option, genesis_time: Option, @@ -186,19 +220,50 @@ pub(super) struct Checks { finalized_checkpoint: Option, proposer_boost_root: Option, get_proposer_head: Option, + /// `[New in Gloas]` the PTC's payload timeliness votes for one block. + payload_timeliness_vote: Option, + /// `[New in Gloas]` the PTC's blob data availability votes for one block. + payload_data_availability_vote: Option, + /// `[New in Gloas]` the payload-aware leaves of the filtered block tree, + /// with their weights, in the generator's own order. + viable_for_head_roots_and_weights: Option>, +} + +/// One leaf of `viable_for_head_roots_and_weights`. +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct ViableLeafCheck { + root: String, + weight: u64, + /// Required on purpose: only gloas cases name this check, so a + /// pre-gloas-shaped leaf without a payload status fails to parse. + payload_status: u8, +} + +/// The expected votes of one block's payload committee, ordered by PTC +/// position, with `null` for a position that has not voted. +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct VoteCheck { + block_root: String, + votes: Vec>, } /// The expected value of [`fork_choice::get_head`], as `checks.head` gives it: /// the root and, redundantly, the slot of the block it names. #[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] struct HeadCheck { slot: u64, root: String, + /// `[New in Gloas]` the head node's payload status, as its discriminant. + payload_status: Option, } /// The expected value of a checkpoint field (`justified_checkpoint` or /// `finalized_checkpoint`). #[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] struct CheckpointCheck { epoch: u64, root: String, @@ -276,7 +341,10 @@ pub(super) fn decode_anchor_block(case: &Case) -> Result Err("gloas not implemented".into()), + ForkName::Gloas => Ok(SignedBeaconBlock::Gloas(gloas::SignedBeaconBlock { + message: decode(case, "anchor_block")?, + signature: Default::default(), + })), ForkName::Lean => lean_is_not_a_fixture_fork("fork_choice"), } } @@ -304,7 +372,7 @@ fn decode_attestation(case: &Case, name: &str) -> Result Ok(fork_choice::Attestation::Phase0(decode(case, name)?)), - ForkName::Gloas => Err("gloas not implemented".into()), + ForkName::Gloas => Ok(fork_choice::Attestation::Gloas(decode(case, name)?)), ForkName::Lean => lean_is_not_a_fixture_fork("fork_choice"), } } @@ -324,7 +392,7 @@ fn decode_attester_slashing( | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb => Ok(fork_choice::AttesterSlashing::Phase0(decode(case, name)?)), - ForkName::Gloas => Err("gloas not implemented".into()), + ForkName::Gloas => Ok(fork_choice::AttesterSlashing::Gloas(decode(case, name)?)), ForkName::Lean => lean_is_not_a_fixture_fork("fork_choice"), } } @@ -461,9 +529,27 @@ fn apply_block( Ok(()) } +/// Turns a handler's outcome into a step's verdict: `valid: true` needs `Ok`, +/// `valid: false` needs `Err`, and either mismatch names the file. +fn expect_outcome( + name: &str, + outcome: Result<(), E>, + valid: bool, +) -> Result<(), String> { + match (outcome, valid) { + (Ok(()), false) => Err(format!( + "{name} was accepted, but the step expects it to be rejected" + )), + (Err(err), true) => Err(format!("{name} was rejected: {err:?}")), + (Ok(()), true) | (Err(_), false) => Ok(()), + } +} + /// Applies one non-`checks` execution step, dispatching on which of /// [`Step::tick`], [`Step::block`], [`Step::attestation`], -/// [`Step::attester_slashing`], or [`Step::pow_block`] is set. +/// [`Step::attester_slashing`], [`Step::pow_block`], +/// [`Step::execution_payload`], or [`Step::payload_attestation_message`] is +/// set. fn apply_execution_step( store: &mut Store, case: &Case, @@ -492,30 +578,20 @@ fn apply_execution_step( if let Some(name) = &step.attestation { let attestation = decode_attestation(case, name)?; - return match ( + return expect_outcome( + name, fork_choice::on_attestation(store, &attestation, false, config, committees), step.valid, - ) { - (Ok(()), false) => Err(format!( - "{name} was accepted, but the step expects it to be rejected" - )), - (Err(err), true) => Err(format!("{name} was rejected: {err:?}")), - _ => Ok(()), - }; + ); } if let Some(name) = &step.attester_slashing { let attester_slashing = decode_attester_slashing(case, name)?; - return match ( + return expect_outcome( + name, fork_choice::on_attester_slashing(store, &attester_slashing), step.valid, - ) { - (Ok(()), false) => Err(format!( - "{name} was accepted, but the step expects it to be rejected" - )), - (Err(err), true) => Err(format!("{name} was rejected: {err:?}")), - _ => Ok(()), - }; + ); } if let Some(name) = &step.pow_block { @@ -528,8 +604,35 @@ fn apply_execution_step( return Ok(()); } + if let Some(name) = &step.execution_payload { + let signed_envelope: gloas::SignedExecutionPayloadEnvelope = decode(case, name)?; + // The format names no sampled columns for this step, so the evidence + // is the empty retrieval, which reads as available. + return expect_outcome( + name, + fork_choice::on_execution_payload_envelope( + store, + &signed_envelope, + config, + &[], + &ExecutionEngine::valid(), + ), + step.valid, + ); + } + + if let Some(name) = &step.payload_attestation_message { + let ptc_message: gloas::PayloadAttestationMessage = decode(case, name)?; + return expect_outcome( + name, + fork_choice::on_payload_attestation_message(store, &ptc_message, false, config), + step.valid, + ); + } + Err( - "step has none of tick, block, attestation, attester_slashing, or pow_block set" + "step has none of tick, block, attestation, attester_slashing, pow_block, \ + execution_payload, or payload_attestation_message set" .to_string(), ) } @@ -603,16 +706,26 @@ fn check_head(expected: &HeadCheck, store: &mut Store, config: &Config) -> Resul })?; let expected_root = parse_root(&expected.root); - if actual_root == expected_root && actual_slot == expected.slot { - Ok(()) - } else { - Err(format!( + if actual_root != expected_root || actual_slot != expected.slot { + return Err(format!( "head: expected {{slot: {}, root: 0x{}}}, got {{slot: {actual_slot}, root: 0x{}}}", expected.slot, hex::encode(expected_root.0), hex::encode(actual_root.0), - )) + )); + } + if let Some(expected_status) = expected.payload_status { + let node = fork_choice::get_head_node(store, config) + .map_err(|err| format!("get_head_node: {err:?}"))?; + if node.payload_status as u8 != expected_status { + return Err(format!( + "head payload_status: expected {expected_status}, got {} (head 0x{})", + node.payload_status as u8, + hex::encode(node.root.0) + )); + } } + Ok(()) } /// Checks `get_proposer_head`, computed the same way the reference test @@ -631,6 +744,85 @@ fn check_get_proposer_head( check_root("get_proposer_head", expected_hex, actual) } +/// Checks one of the two PTC vote fields (`payload_timeliness_vote` or +/// `payload_data_availability_vote`) against the store's votes for the named +/// block, position by position. +fn check_votes( + label: &str, + expected: &VoteCheck, + actual: Option>>, +) -> Result<(), String> { + let block_root = parse_root(&expected.block_root); + let actual = + actual.ok_or_else(|| format!("{label}: no votes for 0x{}", hex::encode(block_root.0)))?; + if actual == expected.votes { + Ok(()) + } else { + Err(format!( + "{label} for 0x{}: expected {:?}, got {actual:?}", + hex::encode(block_root.0), + expected.votes + )) + } +} + +/// Checks `viable_for_head_roots_and_weights`: the leaves of the payload-aware +/// tree under the justified root, each with its weight and payload status. +/// +/// See the module documentation for why the comparison ignores order. +fn check_viable_for_head( + expected: &[ViableLeafCheck], + store: &Store, + config: &Config, + committees: &CommitteeCache, +) -> Result<(), String> { + let index = store.block_index(); + let blocks = fork_choice::get_filtered_block_tree(store, &index, config).map_err(|err| { + format!("viable_for_head_roots_and_weights: get_filtered_block_tree: {err:?}") + })?; + let mut pending = vec![ForkChoiceNode { + root: store.beacon_justified_checkpoint().root, + payload_status: PayloadStatus::Pending, + }]; + let mut actual: Vec<(Root, u64, u8)> = Vec::new(); + while let Some(node) = pending.pop() { + let children = fork_choice::get_node_children(store, &blocks, node).map_err(|err| { + format!("viable_for_head_roots_and_weights: get_node_children: {err:?}") + })?; + if children.is_empty() { + let weight = fork_choice::gloas_get_weight(store, node, config, committees) + .map_err(|err| format!("viable_for_head_roots_and_weights: get_weight: {err:?}"))?; + actual.push((node.root, weight, node.payload_status as u8)); + } else { + pending.extend(children); + } + } + let mut expected: Vec<(Root, u64, u8)> = expected + .iter() + .map(|leaf| (parse_root(&leaf.root), leaf.weight, leaf.payload_status)) + .collect(); + expected.sort(); + actual.sort(); + if actual == expected { + Ok(()) + } else { + let show = |leaves: &[(Root, u64, u8)]| { + leaves + .iter() + .map(|(root, weight, status)| { + format!("{{0x{}, {weight}, {status}}}", hex::encode(root.0)) + }) + .collect::>() + .join(", ") + }; + Err(format!( + "viable_for_head_roots_and_weights: expected [{}], got [{}]", + show(&expected), + show(&actual) + )) + } +} + /// Applies one `checks` step: every field the fixture sets must match. pub(super) fn apply_checks( store: &mut Store, @@ -667,6 +859,17 @@ pub(super) fn apply_checks( if let Some(expected) = &checks.head { check_head(expected, store, config)?; } + if let Some(expected) = &checks.payload_timeliness_vote { + let actual = store.payload_timeliness_vote(&parse_root(&expected.block_root)); + check_votes("payload_timeliness_vote", expected, actual)?; + } + if let Some(expected) = &checks.payload_data_availability_vote { + let actual = store.payload_data_availability_vote(&parse_root(&expected.block_root)); + check_votes("payload_data_availability_vote", expected, actual)?; + } + if let Some(expected) = &checks.viable_for_head_roots_and_weights { + check_viable_for_head(expected, store, config, committees)?; + } if let Some(expected) = &checks.get_proposer_head { check_get_proposer_head(expected, store, config, committees)?; } @@ -719,14 +922,12 @@ fn run_case(case: &Case, config: &Config) -> Result<(), String> { /// case in this suite runs through [`run_case`] the same way regardless of /// which handler it came from, unlike `ssz_static`, which dispatches on it. pub fn trials() -> Vec { - let config = Arc::new(Config::active()); let cases = collect_all_handlers(PRESET, "fork_choice"); let mut trials = vec![super::discovery_trial("fork_choice", cases.len())]; for (_handler, case) in cases { - let config = Arc::clone(&config); - trials.push(super::case_trial("fork_choice", case, move |case| { - run_case(case, &config) + trials.push(super::case_trial("fork_choice", case, |case| { + run_case(case, &super::case_config(case)) })); } diff --git a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs index ce7d310be..b9282e6b6 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs @@ -97,9 +97,9 @@ pub fn fork_active_from_genesis(case: &Case) -> Config { /// The config to process one case against: its own `config.yaml` when it /// ships one, [`fork_active_from_genesis`]'s all-forks-at-genesis default -/// otherwise. Shared by [`gossip`], [`operations`] and [`sanity`], the three -/// runners whose cases can carry a fork schedule (or, for gossip, a blob -/// schedule) of their own. +/// otherwise. Shared by [`gossip`], [`operations`], [`sanity`] and +/// [`fork_choice`], the runners whose cases can carry a fork schedule (or, for +/// gossip, a blob schedule) of their own. /// /// The fork schedule matters in these suites because gloas's `get_ptc`, /// reached through attestation and payload attestation processing, reads @@ -172,6 +172,7 @@ pub const GLOAS_RUNNERS: &[&str] = &[ "finality", "random", "transition", + "fork_choice", ]; /// The root of the extracted fixture tree. From 908e697efb9294e295b31c225c0a67cda096504b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:44:37 -0300 Subject: [PATCH 025/132] test(beacon): run every gloas suite Every gloas runner that has a counterpart at earlier forks already ran through a temporary allow-list, so the fork gate itself was the only thing still holding the rest of the tree back. Declare gloas implemented by raising HIGHEST_IMPLEMENTED_FORK, and remove the allow-list, its check in case_trial, and the per-case handler field that only served it. The custody-group networking suites now run for gloas too. Gloas gossip vectors stay ignored on purpose: the node validates fulu's gossip rules only, and the gossip runner's comment now says so. --- .../tests/beacon_spec/fork.rs | 4 +- .../tests/beacon_spec/gossip.rs | 6 +- .../state_transition/tests/beacon_spec/mod.rs | 64 ++----------------- 3 files changed, 12 insertions(+), 62 deletions(-) diff --git a/crates/blockchain/state_transition/tests/beacon_spec/fork.rs b/crates/blockchain/state_transition/tests/beacon_spec/fork.rs index 7a51b7311..b44332c41 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/fork.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/fork.rs @@ -14,9 +14,9 @@ //! cases whose target this crate does not implement yet are counted as //! skipped rather than silently dropped, the way `ssz_static` counts //! container/fork pairs it does not decode. [`super::HIGHEST_IMPLEMENTED_FORK`] -//! is now fulu, and [`ethlambda_state_transition::beacon::upgrade::upgrade_state`] routes every +//! is now gloas, and [`ethlambda_state_transition::beacon::upgrade::upgrade_state`] routes every //! fork through its own `upgrade_to_*` function, so the two agree by -//! construction all the way to fulu; the two are still checking different +//! construction all the way to gloas; the two are still checking different //! things, though: this gate is "has this crate's state transition caught up //! to this fork at all," which is deliberately conservative, since running a //! fork's upgrade before its state transition exists would let this suite go diff --git a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs index 0ef599b16..cb0fccfe4 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs @@ -375,8 +375,10 @@ pub fn trials() -> Vec { // fork is ignored rather than run, the same "known gap, not a // silent one" treatment `case.in_scope()` already gives a fork // past `HIGHEST_IMPLEMENTED_FORK`. `case.in_scope()` alone would - // pass every fork up to fulu, since the state transition handles - // them all; the gossip rules do not. + // pass every fork up to gloas, since the state transition handles + // them all; the gossip rules do not. In particular the node does + // not validate gloas gossip yet, so a gloas case stays ignored + // here even though its state transition runs. let ignored = !case.in_scope() || case.fork != ForkName::Fulu || SKIPPED.iter().any(|(name, _)| *name == case.name); diff --git a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs index b9282e6b6..f84ae0d5b 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/mod.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/mod.rs @@ -138,42 +138,7 @@ pub fn case_config(case: &Case) -> Config { /// still need its own edit to *map* a fork's new or changed handlers to the /// right function, since that mapping is specific to each runner; only the gate /// is one line. -pub const HIGHEST_IMPLEMENTED_FORK: ForkName = ForkName::Fulu; - -/// Runners (or runner/handler pairs) whose Gloas cases already run, while the -/// fork is only partly implemented. -/// -/// An entry is either a bare runner name (`"ssz_static"`), which enables -/// every handler that runner ships, or `"runner/handler"` -/// (`"operations/withdrawals"`), which enables only that one handler under -/// that runner; `handler` is the directory right under the fork -/// (`gloas///...`), i.e. [`Case::handler`]. This lets a -/// runner whose Gloas support lands handler by handler (`operations`, whose -/// `execution_payload_bid`, `withdrawals`, and `parent_execution_payload` -/// went green well before `attestation` or `payload_attestation`) turn cases -/// on incrementally, without also running every other handler under it that -/// is still unimplemented. -/// -/// [`case_trial`] checks this alongside [`Case::in_scope`], so a case whose -/// runner or runner/handler is named here runs (and counts toward pass/fail) -/// even though [`HIGHEST_IMPLEMENTED_FORK`] has not reached Gloas yet; every -/// other runner's Gloas cases stay ignored. Emptied, and removed along with -/// this check, when [`HIGHEST_IMPLEMENTED_FORK`] becomes `Gloas`. -pub const GLOAS_RUNNERS: &[&str] = &[ - "ssz_static", - "fork", - "epoch_processing", - "rewards", - // Every handler `operations` ships for gloas now runs, so the bare - // runner name serves all of it, the same way `"ssz_static"` already - // does for its own runner. - "operations", - "sanity", - "finality", - "random", - "transition", - "fork_choice", -]; +pub const HIGHEST_IMPLEMENTED_FORK: ForkName = ForkName::Gloas; /// The root of the extracted fixture tree. /// @@ -239,7 +204,6 @@ pub fn collect_crypto(kind: &str, fork_of: impl Fn(&str) -> ForkName) -> Vec<(St Case { path: case_entry.path(), fork, - handler: handler.clone(), suite: handler.clone(), name: case_entry.file_name().to_string_lossy().into_owned(), }, @@ -256,10 +220,6 @@ pub struct Case { pub path: PathBuf, /// The fork whose rules apply. pub fork: ForkName, - /// The handler directory right under the fork's runner directory - /// (`gloas/operations//...`). [`GLOAS_RUNNERS`]'s - /// `"runner/handler"` form is matched against this. - pub handler: String, /// The suite directory name, which groups related cases. pub suite: String, /// The case directory name, which is what test output should identify. @@ -365,7 +325,7 @@ fn collect_in(root: &Path, config: &str, runner: &str, handler: &str) -> Vec Vec<(String, Case)> { // `collect` here would be wrong: `collect` walks every fork itself, // so nesting it inside this fork loop would yield each case once per // fork that happens to ship the runner. - collect_suites(&handler_entry.path(), fork, &handler, &mut |case| { + collect_suites(&handler_entry.path(), fork, &mut |case| { out.push((handler.clone(), case)) }); } @@ -406,10 +366,8 @@ pub fn collect_all_handlers(config: &str, runner: &str) -> Vec<(String, Case)> { /// Walks the suite and case directories under one fork's handler directory. /// /// Shared by both collectors so there is one definition of what a case directory -/// is, and so neither can drift from the other. `handler` is recorded on each -/// emitted [`Case`] as [`Case::handler`], for [`GLOAS_RUNNERS`]'s -/// `"runner/handler"` matching. -fn collect_suites(handler_dir: &Path, fork: ForkName, handler: &str, emit: &mut impl FnMut(Case)) { +/// is, and so neither can drift from the other. +fn collect_suites(handler_dir: &Path, fork: ForkName, emit: &mut impl FnMut(Case)) { if !handler_dir.is_dir() { return; } @@ -420,7 +378,6 @@ fn collect_suites(handler_dir: &Path, fork: ForkName, handler: &str, emit: &mut emit(Case { path: case_entry.path(), fork, - handler: handler.to_string(), suite: suite.clone(), name: case_entry.file_name().to_string_lossy().into_owned(), }); @@ -560,12 +517,6 @@ pub fn fixture_fork_trials() -> Vec { /// itself and print the count; the test harness counts ignored tests already, /// and names each one, which is strictly more than the tally said. /// -/// A Gloas case is the one exception: while Gloas is only partly implemented, -/// [`GLOAS_RUNNERS`] names the runners whose Gloas cases already run, ahead of -/// [`HIGHEST_IMPLEMENTED_FORK`] reaching Gloas, so those cases run (and are -/// counted toward pass/fail) even though [`Case::in_scope`] alone would mark -/// them ignored. -/// /// A panic inside `run` fails this case alone: the harness catches it per test. /// So a fixture that will not decode takes its own case down and no other. pub fn case_trial( @@ -573,10 +524,7 @@ pub fn case_trial( case: Case, run: impl FnOnce(&Case) -> Result<(), String> + Send + 'static, ) -> Trial { - let runner_handler = format!("{runner}/{}", case.handler); - let gloas_enabled = case.fork == ForkName::Gloas - && (GLOAS_RUNNERS.contains(&runner) || GLOAS_RUNNERS.contains(&runner_handler.as_str())); - let ignored = !(case.in_scope() || gloas_enabled); + let ignored = !case.in_scope(); let name = format!("{runner}/{}", case.id()); Trial::test(name, move || run(&case).map_err(Failed::from)).with_ignored_flag(ignored) } From e698f2223e3c3b9b1cbbdda6aca17c548d8d3942 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:37:47 -0300 Subject: [PATCH 026/132] refactor(beacon): name gloas in every fork match Audit every match and comparison over the per-fork enums so a fork is never handled by a rule written for an earlier one. The sites that mattered were on live paths: - A subnet attestation decoded with `fork >= Electra`, so on a gloas network an honest vote (whose `data.index` carries the payload flag) decoded as electra's and was rejected, penalizing the peer. It now answers `UnsupportedFork`, and the attestation topic maps that to `Ignore`, as the aggregate topic does. A handler test pins each topic's ignore arm, since deleting either would penalize honest gloas peers again with nothing failing. - The pool endpoints accepted `Eth-Consensus-Version: gloas` and then applied electra's rules and container to it, and the aggregate query would have labelled a gloas slot's electra-shaped aggregate as gloas. Both refuse gloas by name. - `range_needs_columns`, `fixed_proposer` and the test harness's gossip gate now name their forks, so the next fork has to be placed in each. - `get_head_node` built a fresh committee cache per call on the gloas arm. It runs every slot from the fork epoch, so it now uses the store's shared cache. Also drop references to how the work was scheduled from three doc comments, fix the six intra-doc links that failed `rustdoc::broken_intra_doc_links`, and correct the doc comments that undersold or misattributed what they describe. --- crates/blockchain/src/beacon_engine.rs | 7 +- .../src/beacon/fork_choice.rs | 21 ++-- .../src/beacon/helpers/electra.rs | 4 +- .../src/beacon/helpers/gloas.rs | 6 +- .../src/beacon/helpers/shuffling.rs | 2 +- .../state_transition/src/beacon/precheck.rs | 16 ++- .../src/beacon/stf/capella.rs | 9 +- .../src/beacon/stf/epoch/registry.rs | 6 +- .../tests/beacon_spec/gossip.rs | 13 ++- crates/net/p2p/src/beacon/decode.rs | 33 ++++++- crates/net/p2p/src/gossipsub/handler.rs | 44 +++++++++ crates/net/p2p/src/metrics.rs | 4 +- crates/net/p2p/src/req_resp/handlers.rs | 22 ++++- crates/net/rpc/src/beacon/pool.rs | 99 ++++++++++++++++--- crates/net/rpc/src/lib.rs | 7 +- 15 files changed, 237 insertions(+), 56 deletions(-) diff --git a/crates/blockchain/src/beacon_engine.rs b/crates/blockchain/src/beacon_engine.rs index c643914bc..510edbb9a 100644 --- a/crates/blockchain/src/beacon_engine.rs +++ b/crates/blockchain/src/beacon_engine.rs @@ -45,9 +45,10 @@ pub struct NewPayloadRequest<'a> { /// Gloas is `None` too, genuinely rather than temporarily: ePBS (EIP-7732) /// moves the payload out of the block into a separately gossiped envelope /// (see `containers::gloas::SignedExecutionPayloadEnvelope`), so a gloas -/// block carries nothing this function could ask about at all. A later task -/// answers the engine question gloas actually asks, from that envelope -/// rather than from the block. +/// block carries nothing this function could ask about at all. The engine +/// question gloas does ask, whether a payload is valid, belongs to that +/// envelope rather than to the block, and this follower does not receive +/// envelopes yet, so nothing here asks it. pub fn new_payload_request(block: &SignedBeaconBlock) -> Option> { let inner = match block { SignedBeaconBlock::Electra(inner) | SignedBeaconBlock::Fulu(inner) => inner, diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 13384c1c4..52762e1ce 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -1255,9 +1255,10 @@ pub fn get_proposer_score(store: &Store, config: &Config) -> Result { /// equal, the scored root, and the specification's own contribution for /// such a vote is `0` too. This function is not only /// [`get_weight`]'s own attestation half; [`is_head_weak`] and -/// [`is_parent_strong`] call it too, and both run underneath +/// [`is_parent_strong`] call it too. [`is_head_weak`] runs underneath /// [`should_apply_proposer_boost`], which the gloas head visits on every -/// call whose boosted block's parent is from the previous slot, so a pruned +/// call whose boosted block's parent is from the previous slot, and +/// [`is_parent_strong`] underneath `gloas_get_proposer_head`, so a pruned /// vote reaching this function is not only a pre-gloas concern. pub fn get_attestation_score( store: &Store, @@ -1644,18 +1645,18 @@ fn compute_head( /// one, and says Lighthouse does the same; our FULL rule differs from both. /// Adopt PENDING here if that PR lands. /// -/// A fresh [`CommitteeCache`] on the gloas arm: gloas's own weight -/// (`gloas_get_weight` by way of `should_apply_proposer_boost`) now needs -/// committees a pre-gloas head computation never did, and this is the one -/// entry point both [`get_head`] and the fixture harness reach it through, -/// with no longer-lived cache to hand down instead. See [`gloas_get_weight`]'s -/// own performance note for what that costs repeated across a whole tree -/// walk. +/// The gloas arm reads committees through the store's shared +/// [`Store::committee_cache`], the cache `on_block` and the gossip checks use: +/// gloas's own weight (`gloas_get_weight` by way of +/// `should_apply_proposer_boost`) needs committees a pre-gloas head +/// computation never did, and this runs every slot, so a cache of its own +/// would recompute every committee on every call. See [`gloas_get_weight`]'s +/// own performance note for what that costs across a whole tree walk. pub fn get_head_node(store: &Store, config: &Config) -> Result { let current_slot = get_current_slot(store, config); match config.fork_at_epoch(compute_epoch_at_slot(current_slot)) { ForkName::Gloas => { - let committees = CommitteeCache::default(); + let committees = store.committee_cache(); gloas_get_head(store, config, &committees) } ForkName::Phase0 diff --git a/crates/blockchain/state_transition/src/beacon/helpers/electra.rs b/crates/blockchain/state_transition/src/beacon/helpers/electra.rs index cca5d7bc9..77e292cfa 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/electra.rs @@ -111,14 +111,14 @@ //! `earliest_exit_epoch`, `consolidation_balance_to_consume`, and //! `earliest_consolidation_epoch` are state fields with no fork-invariant //! accessor on [`BeaconState`]. Unlike altair's participation flags (see -//! [`crate::beacon::helpers::altair::altair_state`]'s doc), these fields are not +//! `crate::beacon::helpers::altair::altair_state_ref`'s doc), these fields are not //! electra-only: fulu keeps every one of them unchanged (see the [`fulu`] //! module doc), and gloas keeps the churn cursors and (behind a slice view; //! see [`PendingQueueFields`]'s own doc) the pending queues unchanged too, so //! [`churn_cursors_mut`] and [`pending_queue_fields`]/[`pending_queue_fields_ref`] //! match `BeaconState::Electra`, `BeaconState::Fulu`, and `BeaconState::Gloas` //! rather than only the first. A projection that matched only `Electra`, the way -//! [`crate::beacon::helpers::altair::altair_state`] matches only `Altair`, would +//! `crate::beacon::helpers::altair::altair_state_ref` matches only `Altair`, would //! silently break every one of these functions on a fulu state, which is the //! single easiest mistake to make copying that shape without also copying //! the reasoning behind it. diff --git a/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs b/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs index 14816f09d..8998e7eb3 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs @@ -867,9 +867,9 @@ pub fn initiate_builder_exit( /// [`gloas::BeaconState::builder_pending_withdrawals`] if it carries a real /// amount, then clears the slot. /// -/// Called by `apply_parent_execution_payload` (parent-payload block -/// processing, a later task; SPEC `beacon-chain.md`'s "Settle the builder -/// payment", ~1767-1780), not by +/// Called by [`crate::beacon::stf::gloas::apply_parent_execution_payload`] +/// (parent-payload block processing; gloas `beacon-chain.md`'s "Settle the +/// builder payment"), not by /// [`crate::beacon::stf::epoch::gloas::process_builder_pending_payments`]: that /// epoch step evicts and settles the *older* half of `builder_pending_payments` /// by weight, against [`get_builder_payment_quorum_threshold`]; this settles diff --git a/crates/blockchain/state_transition/src/beacon/helpers/shuffling.rs b/crates/blockchain/state_transition/src/beacon/helpers/shuffling.rs index 2865cceed..787879158 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/shuffling.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/shuffling.rs @@ -93,7 +93,7 @@ pub fn compute_shuffled_index(index: u64, index_count: u64, seed: Bytes32) -> Re /// The algorithm is protolambda's, as lighthouse ships it in /// `swap_or_not_shuffle::shuffle_list` with `forwards = false`. Verified /// position by position against [`compute_shuffled_index`] by -/// [`tests::whole_list_shuffle_matches_the_per_index_shuffle`]: a mistake in +/// `tests::whole_list_shuffle_matches_the_per_index_shuffle`: a mistake in /// the pairing would still yield *a* permutation, quietly wrong, rather than a /// panic. pub fn shuffle_list(mut list: Vec, seed: Bytes32) -> Vec { diff --git a/crates/blockchain/state_transition/src/beacon/precheck.rs b/crates/blockchain/state_transition/src/beacon/precheck.rs index f2c9ee0ae..25d66d032 100644 --- a/crates/blockchain/state_transition/src/beacon/precheck.rs +++ b/crates/blockchain/state_transition/src/beacon/precheck.rs @@ -142,8 +142,20 @@ pub fn precheck_block( /// state, which is the import's job; the specification lets such a block /// through rather than rejecting it, and so does this. pub(crate) fn fixed_proposer(parent_state: &BeaconState, slot: Slot) -> Option { - let BeaconState::Fulu(state) = parent_state else { - return None; + let state = match parent_state { + BeaconState::Fulu(state) => state, + // Gloas keeps the lookahead, but its gossip rules are not applied + // here yet (block and column gossip refuse it before this is asked), + // so no window is read from a gloas parent and the caller lets the + // block through as it does before fulu. + BeaconState::Phase0(_) + | BeaconState::Altair(_) + | BeaconState::Bellatrix(_) + | BeaconState::Capella(_) + | BeaconState::Deneb(_) + | BeaconState::Electra(_) + | BeaconState::Gloas(_) + | BeaconState::Lean(_) => return None, }; let window_start = compute_start_slot_at_epoch(compute_epoch_at_slot(state.slot)); let offset = usize::try_from(slot.checked_sub(window_start)?).ok()?; diff --git a/crates/blockchain/state_transition/src/beacon/stf/capella.rs b/crates/blockchain/state_transition/src/beacon/stf/capella.rs index d5821ed80..9fedd3649 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/capella.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/capella.rs @@ -7,11 +7,10 @@ //! validator a way to get its balance out without ever submitting anything: //! [`process_withdrawals`] sweeps a bounded slice of the validator registry //! on every single block, pays out anyone it finds fully or partially -//! withdrawable, and [`BeaconState::next_withdrawal_index`] / -//! [`BeaconState::next_withdrawal_validator_index`] (reached here through -//! [`capella::BeaconState`]'s own fields) are the cursor that makes each -//! block's share of that sweep bounded regardless of how large the registry -//! grows. [`get_expected_withdrawals`] is the sweep itself; +//! withdrawable, and [`capella::BeaconState::next_withdrawal_index`] / +//! [`capella::BeaconState::next_withdrawal_validator_index`] are the cursor +//! that makes each block's share of that sweep bounded regardless of how +//! large the registry grows. [`get_expected_withdrawals`] is the sweep itself; //! [`process_bls_to_execution_change`] is the one new operation, a //! validator's one-time upgrade from a raw BLS withdrawal credential to an //! execution address, which is what makes it eligible for a payout in the diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/registry.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/registry.rs index 0d115f157..387a28284 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/registry.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/registry.rs @@ -44,8 +44,8 @@ use crate::beacon::primitives::{Epoch, Gwei, ValidatorIndex}; /// wholesale too (it has no validator-count activation cap of its own for /// this function to pick between), so this gate has nothing to serve gloas /// either. `get_activation_churn_limit` belongs to gloas's own -/// `process_pending_deposits` (EIP-8061, a later task), which caps *deposit* -/// processing, not registry activation. +/// `process_pending_deposits` (EIP-8061), which caps *deposit* processing, +/// not registry activation. fn activation_churn_limit(state: &BeaconState, config: &Config) -> Result { let churn_limit = get_validator_churn_limit(state, config); match state.fork_name() { @@ -77,7 +77,7 @@ fn activation_churn_limit(state: &BeaconState, config: &Config) -> Result { /// (EIP-7514) is confined to which cap the third pass dequeues against, so it /// is selected by fork through [`activation_churn_limit`] rather than /// duplicating the three passes around it, the same reuse-by-value pattern -/// [`super::process_slashings`] already uses for its own per-fork multiplier. +/// [`process_slashings`] already uses for its own per-fork multiplier. /// Electra redefines every pass, not just the cap, so it is a wholly separate /// function the crate's electra module owns, and fulu never revives a /// validator-count rule after that; [`activation_churn_limit`] refuses to run diff --git a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs index cb0fccfe4..a3c0a81f2 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs @@ -369,11 +369,14 @@ pub fn trials() -> Vec { cases.len(), )); for case in cases { - // The rules `beacon::gossip::block` and `beacon::gossip::column` - // implement are fulu's own `validate_beacon_block_gossip` and - // `validate_data_column_sidecar_gossip`; a case from any other - // fork is ignored rather than run, the same "known gap, not a - // silent one" treatment `case.in_scope()` already gives a fork + // The block and column rules (`beacon::gossip::{block, column}`) + // are fulu's own `validate_beacon_block_gossip` and + // `validate_data_column_sidecar_gossip`. The aggregate and + // attestation rules (`beacon::gossip::{aggregate, attestation}`) + // are electra's (`p2p-interface.md`), which fulu keeps. Either + // way a case from any other fork is ignored rather than run, the + // same "known gap, not a silent one" treatment `case.in_scope()` + // already gives a fork // past `HIGHEST_IMPLEMENTED_FORK`. `case.in_scope()` alone would // pass every fork up to gloas, since the state transition handles // them all; the gossip rules do not. In particular the node does diff --git a/crates/net/p2p/src/beacon/decode.rs b/crates/net/p2p/src/beacon/decode.rs index 54a223630..6d02fe2c0 100644 --- a/crates/net/p2p/src/beacon/decode.rs +++ b/crates/net/p2p/src/beacon/decode.rs @@ -294,10 +294,23 @@ impl Attestation { /// lighthouse decodes this one on that digest too. `fork` is the fork the /// subscribed digest was computed at. pub fn decode_attestation(fork: ForkName, bytes: &[u8]) -> Result { - if fork >= ForkName::Electra { - electra::SingleAttestation::from_ssz_bytes(bytes).map(Attestation::Electra) - } else { - phase0::Attestation::from_ssz_bytes(bytes).map(Attestation::Phase0) + match fork { + ForkName::Electra | ForkName::Fulu => { + electra::SingleAttestation::from_ssz_bytes(bytes).map(Attestation::Electra) + } + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb => phase0::Attestation::from_ssz_bytes(bytes).map(Attestation::Phase0), + // Gloas's `SingleAttestation` has the same bytes as electra's, but its + // `data.index` carries the payload-availability signal instead of + // being zero, so electra's gossip rules would reject an honest vote. + // `UnsupportedFork`, not `Ssz`: the sender did nothing wrong. + ForkName::Gloas => return Err(DecodeError::UnsupportedFork), + ForkName::Lean => { + unreachable!("a beacon topic's fork is never Lean: it is absent from ForkName::ALL") + } } .map_err(|_| DecodeError::Ssz) } @@ -563,6 +576,18 @@ mod tests { ); } + #[test] + fn a_subnet_attestation_at_gloas_is_unsupported_rather_than_malformed() { + // Gloas reads `data.index` as the payload-availability signal, so + // decoding these bytes with electra's rules would reject honest + // votes. The bytes are valid, so the refusal must not be `Ssz`. + let single = single_attestation(slot_of(10)).to_ssz(); + assert_eq!( + decode_attestation(ForkName::Gloas, &single), + Err(DecodeError::UnsupportedFork) + ); + } + #[test] fn a_truncated_subnet_attestation_is_refused() { let bytes = single_attestation(slot_of(10)).to_ssz(); diff --git a/crates/net/p2p/src/gossipsub/handler.rs b/crates/net/p2p/src/gossipsub/handler.rs index 4e6f28988..3d6b6f5b7 100644 --- a/crates/net/p2p/src/gossipsub/handler.rs +++ b/crates/net/p2p/src/gossipsub/handler.rs @@ -429,6 +429,12 @@ fn triage_attestation( const KIND: &str = beacon_topics::BEACON_ATTESTATION_KIND; let attestation = match beacon_decode::decode_attestation(wire.fork, payload) { Ok(attestation) => attestation, + // As on the aggregate topic: an honest peer on a fork this node has no + // attestation rules for must not be scored as a bad decoder. + Err(beacon_decode::DecodeError::UnsupportedFork) => { + metrics::inc_beacon_gossip(KIND, "unsupported_fork"); + return Dispatch::Report(Outcome::Ignore(IgnoreReason::UnsupportedFork)); + } Err(err) => { metrics::inc_beacon_gossip(KIND, "decode_failed"); debug!(kind = KIND, %err, bytes = payload.len(), "Beacon gossip decode failed"); @@ -963,6 +969,25 @@ mod tests { )); } + /// Gloas's aggregate has no modeled container here, and an honest gloas + /// peer sends exactly this once the clock reaches the fork, so the decode + /// error must not become a `Reject`, which would penalize it. + #[tokio::test] + async fn a_gloas_aggregate_is_ignored_as_an_unsupported_fork() { + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 0); + let server = unconnected_beacon_server(config, 0).await; + let wire = server + .wire + .beacon() + .expect("a beacon server has a beacon wire"); + let payload = electra_aggregate(4, 1).to_ssz(); + + assert!(matches!( + triage_aggregate(&server, wire, &payload, Instant::now()), + Dispatch::Report(Outcome::Ignore(IgnoreReason::UnsupportedFork)) + )); + } + #[tokio::test] async fn garbage_bytes_on_an_attestation_subnet_are_rejected_as_undecodable() { let server = unconnected_beacon_server(Config::mainnet(), 0).await; @@ -1015,6 +1040,25 @@ mod tests { )); } + /// A gloas `SingleAttestation` has electra's bytes, but its `data.index` + /// is the payload flag, so an honest `1` must not reach electra's rules + /// and be rejected as a nonzero index. + #[tokio::test] + async fn a_gloas_attestation_is_ignored_as_an_unsupported_fork() { + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 0); + let server = unconnected_beacon_server(config, 0).await; + let wire = server + .wire + .beacon() + .expect("a beacon server has a beacon wire"); + let payload = electra_single_attestation(4, 1).to_ssz(); + + assert!(matches!( + triage_attestation(&server, wire, &payload, 0), + Dispatch::Report(Outcome::Ignore(IgnoreReason::UnsupportedFork)) + )); + } + #[tokio::test] async fn garbage_bytes_on_a_consumerless_topic_are_rejected_as_undecodable() { let server = unconnected_beacon_server(Config::mainnet(), 0).await; diff --git a/crates/net/p2p/src/metrics.rs b/crates/net/p2p/src/metrics.rs index 3f1c060da..c56f21bbc 100644 --- a/crates/net/p2p/src/metrics.rs +++ b/crates/net/p2p/src/metrics.rs @@ -368,8 +368,8 @@ static LEAN_BEACON_FORK_DIGEST: LazyLock = LazyLock::new(|| { .unwrap() }); -/// Count one gossip message. `result` is `decoded`, `decode_failed`, or -/// `decompress_failed`. +/// Count one gossip message. `result` is `decoded`, `decode_failed`, +/// `decompress_failed`, or `unsupported_fork`. pub fn inc_beacon_gossip(topic: &str, result: &str) { LEAN_BEACON_GOSSIP_MESSAGES_TOTAL .with_label_values(&[topic, result]) diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index fbaecfb69..5cb0f09c5 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -1318,13 +1318,33 @@ fn range_batch_needs_columns(server: &P2PServer, batch: &std::ops::Range) - /// slot is at or after fulu, and the custody set is not empty. Lighthouse's /// range sync skips its custody-peer check before PeerDAS for the same reason: /// a batch with no columns to fetch has no custodian to wait for. +/// +/// Gloas answers yes as fulu does, though its columns have a different shape. +/// A batch ending in gloas can begin in fulu, whose blocks need their columns. +/// A batch lying entirely in gloas also reaches this, since +/// `beacon_fetched_through` advances past the gloas blocks the chain actor +/// refuses. It is then held for custody for nothing, and its column prefetch +/// fails to decode against fulu's shape; that failure logs at debug and +/// penalizes no peer. No gloas column is ever consumed from here. fn range_needs_columns( config: &Config, custody_columns: &[u64], batch: &std::ops::Range, ) -> bool { let last_slot = batch.end.saturating_sub(1); - !custody_columns.is_empty() && fork_at_slot(config, last_slot) >= ForkName::Fulu + let has_columns = match fork_at_slot(config, last_slot) { + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra => false, + ForkName::Fulu | ForkName::Gloas => true, + ForkName::Lean => { + unreachable!("fork_at_slot never returns Lean: it is absent from ForkName::ALL") + } + }; + !custody_columns.is_empty() && has_columns } /// This node's custody columns that no connected peer is known to custody. diff --git a/crates/net/rpc/src/beacon/pool.rs b/crates/net/rpc/src/beacon/pool.rs index 86809218f..63273e0d5 100644 --- a/crates/net/rpc/src/beacon/pool.rs +++ b/crates/net/rpc/src/beacon/pool.rs @@ -90,7 +90,7 @@ async fn post_pool_attestations( headers: HeaderMap, body: Bytes, ) -> Response { - if let Err(err) = require_electra_or_later(&headers) { + if let Err(err) = require_electra_or_fulu(&headers) { return err.into_response(); } let Ok(attestations) = serde_json::from_slice::>(&body) else { @@ -235,19 +235,34 @@ fn validate( }) } -/// The endpoints here take electra's containers, which exist only from that -/// fork on; `Eth-Consensus-Version` is required to say so. -fn require_electra_or_later(headers: &HeaderMap) -> Result<(), ApiError> { +/// The endpoints here take electra's containers, which exist from that fork +/// on and which fulu keeps; `Eth-Consensus-Version` is required to say so. +/// Gloas changed the attestation (a progressive list, and `data.index` as the +/// payload-availability signal) and the pool rules that go with it, neither of +/// which this node models, so it is refused by name rather than read as +/// electra. +fn require_electra_or_fulu(headers: &HeaderMap) -> Result<(), ApiError> { let fork = headers .get("eth-consensus-version") .and_then(|value| value.to_str().ok()) .and_then(ForkName::parse); - if fork.is_some_and(|fork| fork >= ForkName::Electra) { - Ok(()) - } else { - Err(ApiError::BadRequest( - "Eth-Consensus-Version must name electra or a later fork", - )) + match fork { + Some(ForkName::Electra | ForkName::Fulu) => Ok(()), + Some(ForkName::Gloas) => Err(ApiError::BadRequest( + "Eth-Consensus-Version gloas is not supported", + )), + // `ForkName::parse` never returns Lean: it is absent from `ALL`. + Some( + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Lean, + ) + | None => Err(ApiError::BadRequest( + "Eth-Consensus-Version must name electra or fulu", + )), } } @@ -280,7 +295,7 @@ async fn post_aggregate_and_proofs( headers: HeaderMap, body: Bytes, ) -> Response { - if let Err(err) = require_electra_or_later(&headers) { + if let Err(err) = require_electra_or_fulu(&headers) { return err.into_response(); } let Ok(aggregates) = serde_json::from_slice::>(&body) @@ -343,6 +358,26 @@ async fn get_aggregate_attestation( Extension(pool): Extension, Query(query): Query, ) -> Response { + let fork = store + .config() + .fork_at_epoch(compute_epoch_at_slot(query.slot)); + // The pool holds electra-shaped votes, and the response is labelled with + // the slot's fork, so a gloas slot would be served as gloas with a body of + // the wrong shape. Refused by name, as the submit endpoints refuse the + // gloas header. + match fork { + ForkName::Gloas => { + return ApiError::BadRequest("gloas aggregates are not supported").into_response(); + } + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Fulu + | ForkName::Lean => {} + } let aggregate = pool .lock() .expect("attestation pool lock poisoned") @@ -354,9 +389,6 @@ async fn get_aggregate_attestation( let Some(aggregate) = aggregate else { return ApiError::NotFound("no matching attestations to aggregate").into_response(); }; - let fork = store - .config() - .fork_at_epoch(compute_epoch_at_slot(query.slot)); let response = crate::json_response(serde_json::json!({ "version": fork.as_str(), "data": aggregate, @@ -389,6 +421,7 @@ mod tests { accessors::get_beacon_committee, test_state::{sign_for, with_signing_validators_at}, }; + use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::shared::{AttestationData, Checkpoint}; use http_body_util::BodyExt as _; use tower::ServiceExt as _; @@ -743,4 +776,42 @@ mod tests { let response = app.oneshot(request).await.unwrap(); assert_eq!(response.status(), StatusCode::BAD_REQUEST); } + + #[tokio::test] + async fn an_aggregate_query_at_a_gloas_slot_is_refused_by_name() { + let mut fixture = fixture(); + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 0); + let (store, _) = crate::test_utils::beacon_store_with_config(fixture.state.clone(), config); + fixture.store = store; + let (status, json) = get_aggregate(&fixture, Root::ZERO, fixture.state.slot(), 0).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(json["message"], "gloas aggregates are not supported"); + } + + #[tokio::test] + async fn a_gloas_fork_header_is_refused_by_name() { + for path in [ + "/eth/v2/beacon/pool/attestations", + "/eth/v2/validator/aggregate_and_proofs", + ] { + let fixture = fixture(); + let network: RpcToP2PRef = fixture.network.clone(); + let app = routes() + .with_state(fixture.store.clone()) + .layer(Extension(network)) + .layer(Extension(fixture.pool.clone())); + let request = Request::post(path) + .header("eth-consensus-version", "gloas") + .body(Body::from("[]")) + .unwrap(); + let response = app.oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{path}"); + let body = response.into_body().collect().await.unwrap().to_bytes(); + let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!( + json["message"], "Eth-Consensus-Version gloas is not supported", + "{path}" + ); + } + } } diff --git a/crates/net/rpc/src/lib.rs b/crates/net/rpc/src/lib.rs index a2144f7c6..95507c06d 100644 --- a/crates/net/rpc/src/lib.rs +++ b/crates/net/rpc/src/lib.rs @@ -463,13 +463,18 @@ pub(crate) mod test_utils { /// phase0 one whatever `state`'s fork: these endpoints read the state and /// the block's root and slot, never the block's body. pub(crate) fn beacon_store_at(state: BeaconState) -> (Store, H256) { + beacon_store_with_config(state, Config::mainnet()) + } + + /// [`beacon_store_at`], under `config` instead of mainnet's schedule. + pub(crate) fn beacon_store_with_config(state: BeaconState, config: Config) -> (Store, H256) { let slot = state.slot(); let block = phase0_beacon_block(slot, H256::ZERO); let root = block.message_hash_tree_root(); let mut store = Store::init_beacon( Arc::new(InMemoryBackend::default()), 1_606_824_023, - Config::mainnet(), + config, root, Checkpoint { root, slot }, slot, From 9534e6a49bba4333bd661b1fa0d8cc8589fc6117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:37:47 -0300 Subject: [PATCH 027/132] docs: describe gloas in the beacon stf reference The state transition and fork choice now cover gloas, so the reference pages and the session guide still said the module stopped at fulu, that gloas fixtures were ignored, and that gloas had no fork choice. Bring them in line, and record the places where the implementation departs from the spec so a reader comparing the two does not have to rediscover them. - beacon_stf.md: a gloas section (deferred payload, progressive lists and the registry, the narrow views that let electra functions serve gloas, the payload dimension of fork choice), plus status, counts, the ignored-case accounting and the fork directory accounting. - spec_deviations.md: the fulu-to-gloas fork-choice boundary rule (with what Lighthouse, Lodestar and Prysm do there), the tolerances in the gloas head, the cached state root in the envelope check, and the cost of the spec-literal weight walk, which any follower on a network that schedules gloas pays from the fork epoch. - CLAUDE.md: correct the statements that stopped being true and add where to find the gloas code, including that the live follower still refuses gloas. Counts and design stay in the reference pages. - The spec endpoint serves gloas's `Config` keys but not its preset and constant keys; the pool endpoints refuse gloas by name. - Document the `unsupported_fork` gossip reason, the column-wait rule that now names gloas, and the startup refusal of a gloas anchor in checkpoint sync. --- CLAUDE.md | 89 +++++++----- crates/net/rpc/src/beacon/config.rs | 11 +- docs/beacon_stf.md | 218 +++++++++++++++++++++------- docs/beacon_wire.md | 17 ++- docs/checkpoint_sync.md | 1 + docs/metrics.md | 9 +- docs/rpc.md | 11 +- docs/spec_deviations.md | 99 +++++++++++++ 8 files changed, 355 insertions(+), 100 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d3e7a93fe..b24af79a7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -450,9 +450,12 @@ against them. `beacon::tests::the_built_in_network_derives_what_it_always_did` pins the parsed mainnet config to `Config::mainnet()`, so the file and the Rust constant cannot drift apart. Sepolia's config schedules gloas: its `GLOAS_*` keys are claimed (`ForkName::Gloas` exists, and `Config` carries the -fork's schedule, timing and churn fields), but its state transition and fork -choice do not exist yet, so a Sepolia follower stops tracking the chain at -`GLOAS_FORK_EPOCH` regardless. `network::warn_if_gloas_scheduled` (shared by +fork's schedule, timing and churn fields), and its state transition and fork +choice exist, but the live follower refuses gloas blocks +(`process_or_pend_block`) and a gloas anchor (`refuse_unfollowable_fork` in +`main.rs`), since nothing delivers payload envelopes or payload attestations to +it, so a Sepolia follower stops tracking the chain at `GLOAS_FORK_EPOCH` +regardless. `network::warn_if_gloas_scheduled` (shared by `BuiltInNetwork::resolve` and a loaded `NetworkDir`) warns about this explicitly at startup (checked against `config.gloas_fork_epoch`), separately from the ignored-keys warning, which now only ever names heze's keys and @@ -540,7 +543,7 @@ See [`docs/rpc.md`](docs/rpc.md) for the full reference: CLI flags and defaults, ## Beacon Chain types (`crates/common/types/src/beacon/`) `ethlambda-types` carries the **Ethereum Beacon Chain** containers (phase0 -through fulu) alongside lean's own types. The Beacon Chain is a different +through gloas) alongside lean's own types. The Beacon Chain is a different protocol from the Lean consensus this repo implements; the types share a crate so that one `BlockChainServer` can dispatch on a single state type instead of existing once per chain. @@ -553,7 +556,7 @@ existing once per chain. silent wrong answer. - **`ForkName::Lean` is deliberately absent from `ForkName::ALL`.** `ALL` is what `parse`, `previous` and `next` search, so its absence keeps - `parse("lean")` at `None` and `Fulu.next()` at `None`, meaning a fork upgrade + `parse("lean")` at `None` and `Gloas.next()` at `None`, meaning a fork upgrade cannot walk off the end into lean. Lean is not a point on the Beacon Chain's fork timeline. `Lean` is declared *last* so the derived `Ord` puts it after every beacon fork, which is what `fork >= ForkName::X` gating reads. @@ -566,6 +569,9 @@ existing once per chain. - Per-fork containers are plain structs behind an enum, so SSZ stays derived: two of phase0's fields are *replaced* in altair, one field changes type in five separate forks, and the state's merkle tree gains a level at electra. + Gloas's big containers and most lists are progressive (EIP-7688), in + `containers/gloas.rs`, with their own `SignedBeaconBlock::Gloas` body; a + progressive container has no fixed depth. - **`beacon::primitives::Root` *is* `primitives::H256`**, not a second 32-byte hash converted at the boundary, and `beacon::primitives::HashTreeRoot` re-exports lean's convenience trait rather than declaring its own. The @@ -586,20 +592,21 @@ existing once per chain. `lambdaclass/libssz` (for lambdaclass/libssz#33), pinned by `rev` rather than tracking `main`: this is the SSZ encoder and merkleizer behind every `hash_tree_root`, so a routine `cargo update` must not be able to move it. - Return them to a crates.io version once a release carries #33. A git dependency rather than a + Return them to a crates.io version once a release carries #33 and #37. A git dependency rather than a `[patch.crates-io]` override because nothing outside this workspace depends on libssz, so there is no second copy to unify; that also keeps the manifest free of a `[patch]` table, which `shadow/cargo-patch.toml` would collide with. -- The state transition consuming these containers is **not** in this repo yet; - it lives on `feat/beacon-chain-stf`, where these types are verified against - consensus-specs v1.7.0-beta.2. What runs here is the containers' own - round-trip and shape tests. -- **`Validators` and `Balances` are `ethlambda_ssz_tree::List`s**, persistent +- **`Validators` and `Balances` are `ethlambda_ssz_tree::List`s** (gloas: + `ethlambda_ssz_tree::ProgressiveList`, same method set), persistent Merkle trees that cache node hashes and share unchanged subtrees between - states through `Arc`; they have no slices and no `iter_mut`. A leaf holds a - page-sized run of elements rather than one chunk, and an inner node a page of - child pointers spanning several binary levels, so a lookup crosses a handful - of nodes and a rebuilt leaf or node copies one page. Writes are + states through `Arc`; they have no slices and no `iter_mut`. The registry is + therefore reached through element-level `BeaconState` accessors + (`validator`/`validator_mut`, `balance`/`balance_mut`, `push_validator`, + `iter_validators`/`iter_balances`, `validator_count`, `validators_root`), so + no accessor has to return two list types. A leaf holds a page-sized run of + elements rather than one chunk, and an inner node a page of child pointers + spanning several binary levels, so a lookup crosses a handful of nodes and a + rebuilt leaf or node copies one page. Writes are buffered until `BeaconState::apply_pending_mutations`, which the state transition calls before every state-root computation. A state decoded from storage is rebased onto a cached one (`Store::get_state`). @@ -612,7 +619,7 @@ existing once per chain. ## Beacon Chain STF (`crates/blockchain/state_transition/src/beacon/`) -The **Ethereum Beacon Chain** consensus specs (phase0 through fulu), a different +The **Ethereum Beacon Chain** consensus specs (phase0 through gloas), a different protocol from the Lean consensus the rest of this repo implements, live in the `beacon` module of `ethlambda-state-transition` beside lean's own state transition. The module holds the *behavior* (state transition, fork choice, @@ -698,28 +705,34 @@ transitions are in `ethlambda-types`, per the section above. Nothing above release plus `DerefMut`/`IndexMut` on `SszList` (lambdaclass/libssz#33), `as_chunks` in the merkleize fold loop (#35), and the same `DerefMut`/ `IndexMut` for `ProgressiveList` (#37, an unmerged PR branch head for now). - This module needs the `SszList` commit (#33) for the same reason. -- **Status:** all seven forks (phase0 through fulu) have containers, fork - upgrades, state transitions, and epoch processing. As of the v1.6.1 fixture - pin, every case passed on both presets: mainnet is 5705 cases and minimal - 40009. The v1.7.0-beta.2 bump adds failures the fixture release itself - brought: `ssz_static` cases for `NewPayloadRequest` and the four new - `PartialDataColumn*` containers, a fork-choice rule change, and one - `transition` case, each fixed by the commits that follow the bump. (The - bump's new `networking/gossip_*` format is already handled: the gossip - validation vectors come from this same tree now, not the separate - v1.7.0-beta.1 download #38 used to fetch.) The crate's lib - target holds 200 tests with `beacon-spec-tests` on, 185 plus 15 ignored - without; both figures cover lean's own unit tests as well, since the two - chains now share one lib target. Fork choice is fixture-verified too: 150 mainnet - `fork_choice` cases pass, covering bellatrix's `on_merge_block`/terminal-PoW - validation, the `v1.7.0` proposer-boost dependent-root gate and - `get_proposer_head`'s proposer-equivocation branch, deneb's blob data - availability, and fulu's column data availability. -- Nothing is ignored for being unimplemented. Ignored cases are the - `LightClient*` containers (a different layer, out of scope) and the `gloas` - and `heze` fixture trees. Those two do not parse as a `ForkName`, so - `collect` would skip them silently; `UNMODELED_FORKS` names them and + This module needs the `SszList` commit (#33) for the same reason, and gloas's + progressive lists need #37. +- **Status:** all eight forks (phase0 through gloas) have containers, fork + upgrades, state transitions, and epoch processing, and every fixture case + that is not ignored passes on both presets at the pinned consensus-specs + release. The counts live in the Status section of + [`docs/beacon_stf.md`](docs/beacon_stf.md), so they are kept in one place. + Fork choice is fixture-verified too, including bellatrix's terminal-PoW + validation, deneb's and fulu's data availability, and gloas's payload-aware + suites. +- **Gloas** lives in `stf/gloas.rs`, `stf/epoch/gloas.rs`, `helpers/gloas.rs`, + the gloas section of `fork_choice.rs`, and `containers/gloas.rs` in + `ethlambda-types`. **The live follower does not follow gloas**: it refuses + gloas blocks and a gloas anchor until envelopes and payload attestations are + delivered to it. `get_head_node` switches on the clock, so a follower on any + network that schedules gloas (Sepolia does) runs the spec-literal + `gloas_get_head` every slot from `GLOAS_FORK_EPOCH`, whatever it can import. + [`docs/beacon_stf.md`](docs/beacon_stf.md) has the design and + [`docs/spec_deviations.md`](docs/spec_deviations.md) the deliberate departures + from the spec, in particular the fulu-to-gloas fork-choice boundary rule. +- Nothing is ignored for being unimplemented in the state transition or fork + choice. Ignored cases are the `LightClient*` containers (a different layer, + out of scope), `networking/gossip_*` cases outside what the node validates + (every fork's cases for topics it has no validator for, non-fulu cases of the + four it validates, and the fulu vectors in `SKIPPED`, which assume a + bad-block cache), and the `heze` fixture tree. `heze` does + not parse as a `ForkName`, so `collect` would skip it silently; + `UNMODELED_FORKS` names it and `fixture_forks/every_directory_is_accounted_for` fails on any fork directory that is neither parseable nor listed, so a new fork forces a decision. - A fixture case with no `post` state asserts the input must be **rejected**. That diff --git a/crates/net/rpc/src/beacon/config.rs b/crates/net/rpc/src/beacon/config.rs index 3ddb354b0..2defba04f 100644 --- a/crates/net/rpc/src/beacon/config.rs +++ b/crates/net/rpc/src/beacon/config.rs @@ -16,10 +16,13 @@ //! `GENESIS_TIME` is absent by design: it is not a `config.yaml` key, and //! `/eth/v1/beacon/genesis` is where it is reported. //! -//! The preset and constant keys are the ones lighthouse reports, less two -//! kinds: gloas-only keys, since this build cannot process gloas (the same -//! reason `Config` leaves out `GLOAS_*`), and keys the specification does not -//! define at all (`GAS_LIMIT_ADJUSTMENT_FACTOR`, `RESP_TIMEOUT`, +//! The configuration keys include gloas's (`GLOAS_*` and the other gloas +//! timing and churn keys), since `Config` holds them and this serializes the +//! whole of it. The preset and constant keys are the ones lighthouse reports, +//! less two kinds: gloas's own preset and constant keys (`DOMAIN_BEACON_BUILDER` +//! and `DOMAIN_PTC_ATTESTER` among them), which `beacon::preset` and +//! `beacon::constants` hold but this list does not report yet, and keys the +//! specification does not define at all (`GAS_LIMIT_ADJUSTMENT_FACTOR`, `RESP_TIMEOUT`, //! `TTFB_TIMEOUT`). Constants lighthouse leaves out, such as //! `JUSTIFICATION_BITS_LENGTH`, are left out here too. diff --git a/docs/beacon_stf.md b/docs/beacon_stf.md index c91c7c687..db8b9ffd1 100644 --- a/docs/beacon_stf.md +++ b/docs/beacon_stf.md @@ -3,7 +3,7 @@ The `beacon` module of `ethlambda-state-transition` (`crates/blockchain/state_transition/src/beacon/`) implements the Ethereum **Beacon Chain** consensus specification, -[`ethereum/consensus-specs`][specs], phase0 through fulu. +[`ethereum/consensus-specs`][specs], phase0 through gloas. This is not the Lean consensus protocol the rest of the crate implements. The two sit in one crate for the reason their types sit in one crate: a caller @@ -184,16 +184,20 @@ per-fork field lists are not a growing tail: `current_epoch_participation`, which have a different type. - `latest_execution_payload_header` keeps its name from bellatrix on, but is a different container only in bellatrix, capella, and deneb; electra and fulu - reuse deneb's shape unchanged. + reuse deneb's shape unchanged, and gloas drops it for a block hash and the + latest bid. - The field count crosses a power of two at electra, so the state's merkle tree is five levels deep through deneb and six from electra on. The same logical field - has a different generalized index in different forks. + has a different generalized index in different forks. Gloas's state is a + progressive container (EIP-7495), whose tree has no fixed depth at all. - `SignedBeaconBlock::Fulu` wraps `electra::SignedBeaconBlock` rather than a `fulu` type of its own, since fulu changes no field of a block. It still needs to be its own variant: fulu changes how a block is *processed*, since the blob commitment limit becomes epoch-dependent, so code that dispatches on fork still has to tell a fulu block from an electra one even though both - carry the identical payload. + carry the identical payload. Gloas is a different case: its block body + changes (a bid in place of a payload, payload attestations, the parent's + execution requests), so `SignedBeaconBlock::Gloas` has a type of its own. | Fork | State fields | HTR leaves | Depth | |------|--------------|------------|-------| @@ -203,6 +207,7 @@ per-fork field lists are not a growing tail: | capella, deneb | 28 | 32 | 5 | | electra | 37 | 64 | 6 | | fulu | 38 | 64 | 6 | +| gloas | 46 | progressive | none | A single container with fork-conditional serialization would have to reproduce all of that by hand. Derived codecs get it from the struct definition, which is @@ -211,7 +216,7 @@ checked field by field against the spec text and then verified by `ssz_static`. Since SSZ carries no type tag, the fork cannot be recovered from the bytes, so decoding takes it from context: `BeaconState::from_ssz(fork, bytes)`. -### Not duplicating the state transition seven times +### Not duplicating the state transition once per fork The cost of per-fork structs is that a naive implementation would copy every function once per fork. Two things prevent that: @@ -259,9 +264,10 @@ impossible. `validators` and `balances` hold one entry per validator, about 2.4M on mainnet, so they dominate both the cost of a state root and the memory of every -cached state. They are `ethlambda_ssz_tree::List`s rather than `SszList`s: -persistent Merkle trees in the shape of the SSZ one, modeled on the `milhouse` -lists lighthouse keeps its state in. +cached state. They are `ethlambda_ssz_tree::List`s rather than `SszList`s (from +gloas on, `ethlambda_ssz_tree::ProgressiveList`s, described in the Gloas section +below): persistent Merkle trees in the shape of the SSZ one, modeled on the +`milhouse` lists lighthouse keeps its state in. - **Nodes cache their hash, and states share nodes.** A state derived from another shares every subtree the block did not touch through `Arc`, and its @@ -297,6 +303,121 @@ over the registry should walk `iter_validators()`, zipped with candidate for computing once per epoch rather than per call, once it is shown that no block operation changes it mid-epoch. +## Gloas + +Five of gloas's EIPs shape this module: ePBS (EIP-7732), progressive lists and +containers (EIP-7688), proposer selection that skips slashed validators +(EIP-8045), increased exit and consolidation churn (EIP-8061), and builder +deposits and exits (EIP-8282). EIP-7843 and EIP-7928 only add payload fields +(`slot_number`, which `verify_execution_payload_envelope` checks, and +`block_access_list`). It is the one fork whose containers change shape +wholesale, so its code sits beside the pre-gloas code rather than inside it: `stf/gloas.rs` and +`stf/epoch/gloas.rs` (transition), `helpers/gloas.rs` (helpers), the gloas +section of `fork_choice.rs` (fork choice), and `containers/gloas.rs` in +`ethlambda-types`. Every gloas case that is not ignored passes on both presets. + +The live follower does not follow gloas yet. `process_or_pend_block` refuses a +gloas block and `refuse_unfollowable_fork` refuses a gloas anchor at startup, +because nothing delivers payload envelopes or payload attestations to the chain +actor. Gloas gossip is answered with `Ignore(UnsupportedFork)` rather than +scored, so an honest peer past the fork epoch is not penalized. + +### The deferred payload + +A gloas block has no execution payload. Its body commits to a builder's bid +(`process_execution_payload_bid`), and the builder reveals the payload later in +a `SignedExecutionPayloadEnvelope`. The payload is applied by the *next* +block: `process_parent_execution_payload` compares that block's bid against the +parent's committed one to tell whether the parent was full or empty, and for a +full parent `apply_parent_execution_payload` runs the execution-layer requests +carried in `parent_execution_requests` and settles the builder's payment. An +empty parent must carry no requests. + +`verify_execution_payload_envelope` is the checking half that used to sit in +`process_execution_payload`, and it mutates nothing: fork choice calls it on a +stored state when an envelope arrives, and only the next block changes state. +Block processing therefore takes no `ExecutionEngine`, and `process_withdrawals` +takes no payload, since the expected withdrawals are a function of the state +alone. + +Execution-layer requests moved with the payload. Deposit, withdrawal and +consolidation requests, and the two builder requests, are reached only through +`apply_parent_execution_payload`; gloas's `process_operations` no longer calls +the first three. Their behavior did not change, so +`apply_parent_execution_payload` calls the fulu and electra processors instead +of keeping copies. + +### Progressive lists and the registry + +EIP-7688 makes most lists in the gloas state and body unbounded, merkleized as a +chain of growing subtrees (EIP-7916), and makes the big containers progressive +(EIP-7495); a progressive container has no fixed depth. The other progressive +lists are libssz's `ProgressiveList`. The registry is the exception: +`validators` and `balances` stay tree-backed, through +`ethlambda_ssz_tree::ProgressiveList`, which has the same persistent nodes, +update buffer and rebase as the bounded `List` and the same method set, so a +gloas state shares subtrees with its parent the way earlier ones do. + +The registry therefore has two list types across the forks, and an accessor +cannot return either one. The whole-registry accessors were replaced by +element-level ones on `BeaconState`: `validator_count`, `validator`, +`validator_mut`, `balance`, `balance_mut`, `push_validator`, `iter_validators`, +`iter_balances` and `validators_root`. Each dispatches once and hands back an +element, a count or an iterator. Only the few that need the list itself +(`iter_validators`, `iter_balances`, rebasing onto a cached state, pointer +equality) match on the private `Registry` enum. + +### One function for both list types + +The spec modifies a small part of what gloas touches, and the rest is electra's +text run against a differently typed list. Copying it would fork every fix, so +the shared functions take narrow views instead: + +| View | Reaches | +|------|---------| +| `PendingQueueFields` | The deposit balance cursor and the three pending queues, on electra, fulu and gloas | +| `ChurnCursorsMut` | The four exit and consolidation churn cursors, on the same three forks | +| `gloas_state`, `gloas_state_ref` | Gloas's own concrete state, for fields only gloas has | + +Both list types deref to a slice of the same element type, so a view can expose +slices and element-level methods and let electra's `process_withdrawal_request`, +`process_consolidation_request`, exit and slashing paths serve a gloas state +unchanged. The churn limit is the one input that differs: electra's cursor +functions read gloas's own limits on a gloas state, which is a fork dispatch +inside them rather than a second copy. A function gets a gloas copy only where +the spec marks it modified. The rule for choosing between the views and a +projection is the one in "A recurring bug" below: project only when the return +type must be gloas's own. + +### Fork choice + +Gloas gives each block two branches to weigh: *empty*, where no payload was +revealed or attested as timely and available, and *full*, where the builder's +payload was. A gloas fork-choice node is therefore `(root, PayloadStatus)` +with `Empty`, `Full` and `Pending`, where every earlier fork's node is a bare +root. Nodes are derived on demand from the block tree, each block's bid and the +set of verified payloads (`get_node_children` builds them), not stored, which +mirrors the spec's `Store`. + +`ForkRules` (`PreGloas` or `Gloas`) is the other new dimension. It is derived +from an exhaustive match on a fork or a container, and it is what +`validate_on_attestation` and `update_latest_messages` read, since gloas turns +`data.index` into the payload flag and orders votes by slot instead of target +epoch. A vote's rules follow its own container, not the clock. + +The head algorithm follows the current slot's fork: from `GLOAS_FORK_EPOCH` on, +`get_head_node` runs `gloas_get_head` over the whole tree, pre-gloas blocks +included; before it, the unchanged `compute_head` runs. The switch follows the +clock, so any follower on a network that schedules gloas (Sepolia's built-in +config does) runs `gloas_get_head` every slot from that epoch, over its +pre-gloas tree, whether or not it can import a gloas block; its cost is the +subject of [spec_deviations.md](spec_deviations.md). The spec never describes a +tree that crosses the boundary, so that page also records the rule this module +chose. +Two new handlers, `on_execution_payload_envelope` and +`on_payload_attestation_message`, join the spec's list of ways to change the +store. + ## Macros and traits Two `macro_rules!` in the whole crate, both local, both replacing boilerplate that @@ -396,45 +517,40 @@ Two things that arrangement has to be careful about: ## Status -All seven forks, phase0 through fulu, have containers, fork upgrades, state -transitions, and epoch processing. As of the v1.6.1 fixture pin, every case -passed on both presets: - -| Preset | Fixture cases | Ignored | Lib tests | -|--------|---------------|---------|-----------| -| mainnet | 5705, all green | 152 | 195 | -| minimal | 40009, all green | 3692 | 196 | - -The v1.7.0-beta.2 bump (see "Fixture suites" below) adds failures the fixture -release itself brought rather than any change here: `ssz_static` cases for -`NewPayloadRequest` and the four new `PartialDataColumn*` containers, a -fork-choice rule change, and one `transition` case. (A fourth item, the whole -new `networking/gossip_*` format, is fixed: the gossip validation vectors this -crate's `gossip.rs` runner covers now come from this same tree instead of a -separate v1.7.0-beta.1 download, and every other `gossip_*` handler or fork is -ignored by name rather than left unmatched.) The remaining commits fix each of -the rest in turn; this table is stale until they land. - -The lib counts were 244 and 245 while the containers, presets, configuration and -primitives were defined here. Their 48 unit tests moved with them and run in -`ethlambda-types`; the fixture counts, which are what defines correctness here, -are unchanged. +All eight forks, phase0 through gloas, have containers, fork upgrades, state +transitions, and epoch processing. At the pinned fixture release every case +that is not ignored passes on both presets: + +| Preset | Fixture cases passed | Ignored | Lib tests | +|--------|----------------------|---------|-----------| +| mainnet | 7175 | 1285 | 422 passed, 1 ignored | +| minimal | 49261 | 5403 | 423 passed, 1 ignored | + +The lib counts are with `beacon-spec-tests` on, and include lean's own unit +tests, since the two chains share one lib target. The one ignored lib test is +a timing measurement, not a fixture. Minimal runs more cases because the release ships more fixtures for it, and it runs two runners mainnet does not: `genesis`'s `initialization` and `validity`. The release ships no mainnet `genesis` fixtures, so that whole runner is gated -behind the `preset-minimal` feature (`tests/spec/genesis.rs`). +behind the `preset-minimal` feature (`tests/beacon_spec/genesis.rs`). + +The gossip validation vectors this crate's `gossip.rs` runner covers come from +the same tree as every other suite. The node validates four topics under fulu's +rules; every other case is ignored by name rather than left unmatched. -Nothing is ignored for being unimplemented, with one exception now that -`ForkName::Gloas` exists: every gloas case is individually ignored, by the -same `HIGHEST_IMPLEMENTED_FORK`/`Case::in_scope` gate that would ignore any -other fork past the last one this crate implements (`HIGHEST_IMPLEMENTED_FORK` -stays `Fulu` through Part A). Every other ignored case is one of two -deliberate exclusions: +Nothing is ignored for being unimplemented in the state transition or fork +choice: `HIGHEST_IMPLEMENTED_FORK` is gloas, and every gloas case that is not +ignored runs. The ignored cases are deliberate exclusions: - `LightClient*` containers under `ssz_static`, 5 container types across altair - through fulu. The light-client sync protocol is a different layer from the + through gloas. The light-client sync protocol is a different layer from the state transition and fork choice, and is not in this module's scope. +- `networking/gossip_*` cases outside what the node validates: every fork's + cases for topics it has no validator for (`IGNORED_HANDLERS`), non-fulu cases + of the four it validates, and the fulu vectors in `SKIPPED`, which assume a + bad-block cache. A gloas message is refused with `Ignore(UnsupportedFork)` + before any rule runs. - The `heze` fixture tree, one ignored entry. See "Accounting for every fork directory" below. @@ -445,16 +561,14 @@ a name that does not parse is skipped. That skip is silent in a way the `HIGHEST_IMPLEMENTED_FORK` gate is not: the cases never become tests, so they are not counted as ignored either, and nothing in the output says they exist. -`gloas` no longer takes this silent path: `ForkName::parse("gloas")` succeeds, -so its cases become ordinary tests, individually named and ignored through ordinary -`Case::in_scope` gating, the same as fulu's cases would be if -`HIGHEST_IMPLEMENTED_FORK` were dropped back a fork. `heze`, the one fork after -gloas, is still unparseable and still silently skipped without this section's -own accounting. +`gloas` does not take this silent path: `ForkName::parse("gloas")` succeeds, so +its cases become ordinary tests, individually named and run. `heze`, the one +fork after gloas, is still unparseable and would be silently skipped without +this section's own accounting. So `UNMODELED_FORKS` names `heze` alone, it reports as one ignored test, and `fixture_forks/every_directory_is_accounted_for` fails if the tree holds a fork -directory that is neither parseable nor listed. A release that adds a fork now +directory that is neither parseable nor listed. A release that adds a fork forces a decision instead of quietly widening the gap. | Suite | Covers | @@ -471,10 +585,14 @@ forces a decision instead of quietly widening the gap. ### Fork choice is fixture-verified -150 mainnet `fork_choice` cases pass, covering bellatrix's `on_merge_block`/ +205 mainnet `fork_choice` cases pass, covering bellatrix's `on_merge_block`/ terminal-PoW validation, the `v1.7.0` proposer-boost dependent-root gate and `get_proposer_head`'s proposer-equivocation branch, deneb's blob data -availability, and fulu's column data availability. +availability, fulu's column data availability, and gloas's payload-aware +suites: `get_head`, `get_parent_payload_status`, `on_block`, `on_attestation`, +`on_execution_payload_envelope`, `on_payload_attestation_message`, +`payload_data_availability`, `payload_timeliness` and `ex_ante`. Gloas +accounts for 54 of the 205. The release still ships no phase0 `fork_choice` suite: the earliest is altair's, built from altair-shaped states even though altair changes nothing @@ -504,8 +622,8 @@ This caused five separate bugs during implementation: - Capella's withdrawal sweep, projected to capella's own state. - Deneb reusing that same sweep, still projected to capella's state. -- Altair's participation fields, projected to altair's state, so every one of - the five later forks that also carries participation failed. +- Altair's participation fields, projected to altair's state, so every later + fork that also carries participation failed. - Deneb's `process_attestation`, projected to deneb's state. - `slash_validator` calling phase0's `initiate_validator_exit` at electra, which also left electra's EIP-7251 churn cursor unadvanced, mispricing every diff --git a/docs/beacon_wire.md b/docs/beacon_wire.md index c07cba640..6a7d94711 100644 --- a/docs/beacon_wire.md +++ b/docs/beacon_wire.md @@ -149,6 +149,14 @@ reaches it at all, on any outcome: verifying and relaying it is the whole of what this node owes the topic (see above), so there is nothing further for the chain actor to do with one. +The rules above are fulu's. A message from a fork this build has no rules for, +gloas today, is IGNOREd with the reason `unsupported_fork` and is never scored +against the peer: the decoders answer `DecodeError::UnsupportedFork`, not `Ssz`, +for a gloas aggregate, attester slashing or subnet attestation, since an honest +gloas peer sends exactly those once this node's clock reaches the fork. A gloas +subnet attestation has the bytes of an electra one, but its `data.index` is the +payload flag, so decoding it under electra's rules would reject honest votes. + The remaining five global topics are decoded, logged at `debug`, and IGNOREd, since nothing consumes them; an undecodable payload on any topic is REJECTed. Nothing is published on any topic, columns included: nothing this node can @@ -402,9 +410,10 @@ the availability gate. Nothing waits on the answer, so a short, empty or refused one costs nothing and is not retried; the per-block path is the backstop. -Since the column request is aimed by custody, a batch reaching fulu is held -back, blocks included, until every custody column has a known custodian among -the connected peers. Lighthouse's range sync holds its batches the same way. +Since the column request is aimed by custody, a batch whose last slot is fulu +or later is held back, blocks included, until every custody column has a known +custodian among the connected peers. Lighthouse's range sync holds its batches +the same way. Right after startup, custody is known for almost no peer, since a peer's custody arrives with its `metadata/3` answer after it connects. A mainnet follower's first batch after a fresh checkpoint sync went out 5 s after @@ -549,7 +558,7 @@ as a query filter, so a `quic`-only record is invisible to it. | Metric | Meaning | | --- | --- | -| `lean_beacon_gossip_messages_total{topic,result}` | Gossip received, by topic and by `decoded` / `decode_failed` / `decompress_failed` | +| `lean_beacon_gossip_messages_total{topic,result}` | Gossip received, by topic and by `decoded` / `decode_failed` / `decompress_failed` / `unsupported_fork` | | `lean_beacon_gossip_validation_total{kind,outcome,reason}` | Gossip verdicts, now including `beacon_aggregate_and_proof` and `beacon_attestation`; see [metrics.md](./metrics.md#beacon-gossip-validation) | | `lean_beacon_gossip_verdict_expired_total{kind}` | Verdicts that came too late to propagate anything; should stay at zero | | `lean_beacon_status_digest_mismatch_total` | Handshakes seen from another fork digest | diff --git a/docs/checkpoint_sync.md b/docs/checkpoint_sync.md index ff92020b1..48a70c5a4 100644 --- a/docs/checkpoint_sync.md +++ b/docs/checkpoint_sync.md @@ -145,6 +145,7 @@ All checks are performed before a downloaded checkpoint anchor is accepted. The | Justified epoch >= finalized epoch | Justified must be at or after finalized | | Block header slot <= state slot | Block header cannot be ahead of the state | | Block's fork matches state's fork | The fetched block must decode as the same fork the state resolved to | +| Anchor is not gloas (`refuse_unfollowable_fork`) | Nothing delivers payload envelopes or payload attestations to the chain actor yet, so a gloas anchor could not be followed. Runs on both anchor sources, the checkpoint provider's state and a loaded network's own genesis state | | Anchor pairing | See [Anchor Pairing](#anchor-pairing) below | Beacon has no same-slot-checkpoints-matching-roots check: nothing here computes two separate checkpoint roots to compare, since a trusted checkpoint-synced anchor is finalized by fiat, and the beacon spec's own construction applies that single checkpoint to all four of the store's justified/finalized slots at once. diff --git a/docs/metrics.md b/docs/metrics.md index bf215a3c6..83d0627f2 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -381,7 +381,14 @@ only), `not_aggregator` (aggregate only), `not_in_committee`, `unknown_validator`, `selection_proof` (aggregate only), `aggregator_signature` (aggregate only), `aggregate_signature` (aggregate only), `target_not_ancestor`, `wrong_subnet` (attestation only) on the reject -side. `already_seen` and `overloaded` are shared with every other topic. +side. `already_seen`, `overloaded` and `unsupported_fork` are shared with the +other topics: `unsupported_fork` is an ignore reason for a message of a fork this +build has no gossip rules for (gloas today), on the block, data column, +aggregate, attestation and attester-slashing topics, so an honest peer past the +fork epoch is not scored as a bad decoder. The same label value appears as +`result` on `lean_beacon_gossip_messages_total` for every one of those but the +block topic: a gloas block decodes, so it counts as `decoded` there and +`unsupported_fork` shows only as a verdict reason. Two permit pools bound the blocking-thread half of validation: `gossip_validation_permits` for blocks and columns, diff --git a/docs/rpc.md b/docs/rpc.md index cd0f22595..ee355cb8f 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -273,7 +273,9 @@ the chain actor writes, so no request waits on the actor. back in an `IndexedErrorMessage` with its position; the valid ones in the same batch are still published. There is no seen-attestation cache. Each accepted attestation also goes into the node's **attestation pool**, since - gossip never delivers a node its own messages. + gossip never delivers a node its own messages. `Eth-Consensus-Version: + gloas` is refused with a `400` here and on `aggregate_and_proofs`, as is a + gloas slot on the aggregate query below: the pool holds electra-shaped votes. - **`beacon_committee_subscriptions`**: each aggregator's entry makes the node join its committee's attestation subnet until the end of that slot, so the committee's votes from other validators reach the pool too: every @@ -350,8 +352,11 @@ the specification's constants, as the Beacon API asks. Validator clients depend on it: lighthouse's refuses a beacon node whose `PRESET_BASE` does not match its own, and treats an absent key as a mismatch. -The key set is lighthouse's, less gloas-only keys (this build cannot process -gloas) and three keys the specification does not define +The key set is lighthouse's, less gloas's preset and constant keys +(`beacon::preset` and `beacon::constants` hold them, such as +`DOMAIN_BEACON_BUILDER` and `DOMAIN_PTC_ATTESTER`, but this list does not +report them yet; gloas's schedule, timing and churn keys are reported, since +they are `Config` fields) and three keys the specification does not define (`GAS_LIMIT_ADJUSTMENT_FACTOR`, `RESP_TIMEOUT`, `TTFB_TIMEOUT`). Domain types and withdrawal prefixes are `0x`-prefixed hex; `VERSIONED_HASH_VERSION_KZG` is a decimal, as lighthouse reports it. `GENESIS_TIME` is absent: diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 7ff2d24e2..f1ffe819a 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -131,3 +131,102 @@ answers `404`, since no candidate matches. This was found by running a mainnet follower and pointing a second one at its API: before the fallback existed, the second died with `peer served no block at the anchor slot 15265888`. + +## The fulu-to-gloas fork-choice boundary + +The gloas fork-choice text describes only a store anchored at a gloas block, so +it says nothing about a tree that crosses from fulu into gloas, which is the +tree every live follower holds at the fork epoch. + +- **The specification:** `get_parent_payload_status` reads the parent block's + execution payload bid, which a fulu parent does not have. Consensus-specs + issue #5096 asked what a pre-gloas parent should answer and was closed as + not planned. Pull request #5125 is open, as of this writing, and proposes + `PENDING`. +- **ethlambda:** a pre-gloas block is a single node whose payload status is + `FULL`, since its payload ran inside the block and never had an empty branch. + Concretely, `get_parent_payload_status` of a gloas block with a pre-gloas + parent answers `FULL`; a pre-gloas block's only child (`get_node_children`) is + its `FULL` node; a vote for one (`get_supported_node`) supports that node + whatever its `payload_present` says; and `gloas_get_ancestor` stepping onto + one lands on `FULL`. Its payload also counts as verified, timely and available + (`is_payload_verified`, `payload_timeliness`, `payload_data_availability`), so + the first gloas block passes `on_block` and an `index == 1` vote for the last + fulu block passes `validate_on_attestation`. +- **Which head runs:** `get_head_node` follows the current slot's fork, not the + head block's or the justified checkpoint's. From `GLOAS_FORK_EPOCH` on, + `gloas_get_head` runs over the whole tree, pre-gloas blocks included; before + it, the unchanged `compute_head` runs. +- **Other clients:** Lodestar and Prysm also treat a pre-gloas parent as + `FULL`. Lighthouse (v8.2.2) answers `EMPTY` for a pre-gloas parent, although + #5125 describes its handling as `PENDING`. +- **Consequence:** if #5125 lands, only the answers above change, and each is + named in the header comment of `fork_choice.rs`'s gloas section. + +## The gloas head tolerates state the specification asserts on + +Three places in the gloas head read something the specification would abort on, +because a live node holds gaps a fixture tree never has. Each is marked +"Implementation choice, not spec text" at its definition. + +- **Pruned votes** (`get_attestation_score`, and through it `is_head_weak` and + `is_parent_strong`; the gloas head reaches `is_head_weak` from + `should_apply_proposer_boost`, and the gloas proposer head reaches + `is_parent_strong`). A vote whose ancestor walk meets a root missing + from the block index contributes nothing, where `get_ancestor` would raise. + Pruning removes only entries below the finalized slot, and every root scored is + indexed at or above it, so such a walk cannot descend from the scored root and + the specification's own contribution for that vote is zero too. +- **Unknown voters** (`gloas_get_attestation_score`). A vote for a root the store + never held is skipped, where `get_supported_node` would raise, so one stale + voter cannot abort a head computation. A block pruned from the index but still + in the store is not skipped: its vote walks the block table and contributes + nothing. +- **Missing timeliness** (`should_apply_proposer_boost`). A candidate with no + `block_timeliness` entry reads as not timely by either deadline, where the + specification indexes the entry directly. The entries are in memory only, so a + restart empties them. The reading can only miss withholding a boost from an + early equivocation, never withhold one wrongly. +- **Equivalence:** each is the specification's answer wherever the specification + has one; they differ only where it would have raised. + +## The envelope check reads the cached state root + +`verify_execution_payload_envelope` compares `envelope.beacon_block_root` with +the root of the state's latest block header. + +- **The specification:** sets `header.state_root = hash_tree_root(state)` on a + copy of the header. That is exact for a state fresh out of block processing, + whose `latest_block_header.state_root` is still zero. +- **ethlambda:** uses `BeaconState::compute_state_root`. The function's caller + hands it a stored state, and this repository writes the real root into that + header field as soon as the block's transition returns. Hashing such a state + would hash a header whose `state_root` is already set, a different value from + the one the header committed to, and every stored state's envelope would fail. + `compute_state_root` returns the cached value only while the state is still + at that block's slot and the field is set, and hashes otherwise, so both + shapes give the same root. +- **Equivalence:** the same check, on both kinds of state. + +## Gloas `get_weight` is the specification's walk, with its cost + +`gloas_get_weight` transcribes the specification statement by statement, where +the pre-gloas head folds weights bottom-up once per tree. + +- **ethlambda:** every node the head visits re-walks every latest message, each + ancestor step decodes signed blocks (the node's own and its parent's, and a + third per voter in `get_supported_node`), telling a pre-gloas root from a gloas + one costs a decode the pre-gloas tree never paid, and + `should_apply_proposer_boost` reruns a full `is_head_weak` score although its + answer does not depend on the node. `get_head_node` passes the store's shared + committee cache, so committees are not rebuilt per call. +- **Why:** the port is checked against the fixtures statement by statement, and + a faster form is a second implementation to keep in step with the spec. +- **Consequence:** fine for fixture trees, which are a handful of blocks. + `get_head_node` switches on the clock, so any follower on a network that + schedules gloas (Sepolia's built-in config does) runs `gloas_get_head` every + slot from `GLOAS_FORK_EPOCH`, over its pre-gloas tree, whether or not it can + import a gloas block. The remaining costs need a bottom-up form before that + epoch, not only before a gloas follower is deployed. The pre-gloas + form's own measurement, in `compute_weights`' documentation, is the estimate + of what the walk costs on a mainnet tree. From b9c9dd0206540e02f59d96e6b0181b9fbc7f3f42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:50:02 -0300 Subject: [PATCH 028/132] perf(beacon): one bottom-up head computation for every fork From GLOAS_FORK_EPOCH the head ran the specification's get_head, which weighs every candidate node by walking every latest message, with a block decode at each ancestor hop and in get_supported_node. On a Sepolia follower that work runs on every tick and every import, and grows with the validator count times the candidate nodes, which would wedge the chain actor. One walk now serves every fork. Every block is a PENDING node with EMPTY and FULL nodes under it, and a child hangs off the payload node its bid builds on; a pre-gloas block is the degenerate case with only FULL, the fulu-to-gloas rule stated once. Votes are summed at their supported node in one pass, then each block's total is folded into the parent payload node it builds on, highest slot first, so the work is proportional to messages plus blocks. The fold equals the specification's get_attestation_score for every node; the argument from is_ancestor is in the fork_choice.rs section that introduces it. The current slot's fork picks the two rules that differ: how a block's payload dimension is read (collapsed to a single FULL node before the fork epoch, as the pre-gloas head treated it) and whether the proposer boost is gated by should_apply_proposer_boost. That gate reads the parent's score from the same table, and the boost is added along the boosted block's chain afterwards. The walk stops at the least of the finalized block's slot, the justified block's slot and the boosted block's parent's slot, which on a live node is the finalized block's. The descent compares only descendants of the justified root, a vote for a block below the bound supports only nodes below it, and the boost gate reads a parent at or above it, so the bound changes no answer. A beacon store never prunes its block index, so without the bound each head computation would cover the whole history since the anchor rather than the unfinalized window. A block's fork and parent payload status are recorded as a BlockPayloadLink when on_block imports it, so the walk decodes no block in the ordinary case. The child-versus-parent bid comparison is one function shared with get_parent_payload_status, and a bid error is an error, not EMPTY. Links carry their block's slot and are pruned below the finalized block with the execution-hash cache. A link lost to a restart is derived once by decoding; the actor records the finalized block's own link before pruning, since the prune bound is read from it. The weak-head check takes the parent's slot from the block index rather than decoding it. The tiebreaker and should_extend_payload answer from the index and the links for the same reason. compute_head and the spec-literal gloas_get_head and gloas_get_weight stay, public, as references, and the public should_extend_payload keeps the specification's is_parent_node_full, so the tests never compare the walk with itself. The fork-choice fixture harness compares the walk with the references at every head check and every viable leaf weight. Seeded randomized tests do the same over trees that cross the fork boundary, with random bids, verified payloads, votes, equivocators and boost roots, a justified checkpoint above the anchor, and a finalized checkpoint that names a real block; others show that a second head computation, a prune at finality, and the actor's prune cycle after a restart read no block. --- CLAUDE.md | 11 +- crates/blockchain/src/lib.rs | 28 +- .../src/beacon/fork_choice.rs | 1882 +++++++++++++++-- .../tests/beacon_spec/fork_choice.rs | 105 +- crates/common/types/src/beacon/fork_choice.rs | 61 +- crates/storage/src/store.rs | 135 +- docs/beacon_engine.md | 8 +- docs/beacon_stf.md | 21 +- docs/spec_deviations.md | 116 +- 9 files changed, 2109 insertions(+), 258 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index b24af79a7..4b7b5f80f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -719,9 +719,14 @@ transitions are in `ethlambda-types`, per the section above. Nothing above the gloas section of `fork_choice.rs`, and `containers/gloas.rs` in `ethlambda-types`. **The live follower does not follow gloas**: it refuses gloas blocks and a gloas anchor until envelopes and payload attestations are - delivered to it. `get_head_node` switches on the clock, so a follower on any - network that schedules gloas (Sepolia does) runs the spec-literal - `gloas_get_head` every slot from `GLOAS_FORK_EPOCH`, whatever it can import. + delivered to it. `get_head_node` is one bottom-up walk for every fork, with + the current slot's fork selecting the payload rules and the boost gate, so a + follower on any network that schedules gloas (Sepolia does) pays work + proportional to its votes plus blocks every slot and decodes no block in the + ordinary case (each block's payload link is recorded at import; only the rare + weak-parent equivocation scan and the one-time derivation of a link lost to a + restart decode, see `docs/spec_deviations.md`). `compute_head` and the + spec-literal `gloas_get_head` are the references its tests compare it with. [`docs/beacon_stf.md`](docs/beacon_stf.md) has the design and [`docs/spec_deviations.md`](docs/spec_deviations.md) the deliberate departures from the spec, in particular the fulu-to-gloas fork-choice boundary rule. diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 35e4c3529..db30a58e1 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -3128,16 +3128,16 @@ impl BlockChainServer { /// import — the two places beacon finality can move — bounds that by the /// unfinalized window rather than by this node's uptime. /// - /// Also bounds the two beacon scratch caches with the same horizon, the - /// execution-hash cache and the optimistic-root set, which is why the - /// finalized slot is read before the "nothing is held" early return rather - /// than after it: all three share a horizon and these two call sites, but - /// the caches fill on every beacon import whether or not anything is being - /// held for its columns. + /// Also bounds the three beacon scratch caches with the same horizon, the + /// execution-hash cache, the payload-link cache and the optimistic-root + /// set, which is why the finalized slot is read before the "nothing is + /// held" early return rather than after it: all four share a horizon and + /// these two call sites, but the caches fill on every beacon import + /// whether or not anything is being held for its columns. /// /// The held-block half is a no-op whenever nothing is held, which is always /// true on lean; the cache halves are no-ops there too, since only a beacon - /// import ever writes either. + /// import ever writes to them. fn evict_held_blocks_at_or_below_finality(&mut self) { let finalized = self .store @@ -3154,6 +3154,20 @@ impl BlockChainServer { self.store .prune_beacon_el_block_hashes(finalized_slot, finalized.root); + // The head walk's per-block payload links share that horizon: the walk + // weighs only blocks at or above the finalized block, so a link below + // it is never read again. The prune keys its bound on the finalized + // block's own link, which a restart loses, so it is recorded first. + if self.store.chain() == Chain::Beacon { + fork_choice::ensure_payload_link(&self.store, finalized.root) + .inspect_err(|err| { + warn!(?err, "Could not record the finalized block's payload link") + }) + .ok(); + } + self.store + .prune_beacon_payload_links(finalized_slot, finalized.root); + // Same horizon, same reason. An execution client doing a long state // sync answers `NOT_VALIDATED` to every block, so the optimistic set // takes one root per import and neither `mark_validated` nor diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 52762e1ce..a3ebca083 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -210,7 +210,8 @@ use crate::beacon::stf; // keyed on plain `Root`s, and nothing about either type needs storage of its // own beyond what `BeaconScratch` already keeps. pub use ethlambda_types::beacon::fork_choice::{ - ForkChoiceNode, LatestMessage, PayloadStatus, PayloadStatusEnum, PayloadStatusV1, PowBlock, + BlockPayloadLink, ForkChoiceNode, LatestMessage, PayloadStatus, PayloadStatusEnum, + PayloadStatusV1, PowBlock, }; // --------------------------------------------------------------------------- @@ -275,6 +276,18 @@ impl ForkRules { ForkName::Lean => lean_fork_unreachable("ForkRules::of"), } } + + /// The [`BlockPayloadLink`] of a block handled under these rules, given + /// the parent payload branch it builds on where the rules record one. The + /// one place the rules' payload shape is stated: a pre-gloas block is a + /// single FULL node whose payload ran inside it, so it has no payload + /// branch of its parent to choose, and `parent_status` is ignored. + fn payload_link(self, parent_status: Option) -> BlockPayloadLink { + match self { + ForkRules::PreGloas => BlockPayloadLink::PreGloas, + ForkRules::Gloas => BlockPayloadLink::Gloas { parent_status }, + } + } } impl Attestation { @@ -753,10 +766,10 @@ pub fn resolve_invalid_block( /// blocks MUST NOT be applied to any `VALID` or `NOT_VALIDATED` ancestors." /// Deleting the `LiveChain` rows satisfies both halves at once, because /// `Store::block_index` is the only source fork choice reads: -/// [`compute_weights`] folds a subtree total only into parents it finds in that -/// index and gates the proposer-boost walk on the same membership, so a vote -/// naming a removed root seeds an entry that is never folded anywhere; -/// [`filter_block_tree`] and [`get_head`] are index-derived too. +/// [`compute_node_weights`] folds a subtree total only into parents it finds in +/// that index and gates the proposer-boost walk on the same membership, so a +/// vote naming a removed root seeds an entry that is never folded anywhere; +/// [`filter_block_tree`] and [`walk_head`] are index-derived too. /// /// One index scan builds the whole child map rather than rescanning per level: /// the descendants of one root are a tiny fraction of the tree, but finding @@ -769,14 +782,14 @@ pub fn resolve_invalid_block( /// whose freshest vote named a branch the execution layer has since rejected /// keeps pointing at a root no longer in the index, until it attests again. /// -/// [`compute_weights`] already drops such a vote rather than raising, so +/// [`compute_node_weights`] already drops such a vote rather than raising, so /// [`get_head`] is unaffected. [`get_weight`] deliberately does not: it is the /// specification's own version, and /// `tests::a_vote_for_a_pruned_block_weighs_nothing_instead_of_failing` pins /// that divergence on purpose. Nothing on this node's paths calls it today /// ([`get_proposer_head`] has no callers outside this file), but wiring up a /// beacon proposer duty would make it reachable, and it should get -/// `compute_weights`' treatment first. +/// `compute_node_weights`' treatment first. pub fn invalidate_subtree(store: &mut Store, invalid_root: Root) -> usize { let index = store.block_index(); @@ -1016,6 +1029,7 @@ pub fn get_forkchoice_store( let anchor_root = anchor_block.message_hash_tree_root(); let anchor_rules = ForkRules::of(anchor_state.fork_name()); + let anchor_slot = anchor_block.slot(); // The specification asserts `anchor_block.state_root == // hash_tree_root(anchor_state)`, which holds only when the anchor state is @@ -1112,6 +1126,12 @@ pub fn get_forkchoice_store( // (nothing to insert: the anchor's own payload is not verified until an // envelope arrives for it). Earlier forks record none of these for the // anchor, and nothing reads them there. + // + // The anchor's payload link is recorded here as well. Its parent is not in + // this store, so a gloas anchor has no parent status to compare bids for; + // nothing reads one, since the head walk only asks for a status when the + // parent is indexed. + store.set_payload_link(anchor_root, anchor_slot, anchor_rules.payload_link(None)); match anchor_rules { ForkRules::Gloas => { store.set_block_timeliness(anchor_root, [true, true]); @@ -1255,11 +1275,10 @@ pub fn get_proposer_score(store: &Store, config: &Config) -> Result { /// equal, the scored root, and the specification's own contribution for /// such a vote is `0` too. This function is not only /// [`get_weight`]'s own attestation half; [`is_head_weak`] and -/// [`is_parent_strong`] call it too. [`is_head_weak`] runs underneath -/// [`should_apply_proposer_boost`], which the gloas head visits on every -/// call whose boosted block's parent is from the previous slot, and -/// [`is_parent_strong`] underneath `gloas_get_proposer_head`, so a pruned -/// vote reaching this function is not only a pre-gloas concern. +/// [`is_parent_strong`] call it too, [`is_head_weak`] underneath the public +/// [`should_apply_proposer_boost`] and [`is_parent_strong`] underneath +/// `gloas_get_proposer_head`. The head walk does not: it reads the same scores +/// from [`compute_node_weights`], which drops such a vote too. pub fn get_attestation_score( store: &Store, index: &HashMap, @@ -1293,8 +1312,9 @@ pub fn get_attestation_score( /// justified checkpoint's state, plus the proposer boost if it applies. /// /// The specification's own per-root definition, kept as written. [`get_head`] -/// calls [`compute_weights`] instead, which produces the same numbers for the -/// whole tree at once; this is what that is tested against. +/// weighs with [`compute_node_weights`] instead, which produces the same +/// numbers for the whole tree at once; this is what [`compute_weights`], its +/// pre-gloas reference, is tested against. pub fn get_weight( store: &Store, index: &HashMap, @@ -1557,28 +1577,27 @@ pub fn get_filtered_block_tree( Ok(blocks) } -/// The LMD GHOST head: starting from the justified checkpoint, repeatedly -/// step to the child with the greatest weight until a leaf is reached. +/// The pre-gloas LMD GHOST head: starting from the justified checkpoint, +/// repeatedly step to the child with the greatest weight until a leaf is +/// reached. +/// +/// **A reference, not the node's head computation.** [`get_head_node`] runs +/// [`walk_head`] for every fork, and on a pre-gloas tree the two agree; this +/// is kept, public, as the independent [`compute_weights`] algorithm the walk +/// is tested against (the fork-choice fixture harness and the randomized +/// differential test both call it). /// /// The children scan in the loop below reads `blocks`, [`get_filtered_block_tree`]'s /// already-filtered, already in-memory result, not [`Store::block_index`] /// itself: it is the specification's own second whole-`Dict` scan the module /// documentation calls out, but it never costs a further backend round trip. /// -/// Takes `&Store` rather than `&mut Store`: unlike [`get_head`], which records -/// the head it finds, this is the read-only walk underneath it. [`on_block`] -/// is the other caller, which needs the head *before* the block it is -/// importing joins the store (see `update_proposer_boost_root` in the -/// specification's `on_block`) and must not record that transient answer as -/// the store's own head. +/// Takes `&Store` rather than `&mut Store`: nothing here records the head it +/// finds. No node code calls it; only tests do. /// /// Takes `index` rather than building it, matching [`get_filtered_block_tree`] -/// and [`compute_weights`], which this hands the same one to: [`get_head`] -/// has nothing else to build it from, but [`on_block`] already scanned -/// `LiveChain` once for its own finalized-descendant check by the time it -/// gets here, and passes that scan's result on rather than paying for a -/// second one just to compute the head. -fn compute_head( +/// and [`compute_weights`], which this hands the same one to. +pub fn compute_head( store: &Store, index: &HashMap, config: &Config, @@ -1617,17 +1636,489 @@ fn compute_head( Ok(head) } -/// [`compute_head`] pre-gloas, gloas's own [`gloas_get_head`] once the chain -/// has upgraded, as a full [`ForkChoiceNode`] rather than just a root. +// --------------------------------------------------------------------------- +// The head computation: one bottom-up walk for every fork +// --------------------------------------------------------------------------- +// +// One weight table and one descent serve both kinds of block. Every block is +// a `PENDING` node with two payload nodes under it (`EMPTY`, and `FULL` once +// its payload is verified), and a child block hangs off whichever of those its +// bid says it builds on. A pre-gloas block is the degenerate case: its only +// payload node is `FULL` (its payload ran inside the block), and every child +// builds on it. That is the fulu-to-gloas rule this crate decided on, stated +// once, in [`BlockPayloadLink`], instead of being repeated in a second head +// algorithm. +// +// # Why one pass over the votes gives the specification's weight +// +// The specification's `get_attestation_score(node)` adds the balance of every +// vote whose supported node `is_ancestor` of `node` matches: the chain from +// the supported node up through `get_ancestor`, where the wildcard `PENDING` +// ancestor matches either payload branch. +// +// Follow that chain from a vote's supported node `S = (r, s)`. `get_ancestor` +// at the slot of `node.root` returns `S` itself while `r`'s slot is at or +// below it, so `S` matches `node` exactly when `node.root == r` and either the +// statuses agree or `node` is `PENDING`. When `r`'s slot is above it, the +// descent replaces `S` by `(parent(r), status of r's parent payload)` and +// asks again. So the nodes a vote counts for are exactly those on this chain, +// in the node tree of the section above: +// +// ```text +// S = (r, s) -> PENDING(r) -> (parent(r), ps(r)) -> PENDING(parent(r)) -> ... +// ``` +// +// where `ps(r)` is `get_parent_payload_status` of `r`, the first arrow exists +// only for `s` of `EMPTY` or `FULL`, and `PENDING(r)` is the last node of `r` +// on the chain (a vote for `PENDING(r)` counts for no `EMPTY` or `FULL` node of +// `r`, since the statuses differ and `node` is not the wildcard). The score of +// a node is therefore the sum, over the votes whose chain contains it, of the +// voter's balance; adding each vote at `S` and folding every node's total into +// the next node on its chain, children before parents, computes that sum for +// every node at once. `compute_weights` does the same for the pre-gloas tree, +// where the chain is the block chain. +// +// Children before parents needs a topological order, and processing blocks +// from the highest slot down gives one: a parent is at a strictly earlier +// slot, and a block's own `EMPTY` and `FULL` nodes are complete before its +// `PENDING` node reads them, since everything folded into them comes from a +// later slot. +// +// The proposer boost is the same sum with one extra voter at `PENDING(boost)`: +// `is_ancestor(PENDING(boost), node)` holds for exactly the nodes on that +// node's chain. It cannot be folded with the votes, because whether it applies +// depends on the parent's attestation score, which the fold produces; it is +// added along the chain afterwards. + +/// How a head computation reads a block's payload dimension. +/// +/// From `GLOAS_FORK_EPOCH` on, every block's payload dimension is read from +/// its [`BlockPayloadLink`]. Before it, no payload dimension exists: every +/// block is a single `FULL` node, whatever its bytes are, exactly as the +/// pre-gloas head algorithm treated it. Reading the links would change +/// nothing on a real chain (a gloas block cannot exist before the fork epoch), +/// but a store holding one anyway must give the answer the pre-gloas rules +/// give, so the rules, not the block, decide. +#[derive(Clone, Copy)] +struct PayloadTree<'a> { + store: &'a Store, + rules: ForkRules, +} + +impl PayloadTree<'_> { + /// `root`'s payload link under this tree's rules. + fn link(&self, root: Root) -> Result { + match self.rules { + ForkRules::PreGloas => Ok(self.rules.payload_link(None)), + ForkRules::Gloas => block_payload_link(self.store, root)? + .ok_or(Error::SpecAssert("root in store.blocks")), + } + } + + /// Which of its parent's payload branches `root` builds on. + fn parent_status(&self, root: Root) -> Result { + self.link(root)? + .parent_status() + .ok_or(Error::SpecAssert("block.parent_root in store.blocks")) + } +} + +/// Every node's LMD GHOST weight, from one pass over the votes and one over the +/// blocks; see the section documentation above for why this equals the +/// specification's per-node `get_weight`. +/// +/// Work is proportional to the number of latest messages plus the number of +/// indexed blocks at or above the finalized block (times two lookups each), +/// which is the unfinalized window, not the history since the anchor (see +/// [`compute_node_weights`]'s bound), against the specification's +/// messages times candidate nodes, each with block decodes along the ancestor +/// walk. Once the payload links are recorded, the only decodes left are the +/// equivocation scan under a weak previous-slot parent and the one-time +/// derivation of a link that was never recorded (after a restart); see +/// `docs/spec_deviations.md`. +/// +/// A vote for a block no longer in `index` is dropped rather than raising, the +/// tolerance [`compute_weights`] documents; a missing `block_timeliness` entry +/// reads as not timely, the one [`should_apply_proposer_boost`] documents. +#[derive(Debug)] +pub struct NodeWeights { + rules: ForkRules, + current_slot: Slot, + weights: HashMap, +} + +impl NodeWeights { + /// `node`'s weight, `block_slot` being its block's slot. + /// + /// Under gloas rules a `EMPTY` or `FULL` node of the previous slot's block + /// weighs `0`: its votes are still arriving, so only the tiebreaker + /// decides between the two (`is_previous_slot_payload_decision`). The + /// pre-gloas rules have no such node. + pub fn weight(&self, node: ForkChoiceNode, block_slot: Slot) -> Gwei { + if self.rules == ForkRules::Gloas + && is_payload_decision_at(block_slot, node.payload_status, self.current_slot) + { + return 0; + } + self.raw(node) + } + + /// `node`'s weight before the previous-slot rule, the value the votes and + /// the boost add up to. + fn raw(&self, node: ForkChoiceNode) -> Gwei { + self.weights.get(&node).copied().unwrap_or_default() + } + + fn set(&mut self, node: ForkChoiceNode, weight: Gwei) { + self.weights.insert(node, weight); + } + + fn add(&mut self, node: ForkChoiceNode, amount: Gwei) { + let entry = self.weights.entry(node).or_default(); + *entry = entry.saturating_add(amount); + } +} + +/// The node a block's own `PENDING` total is folded into: the parent block's +/// payload node that this block builds on. +fn parent_node(parent_root: Root, parent_status: PayloadStatus) -> ForkChoiceNode { + ForkChoiceNode { + root: parent_root, + payload_status: parent_status, + } +} + +/// The lowest slot [`compute_node_weights`] weighs: the least of the finalized +/// block's slot, the justified block's slot and, when a proposer boost root is +/// set and indexed, its parent's slot. All three come from `index`. +/// +/// A root that is not indexed gives `0` for the finalized and justified +/// blocks, which weighs everything. Why each term is there, and why the least +/// is the finalized block's slot on a live node, is argued once, in +/// [`compute_node_weights`]'s `# The bound` section. +fn walk_bound(store: &Store, index: &HashMap) -> Slot { + let slot_of = |root: Root| index.get(&root).map_or(0, |&(slot, _)| slot); + let mut bound = slot_of(store.beacon_finalized_checkpoint().root) + .min(slot_of(store.beacon_justified_checkpoint().root)); + let boost_root = store.proposer_boost_root(); + if let Some(&(_, parent_root)) = index.get(&boost_root) + && let Some(&(parent_slot, _)) = index.get(&parent_root) + { + bound = bound.min(parent_slot); + } + bound +} + +/// [`NodeWeights`] for `rules`, on the block tree `index`. +/// +/// `rules` selects the two fork-dependent rules: how a block's payload +/// dimension is read (see [`PayloadTree`]) and whether the proposer boost is +/// gated by `should_apply_proposer_boost` (gloas) or applies whenever a boost +/// root is set (pre-gloas). +/// +/// # The bound +/// +/// Vote placement, the fold and the boost chain all stop at [`walk_bound`]'s +/// slot: a block below it gets no weight and is never read, so a head +/// computation covers the unfinalized window, not the history since the +/// anchor. (A beacon store never prunes its block index, so without the bound +/// every call would fold, and read the payload link of, every block down to +/// the anchor.) The bound is the least of the finalized block's slot, the +/// justified block's slot and the boosted block's parent's slot, and it +/// changes no answer the head reads, whichever of the three is least: +/// - The descent compares only descendants of the justified root, so every +/// node it weighs is at or above the justified block's slot, hence at or +/// above the bound. +/// - A vote for a block below the bound supports only nodes at or below that +/// block, since a vote counts for the nodes on its chain toward the root, so +/// it adds nothing to a node at or above the bound. +/// - The boost chain from a block adds to that block's ancestors, and it is +/// walked down to the bound, so every node at or above the bound gets what +/// the specification gives it. The gloas gate reads only the boosted block's +/// parent's score, and that parent is at or above the bound by +/// construction. +/// +/// On a live node the least of the three is the finalized block's slot: the +/// justified block is at or above it, and the boosted block is a current-slot +/// block, which descends from the finalized root, so its parent is too. That +/// is what keeps the walk to the unfinalized window. +/// +/// The weight of a node below the bound is not computed, and reads as `0`. +pub fn compute_node_weights( + store: &Store, + index: &HashMap, + config: &Config, + committees: &CommitteeCache, + rules: ForkRules, +) -> Result { + let tree = PayloadTree { store, rules }; + let justified_checkpoint = store.beacon_justified_checkpoint(); + let state = checkpoint_state(store, &justified_checkpoint, config)?; + let current_epoch = get_current_epoch(&state); + let bound = walk_bound(store, index); + let in_window = |root: &Root| index.get(root).is_some_and(|&(slot, _)| slot >= bound); + + // Votes, summed by what determines the node they support. Nothing that + // reads the store's own scratch may run inside the closure: see + // `for_each_non_equivocating_latest_message` for why asking it per voter + // from in here would deadlock, and a payload link is such a read. + let mut votes: HashMap<(Root, Slot, bool), Gwei> = HashMap::new(); + store.for_each_non_equivocating_latest_message(|validator_index, message| { + // Not `get_active_validator_indices`, for the reason + // `compute_weights` gives; a validator past this state's registry + // did not exist at the justified checkpoint, which is a skip. + let Ok(validator) = state.validator(validator_index) else { + return; + }; + if validator.slashed || !is_active_validator(validator, current_epoch) { + return; + } + let entry = votes + .entry((message.root, message.slot, message.payload_present)) + .or_default(); + *entry = entry.saturating_add(validator.effective_balance); + }); + + let mut weights = NodeWeights { + rules, + current_slot: get_current_slot(store, config), + weights: HashMap::new(), + }; + for ((root, message_slot, payload_present), balance) in votes { + let Some(&(block_slot, _)) = index.get(&root) else { + continue; + }; + if block_slot < bound { + continue; + } + // `get_supported_node`: a vote made after its block's own slot names + // one of the block's payload branches, and a pre-gloas block has only + // the full one. + let payload_status = if block_slot < message_slot { + if !tree.link(root)?.is_gloas() || payload_present { + PayloadStatus::Full + } else { + PayloadStatus::Empty + } + } else { + PayloadStatus::Pending + }; + weights.add( + ForkChoiceNode { + root, + payload_status, + }, + balance, + ); + } + + // Fold, highest slot first. A block's total is its own votes plus what its + // two payload nodes gathered from later blocks; it then moves into the + // payload node of its parent that it builds on. + let mut blocks: Vec<(Slot, Root, Root)> = index + .iter() + .filter(|&(_, &(slot, _))| slot >= bound) + .map(|(root, (slot, parent_root))| (*slot, *root, *parent_root)) + .collect(); + blocks.sort_unstable_by_key(|block| std::cmp::Reverse(block.0)); + for &(_slot, root, parent_root) in &blocks { + let pending = ForkChoiceNode { + root, + payload_status: PayloadStatus::Pending, + }; + let total = [ + pending, + ForkChoiceNode { + root, + payload_status: PayloadStatus::Empty, + }, + ForkChoiceNode { + root, + payload_status: PayloadStatus::Full, + }, + ] + .into_iter() + .fold(0, |sum: Gwei, node| sum.saturating_add(weights.raw(node))); + if total == 0 { + continue; + } + weights.set(pending, total); + // Only into a parent that is still indexed and at or above the bound: + // the finalized block's own parent is below it, and the anchor's is + // below the retained window; there is nothing there to weigh. + if in_window(&parent_root) { + let parent_status = tree.parent_status(root)?; + weights.add(parent_node(parent_root, parent_status), total); + } + } + + // The boost. Gloas asks `should_apply_proposer_boost`, fed the parent's + // score from the table just built; before gloas it applies whenever a boost + // root is set and still indexed, the tolerance `compute_weights` documents. + let boost_root = store.proposer_boost_root(); + let boost_applies = match rules { + ForkRules::PreGloas => !boost_root.is_zero() && in_window(&boost_root), + ForkRules::Gloas => { + should_apply_proposer_boost_with(store, config, committees, index, &state, |parent| { + Ok(weights.raw(ForkChoiceNode { + root: parent, + payload_status: PayloadStatus::Pending, + })) + })? + } + }; + if boost_applies { + let proposer_score = get_proposer_score(store, config)?; + let mut root = boost_root; + loop { + weights.add( + ForkChoiceNode { + root, + payload_status: PayloadStatus::Pending, + }, + proposer_score, + ); + let Some(&(_, parent_root)) = index.get(&root) else { + break; + }; + if !in_window(&parent_root) { + break; + } + let parent_status = tree.parent_status(root)?; + weights.add(parent_node(parent_root, parent_status), proposer_score); + root = parent_root; + } + } + + Ok(weights) +} + +/// The nodes directly under `node` in the payload-aware tree, for the head +/// descent: [`get_node_children`], answered from `children_of` (the filtered +/// tree's parent-to-children map) and the payload links instead of decoding +/// every block it touches. +fn walk_children( + tree: PayloadTree, + children_of: &HashMap>, + node: ForkChoiceNode, +) -> Result> { + if node.payload_status == PayloadStatus::Pending { + if !tree.link(node.root)?.is_gloas() { + return Ok(vec![ForkChoiceNode { + root: node.root, + payload_status: PayloadStatus::Full, + }]); + } + let mut children = vec![ForkChoiceNode { + root: node.root, + payload_status: PayloadStatus::Empty, + }]; + if tree.store.has_verified_payload(&node.root) { + children.push(ForkChoiceNode { + root: node.root, + payload_status: PayloadStatus::Full, + }); + } + return Ok(children); + } + let mut children = Vec::new(); + for &child in children_of.get(&node.root).into_iter().flatten() { + if tree.parent_status(child)? == node.payload_status { + children.push(ForkChoiceNode { + root: child, + payload_status: PayloadStatus::Pending, + }); + } + } + Ok(children) +} + +/// The LMD GHOST head as a [`ForkChoiceNode`], for `rules`: the descent from +/// the justified checkpoint that both [`compute_head`] and [`gloas_get_head`] +/// perform, over the one weight table of [`compute_node_weights`]. +/// +/// At every step the heaviest child wins, ties going to the higher root and +/// then to [`get_payload_status_tiebreaker`], the key gloas's `get_head` sorts +/// by. Before gloas a step has one payload node and siblings always differ by +/// root, so neither of the last two keys can decide, and the answer is the +/// pre-gloas head, reported as `(root, Full)`. /// -/// Dispatches on the current slot's own fork, via [`Config::fork_at_epoch`], -/// not on the head's fork or the justified checkpoint's: from -/// `GLOAS_FORK_EPOCH` on, every call runs [`gloas_get_head`] over the whole -/// tree, pre-gloas blocks included, so the payload-aware algorithm takes -/// over as soon as the chain could hold a gloas block at all, rather than -/// waiting for a block, or justification, to actually reach one. Before that -/// epoch, the pre-gloas [`compute_head`]/[`compute_weights`] path runs -/// exactly as it did before gloas existed. +/// Takes `index` rather than building it, like [`get_filtered_block_tree`]: +/// [`on_block`] already holds one. +fn walk_head( + store: &Store, + index: &HashMap, + config: &Config, + committees: &CommitteeCache, + rules: ForkRules, +) -> Result { + let tree = PayloadTree { store, rules }; + let blocks = get_filtered_block_tree(store, index, config)?; + let weights = compute_node_weights(store, index, config, committees, rules)?; + let current_slot = get_current_slot(store, config); + + let mut children_of: HashMap> = HashMap::new(); + for (&root, &(_, parent_root)) in &blocks { + children_of.entry(parent_root).or_default().push(root); + } + let block_entry = |root: Root| index.get(&root).copied(); + + let mut head = ForkChoiceNode { + root: store.beacon_justified_checkpoint().root, + payload_status: PayloadStatus::Pending, + }; + loop { + let children = walk_children(tree, &children_of, head)?; + if children.len() <= 1 { + let Some(only) = children.first() else { + return Ok(head); + }; + head = *only; + continue; + } + + let mut best: Option<((Gwei, Root, u8), ForkChoiceNode)> = None; + for child in children { + let (block_slot, _) = + block_entry(child.root).ok_or(Error::SpecAssert("node.root in store.blocks"))?; + let tiebreak = match rules { + ForkRules::Gloas => payload_status_tiebreaker_with( + store, + child, + current_slot, + block_entry, + proposer_parent_is_full_from_link, + )?, + ForkRules::PreGloas => child.payload_status as u8, + }; + let key = (weights.weight(child, block_slot), child.root, tiebreak); + if best.is_none_or(|(best_key, _)| key > best_key) { + best = Some((key, child)); + } + } + head = best.expect("more than one child, checked above").1; + } +} + +/// The LMD GHOST head as a full [`ForkChoiceNode`] rather than just a root, +/// from the one bottom-up walk ([`compute_node_weights`] and the descent over +/// it) that serves every fork. +/// +/// The current slot's own fork, via [`Config::fork_at_epoch`], not the head's +/// or the justified checkpoint's, selects the two rules that differ by fork +/// (see [`walk_head`]): from `GLOAS_FORK_EPOCH` on, blocks carry the payload +/// dimension their bids give them and the proposer boost is gated by +/// `should_apply_proposer_boost`, so the payload-aware rules take over as soon +/// as the chain could hold a gloas block at all, rather than waiting for a +/// block, or justification, to actually reach one. Before that epoch every +/// block is a single full node and the boost applies whenever it is set, +/// which is the head algorithm every pre-gloas fork uses. +/// +/// [`gloas_get_head`] is the spec-literal algorithm this walk replaced, and +/// [`compute_head`] the [`compute_weights`] fold kept as the pre-gloas +/// reference. Neither is on the node's path any more: they are the references +/// its tests compare it with, at every `head` check of every fork-choice +/// fixture and on randomized trees. /// /// A pre-gloas block has no payload dimension of its own: this crate treats /// it as a single-variant node whose status is always @@ -1645,35 +2136,16 @@ fn compute_head( /// one, and says Lighthouse does the same; our FULL rule differs from both. /// Adopt PENDING here if that PR lands. /// -/// The gloas arm reads committees through the store's shared -/// [`Store::committee_cache`], the cache `on_block` and the gossip checks use: -/// gloas's own weight (`gloas_get_weight` by way of -/// `should_apply_proposer_boost`) needs committees a pre-gloas head -/// computation never did, and this runs every slot, so a cache of its own -/// would recompute every committee on every call. See [`gloas_get_weight`]'s -/// own performance note for what that costs across a whole tree walk. +/// Committees are read through the store's shared [`Store::committee_cache`], +/// the cache `on_block` and the gossip checks use: gloas's boost gate reads +/// the committees of a weak parent's slot, and this runs every slot, so a +/// cache of its own would recompute every committee on every call. pub fn get_head_node(store: &Store, config: &Config) -> Result { let current_slot = get_current_slot(store, config); - match config.fork_at_epoch(compute_epoch_at_slot(current_slot)) { - ForkName::Gloas => { - let committees = store.committee_cache(); - gloas_get_head(store, config, &committees) - } - ForkName::Phase0 - | ForkName::Altair - | ForkName::Bellatrix - | ForkName::Capella - | ForkName::Deneb - | ForkName::Electra - | ForkName::Fulu => { - let index = store.block_index(); - Ok(ForkChoiceNode { - root: compute_head(store, &index, config)?, - payload_status: PayloadStatus::Full, - }) - } - ForkName::Lean => lean_fork_unreachable("fork_choice::get_head_node"), - } + let rules = ForkRules::of(config.fork_at_epoch(compute_epoch_at_slot(current_slot))); + let index = store.block_index(); + let committees = store.committee_cache(); + walk_head(store, &index, config, &committees, rules) } /// [`get_head_node`], recorded as the store's own head. @@ -1876,19 +2348,49 @@ pub fn is_head_weak( ) -> Result { let justified_checkpoint = store.beacon_justified_checkpoint(); let justified_state = checkpoint_state(store, &justified_checkpoint, config)?; - let reorg_threshold = - calculate_committee_fraction(&justified_state, config.reorg_head_weight_threshold)?; - let index = store.block_index(); - let mut head_weight = get_attestation_score(store, &index, head_root, &justified_state)?; + let attestation_score = get_attestation_score(store, &index, head_root, &justified_state)?; + let &(head_slot, _) = index + .get(&head_root) + .ok_or(Error::SpecAssert("head_root in store.blocks"))?; + is_head_weak_with( + store, + head_root, + head_slot, + config, + committees, + &justified_state, + attestation_score, + ) +} + +/// [`is_head_weak`], given `head_root`'s [`get_attestation_score`] and its slot +/// instead of computing them. +/// +/// The score is the one piece of [`is_head_weak`] that walks every vote, and +/// the head computation already holds it for every node in its weight table, +/// so [`should_apply_proposer_boost_with`] hands it in rather than paying for a +/// second pass over the votes. The slot comes from the same block index, which +/// spares the block decode that +/// [`Store::block_entry`](ethlambda_storage::Store::block_entry) costs on a +/// beacon store. +fn is_head_weak_with( + store: &Store, + head_root: Root, + head_slot: Slot, + config: &Config, + committees: &CommitteeCache, + justified_state: &BeaconState, + attestation_score: Gwei, +) -> Result { + let reorg_threshold = + calculate_committee_fraction(justified_state, config.reorg_head_weight_threshold)?; + let mut head_weight = attestation_score; let head_state = store .get_state(&head_root) .expect("get") .ok_or(Error::SpecAssert("head_root in store.block_states"))?; - let (head_slot, _) = store - .block_entry(&head_root) - .ok_or(Error::SpecAssert("head_root in store.blocks"))?; let epoch = compute_epoch_at_slot(head_slot); let epoch_committees = committees.committees(&head_state, epoch); for committee_index in 0..epoch_committees.committees_per_slot() { @@ -2119,8 +2621,9 @@ pub fn get_proposer_head( /// Every function in this section assumes `block` is itself gloas-shaped, /// which does not hold at the fulu-to-gloas boundary on its own: the first /// gloas block's own parent is a fulu block with no bid at all. -/// [`get_parent_payload_status`] and [`on_execution_payload_envelope`] call -/// this. The first never reaches here for a pre-gloas block though: it +/// [`parent_payload_status_of`] (behind both [`get_parent_payload_status`] +/// and [`derive_payload_link`]) and [`on_execution_payload_envelope`] call +/// this. The first never reaches here for a pre-gloas parent though: it /// answers [`PayloadStatus::Full`] for one directly, per this crate's decided /// rule for that boundary (see the module documentation's "Gloas: /// payload-aware fork choice" section). A non-gloas block still reaching here @@ -2153,20 +2656,74 @@ fn gloas_bid(block: &SignedBeaconBlock) -> Result<&gloas::ExecutionPayloadBid> { /// [`is_payload_verified`] `true` for a root the specification would never /// call verified. /// -/// Decodes `root`'s full signed block to answer, which every caller here -/// otherwise avoids doing more than once: [`is_payload_verified`], -/// [`payload_timeliness`] and [`payload_data_availability`] call this first -/// and then read [`Store::has_verified_payload`](ethlambda_storage::Store::has_verified_payload) -/// directly rather than back through [`is_payload_verified`], and -/// [`get_node_children`]'s own decode of a `Pending` node's block answers -/// this same question without a second call here. +/// Needs only the block's fork, so it reads the recorded [`BlockPayloadLink`] +/// when there is one and otherwise decodes the block, without deriving (or +/// recording) the parent comparison a full link needs. fn is_known_pre_gloas_block(store: &Store, root: Root) -> bool { + if let Some(link) = store.payload_link(&root) { + return !link.is_gloas(); + } matches!( store.get_signed_block(&root).expect("get"), Some(block) if is_pre_gloas(&block) ) } +/// `block`'s [`BlockPayloadLink`], derived from the block and, for a gloas +/// block, its parent's bid through [`parent_payload_status_of`]. +/// +/// The parent's status is `None` only when the parent block is not in the +/// store. A gloas block that is not the anchor always has its parent there +/// (`on_block` refuses it otherwise), and `on_block` records the link with the +/// status [`get_parent_payload_status`] returned, so the two agree by +/// construction; `the_recorded_payload_links_match_the_specifications_parent_status` +/// checks that they do. +fn derive_payload_link(store: &Store, block: &SignedBeaconBlock) -> Result { + let rules = ForkRules::of(block.fork_name()); + let parent_status = match rules { + ForkRules::PreGloas => None, + ForkRules::Gloas => store + .get_signed_block(&block.parent_root()) + .expect("get") + .map(|parent| parent_payload_status_of(block, &parent)) + .transpose()?, + }; + Ok(rules.payload_link(parent_status)) +} + +/// `root`'s [`BlockPayloadLink`]: the entry `on_block` recorded, or, for a +/// block imported before the last restart (the scratch is in memory only) or +/// whose entry was pruned, one derived by decoding the block and recorded for +/// next time. +/// +/// `None` when the store does not hold `root` at all. A link whose parent +/// status is unknown is recorded like any other: it is the anchor's, whose +/// parent is not in the store by construction, and re-deriving it on every +/// read would decode two blocks each time. A block persisted ahead of its +/// parent is not read here before `on_block` imports it, and `on_block` +/// records its own link over whatever is there. +fn block_payload_link(store: &Store, root: Root) -> Result> { + if let Some(link) = store.payload_link(&root) { + return Ok(Some(link)); + } + let Some(block) = store.get_signed_block(&root).expect("get") else { + return Ok(None); + }; + let link = derive_payload_link(store, &block)?; + store.set_payload_link(root, block.slot(), link); + Ok(Some(link)) +} + +/// Makes sure `root`'s [`BlockPayloadLink`] is recorded, deriving it by +/// decoding the block if it is not (after a restart, which loses the scratch). +/// +/// For the actor to call before pruning links at the finalized block, whose +/// own link the prune keys its bound on. Costs at most one derivation for a +/// finalized root that was imported before the restart. +pub fn ensure_payload_link(store: &Store, root: Root) -> Result<()> { + block_payload_link(store, root).map(|_| ()) +} + /// Whether `block` is handled under the pre-gloas rules, from an exhaustive /// match on [`ForkRules`], so a fork added after gloas has to be placed there /// rather than being read as "not gloas, therefore pre-gloas" by each caller. @@ -2272,11 +2829,22 @@ pub fn get_parent_payload_status( .get_signed_block(&block.parent_root()) .expect("get") .ok_or(Error::SpecAssert("block.parent_root in store.blocks"))?; - if is_pre_gloas(&parent) { + parent_payload_status_of(block, &parent) +} + +/// [`get_parent_payload_status`]'s comparison, given the parent block: the one +/// place a child's bid is read against its parent's, shared with +/// [`derive_payload_link`] so a recorded link cannot disagree with the +/// specification's function. +fn parent_payload_status_of( + block: &SignedBeaconBlock, + parent: &SignedBeaconBlock, +) -> Result { + if is_pre_gloas(parent) { return Ok(PayloadStatus::Full); } let parent_block_hash = gloas_bid(block)?.parent_block_hash; - let message_block_hash = gloas_bid(&parent)?.block_hash; + let message_block_hash = gloas_bid(parent)?.block_hash; Ok(if parent_block_hash == message_block_hash { PayloadStatus::Full } else { @@ -2423,12 +2991,24 @@ pub fn is_previous_slot_payload_decision( let (block_slot, _) = store .block_entry(&node.root) .ok_or(Error::SpecAssert("node.root in store.blocks"))?; - let is_previous_slot = block_slot.checked_add(1) == Some(get_current_slot(store, config)); - let is_payload_decision = matches!( + Ok(is_payload_decision_at( + block_slot, node.payload_status, - PayloadStatus::Empty | PayloadStatus::Full - ); - Ok(is_previous_slot && is_payload_decision) + get_current_slot(store, config), + )) +} + +/// [`is_previous_slot_payload_decision`]'s test, with the block's slot and the +/// current slot already in hand, for the head computation that reads the slot +/// from its block index instead of decoding the block. +fn is_payload_decision_at( + block_slot: Slot, + payload_status: PayloadStatus, + current_slot: Slot, +) -> bool { + let is_previous_slot = block_slot.checked_add(1) == Some(current_slot); + let is_payload_decision = matches!(payload_status, PayloadStatus::Empty | PayloadStatus::Full); + is_previous_slot && is_payload_decision } /// `should_build_on_full` (gloas `fork-choice.md`): whether a proposer at @@ -2462,11 +3042,33 @@ pub fn should_build_on_full(store: &Store, head: ForkChoiceNode, slot: Slot) -> /// build empty, favoring extension unless the PTC view is against it and the /// current proposer-boosted block itself chose empty. pub fn should_extend_payload(store: &Store, root: Root, config: &Config) -> Result { - let (block_slot, _) = store - .block_entry(&root) - .ok_or(Error::SpecAssert("root in store.blocks"))?; + should_extend_payload_with( + store, + root, + get_current_slot(store, config), + |block_root| store.block_entry(&block_root), + proposer_parent_is_full_from_blocks, + ) +} + +/// [`should_extend_payload`] with the current slot, a block's +/// `(slot, parent_root)` lookup and the answer to the specification's +/// `is_parent_node_full` for the proposer-boosted block supplied by the +/// caller. The head computation answers both from its block index and the +/// payload links rather than decoding blocks +/// ([`proposer_parent_is_full_from_link`]); the public function keeps the +/// specification's own decode ([`proposer_parent_is_full_from_blocks`]), so +/// a test comparing the two does not compare the walk with itself on that leg. +fn should_extend_payload_with( + store: &Store, + root: Root, + current_slot: Slot, + block_entry: impl Fn(Root) -> Option<(Slot, Root)>, + proposer_parent_is_full: impl Fn(&Store, Root) -> Result, +) -> Result { + let (block_slot, _) = block_entry(root).ok_or(Error::SpecAssert("root in store.blocks"))?; verify( - block_slot.checked_add(1) == Some(get_current_slot(store, config)), + block_slot.checked_add(1) == Some(current_slot), "store.blocks[root].slot + 1 == get_current_slot(store)", )?; if !is_payload_verified(store, root) { @@ -2481,6 +3083,18 @@ pub fn should_extend_payload(store: &Store, root: Root, config: &Config) -> Resu if proposer_root.is_zero() { return Ok(true); } + let (_, proposer_parent_root) = block_entry(proposer_root).ok_or(Error::SpecAssert( + "store.proposer_boost_root in store.blocks", + ))?; + if proposer_parent_root != root { + return Ok(true); + } + proposer_parent_is_full(store, proposer_root) +} + +/// `is_parent_node_full(store, store.blocks[proposer_root])`, the +/// specification's own call: both blocks decoded. +fn proposer_parent_is_full_from_blocks(store: &Store, proposer_root: Root) -> Result { let proposer_block = store .get_signed_block(&proposer_root) @@ -2488,12 +3102,21 @@ pub fn should_extend_payload(store: &Store, root: Root, config: &Config) -> Resu .ok_or(Error::SpecAssert( "store.proposer_boost_root in store.blocks", ))?; - if proposer_block.parent_root() != root { - return Ok(true); - } is_parent_node_full(store, &proposer_block) } +/// The same answer from the block's [`BlockPayloadLink`], which records what +/// [`is_parent_node_full`] would compute; see [`derive_payload_link`]. +fn proposer_parent_is_full_from_link(store: &Store, proposer_root: Root) -> Result { + let link = block_payload_link(store, proposer_root)?.ok_or(Error::SpecAssert( + "store.proposer_boost_root in store.blocks", + ))?; + let parent_status = link + .parent_status() + .ok_or(Error::SpecAssert("block.parent_root in store.blocks"))?; + Ok(parent_status == PayloadStatus::Full) +} + /// `get_payload_status_tiebreaker` (gloas `fork-choice.md`): [`gloas_get_head`]'s /// tiebreaker between a block's full and empty branches, once their weights /// alone cannot decide (their weight is `0` while @@ -2503,11 +3126,38 @@ pub fn get_payload_status_tiebreaker( node: ForkChoiceNode, config: &Config, ) -> Result { - if is_previous_slot_payload_decision(store, node, config)? { + let current_slot = get_current_slot(store, config); + payload_status_tiebreaker_with( + store, + node, + current_slot, + |root| store.block_entry(&root), + proposer_parent_is_full_from_blocks, + ) +} + +/// [`get_payload_status_tiebreaker`] with the current slot and the block +/// lookup supplied by the caller; see [`should_extend_payload_with`]. +fn payload_status_tiebreaker_with( + store: &Store, + node: ForkChoiceNode, + current_slot: Slot, + block_entry: impl Fn(Root) -> Option<(Slot, Root)>, + proposer_parent_is_full: impl Fn(&Store, Root) -> Result, +) -> Result { + let (block_slot, _) = + block_entry(node.root).ok_or(Error::SpecAssert("node.root in store.blocks"))?; + if is_payload_decision_at(block_slot, node.payload_status, current_slot) { if node.payload_status == PayloadStatus::Empty { return Ok(1); } - if should_extend_payload(store, node.root, config)? { + if should_extend_payload_with( + store, + node.root, + current_slot, + &block_entry, + proposer_parent_is_full, + )? { return Ok(2); } Ok(0) @@ -2537,19 +3187,53 @@ pub fn should_apply_proposer_boost( store: &Store, config: &Config, committees: &CommitteeCache, +) -> Result { + if store.proposer_boost_root().is_zero() { + return Ok(false); + } + let justified_checkpoint = store.beacon_justified_checkpoint(); + let justified_state = checkpoint_state(store, &justified_checkpoint, config)?; + let index = store.block_index(); + should_apply_proposer_boost_with( + store, + config, + committees, + &index, + &justified_state, + |parent_root| get_attestation_score(store, &index, parent_root, &justified_state), + ) +} + +/// [`should_apply_proposer_boost`] with the block index, the justified state +/// and the parent's attestation score supplied by the caller. +/// +/// The decision does not depend on which node is being weighed, and its only +/// expensive input is the parent's attestation score. The head computation +/// already holds that score for every node it weighs, so it passes a lookup +/// into its own table here, where the specification's per-node `get_weight` +/// recomputes it from the votes on every call. +/// +/// `parent_score` is called at most once, with the parent's root, and only +/// when the parent is from the previous slot. It must return the same number +/// [`get_attestation_score`] does for that root. +fn should_apply_proposer_boost_with( + store: &Store, + config: &Config, + committees: &CommitteeCache, + index: &HashMap, + justified_state: &BeaconState, + parent_score: impl FnOnce(Root) -> Result, ) -> Result { let proposer_boost_root = store.proposer_boost_root(); if proposer_boost_root.is_zero() { return Ok(false); } - let (slot, parent_root) = store - .block_entry(&proposer_boost_root) - .ok_or(Error::SpecAssert( - "store.proposer_boost_root in store.blocks", - ))?; - let (parent_slot, _) = store - .block_entry(&parent_root) + let &(slot, parent_root) = index.get(&proposer_boost_root).ok_or(Error::SpecAssert( + "store.proposer_boost_root in store.blocks", + ))?; + let &(parent_slot, _) = index + .get(&parent_root) .ok_or(Error::SpecAssert("parent_root in store.blocks"))?; // Apply proposer boost if `parent` is not from the previous slot. @@ -2559,7 +3243,16 @@ pub fn should_apply_proposer_boost( // Apply proposer boost if `parent` is not weak. Scored through the // pre-gloas `is_head_weak`, not a gloas copy: see `is_ancestor`'s own // doc for why a `Pending` node's score is the same either way. - if !is_head_weak(store, parent_root, config, committees)? { + let parent_attestation_score = parent_score(parent_root)?; + if !is_head_weak_with( + store, + parent_root, + parent_slot, + config, + committees, + justified_state, + parent_attestation_score, + )? { return Ok(true); } @@ -2569,9 +3262,8 @@ pub fn should_apply_proposer_boost( .get_signed_block(&parent_root) .expect("get") .ok_or(Error::SpecAssert("parent_root in store.blocks"))?; - let index = store.block_index(); let mut has_equivocation = false; - for (&candidate_root, &(candidate_slot, _)) in &index { + for (&candidate_root, &(candidate_slot, _)) in index { if candidate_root == parent_root { continue; } @@ -2646,40 +3338,24 @@ fn gloas_get_attestation_score( /// `get_weight` (gloas `fork-choice.md`, modified). /// -/// **Performance note**: this is spec-literal, not the bottom-up form -/// pre-gloas [`get_weight`]/[`compute_weights`] moved to, and carries -/// several costs a later change must close before any live use, not only -/// the walk [`compute_weights`]'s own doc measured at 62% of a live mainnet -/// follower's CPU before that fold replaced repeated per-root calls: -/// - every latest message is walked afresh for every node [`gloas_get_head`] -/// visits, rather than folded bottom-up once per tree; -/// - each [`gloas_get_ancestor`] hop decodes two signed blocks (the node's -/// own, then its parent's inside [`get_parent_payload_status`]), and -/// [`get_supported_node`] decodes a third, once per voter, where the -/// pre-gloas, index-only [`get_ancestor`] decodes none; -/// - telling a pre-gloas root from a gloas one is itself a decode a -/// pre-gloas tree never needed at all, paid by [`is_payload_verified`], -/// [`payload_timeliness`], [`payload_data_availability`], and -/// [`get_node_children`]'s own check on every `Pending` node it visits; -/// - [`should_apply_proposer_boost`] reruns a full [`is_head_weak`] score -/// and its own `store.block_index()` scan on every call, even though its -/// answer does not depend on `node` at all and so is identical for every -/// candidate one [`gloas_get_head`] call weighs. -/// -/// One cost this list used to carry is closed: a vote for a block this -/// store has pruned below its finalized anchor no longer aborts the whole -/// head computation with `Error::SpecAssert`, the failure -/// [`compute_weights`]'s own doc describes pre-gloas `get_weight` having. -/// Closing it took two fixes, not one: [`gloas_get_attestation_score`]'s -/// own doc has the direct one; the other is reached indirectly, through -/// [`should_apply_proposer_boost`] (called above regardless of `node`) into -/// pre-gloas [`is_head_weak`]'s own [`get_attestation_score`], which fires -/// whenever the boosted block's parent is from the previous slot, the -/// ordinary case rather than a corner one; [`get_attestation_score`]'s own -/// doc has that fix. -/// -/// Fine for spec tests, where the tree is a handful of blocks and never -/// prunes; live use needs the remaining costs addressed. +/// **A reference, not the node's weight.** The node weighs with +/// [`compute_node_weights`], one bottom-up pass; this stays spec-literal, as the +/// per-node definition that table is tested against (the fork-choice harness +/// compares every `viable_for_head_roots_and_weights` leaf, and the randomized +/// differential test every node). Its costs are why the node does not call it: +/// - every latest message is walked afresh for every node visited; +/// - each [`gloas_get_ancestor`] hop decodes two signed blocks, and +/// [`get_supported_node`] a third, once per voter; +/// - telling a pre-gloas root from a gloas one is itself a decode; +/// - [`should_apply_proposer_boost`] reruns a full [`is_head_weak`] score on +/// every call, although its answer does not depend on `node`. +/// +/// A vote for a block this store has pruned below its finalized anchor does not +/// abort the computation with `Error::SpecAssert`, the failure +/// [`compute_weights`]'s own doc describes pre-gloas `get_weight` having: +/// [`gloas_get_attestation_score`]'s own doc has the direct fix, and +/// [`get_attestation_score`]'s the one reached through +/// [`should_apply_proposer_boost`] into [`is_head_weak`]. pub fn gloas_get_weight( store: &Store, node: ForkChoiceNode, @@ -2725,11 +3401,10 @@ pub fn gloas_get_weight( /// section). The specification's own gloas `fork-choice.md` never describes /// this case, since it only ever anchors a gloas `Store`. /// -/// **Performance note**: telling a pre-gloas `node` apart from a gloas one -/// costs a full block decode, on every `Pending` node this walks, that a -/// pre-gloas tree never needed. See [`gloas_get_weight`]'s own performance -/// note for the other costs this section's spec-literal walk carries above -/// pre-gloas [`compute_weights`]'s bottom-up fold. +/// **A reference, not the node's descent.** Telling a pre-gloas `node` apart +/// from a gloas one costs a full block decode here, on every `Pending` node. +/// The head walk answers the same question from `walk_children`, which reads +/// the payload links instead; see [`gloas_get_weight`] for the rest. /// /// `blocks` is [`Store::block_index`]'s own shape, not the specification's /// `Dict[Root, BeaconBlock]`: see the section documentation above for why @@ -2793,8 +3468,11 @@ pub fn get_node_children( /// /// Reuses the pre-gloas, index-only [`get_filtered_block_tree`] for the /// candidate tree: see [`gloas_get_checkpoint_block`]'s own doc for why that -/// is sound rather than a shortcut. [`get_head_node`] is the dispatching -/// entry point this and pre-gloas's own [`compute_head`] share. +/// is sound rather than a shortcut. +/// +/// **A reference, not the node's head computation.** [`get_head_node`] runs +/// [`walk_head`], which reaches the same node from one bottom-up weight table; +/// this is the spec-literal transcription its tests compare it with. pub fn gloas_get_head( store: &Store, config: &Config, @@ -3458,8 +4136,8 @@ pub fn get_shuffling_dependent_root( /// the boost cannot itself be the thing that drags the head onto a branch with /// a different, no-longer-relevant view of who was supposed to propose it. /// -/// `head` must be the head [`compute_head`] (pre-gloas) or [`gloas_get_head`] -/// (gloas) found *before* `root` joined the store: [`on_block`] is this +/// `head` must be the head [`walk_head`] found *before* `root` joined the +/// store: [`on_block`] is this /// function's only caller, and it passes exactly that. `index` should include /// `root`'s own entry (`on_block` extends its own pre-insertion index with it /// rather than re-scanning `LiveChain`), or the walk below falls back to @@ -3819,18 +4497,26 @@ pub fn on_block( // that payload must have been verified by `on_execution_payload_envelope`. // A pre-gloas parent is a full node whose payload is verified by // definition (see `is_payload_verified`), so the first gloas block passes. + // + // The parent status is also what the head computation needs to place this + // block in the payload tree, so it is kept as this block's payload link + // (recorded once the block is stored, below) rather than decoded again on + // every head computation. let rules = ForkRules::of(signed_block.fork_name()); - match rules { + let parent_status = match rules { ForkRules::Gloas => { - if is_parent_node_full(store, &signed_block)? { + let parent_status = get_parent_payload_status(store, &signed_block)?; + if parent_status == PayloadStatus::Full { verify( is_payload_verified(store, parent_root), "is_payload_verified(store, block.parent_root)", )?; } + Some(parent_status) } - ForkRules::PreGloas => {} - } + ForkRules::PreGloas => None, + }; + let payload_link = rules.payload_link(parent_status); // Blocks cannot be in the future. If they are, their consideration must // be delayed until they are in the past. @@ -3854,11 +4540,11 @@ pub fn on_block( // // `mut`, and kept alive for the rest of this function: the pre-import // head computation and the proposer-boost shuffling check further down - // both want this same pre-insertion snapshot of `LiveChain` (`compute_head` - // takes it as a parameter, matching `get_filtered_block_tree`/ - // `compute_weights`), so it is built once here rather than three times - // over. Once `block_root` itself joins the store below, one `insert` - // keeps this index in step with it instead of re-scanning for that alone. + // both want this same pre-insertion snapshot of `LiveChain` (`walk_head` + // takes it as a parameter, matching `get_filtered_block_tree`), so it is + // built once here rather than three times over. Once `block_root` itself + // joins the store below, one `insert` keeps this index in step with it + // instead of re-scanning for that alone. let mut index = store.block_index(); let finalized_checkpoint_block = get_checkpoint_block(&index, parent_root, finalized_checkpoint.epoch)?; @@ -4008,28 +4694,24 @@ pub fn on_block( // `is_timely and is_first_block and `; the first two conditions are cheap and already decide most // blocks, especially every block a follower receives while syncing, which - // is never timely. Computing that head (`compute_head`: a `LiveChain` - // scan, a filtered-tree walk, and a pass over every latest message; see - // `compute_weights` for why the last of those matters at scale) is not + // is never timely. Computing that head (`walk_head`: a filtered-tree walk + // and a pass over every latest message and every block; see + // `compute_weights` for why the votes matter at scale) is not // worth paying for on a block the shuffling check could not change the // answer for anyway. // - // Gloas gates on its own attestation deadline, and computes the head with - // its payload-aware algorithm. When it is computed, its `get_head` failing - // fails the import, as the pre-gloas head computation does. The head is - // only computed for a block that is timely, which puts it in the current - // slot, so `get_head_node`'s dispatch on the current slot's fork would - // pick gloas's algorithm here anyway; calling it directly reuses - // `committees` instead of building a fresh cache. + // Gloas gates on its own attestation deadline. The head is computed by the + // one walk under the block's own rules; when it is computed, its failing + // fails the import. The head is only computed for a block that is timely, + // which puts it in the current slot, so `get_head_node`'s choice of rules + // by the current slot's fork would be the same here; calling the walk + // directly reuses `committees` and `index` instead of building them. let block_slot = signed_block.slot(); let timeliness = block_timeliness(store, block_slot, rules, config); let is_timely = timeliness[constants::ATTESTATION_TIMELINESS_INDEX]; let is_first_block = store.proposer_boost_root().is_zero(); let pre_block_head = if is_timely && is_first_block { - Some(match rules { - ForkRules::Gloas => gloas_get_head(store, config, committees)?.root, - ForkRules::PreGloas => compute_head(store, &index, config)?, - }) + Some(walk_head(store, &index, config, committees, rules)?.root) } else { None }; @@ -4055,6 +4737,7 @@ pub fn on_block( .insert_signed_block(block_root, signed_block) .expect("insert"); store.insert_state(block_root, state).expect("insert"); + store.set_payload_link(block_root, block_slot, payload_link); // [New in Gloas:EIP7732] A new payload timeliness committee vote for this // block, for each of the two questions it votes on. match rules { @@ -4525,7 +5208,12 @@ mod tests { /// `BlockRoots` index across the two, so a store cannot be pointed at a /// root whose block it does not hold yet. fn store_anchored_at(root: Root) -> Store { - let backend = Arc::new(InMemoryBackend::new()); + store_anchored_at_on(Arc::new(InMemoryBackend::new()), root) + } + + /// [`store_anchored_at`] over a backend the caller supplies, for tests + /// that watch what the store reads. + fn store_anchored_at_on(backend: Arc, root: Root) -> Store { let anchor = Checkpoint { epoch: constants::GENESIS_EPOCH, root, @@ -7211,4 +7899,850 @@ mod tests { ); assert!(is_data_available_blobs(&[], &DataAvailability::Columns(Vec::new())).is_err()); } + + // ------------------------------------------------------------------ + // The bottom-up head walk against the spec-literal references + // ------------------------------------------------------------------ + + /// SplitMix64: a seeded generator small enough to live in a test, so the + /// crate needs no `rand` dependency and a failing seed reproduces. + struct TestRng(u64); + + impl TestRng { + fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + fn below(&mut self, bound: u64) -> u64 { + self.next_u64() % bound + } + + fn chance(&mut self, numerator: u64, denominator: u64) -> bool { + self.below(denominator) < numerator + } + + fn root(&mut self) -> Root { + let mut bytes = [0u8; 32]; + for chunk in bytes.chunks_mut(8) { + chunk.copy_from_slice(&self.next_u64().to_le_bytes()); + } + Root::from_slice(&bytes) + } + } + + const TREE_VALIDATORS: usize = 12; + + /// A random block tree over the fulu-to-gloas boundary, written straight + /// into a store: a fulu prefix, gloas blocks whose bids make each parent + /// status vary, some verified payloads, PTC votes of every shape, latest + /// messages with random `payload_present`, equivocators and a boost root. + /// + /// Two seeds in three also move the justified checkpoint above the anchor, + /// with each block's unrealized justification random, so that + /// `filter_block_tree` has branches to drop once the store is far enough + /// past them and some votes sit below the justified block. One of those + /// two also stores blocks beside the anchor (at its slot) that the block + /// index does not hold, as finality pruning leaves them, and some votes + /// name them. + struct RandomTree { + store: Store, + config: Config, + roots: Vec, + slots: Vec, + first_gloas_slot: Slot, + justified_moved: bool, + } + + fn random_tree(seed: u64, backend: Arc) -> RandomTree { + let mut rng = TestRng(seed); + let config = Config::active().with_fork_epoch(ForkName::Gloas, 1); + let first_gloas_slot = compute_start_slot_at_epoch(1); + let slot_cap = 2 * preset::SLOTS_PER_EPOCH - 1; + // The anchor stays at slot 0, the finalized slot every walk stops at; + // the first block after it jumps to just before the fork epoch, so a + // short chain crosses the boundary. + let anchor_slot: Slot = 0; + let phase = seed % 3; + let justified_moved = phase != 0; + let prunes_index = phase == 2; + + let anchor_root = rng.root(); + let mut store = store_anchored_at_on(backend, anchor_root); + let fulu_state = test_state::with_validators_at(ForkName::Fulu, TREE_VALIDATORS); + let gloas_state = test_state::with_validators_at(ForkName::Gloas, TREE_VALIDATORS); + store + .insert_signed_block(anchor_root, fulu_block(anchor_slot, Root::ZERO)) + .unwrap(); + store.insert_state(anchor_root, fulu_state.clone()).unwrap(); + store.set_unrealized_justification(anchor_root, Checkpoint::default()); + + let mut roots = vec![anchor_root]; + let mut slots: Vec = vec![anchor_slot]; + let mut hashes: Vec> = vec![None]; + let mut parent_indices: Vec = vec![0]; + // The tree is built ancestors first, so a block's parent is always + // earlier in these vectors. + let count = 8 + rng.below(12) as usize; + for i in 0..count { + // Mostly extend a recent block so the tree reaches the gloas + // epoch, and now and then add a same-slot sibling of an existing + // block, which is what an early equivocation looks like. + let (parent_index, slot) = if roots.len() > 1 && rng.chance(1, 4) { + let sibling = 1 + rng.below(roots.len() as u64 - 1) as usize; + (parent_indices[sibling], slots[sibling]) + } else { + let recent = rng.below(roots.len().min(3) as u64) as usize; + let parent_index = roots.len() - 1 - recent; + let slot = if parent_index == 0 { + first_gloas_slot - 3 + rng.below(3) + } else { + slots[parent_index] + 1 + rng.below(4) + }; + (parent_index, slot) + }; + if slot > slot_cap { + continue; + } + let root = rng.root(); + let parent_root = roots[parent_index]; + if slot >= first_gloas_slot { + let hash = ExecutionBlockHash::repeat_byte(1 + i as u8); + let parent_hash = match hashes[parent_index] { + Some(parent_hash) if rng.chance(1, 2) => parent_hash, + _ => ExecutionBlockHash::repeat_byte(0xf0 + rng.below(8) as u8), + }; + store + .insert_signed_block(root, gloas_block(slot, parent_root, parent_hash, hash)) + .unwrap(); + store.insert_state(root, gloas_state.clone()).unwrap(); + if rng.chance(1, 2) { + store.insert_verified_payload(root); + } + for timely_votes in [true, false] { + let votes = match rng.below(4) { + 0 => vec![None; preset::PTC_SIZE], + 1 => vec![Some(true); preset::PTC_SIZE], + 2 => vec![Some(false); preset::PTC_SIZE], + _ => (0..preset::PTC_SIZE) + .map(|_| rng.chance(2, 3).then(|| rng.chance(1, 2))) + .collect(), + }; + if timely_votes { + store.set_payload_timeliness_vote(root, votes); + } else { + store.set_payload_data_availability_vote(root, votes); + } + } + hashes.push(Some(hash)); + } else { + store + .insert_signed_block(root, fulu_block(slot, parent_root)) + .unwrap(); + store.insert_state(root, fulu_state.clone()).unwrap(); + hashes.push(None); + } + let unrealized_epoch = if justified_moved { rng.below(2) } else { 0 }; + store.set_unrealized_justification( + root, + Checkpoint { + epoch: unrealized_epoch, + root: Root::ZERO, + }, + ); + if rng.chance(3, 4) { + store.set_block_timeliness(root, [rng.chance(1, 2), rng.chance(1, 2)]); + } + roots.push(root); + slots.push(slot); + parent_indices.push(parent_index); + } + + // Justified above the anchor: the store follows the justified block's + // own state, cached under the checkpoint so that no slot processing + // is needed to reach it. + if justified_moved && roots.len() > 1 { + let pick = 1 + rng.below(roots.len() as u64 - 1) as usize; + let justified = Checkpoint { + epoch: 1, + root: roots[pick], + }; + let justified_state = store.get_state(&justified.root).unwrap().unwrap(); + store.cache_state( + CacheKey::CheckpointState { + epoch: justified.epoch, + root: justified.root, + }, + justified_state, + ); + let finalized = store.beacon_finalized_checkpoint(); + update_checkpoints(&mut store, justified, finalized); + } + + // Blocks the store holds but the index does not: what pruning at the + // finalized anchor leaves behind, and what a stale vote can name. + let mut outside_index: Vec = Vec::new(); + if prunes_index { + for _ in 0..1 + rng.below(3) { + let root = rng.root(); + store + .insert_signed_block(root, fulu_block(anchor_slot, Root::ZERO)) + .unwrap(); + store.insert_state(root, fulu_state.clone()).unwrap(); + store.delete_live_chain_entries(&[(anchor_slot, root)]); + outside_index.push(root); + } + } + + for validator_index in 0..TREE_VALIDATORS as u64 { + if rng.chance(1, 6) { + continue; + } + let (root, block_slot) = if !outside_index.is_empty() && rng.chance(1, 4) { + ( + outside_index[rng.below(outside_index.len() as u64) as usize], + anchor_slot, + ) + } else if rng.chance(1, 12) { + (rng.root(), 0) + } else { + let pick = rng.below(roots.len() as u64) as usize; + (roots[pick], slots[pick]) + }; + store.set_latest_message( + validator_index, + LatestMessage { + epoch: 0, + slot: block_slot + rng.below(3), + root, + payload_present: rng.chance(1, 2), + }, + ); + if rng.chance(1, 8) { + store.insert_equivocating_index(validator_index); + } + } + if roots.len() > 1 && rng.chance(2, 3) { + let pick = 1 + rng.below(roots.len() as u64 - 1) as usize; + store.set_proposer_boost_root(roots[pick]); + } + + RandomTree { + store, + config, + roots, + slots, + first_gloas_slot, + justified_moved, + } + } + + /// What the differential test saw, so it can insist it was not vacuous. + #[derive(Default, Debug)] + struct WalkStats { + full_heads: usize, + empty_heads: usize, + boost_applied: usize, + boost_withheld: usize, + previous_slot_decisions: usize, + pre_fork_weights: usize, + branches_filtered_out: usize, + } + + /// Whether `root` is `justified` or a block below it in `index`. + fn descends_from_justified( + index: &HashMap, + root: Root, + justified: Root, + ) -> bool { + let justified_slot = index[&justified].0; + let mut cursor = root; + while index[&cursor].0 > justified_slot { + cursor = index[&cursor].1; + if !index.contains_key(&cursor) { + return false; + } + } + cursor == justified + } + + /// Runs the bottom-up walk and both references at `current_slot` and + /// panics, naming the seed, on any disagreement. + /// + /// Node weights are compared only for nodes at or above `walk_bound`, which + /// can be lower than the finalized slot: the walk weighs nothing below it + /// (see `compute_node_weights`' bound), so those nodes read as `0` where + /// the references count their votes. + fn check_walk_against_references( + seed: u64, + tree: &mut RandomTree, + current_slot: Slot, + stats: &mut WalkStats, + ) { + let config = tree.config.clone(); + tree.store + .set_time_ms(current_slot * config.slot_duration_ms) + .unwrap(); + let store = &tree.store; + let committees = store.committee_cache(); + let index = store.block_index(); + let walked = get_head_node(store, &config) + .unwrap_or_else(|err| panic!("seed {seed} slot {current_slot}: walk failed: {err:?}")); + let justified = store.beacon_justified_checkpoint().root; + let bound = walk_bound(store, &index); + let justified_subtree = tree + .roots + .iter() + .filter(|&&root| descends_from_justified(&index, root, justified)) + .count(); + let kept = get_filtered_block_tree(store, &index, &config) + .unwrap() + .len(); + if kept < justified_subtree { + stats.branches_filtered_out += 1; + } + + if current_slot >= tree.first_gloas_slot { + let reference = gloas_get_head(store, &config, &committees).unwrap_or_else(|err| { + panic!("seed {seed} slot {current_slot}: gloas_get_head failed: {err:?}") + }); + assert_eq!( + walked, reference, + "seed {seed} slot {current_slot}: head differs from gloas_get_head" + ); + + let weights = + compute_node_weights(store, &index, &config, &committees, ForkRules::Gloas) + .unwrap_or_else(|err| panic!("seed {seed}: weights failed: {err:?}")); + for (&root, &slot) in tree.roots.iter().zip(&tree.slots) { + if slot < bound { + continue; + } + for payload_status in [ + PayloadStatus::Pending, + PayloadStatus::Empty, + PayloadStatus::Full, + ] { + let node = ForkChoiceNode { + root, + payload_status, + }; + let expected = gloas_get_weight(store, node, &config, &committees) + .unwrap_or_else(|err| panic!("seed {seed}: reference weight: {err:?}")); + assert_eq!( + weights.weight(node, slot), + expected, + "seed {seed} slot {current_slot}: weight of {node:?} differs" + ); + if is_payload_decision_at(slot, payload_status, current_slot) { + stats.previous_slot_decisions += 1; + } + } + } + + match walked.payload_status { + PayloadStatus::Full => stats.full_heads += 1, + PayloadStatus::Empty => stats.empty_heads += 1, + PayloadStatus::Pending => {} + } + if !store.proposer_boost_root().is_zero() { + if should_apply_proposer_boost(store, &config, &committees).unwrap() { + stats.boost_applied += 1; + } else { + stats.boost_withheld += 1; + } + } + } else { + let root = compute_head(store, &index, &config).unwrap(); + assert_eq!( + walked, + ForkChoiceNode { + root, + payload_status: PayloadStatus::Full, + }, + "seed {seed} slot {current_slot}: head differs from compute_head" + ); + + // Weights too, not only the head. `compute_weights` gives the boost + // to a chain only down to the justified block, so the comparison + // covers the blocks at or above it that descend from it. + let reference = compute_weights(store, &index, &config).unwrap(); + let weights = + compute_node_weights(store, &index, &config, &committees, ForkRules::PreGloas) + .unwrap_or_else(|err| panic!("seed {seed}: weights failed: {err:?}")); + for (&root, &slot) in tree.roots.iter().zip(&tree.slots) { + if slot < bound || !descends_from_justified(&index, root, justified) { + continue; + } + let node = ForkChoiceNode { + root, + payload_status: PayloadStatus::Pending, + }; + assert_eq!( + weights.weight(node, slot), + reference.get(&root).copied().unwrap_or_default(), + "seed {seed} slot {current_slot}: weight of {root:?} differs from compute_weights" + ); + stats.pre_fork_weights += 1; + } + } + } + + /// The walk agrees with the spec-literal references on random trees, for a + /// current slot before the fork epoch (against `compute_head`, gloas blocks + /// in the tree read as single full nodes) and after it (against + /// `gloas_get_head`, and node by node against `gloas_get_weight`). + #[test] + fn the_bottom_up_walk_matches_the_spec_literal_references_on_random_trees() { + let mut stats = WalkStats::default(); + for seed in 0..400 { + let mut tree = random_tree(seed, Arc::new(InMemoryBackend::new())); + let max_slot = tree.slots.iter().copied().max().unwrap(); + let first = tree.first_gloas_slot; + let mut current_slots = vec![ + first - 1, + max_slot.max(first), + max_slot.max(first) + 1, + max_slot.max(first) + 2, + max_slot.max(first) + 5, + ]; + if tree.justified_moved { + // Far enough past every block that a branch whose voting + // source is stale is dropped from the filtered tree. + current_slots.push(first + 3 * preset::SLOTS_PER_EPOCH); + } + for current_slot in current_slots { + check_walk_against_references(seed, &mut tree, current_slot, &mut stats); + } + } + assert!(stats.full_heads > 0, "no full head was chosen: {stats:?}"); + assert!(stats.empty_heads > 0, "no empty head was chosen: {stats:?}"); + assert!( + stats.boost_applied > 0, + "the boost never applied: {stats:?}" + ); + assert!( + stats.boost_withheld > 0, + "the boost was never withheld: {stats:?}" + ); + assert!( + stats.previous_slot_decisions > 0, + "no previous-slot decision was weighed: {stats:?}" + ); + assert!( + stats.pre_fork_weights > 0, + "no weight was compared before the fork: {stats:?}" + ); + assert!( + stats.branches_filtered_out > 0, + "the block filter never dropped a branch: {stats:?}" + ); + } + + /// The payload link derived lazily from the blocks is the specification's + /// parent payload status. + /// + /// `random_tree` writes its blocks straight into the store instead of + /// importing them, so this covers the derivation only; the fork-choice + /// fixture harness checks the links `on_block` records. + #[test] + fn the_recorded_payload_links_match_the_specifications_parent_status() { + for seed in 0..100 { + let tree = random_tree(seed, Arc::new(InMemoryBackend::new())); + for &root in &tree.roots { + let block = tree.store.get_signed_block(&root).unwrap().unwrap(); + let link = block_payload_link(&tree.store, root).unwrap().unwrap(); + assert_eq!(link.is_gloas(), !is_pre_gloas(&block), "seed {seed}"); + if root == tree.roots[0] { + assert_eq!(link.parent_status(), Some(PayloadStatus::Full)); + } else { + assert_eq!( + link.parent_status(), + Some(get_parent_payload_status(&tree.store, &block).unwrap()), + "seed {seed}" + ); + } + assert_eq!(tree.store.payload_link(&root), Some(link), "seed {seed}"); + } + } + } + + type StorageError = Box; + + thread_local! { + /// Block-table reads made by the current thread. Per thread, not + /// per backend: the store's background state writer reads the same + /// table from its own thread at times of its choosing, and only the + /// reads the head computation itself makes are in question. + static BLOCK_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; + } + + /// A backend that counts reads of the block table, to show the head walk + /// decodes no block once the payload links are recorded. + struct CountingBackend { + inner: InMemoryBackend, + } + + struct CountingView<'a> { + inner: Box, + } + + impl StorageBackend for CountingBackend { + fn begin_read( + &self, + ) -> std::result::Result, StorageError> + { + Ok(Box::new(CountingView { + inner: self.inner.begin_read()?, + })) + } + + fn begin_write( + &self, + ) -> std::result::Result< + Box, + StorageError, + > { + self.inner.begin_write() + } + } + + impl ethlambda_storage::StorageReadView for CountingView<'_> { + // Counted in `read` rather than `get`: every value read, `get` and + // `read_with` alike, goes through it. + fn read( + &self, + table: ethlambda_storage::Table, + key: &[u8], + read_fn: &mut dyn FnMut(&[u8]) -> std::result::Result<(), StorageError>, + ) -> std::result::Result { + if table == ethlambda_storage::Table::BlockHeaders { + BLOCK_READS.with(|reads| reads.set(reads.get() + 1)); + } + self.inner.read(table, key, read_fn) + } + + // Forwarded rather than left to the default, which goes through + // `read`: an existence check decodes no block, so it is not counted. + fn contains( + &self, + table: ethlambda_storage::Table, + key: &[u8], + ) -> std::result::Result { + self.inner.contains(table, key) + } + + fn prefix_iterator( + &self, + table: ethlambda_storage::Table, + prefix: &[u8], + ) -> std::result::Result< + Box< + dyn Iterator, Box<[u8]>), StorageError>> + '_, + >, + StorageError, + > { + self.inner.prefix_iterator(table, prefix) + } + } + + /// The first head computation after a restart fills the payload links by + /// decoding each block once; every later one decodes none, however many + /// votes and blocks it weighs. + /// + /// The proposer boost stays set, and the current slot is the one after the + /// latest block's, so the previous-slot payload decision, the tiebreaker + /// and `should_extend_payload_with` all run. The one branch that decodes + /// blocks is the equivocation scan under a weak previous-slot parent, + /// rare and bounded by the candidates at one slot; a seed whose boost gate + /// reaches it is left out of the comparison, and the test insists that + /// enough seeds keep the boost set without reaching it. + #[test] + fn the_head_walk_decodes_no_block_once_the_payload_links_are_recorded() { + let mut checked = 0; + let mut boosted = 0; + for seed in 0..40 { + let backend = Arc::new(CountingBackend { + inner: InMemoryBackend::new(), + }); + let mut tree = random_tree(seed, backend); + let max_slot = tree.slots.iter().copied().max().unwrap(); + let latest = max_slot.max(tree.first_gloas_slot); + let scans = boost_gate_scans_for_equivocation(&tree); + for (attempt, current_slot) in [latest + 1, latest + 2].into_iter().enumerate() { + tree.store + .set_time_ms(current_slot * tree.config.slot_duration_ms) + .unwrap(); + + let before = BLOCK_READS.with(|reads| reads.get()); + let first = get_head_node(&tree.store, &tree.config).unwrap(); + let after_first = BLOCK_READS.with(|reads| reads.get()); + let second = get_head_node(&tree.store, &tree.config).unwrap(); + let after_second = BLOCK_READS.with(|reads| reads.get()); + assert_eq!(first, second); + if scans { + continue; + } + assert_eq!( + after_second, after_first, + "seed {seed} slot {current_slot}: the second head computation read a block" + ); + if attempt == 0 && after_first > before { + checked += 1; + if !tree.store.proposer_boost_root().is_zero() { + boosted += 1; + } + } + } + } + assert!(checked > 0, "the counter never moved, so it proves nothing"); + assert!(boosted > 0, "no seed kept the boost without the scan"); + } + + /// Moves finality of `tree`'s store to a checkpoint that names its + /// justified block: the first epoch that starts at or after that block's + /// slot. Returns the justified checkpoint, whose root is now the finalized + /// root too. + fn finalize_at_justified(tree: &mut RandomTree) -> Checkpoint { + let justified = tree.store.beacon_justified_checkpoint(); + let (justified_slot, _) = tree.store.block_index()[&justified.root]; + let finalized = Checkpoint { + epoch: justified_slot.div_ceil(preset::SLOTS_PER_EPOCH), + root: justified.root, + }; + let before = tree.store.beacon_finalized_checkpoint(); + update_checkpoints(&mut tree.store, justified, finalized); + assert_ne!(finalized, before); + justified + } + + /// Clears a boost root that a live node could not hold: the boosted block + /// is a current-slot block, so it descends from the finalized block and is + /// not that block itself. + fn keep_only_a_live_boost(tree: &mut RandomTree, finalized_root: Root) { + let boost_root = tree.store.proposer_boost_root(); + let index = tree.store.block_index(); + if boost_root == finalized_root + || (!boost_root.is_zero() + && !descends_from_justified(&index, boost_root, finalized_root)) + { + tree.store.set_proposer_boost_root(Root::ZERO); + } + } + + /// The current slots the bounded walk is compared at: before the fork + /// epoch, just after the latest block, and far enough on for the block + /// filter to drop stale branches. + fn comparison_slots(tree: &RandomTree) -> [Slot; 4] { + let latest = tree + .slots + .iter() + .copied() + .max() + .unwrap() + .max(tree.first_gloas_slot); + [ + tree.first_gloas_slot - 1, + latest + 1, + latest + 2, + tree.first_gloas_slot + 3 * preset::SLOTS_PER_EPOCH, + ] + } + + /// Once finality has moved, the actor's prune of the payload links below the + /// finalized block costs the head computation nothing: it weighs only blocks + /// at or above the bound, so it neither reads the pruned links nor derives + /// them again by decoding. The bounded walk still names the head, and gives + /// every node at or above the bound the weight, that the unbounded + /// references do, with a nonzero bound. + #[test] + fn pruning_links_below_the_finalized_block_changes_nothing_the_head_reads() { + let mut stats = WalkStats::default(); + let mut pruned_some = 0; + let mut read_nothing = 0; + for seed in 0..200 { + let backend = Arc::new(CountingBackend { + inner: InMemoryBackend::new(), + }); + let mut tree = random_tree(seed, backend); + if !tree.justified_moved { + continue; + } + let justified = finalize_at_justified(&mut tree); + keep_only_a_live_boost(&mut tree, justified.root); + if boost_gate_scans_for_equivocation(&tree) { + continue; + } + + // Every block's link is recorded, as `on_block` would have. + for &root in &tree.roots { + block_payload_link(&tree.store, root).unwrap().unwrap(); + } + let linked = |tree: &RandomTree| { + tree.roots + .iter() + .filter(|&&root| tree.store.payload_link(&root).is_some()) + .count() + }; + let before_prune = linked(&tree); + + // The actor's own prune. + let checkpoint = tree.store.latest_finalized().unwrap(); + ensure_payload_link(&tree.store, checkpoint.root).unwrap(); + tree.store + .prune_beacon_payload_links(checkpoint.slot, checkpoint.root); + assert!( + linked(&tree) < before_prune, + "seed {seed}: nothing was pruned" + ); + pruned_some += 1; + + let current_slot = comparison_slots(&tree)[2]; + tree.store + .set_time_ms(current_slot * tree.config.slot_duration_ms) + .unwrap(); + let before = BLOCK_READS.with(|reads| reads.get()); + get_head_node(&tree.store, &tree.config).unwrap(); + let after = BLOCK_READS.with(|reads| reads.get()); + assert_eq!(after, before, "seed {seed}: a pruned link was read again"); + read_nothing += 1; + + for current_slot in comparison_slots(&tree) { + check_walk_against_references(seed, &mut tree, current_slot, &mut stats); + } + } + assert!(pruned_some > 0, "no link was pruned, so nothing was tested"); + assert!(read_nothing > 0, "no seed checked that no block was read"); + assert!( + stats.pre_fork_weights > 0, + "no weight was compared: {stats:?}" + ); + } + + /// The bound is the least of three slots so that it holds without assuming + /// the boosted block descends from the finalized one: with the boost on the + /// finalized block itself, the boost gate reads the score of a parent below + /// the finalized slot, and the walk still agrees with the unbounded + /// references on the head and on every node at or above the bound. + #[test] + fn the_bound_covers_a_boosted_finalized_block() { + let mut stats = WalkStats::default(); + let mut boosted_finalized = 0; + for seed in 0..200 { + let mut tree = random_tree(seed, Arc::new(InMemoryBackend::new())); + if !tree.justified_moved { + continue; + } + let justified = finalize_at_justified(&mut tree); + tree.store.set_proposer_boost_root(justified.root); + if boost_gate_scans_for_equivocation(&tree) { + continue; + } + boosted_finalized += 1; + for current_slot in comparison_slots(&tree) { + check_walk_against_references(seed, &mut tree, current_slot, &mut stats); + } + } + assert!( + boosted_finalized > 0, + "the finalized block was never boosted" + ); + } + + /// After a restart no link is recorded. Each cycle of the actor's sequence + /// (record the finalized root's link, prune, compute the head) derives what + /// it finds missing once, so the first cycle reads blocks and the next two + /// read none. + #[test] + fn the_actors_prune_cycle_derives_links_once_after_a_restart() { + let mut derived_first = 0; + for seed in 0..60 { + let backend = Arc::new(CountingBackend { + inner: InMemoryBackend::new(), + }); + let mut tree = random_tree(seed, backend); + if !tree.justified_moved { + continue; + } + let justified = finalize_at_justified(&mut tree); + keep_only_a_live_boost(&mut tree, justified.root); + if boost_gate_scans_for_equivocation(&tree) { + continue; + } + let current_slot = comparison_slots(&tree)[2]; + tree.store + .set_time_ms(current_slot * tree.config.slot_duration_ms) + .unwrap(); + + let mut reads_per_cycle = Vec::new(); + for _ in 0..3 { + let before = BLOCK_READS.with(|reads| reads.get()); + let checkpoint = tree.store.latest_finalized().unwrap(); + ensure_payload_link(&tree.store, checkpoint.root).unwrap(); + tree.store + .prune_beacon_payload_links(checkpoint.slot, checkpoint.root); + get_head_node(&tree.store, &tree.config).unwrap(); + reads_per_cycle.push(BLOCK_READS.with(|reads| reads.get()) - before); + } + assert_eq!( + reads_per_cycle[1..], + [0, 0], + "seed {seed}: a later cycle read a block, reads {reads_per_cycle:?}" + ); + if reads_per_cycle[0] > 0 { + derived_first += 1; + } + } + assert!( + derived_first > 0, + "no seed derived a link after the restart" + ); + } + + /// Whether the boost gate of `tree`'s store reaches the equivocation scan: + /// a boosted block whose parent is from the previous slot and weak. + fn boost_gate_scans_for_equivocation(tree: &RandomTree) -> bool { + let store = &tree.store; + let boost_root = store.proposer_boost_root(); + if boost_root.is_zero() { + return false; + } + let index = store.block_index(); + let (slot, parent_root) = index[&boost_root]; + if index[&parent_root].0 + 1 < slot { + return false; + } + let committees = store.committee_cache(); + is_head_weak(store, parent_root, &tree.config, &committees).unwrap() + } + + /// A gloas block whose parent the store does not hold, the anchor of a + /// resumed directory, gets its link derived once and then recorded, + /// although its parent status is unknown. + #[test] + fn the_link_of_a_gloas_block_without_its_parent_is_recorded_once() { + let mut store = empty_store(); + let root = Root::repeat_byte(0xa1); + let block = gloas_block( + 5, + Root::repeat_byte(0xee), + ExecutionBlockHash::repeat_byte(1), + ExecutionBlockHash::repeat_byte(2), + ); + store.insert_signed_block(root, block).unwrap(); + assert_eq!(store.payload_link(&root), None); + + let link = block_payload_link(&store, root).unwrap().unwrap(); + + assert_eq!( + link, + BlockPayloadLink::Gloas { + parent_status: None + } + ); + assert_eq!(store.payload_link(&root), Some(link)); + } } diff --git a/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs b/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs index c0835edad..fb901a2ef 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/fork_choice.rs @@ -127,7 +127,7 @@ use ethlambda_state_transition::beacon::containers::{ gloas, phase0, }; use ethlambda_state_transition::beacon::fork_choice::{ - self, DataAvailability, ForkChoiceNode, PayloadStatus, Store, + self, DataAvailability, ForkChoiceNode, ForkRules, PayloadStatus, Store, }; use ethlambda_state_transition::beacon::helpers::accessors::CommitteeCache; use ethlambda_state_transition::beacon::preset; @@ -688,12 +688,91 @@ fn check_checkpoint( } } +/// Checks that the node's head computation ([`fork_choice::get_head_node`], the +/// one bottom-up walk) names the same node as the spec-literal reference for +/// the current slot's fork: [`fork_choice::gloas_get_head`] from +/// `GLOAS_FORK_EPOCH` on, the pre-gloas [`fork_choice::compute_head`] (reported +/// as a full node) before it. Run at every `head` check, so every step of every +/// fixture is also a differential test of the walk. +fn check_head_against_reference(store: &Store, config: &Config) -> Result<(), String> { + let walked = fork_choice::get_head_node(store, config) + .map_err(|err| format!("get_head_node: {err:?}"))?; + let current_slot = fork_choice::get_current_slot(store, config); + let fork = config.fork_at_epoch(current_slot / preset::SLOTS_PER_EPOCH); + let reference = match ForkRules::of(fork) { + ForkRules::Gloas => { + let committees = store.committee_cache(); + fork_choice::gloas_get_head(store, config, &committees) + .map_err(|err| format!("reference gloas_get_head: {err:?}"))? + } + ForkRules::PreGloas => { + let index = store.block_index(); + let root = fork_choice::compute_head(store, &index, config) + .map_err(|err| format!("reference compute_head: {err:?}"))?; + ForkChoiceNode { + root, + payload_status: PayloadStatus::Full, + } + } + }; + if walked == reference { + Ok(()) + } else { + Err(format!( + "the bottom-up walk chose 0x{} ({:?}) but the reference chose 0x{} ({:?})", + hex::encode(walked.root.0), + walked.payload_status, + hex::encode(reference.root.0), + reference.payload_status, + )) + } +} + +/// Checks that every payload link `on_block` recorded is what decoding the block +/// and its parent gives: the walk reads the link instead of the blocks. +fn check_payload_links(store: &Store) -> Result<(), String> { + for root in store.block_index().into_keys() { + let Some(link) = store.payload_link(&root) else { + continue; + }; + let block = store + .get_signed_block(&root) + .expect("get") + .ok_or("a linked block is missing from the store")?; + let is_gloas = match ForkRules::of(block.fork_name()) { + ForkRules::Gloas => true, + ForkRules::PreGloas => false, + }; + if link.is_gloas() != is_gloas { + return Err(format!( + "payload link of 0x{} has the wrong fork", + hex::encode(root.0) + )); + } + if let Some(parent_status) = link.parent_status() + && store.has_block(&block.parent_root()) + { + let expected = fork_choice::get_parent_payload_status(store, &block) + .map_err(|err| format!("get_parent_payload_status: {err:?}"))?; + if parent_status != expected { + return Err(format!( + "payload link of 0x{} says {parent_status:?}, the blocks say {expected:?}", + hex::encode(root.0) + )); + } + } + } + Ok(()) +} + /// Checks `head` against [`fork_choice::get_head`]'s root, and that root's /// slot (read through [`fork_choice::get_forkchoice_store`]'s store) against /// the fixture's redundant `slot` field. fn check_head(expected: &HeadCheck, store: &mut Store, config: &Config) -> Result<(), String> { let actual_root = fork_choice::get_head(store, config).map_err(|err| format!("get_head: {err:?}"))?; + check_head_against_reference(store, config)?; + check_payload_links(store)?; let actual_slot = store .get_signed_block(&actual_root) .expect("get") @@ -785,6 +864,17 @@ fn check_viable_for_head( payload_status: PayloadStatus::Pending, }]; let mut actual: Vec<(Root, u64, u8)> = Vec::new(); + // The bottom-up table the node's head computation weighs with, checked + // against the spec-literal weight at every leaf. This check only runs in + // gloas cases, so the current slot's rules are gloas's. + let walked_weights = fork_choice::compute_node_weights( + store, + &index, + config, + committees, + fork_choice::ForkRules::Gloas, + ) + .map_err(|err| format!("viable_for_head_roots_and_weights: compute_node_weights: {err:?}"))?; while let Some(node) = pending.pop() { let children = fork_choice::get_node_children(store, &blocks, node).map_err(|err| { format!("viable_for_head_roots_and_weights: get_node_children: {err:?}") @@ -792,6 +882,19 @@ fn check_viable_for_head( if children.is_empty() { let weight = fork_choice::gloas_get_weight(store, node, config, committees) .map_err(|err| format!("viable_for_head_roots_and_weights: get_weight: {err:?}"))?; + let block_slot = index + .get(&node.root) + .map(|&(slot, _)| slot) + .ok_or("viable_for_head_roots_and_weights: a leaf is not in the block index")?; + let walked = walked_weights.weight(node, block_slot); + if walked != weight { + return Err(format!( + "viable_for_head_roots_and_weights: the bottom-up weight of 0x{} ({}) is {walked}, \ + the reference is {weight}", + hex::encode(node.root.0), + node.payload_status as u8, + )); + } actual.push((node.root, weight, node.payload_status as u8)); } else { pending.extend(children); diff --git a/crates/common/types/src/beacon/fork_choice.rs b/crates/common/types/src/beacon/fork_choice.rs index 91890d2f1..f7515cf0d 100644 --- a/crates/common/types/src/beacon/fork_choice.rs +++ b/crates/common/types/src/beacon/fork_choice.rs @@ -1,18 +1,20 @@ //! Fork-choice-adjacent data that is neither a block nor a state. //! -//! Three kinds share this file. `LatestMessage` and `PowBlock` are SSZ +//! Four kinds share this file. `LatestMessage` and `PowBlock` are SSZ //! consensus containers moved out of the `beacon::fork_choice` module of //! `ethlambda-state-transition`, which re-exports both at their old paths so //! every use site inside it is unchanged. `PayloadStatusEnum` and //! `PayloadStatusV1` are plain Engine-API-shaped data with no such former //! home. `PayloadStatus` and `ForkChoiceNode` are gloas's own fork-choice //! node types, new here rather than moved, since nothing named them before -//! gloas. All six live here for the same reason: the DB-backed -//! `ethlambda_storage::Store` holds `LatestMessage`, `PowBlock` and -//! `PayloadStatusV1`, and `ethlambda-storage` cannot depend on -//! `ethlambda-state-transition`, which pulls in `blst` and `c-kzg`; -//! `PayloadStatus` and `ForkChoiceNode` join them here rather than living -//! beside `ethlambda-state-transition`'s own fork choice, so that a +//! gloas. `BlockPayloadLink` is not a specification type at all: it is the +//! per-block record the head walk reads instead of decoding the block. All +//! seven live here for the same reason: the DB-backed +//! `ethlambda_storage::Store` holds `LatestMessage`, `PowBlock`, +//! `PayloadStatusV1` and `BlockPayloadLink`, and `ethlambda-storage` cannot +//! depend on `ethlambda-state-transition`, which pulls in `blst` and +//! `c-kzg`; `PayloadStatus` and `ForkChoiceNode` join them here rather than +//! living beside `ethlambda-state-transition`'s own fork choice, so that a //! `LatestMessage`'s `payload_present` field and a stored node's own //! `PayloadStatus` share one crate with no dependency to cross. @@ -69,6 +71,51 @@ pub enum PayloadStatus { Pending = 2, } +/// What fork choice needs to know about a block's payload dimension, recorded +/// when the block is imported so that walking the tree never decodes a block +/// to learn it. +/// +/// Two facts, both read off a block's own bytes and both fixed for the life of +/// the block: which rules it is handled under, and which of its parent's +/// payload branches it builds on (`get_parent_payload_status`). Neither +/// changes after import, so the record is a cache that is correct whenever it +/// is present and can be rederived by decoding the block when it is not. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BlockPayloadLink { + /// A pre-gloas block: a single FULL node whose payload ran inside the + /// block itself. It builds on its parent's FULL branch, the + /// fulu-to-gloas boundary rule. + PreGloas, + /// A gloas block. + Gloas { + /// The parent payload branch the block builds on, `None` when the + /// store does not hold the parent (a gloas anchor, whose parent is + /// below the retained window, so there is no bid to compare against). + /// + /// `Some(Full)` for a block whose parent is pre-gloas, by the same + /// boundary rule. + parent_status: Option, + }, +} + +impl BlockPayloadLink { + /// Whether the block is a gloas block. + pub fn is_gloas(self) -> bool { + match self { + BlockPayloadLink::PreGloas => false, + BlockPayloadLink::Gloas { .. } => true, + } + } + + /// The parent payload branch the block builds on, `None` when unknown. + pub fn parent_status(self) -> Option { + match self { + BlockPayloadLink::PreGloas => Some(PayloadStatus::Full), + BlockPayloadLink::Gloas { parent_status } => parent_status, + } + } +} + /// A gloas fork-choice node (`fork-choice.md`'s modified `ForkChoiceNode`): a /// block, and which of its payload branches. /// diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index a6d472bd8..87e89df11 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -20,7 +20,7 @@ use ethlambda_types::{ constants::NUM_BLOCK_TIMELINESS_DEADLINES, containers::{BeaconState, Checkpoint as BeaconCheckpoint, SignedBeaconBlock}, fork::ForkName, - fork_choice::{LatestMessage, PayloadStatusV1, PowBlock}, + fork_choice::{BlockPayloadLink, LatestMessage, PayloadStatusV1, PowBlock}, preset::{Preset, SLOTS_PER_EPOCH}, primitives::ExecutionBlockHash, }, @@ -654,8 +654,8 @@ impl GossipSignatureBuffer { /// copy of a block's post-state, which a node resuming from an anchor does /// anyway as it re-imports the unfinalized window. `optimistic_roots` and /// `payload_statuses` are likewise answers an execution client can be asked -/// for again, and `el_block_hashes` is a cache over data already decodable -/// from the block itself. +/// for again, and `el_block_hashes` and `payload_links` are caches over data +/// already decodable from the block itself. /// /// Gloas's own four (`verified_payloads`, `payload_timeliness_vote`, /// `payload_data_availability_vote`, and `block_timeliness`'s widened shape) @@ -680,13 +680,14 @@ impl GossipSignatureBuffer { /// Most of this is uncapped: the per-validator maps are bounded by the /// validator set, and the per-block ones (`block_timeliness`, /// `unrealized_justifications`, and gloas's own three) grow with the blocks -/// this process has imported. Two are the exception, `el_block_hashes` and -/// `optimistic_roots`, each pruned to the unfinalized window by its own -/// `prune_*` method. Both fill on a path that runs for the whole life of the -/// process and has no other way of emptying them: `forkchoiceUpdated` reads -/// the first once per head move, and an execution client doing a long state -/// sync answers `NOT_VALIDATED` to every block, which writes the second once -/// per import. +/// this process has imported. Three are the exception, `el_block_hashes`, +/// `payload_links` and `optimistic_roots`, each pruned to the unfinalized +/// window by its own `prune_*` method. All three fill on a path that runs for +/// the whole life of the process and has no other way of emptying them: +/// `forkchoiceUpdated` reads the first once per head move, the head walk +/// reads the second for every block it visits, and an execution client doing a +/// long state sync answers `NOT_VALIDATED` to every block, which writes the +/// third once per import. #[derive(Default)] pub(crate) struct BeaconScratch { pub(crate) proposer_boost_root: H256, @@ -710,6 +711,18 @@ pub(crate) struct BeaconScratch { /// struct's own doc) `verified_payloads` needs to survive a restart and /// nothing else here does. pub(crate) verified_payloads: HashSet, + /// Each imported block's payload link (which rules it is handled under, + /// and which of its parent's payload branches it builds on), so the head + /// computation reads both without decoding a block. Written by `on_block`, + /// and by the head computation itself for a block it finds missing (a block + /// imported before a restart), the one decode that block ever costs. + /// + /// A cache over data already decodable from the block, like + /// `el_block_hashes`, and bounded the same way: each entry carries its + /// block's slot, and `prune_beacon_payload_links` drops the ones below + /// finality. A pruned link is derived again by decoding if anything asks + /// for it. + pub(crate) payload_links: HashMap, /// Gloas: the payload timeliness committee's per-member vote on whether /// the block's payload showed up on time, one slot per `PTC_SIZE` index, /// `None` until that member votes. Keyed by beacon block root, like @@ -2050,8 +2063,8 @@ impl Store { /// /// Dropping the row is the whole of "remove this block from fork choice": /// [`block_index`](Self::block_index) is the only source - /// `filter_block_tree`, `compute_weights` and `get_head` read, so a root - /// with no row contributes no weight to any ancestor and can never be + /// `filter_block_tree`, `compute_node_weights` and `walk_head` read, so a + /// root with no row contributes no weight to any ancestor and can never be /// walked to. /// /// The block and its state stay in their own tables. Nothing reads them @@ -3290,6 +3303,60 @@ impl Store { self.beacon.lock().unwrap().verified_payloads.insert(root); } + /// `root`'s payload link, if one has been recorded. See + /// [`BlockPayloadLink`] for what it holds and why a missing entry is not + /// an error. + pub fn payload_link(&self, root: &H256) -> Option { + self.beacon + .lock() + .unwrap() + .payload_links + .get(root) + .map(|(_slot, link)| *link) + } + + /// Records `root`'s payload link, against the block's `slot` that the + /// unfinalized-window bound prunes it by. + /// + /// `&self`, unlike the other setters here: the head computation records the + /// link of a block it had to decode and only has a shared reference to the + /// store, and the scratch is behind its own lock either way. + pub fn set_payload_link(&self, root: H256, slot: u64, link: BlockPayloadLink) { + self.beacon + .lock() + .unwrap() + .payload_links + .insert(root, (slot, link)); + } + + /// Drops payload links strictly below `finalized_slot`, always keeping + /// `keep`. + /// + /// The same bound and exemption as + /// [`Store::prune_beacon_el_block_hashes`], with one refinement. The head + /// walk weighs only blocks at or above the finalized block, so nothing + /// below that block's own slot is read again, and a link asked for after + /// all is derived again. That slot can be below `finalized_slot` (the + /// checkpoint's stored slot is its epoch's start, and a skipped boundary + /// leaves the block earlier), so the bound is the lower of the two, read + /// from `keep`'s own recorded link. + /// + /// Prunes nothing when `keep` has no recorded link, since the block's own + /// slot is then unknown and a bound taken from the checkpoint could drop + /// links the walk still reads. A caller records that link first (see + /// `fork_choice::ensure_payload_link`); it is missing after this method + /// only when the block is not stored at all. + pub fn prune_beacon_payload_links(&mut self, finalized_slot: u64, keep: H256) { + let mut beacon = self.beacon.lock().unwrap(); + let Some(&(keep_slot, _link)) = beacon.payload_links.get(&keep) else { + return; + }; + let bound = keep_slot.min(finalized_slot); + beacon + .payload_links + .retain(|root, (slot, _link)| *slot >= bound || *root == keep); + } + /// The latest attestation recorded for validator `index`, if any. pub fn latest_message(&self, index: u64) -> Option { self.beacon @@ -7141,6 +7208,50 @@ mod tests { ); } + /// Payload links are pruned like the execution hashes: strictly below the + /// finalized slot, keeping the finalized root even when its block sits + /// below that slot. + #[test] + fn payload_links_prune_strictly_below_the_finalized_slot_and_keep_its_root() { + let mut store = Store::test_store(); + let finalized = H256::repeat_byte(1); + let stale = H256::repeat_byte(2); + let at = H256::repeat_byte(3); + let above = H256::repeat_byte(4); + let link = BlockPayloadLink::PreGloas; + store.set_payload_link(finalized, 30, link); + let between = H256::repeat_byte(5); + store.set_payload_link(stale, 29, link); + // Between the finalized block (30) and the checkpoint's slot (32): a + // fork block the head walk still reads. + store.set_payload_link(between, 31, link); + store.set_payload_link(at, 32, link); + store.set_payload_link(above, 33, link); + + store.prune_beacon_payload_links(32, finalized); + + assert_eq!(store.payload_link(&finalized), Some(link)); + assert_eq!(store.payload_link(&stale), None); + assert_eq!(store.payload_link(&between), Some(link)); + assert_eq!(store.payload_link(&at), Some(link)); + assert_eq!(store.payload_link(&above), Some(link)); + } + + /// Without a link for the finalized root its own slot is unknown, so the + /// prune leaves every link alone rather than bounding on the checkpoint. + #[test] + fn payload_links_are_not_pruned_when_the_finalized_root_has_none() { + let mut store = Store::test_store(); + let finalized = H256::repeat_byte(1); + let stale = H256::repeat_byte(2); + let link = BlockPayloadLink::PreGloas; + store.set_payload_link(stale, 29, link); + + store.prune_beacon_payload_links(32, finalized); + + assert_eq!(store.payload_link(&stale), Some(link)); + } + #[test] fn a_fresh_beacon_store_seeds_every_key_its_accessors_read() { let backend = Arc::new(InMemoryBackend::new()); diff --git a/docs/beacon_engine.md b/docs/beacon_engine.md index 78ff476ac..d7c557baa 100644 --- a/docs/beacon_engine.md +++ b/docs/beacon_engine.md @@ -96,10 +96,10 @@ head to fall back when the `SYNCING` branch turns out to be invalid. An invalidated block leaves fork choice by having its `LiveChain` index row deleted. That is sufficient because `Store::block_index()` is the only source -`filter_block_tree`, `compute_weights` and `get_head` read: a root with no row -contributes no weight to any ancestor and can never be walked to. The block and -its state stay in their own tables, so an operator can still inspect what was -rejected. +`filter_block_tree`, `compute_node_weights` and `walk_head` read: a root with no +row contributes no weight to any ancestor and can never be walked to. The block +and its state stay in their own tables, so an operator can still inspect what +was rejected. That an `INVALIDATED` block is never imported is enforced on the verdict itself, in `on_block`, not on the state transition having failed. The transition still diff --git a/docs/beacon_stf.md b/docs/beacon_stf.md index db8b9ffd1..e2cee346c 100644 --- a/docs/beacon_stf.md +++ b/docs/beacon_stf.md @@ -405,15 +405,18 @@ from an exhaustive match on a fork or a container, and it is what `data.index` into the payload flag and orders votes by slot instead of target epoch. A vote's rules follow its own container, not the clock. -The head algorithm follows the current slot's fork: from `GLOAS_FORK_EPOCH` on, -`get_head_node` runs `gloas_get_head` over the whole tree, pre-gloas blocks -included; before it, the unchanged `compute_head` runs. The switch follows the -clock, so any follower on a network that schedules gloas (Sepolia's built-in -config does) runs `gloas_get_head` every slot from that epoch, over its -pre-gloas tree, whether or not it can import a gloas block; its cost is the -subject of [spec_deviations.md](spec_deviations.md). The spec never describes a -tree that crosses the boundary, so that page also records the rule this module -chose. +One head computation serves every fork: `get_head_node` builds a weight table +for the node tree in one bottom-up pass and descends over it. The current +slot's fork selects the two rules that differ: from `GLOAS_FORK_EPOCH` on, +blocks carry the payload dimension their bids give them and the proposer boost +is gated by `should_apply_proposer_boost`; before it, every block is a single +full node and the boost applies whenever it is set. A block's fork and parent +payload status are recorded at import (`BlockPayloadLink`), so the walk decodes +no block in the ordinary case; the two exceptions are in +[spec_deviations.md](spec_deviations.md). `compute_head` and the spec-literal +`gloas_get_head` are kept as the references it is tested against; the cost +argument is in the same page. The spec never describes a tree that +crosses the boundary, so that page also records the rule this module chose. Two new handlers, `on_execution_payload_envelope` and `on_payload_attestation_message`, join the spec's list of ways to change the store. diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index f1ffe819a..2ea512cc8 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -153,10 +153,14 @@ tree every live follower holds at the fork epoch. (`is_payload_verified`, `payload_timeliness`, `payload_data_availability`), so the first gloas block passes `on_block` and an `index == 1` vote for the last fulu block passes `validate_on_attestation`. -- **Which head runs:** `get_head_node` follows the current slot's fork, not the - head block's or the justified checkpoint's. From `GLOAS_FORK_EPOCH` on, - `gloas_get_head` runs over the whole tree, pre-gloas blocks included; before - it, the unchanged `compute_head` runs. +- **Which rules the head uses:** `get_head_node` runs one bottom-up walk for + every fork, and the current slot's fork, not the head block's or the + justified checkpoint's, selects its two fork-dependent rules. From + `GLOAS_FORK_EPOCH` on, blocks carry the payload dimension their bids give + them and the proposer boost is gated by `should_apply_proposer_boost`; before + it, every block is a single full node and the boost applies whenever it is + set. `compute_head` and `gloas_get_head` are the references the walk is + tested against. - **Other clients:** Lodestar and Prysm also treat a pre-gloas parent as `FULL`. Lighthouse (v8.2.2) answers `EMPTY` for a pre-gloas parent, although #5125 describes its handling as `PENDING`. @@ -167,21 +171,25 @@ tree every live follower holds at the fork epoch. Three places in the gloas head read something the specification would abort on, because a live node holds gaps a fixture tree never has. Each is marked -"Implementation choice, not spec text" at its definition. - -- **Pruned votes** (`get_attestation_score`, and through it `is_head_weak` and - `is_parent_strong`; the gloas head reaches `is_head_weak` from - `should_apply_proposer_boost`, and the gloas proposer head reaches - `is_parent_strong`). A vote whose ancestor walk meets a root missing - from the block index contributes nothing, where `get_ancestor` would raise. - Pruning removes only entries below the finalized slot, and every root scored is - indexed at or above it, so such a walk cannot descend from the scored root and - the specification's own contribution for that vote is zero too. -- **Unknown voters** (`gloas_get_attestation_score`). A vote for a root the store - never held is skipped, where `get_supported_node` would raise, so one stale - voter cannot abort a head computation. A block pruned from the index but still - in the store is not skipped: its vote walks the block table and contributes - nothing. +"Implementation choice, not spec text" at its definition. The head the node +runs is `walk_head` (through `get_head_node`) over the weights of +`compute_node_weights`; the spec-literal references carry the same tolerances so +that the tests compare like with like. + +- **Pruned votes** (`compute_node_weights`; in the references, + `get_attestation_score`, and through it `is_head_weak` and + `is_parent_strong`, the latter reached from the gloas proposer head). A vote + for a root missing from the block index contributes nothing, where + `get_ancestor` would raise. Pruning removes only entries below the finalized + slot, and every root scored is indexed at or above it, so such a vote cannot + descend from the scored root and the specification's own contribution for it + is zero too. +- **Unknown voters** (`compute_node_weights`; `gloas_get_attestation_score` in + the reference). A vote for a root the store never held is skipped, where + `get_supported_node` would raise, so one stale voter cannot abort a head + computation. The walk drops a block pruned from the index but still in the + store the same way, with the pruned votes above; the reference does not skip + it, and its vote walks the block table and contributes nothing. - **Missing timeliness** (`should_apply_proposer_boost`). A candidate with no `block_timeliness` entry reads as not timely by either deadline, where the specification indexes the entry directly. The entries are in memory only, so a @@ -208,25 +216,51 @@ the root of the state's latest block header. shapes give the same root. - **Equivalence:** the same check, on both kinds of state. -## Gloas `get_weight` is the specification's walk, with its cost - -`gloas_get_weight` transcribes the specification statement by statement, where -the pre-gloas head folds weights bottom-up once per tree. - -- **ethlambda:** every node the head visits re-walks every latest message, each - ancestor step decodes signed blocks (the node's own and its parent's, and a - third per voter in `get_supported_node`), telling a pre-gloas root from a gloas - one costs a decode the pre-gloas tree never paid, and - `should_apply_proposer_boost` reruns a full `is_head_weak` score although its - answer does not depend on the node. `get_head_node` passes the store's shared - committee cache, so committees are not rebuilt per call. -- **Why:** the port is checked against the fixtures statement by statement, and - a faster form is a second implementation to keep in step with the spec. -- **Consequence:** fine for fixture trees, which are a handful of blocks. - `get_head_node` switches on the clock, so any follower on a network that - schedules gloas (Sepolia's built-in config does) runs `gloas_get_head` every - slot from `GLOAS_FORK_EPOCH`, over its pre-gloas tree, whether or not it can - import a gloas block. The remaining costs need a bottom-up form before that - epoch, not only before a gloas follower is deployed. The pre-gloas - form's own measurement, in `compute_weights`' documentation, is the estimate - of what the walk costs on a mainnet tree. +## The head is one bottom-up walk, not the specification's per-node `get_weight` + +The specification's gloas `get_head` calls `get_weight` for every candidate +node, and each call walks every latest message and every ancestor step. + +- **ethlambda:** `get_head_node` computes every node's attestation score in one + pass over the latest messages and one over the blocks (highest slot first), + folding each block's total into the payload node of its parent that it builds + on, then descends from the justified checkpoint over that table. The proof + that the fold equals `get_attestation_score` for every node, from + `is_ancestor`'s definition, is in the `fork_choice.rs` section "The head + computation". The proposer boost is added along the boosted block's chain + after `should_apply_proposer_boost` has read the parent's score from the same + table, so that gate no longer re-walks the votes either. +- **Few decodes:** a block's fork and parent payload status are recorded as a + `BlockPayloadLink` when `on_block` imports it, and the head's slot reads + come from the block index, so a head computation decodes no block in the + ordinary case. Two cases still decode. A block imported before a restart has + no entry (the scratch is in memory), and the walk derives it once by + decoding the block and its parent and records it; the actor does the same for + each new finalized root before pruning links. And a weak boosted-block parent + from the previous slot scans same-slot candidates for an early equivocation, + which needs their proposer indexes; that scan is rare and bounded by the + candidates at one slot. +- **Bounded at the finalized block:** vote placement, the fold and the boost + chain stop at the least of the finalized block's slot, the justified block's + slot and the boosted block's parent's slot, which on a live node is the + finalized block's. The other two terms make the bound safe without assuming + that the justified block and the boosted block's parent are at or above the + finalized block. The block index of a beacon store is never pruned, so + without the bound each head computation would fold, and read the link of, + every block since the anchor. Links below the finalized block are pruned + because nothing reads them. +- **Cost:** work is proportional to the latest messages plus the indexed + blocks at or above the finalized block, the unfinalized window. The + spec-literal form is proportional to the messages times the + candidate nodes, with block decodes at each ancestor step and for each voter, + which would wedge the chain actor on a large tree. +- **Equivalence:** the walk names the head and gives every node at or above the + finalized block the weight the specification's `get_head` and `get_weight` + do, wherever they do not raise; the tolerances above are the only + differences. The fixture harness checks the head at every `head` step and + every viable leaf's weight, and a seeded randomized test checks both over + trees that cross the fork boundary. +- **Why still tested against the spec-literal form:** `gloas_get_head` and + `gloas_get_weight` remain, as the references the walk is compared with, at + every `head` check of every fork-choice fixture and on randomized trees; + `compute_head` is the pre-gloas reference. From af123e13e25e74c4ab68f825a2496c04f3411829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 12:17:48 -0300 Subject: [PATCH 029/132] fix(beacon): refuse gloas where the node cannot answer it The follower does not follow gloas, but several surfaces still answered as if it did: - attestation_data and both duty endpoints answered gloas slots from the frozen fulu head, so a validator client would sign a wrong-fork vote and publish it through a fallback node. They now refuse gloas epochs by name through one helper, which the aggregate endpoint shares. - block production and publication advanced a state through upgrade_to_gloas before refusing the slot. Both now check the slot's fork first, before the engine is needed, so a gloas slot is a 400 even on a node with no execution client. - the sync tracker reported synced for good once the network looked stalled, which is what a node sees past a fork it does not follow: gossip blocks from that fork are ignored in p2p, and post-fork peers' Status is dropped for its fork digest. With the clock past a fork the node does not follow and the head from one it does, the network is now treated as fresh, so head lag decides and the node reports syncing. - a beacon block that fails to decode at a gloas slot is answered with Ignore(UnsupportedFork) instead of Reject(Decode), read from the block's own slot as the other decoders do, so a later change to a gloas container cannot penalize honest peers, while garbage and malformed fulu blocks are still rejected. - decode::current_fork has no caller outside the crate; narrow it. Which forks this node follows was spelled out, with the same answer, at startup, in block import, in the gossip block check, in two gossip decoders, in the sync tracker and in the validator API. It is now one named-arm predicate, ForkName::is_followed, so following gloas is one edit there, plus whatever each surface needs to answer at that fork. --- CLAUDE.md | 5 +- bin/ethlambda/src/main.rs | 14 +- bin/ethlambda/src/network/built_in.rs | 4 +- bin/ethlambda/src/network/dir.rs | 2 +- bin/ethlambda/src/network/mod.rs | 53 ++++--- crates/blockchain/src/lib.rs | 150 +++++++++++++++++- .../src/beacon/gossip/block.rs | 7 +- crates/common/types/src/beacon/fork.rs | 40 +++++ crates/net/p2p/src/beacon/decode.rs | 34 +++- crates/net/p2p/src/gossipsub/handler.rs | 69 +++++++- crates/net/rpc/src/beacon/mod.rs | 21 +++ crates/net/rpc/src/beacon/pool.rs | 23 +-- crates/net/rpc/src/beacon/proposal.rs | 117 +++++++++++++- crates/net/rpc/src/beacon/validator.rs | 99 +++++++++++- docs/beacon_wire.md | 3 +- docs/rpc.md | 14 +- 16 files changed, 578 insertions(+), 77 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 4b7b5f80f..1867d5dc5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -455,9 +455,10 @@ choice exist, but the live follower refuses gloas blocks (`process_or_pend_block`) and a gloas anchor (`refuse_unfollowable_fork` in `main.rs`), since nothing delivers payload envelopes or payload attestations to it, so a Sepolia follower stops tracking the chain at `GLOAS_FORK_EPOCH` -regardless. `network::warn_if_gloas_scheduled` (shared by +regardless. `network::warn_if_unfollowed_fork_scheduled` (shared by `BuiltInNetwork::resolve` and a loaded `NetworkDir`) warns about this -explicitly at startup (checked against `config.gloas_fork_epoch`), separately +explicitly at startup, once per scheduled fork for which `ForkName::is_followed` +answers no (the one place that says which forks the node follows), separately from the ignored-keys warning, which now only ever names heze's keys and `GAS_LIMIT_SCHEDULE`/`INCLUSION_LIST_DUE_BPS`. diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index 2ce1be2b9..8ad84ce64 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -1710,16 +1710,10 @@ fn first_config_difference(persisted: &Config, supplied: &Config) -> Option Result<(), checkpoint_sync::CheckpointSyncError> { - match fork { - ForkName::Gloas => Err(checkpoint_sync::CheckpointSyncError::UnsupportedFork { fork }), - ForkName::Phase0 - | ForkName::Altair - | ForkName::Bellatrix - | ForkName::Capella - | ForkName::Deneb - | ForkName::Electra - | ForkName::Fulu => Ok(()), - ForkName::Lean => unreachable!("a beacon anchor is never ForkName::Lean"), + if fork.is_followed() { + Ok(()) + } else { + Err(checkpoint_sync::CheckpointSyncError::UnsupportedFork { fork }) } } diff --git a/bin/ethlambda/src/network/built_in.rs b/bin/ethlambda/src/network/built_in.rs index 6a6ee6d64..1d389d5a8 100644 --- a/bin/ethlambda/src/network/built_in.rs +++ b/bin/ethlambda/src/network/built_in.rs @@ -66,7 +66,7 @@ impl BuiltInNetwork { // but whose state transition it does not implement yet (gloas), or // one whose keys it does not even claim (heze). The ignored-keys // warning below only ever said the second half of that; it used to - // say both, back when GLOAS_* were unclaimed too. `warn_if_gloas_scheduled` + // say both, back when GLOAS_* were unclaimed too. `warn_if_unfollowed_fork_scheduled` // says the first half explicitly instead of leaving it to be // discovered as a stall. parsed.warn_about_ignored_keys(); @@ -74,7 +74,7 @@ impl BuiltInNetwork { let genesis = chain.genesis(); let mut config = parsed.config; super::derive_genesis_fields(&mut config, genesis.genesis_time); - super::warn_if_gloas_scheduled(self.name(), &config); + super::warn_if_unfollowed_fork_scheduled(self.name(), &config); Ok(BuiltIn { config, diff --git a/bin/ethlambda/src/network/dir.rs b/bin/ethlambda/src/network/dir.rs index 8505cf062..6bde684cc 100644 --- a/bin/ethlambda/src/network/dir.rs +++ b/bin/ethlambda/src/network/dir.rs @@ -129,7 +129,7 @@ impl NetworkDir { // once, before anything reads the config. let mut config = parsed.config; super::derive_genesis_fields(&mut config, genesis_state.genesis_time()); - super::warn_if_gloas_scheduled(config.config_name.as_str(), &config); + super::warn_if_unfollowed_fork_scheduled(config.config_name.as_str(), &config); Ok(Self { config, diff --git a/bin/ethlambda/src/network/mod.rs b/bin/ethlambda/src/network/mod.rs index 29f2cbf49..a1f36424e 100644 --- a/bin/ethlambda/src/network/mod.rs +++ b/bin/ethlambda/src/network/mod.rs @@ -232,33 +232,38 @@ fn derive_genesis_fields(config: &mut Config, genesis_time: u64) { config.slot_duration_ms = config.seconds_per_slot * 1_000; } -/// Warn at startup when `config` schedules gloas, whichever way this network -/// was resolved. +/// Warn at startup when `config` schedules a fork this build does not follow +/// ([`ForkName::is_followed`](ethlambda_types::beacon::fork::ForkName::is_followed)), +/// whichever way this network was resolved. /// /// Shared by a loaded directory and the built-in networks for the same /// reason [`derive_genesis_fields`] is. The ignored-keys warning -/// (`ConfigFile::warn_about_ignored_keys`) no longer implies this the way it -/// used to, now that `GLOAS_*` keys are claimed rather than reported as -/// unknown, so this says it explicitly instead of leaving it to be -/// discovered as a stall: the chain actor refuses every gloas block -/// (`process_or_pend_block`), because nothing delivers payload envelopes or -/// payload attestations to it yet, so a follower stops making progress there -/// regardless of how cleanly its config parsed. A loaded -/// network reaches this the same as a built-in one, and can even schedule -/// gloas at epoch 0, in which case its own genesis state already decodes as -/// gloas and startup refuses it outright (`refuse_unfollowable_fork` in -/// `main.rs`, since fork choice itself accepts a gloas anchor); this warning -/// fires first either way. -fn warn_if_gloas_scheduled(network: &str, config: &Config) { - if config.gloas_fork_epoch != ethlambda_types::beacon::constants::FAR_FUTURE_EPOCH { - tracing::warn!( - network, - gloas_fork_epoch = config.gloas_fork_epoch, - "This build stops following this chain at its gloas fork epoch; \ - crossing it is not a stall a restart clears, since the chain actor \ - refuses every gloas block until a later release delivers payload \ - envelopes and payload attestations to it" - ); +/// (`ConfigFile::warn_about_ignored_keys`) does not say this: it names keys +/// the build does not claim, and the keys of a fork it does claim parse +/// cleanly. So this says it explicitly instead of leaving it to be discovered +/// as a stall: from that fork's epoch the chain actor refuses every block, so +/// a follower stops making progress there regardless of how cleanly its +/// config parsed. A loaded network reaches this the same as a built-in one, +/// and can even schedule such a fork at epoch 0, in which case its own genesis +/// state already decodes as that fork and startup refuses it outright +/// (`refuse_unfollowable_fork` in `main.rs`, since fork choice itself accepts +/// a gloas anchor); this warning fires first either way. +fn warn_if_unfollowed_fork_scheduled(network: &str, config: &Config) { + let unfollowed = ethlambda_types::beacon::fork::ForkName::ALL + .into_iter() + .filter(|fork| !fork.is_followed()); + for fork in unfollowed { + let fork_epoch = config.fork_epoch(fork); + if fork_epoch != ethlambda_types::beacon::constants::FAR_FUTURE_EPOCH { + tracing::warn!( + network, + fork = fork.as_str(), + fork_epoch, + "This build stops following this chain at the epoch of a fork it does \ + not follow; crossing it is not a stall a restart clears, since the \ + chain actor refuses every block of that fork" + ); + } } } diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index db30a58e1..abf68a754 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -18,6 +18,7 @@ use ethlambda_types::{ containers::{SignedAggregateAndProof, SignedBeaconBlock, fulu}, preset, primitives::ValidatorIndex, + signing::compute_epoch_at_slot, }, block::SignedBlock, chain_config::ChainConfig, @@ -2608,7 +2609,14 @@ impl BlockChainServer { // children already queued under it before this landed; logged once, // here, rather than once per retry, since after this there is no // retry. - if matches!(signed_block, SignedBeaconBlock::Gloas(_)) { + // + // Decided by chain first: a lean block is not a point on the beacon + // fork schedule, and `ForkName::is_followed` refuses to answer for it. + let refused = match self.store.chain() { + Chain::Lean => false, + Chain::Beacon => !signed_block.fork_name().is_followed(), + }; + if refused { warn!( %slot, block_root = %ShortRoot(&block_root.0), @@ -3686,15 +3694,29 @@ impl BlockChainServer { /// /// Reads the head through [`Self::head_slot`], which is where the /// per-chain part of that lives (lean's `Store::head_slot` and beacon's - /// `Store::beacon_head` decode different tables); everything past that - /// point is chain-agnostic. + /// `Store::beacon_head` decode different tables). The tracker itself is + /// chain-agnostic, but the freshest block seen is not: on a beacon + /// follower, once the clock is past a fork the node does not follow (see + /// [`Self::clock_is_past_followed_forks`]) the network is treated as fresh. fn update_sync_status(&mut self, current_slot: u64) { let head_slot = self.head_slot(); - let max_seen_slot = self + let live_max_slot = self .store .max_live_chain_slot() .expect("max live chain slot exists") .unwrap_or(head_slot); + let max_seen_slot = match self.store.chain() { + Chain::Beacon + if Self::clock_is_past_followed_forks( + &self.store.config(), + current_slot, + head_slot, + ) => + { + current_slot + } + Chain::Beacon | Chain::Lean => live_max_slot, + }; let status = self .sync_status .update(current_slot, head_slot, max_seen_slot); @@ -3702,6 +3724,23 @@ impl BlockChainServer { self.sync_status_controller.set(status); } + /// Whether the clock is at a fork this node does not follow while the + /// node's own head is still from one it does. + /// + /// Blocks past the fork never reach the live chain, so the freshest block + /// the store knows stays at the head and the sync tracker would read the + /// network as stalled `NETWORK_STALL_THRESHOLD` slots after the fork and + /// report `Synced` for good on a node that follows nothing. Nothing else + /// tells the node the network is ahead: gossip blocks past the fork are + /// ignored in p2p, and a node that runs across the fork keeps the fork + /// digest it computed at startup, so peers' Status messages are dropped + /// and no range session starts. The clock is the one signal that is always + /// there, and treating the network as fresh lets head lag decide. + fn clock_is_past_followed_forks(config: &Config, current_slot: u64, head_slot: u64) -> bool { + let fork_at = |slot: u64| config.fork_at_epoch(compute_epoch_at_slot(slot)); + !fork_at(current_slot).is_followed() && fork_at(head_slot).is_followed() + } + /// Milliseconds until this actor's next tick, dispatched by chain: lean's /// interval grid via [`ms_until_next_interval`], beacon's once-per-slot /// cadence via [`ms_until_next_beacon_slot`]. @@ -5238,6 +5277,109 @@ mod tests { assert!(!server.blocks_awaiting_columns.contains_key(&parent_root)); } + /// A beacon server scheduling gloas at `gloas_epoch` (never, for `None`), + /// holding nothing past its anchor: no block of any fork arrives in these + /// tests. + fn syncing_server(gloas_epoch: Option) -> BlockChainServer { + let config = match gloas_epoch { + Some(epoch) => Config::mainnet().with_fork_epoch(ForkName::Gloas, epoch), + None => Config::mainnet(), + }; + beacon_server(beacon_store_with_config(GENESIS_TIME, 0, config)) + } + + #[test] + fn the_node_reports_syncing_once_the_clock_passes_the_fork_it_does_not_follow() { + // No block arrives at all: gossip blocks past the fork are ignored in + // p2p and peers' Status messages are dropped, so the store's freshest + // block stays at the head. The clock alone must keep the node from + // reading the network as stalled. + let mut server = syncing_server(Some(1)); + let past_the_fork = preset::SLOTS_PER_EPOCH + 8; + server.update_sync_status(past_the_fork); + assert_eq!( + server.sync_status_controller.get(), + crate::metrics::SyncStatus::Syncing + ); + } + + #[test] + fn a_stalled_network_with_no_fork_ahead_still_reads_as_synced() { + // The same silence with no gloas scheduled is a stalled network, which + // the tracker deliberately reports as synced. + let mut server = syncing_server(None); + server.update_sync_status(preset::SLOTS_PER_EPOCH + 8); + assert_eq!( + server.sync_status_controller.get(), + crate::metrics::SyncStatus::Synced + ); + } + + /// Gloas at epoch 1, so slot `SLOTS_PER_EPOCH` is the first gloas slot. + fn gloas_at_epoch_one() -> Config { + Config::mainnet().with_fork_epoch(ForkName::Gloas, 1) + } + + #[test] + fn the_clock_rule_holds_for_a_followed_head_once_the_clock_is_at_gloas() { + let first_gloas_slot = preset::SLOTS_PER_EPOCH; + assert!(BlockChainServer::clock_is_past_followed_forks( + &gloas_at_epoch_one(), + first_gloas_slot + 8, + 0 + )); + } + + #[test] + fn the_clock_rule_does_not_hold_before_the_clock_reaches_gloas() { + let first_gloas_slot = preset::SLOTS_PER_EPOCH; + assert!(!BlockChainServer::clock_is_past_followed_forks( + &gloas_at_epoch_one(), + first_gloas_slot - 1, + 0 + )); + } + + #[test] + fn the_clock_rule_does_not_hold_for_a_head_already_past_the_fork() { + let first_gloas_slot = preset::SLOTS_PER_EPOCH; + assert!(!BlockChainServer::clock_is_past_followed_forks( + &gloas_at_epoch_one(), + first_gloas_slot + 8, + first_gloas_slot + )); + } + + #[tokio::test] + async fn a_lean_block_is_not_refused_as_an_unfollowed_fork() { + // The import path serves both chains, so a lean block reaches the + // fork check, and `ForkName::Lean` has no answer to "is it followed". + // With an unknown parent the block is parked, which is all that has to + // happen: the point is that it gets that far without a panic. + let backend = Arc::new(InMemoryBackend::default()); + let store = Store::from_anchor_state( + backend, + State::from_genesis(0, Vec::new()), + DEFAULT_MILLISECONDS_PER_SLOT, + ); + let mut server = beacon_server(store); + let block = SignedBeaconBlock::Lean(ethlambda_types::block::SignedBlock { + message: ethlambda_types::block::Block { + slot: 5, + proposer_index: 0, + parent_root: H256::repeat_byte(9), + state_root: H256::ZERO, + body: ethlambda_types::block::BlockBody::default(), + }, + proof: ethlambda_types::block::MultiMessageAggregate::default(), + }); + let mut queue = VecDeque::new(); + let outcome = server + .process_or_pend_block(block, ImportTimings::default(), &mut queue) + .await; + assert_eq!(outcome, None); + } + #[tokio::test] async fn the_descendants_of_a_held_block_wait_for_it_without_re_importing_it() { // A range batch hands the actor a whole chain at once. When its first diff --git a/crates/blockchain/state_transition/src/beacon/gossip/block.rs b/crates/blockchain/state_transition/src/beacon/gossip/block.rs index 163125ee4..6600e2ae6 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/block.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/block.rs @@ -8,7 +8,7 @@ use super::{ finalized_start_slot, is_future_slot, }; use crate::beacon::containers::SignedBeaconBlock; -use crate::beacon::fork_choice::{ForkRules, Store}; +use crate::beacon::fork_choice::Store; use crate::beacon::helpers::misc::compute_epoch_at_slot; use crate::beacon::precheck::{self, PrecheckError, Reference, precheck_block}; use crate::beacon::primitives::Root; @@ -35,9 +35,8 @@ pub fn cheap_checks( // builder's bid) but whose verdict would misdescribe it. `Ignore`, not // `Reject`: the sender did nothing wrong, and every honest peer sends // exactly this once this node's own clock reaches gloas. - match ForkRules::of(block.fork_name()) { - ForkRules::Gloas => return Err(Outcome::Ignore(IgnoreReason::UnsupportedFork)), - ForkRules::PreGloas => {} + if !block.fork_name().is_followed() { + return Err(Outcome::Ignore(IgnoreReason::UnsupportedFork)); } // [IGNORE] The block is not from a future slot. if is_future_slot(&config, slot, now_ms) { diff --git a/crates/common/types/src/beacon/fork.rs b/crates/common/types/src/beacon/fork.rs index be895bd6d..79584df4c 100644 --- a/crates/common/types/src/beacon/fork.rs +++ b/crates/common/types/src/beacon/fork.rs @@ -110,6 +110,33 @@ impl ForkName { ForkName::ALL.get(index + 1).copied() } + /// Whether this node follows a chain that has reached this fork. + /// + /// Phase0 through fulu are followed. Gloas is not: nothing delivers the + /// payload envelopes or payload attestations a gloas chain needs, so the + /// node refuses gloas blocks and anchors, ignores gloas gossip and reports + /// syncing once the clock reaches the fork. This is the one place to + /// change when the node starts following gloas; every site whose rule is + /// "does this node follow the fork" calls it. + /// + /// # Panics + /// + /// On [`ForkName::Lean`], which is not a point on the beacon fork + /// schedule. + pub fn is_followed(self) -> bool { + match self { + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Fulu => true, + ForkName::Gloas => false, + ForkName::Lean => super::lean_fork_unreachable("ForkName::is_followed"), + } + } + /// The one-byte tag this fork is stored under in a `States` value. /// /// Spelled out rather than `self as u8`. The variant order is already @@ -272,6 +299,19 @@ mod tests { ); } + #[test] + fn every_beacon_fork_through_fulu_is_followed_and_gloas_is_not() { + for fork in ForkName::ALL { + assert_eq!(fork.is_followed(), fork != ForkName::Gloas, "{fork:?}"); + } + } + + #[test] + #[should_panic(expected = "ForkName::Lean reached a Beacon Chain function")] + fn lean_is_not_a_fork_the_node_can_follow() { + ForkName::Lean.is_followed(); + } + #[test] fn selectors_are_pinned_to_their_on_disk_values() { // These bytes are a storage format: changing one makes every existing diff --git a/crates/net/p2p/src/beacon/decode.rs b/crates/net/p2p/src/beacon/decode.rs index 6d02fe2c0..84da94623 100644 --- a/crates/net/p2p/src/beacon/decode.rs +++ b/crates/net/p2p/src/beacon/decode.rs @@ -194,7 +194,7 @@ pub fn fork_at_slot(config: &Config, slot: Slot) -> ForkName { /// reads the live clock instead of whatever fork the topic was actually /// built for; see [`decode_data_column_sidecar`]'s own doc and its caller in /// `gossipsub::handler::triage_data_column`. -pub fn current_fork(config: &Config) -> ForkName { +pub(crate) fn current_fork(config: &Config) -> ForkName { let slot = unix_now_ms() .saturating_sub(config.genesis_time_ms()) .checked_div(config.slot_duration_ms) @@ -207,9 +207,21 @@ pub fn current_fork(config: &Config) -> ForkName { /// Separate from [`decode_gossip`] because the two topics worth logging in /// detail are dispatched by name, and a handler that already knows it is /// holding a block should not have to unwrap a [`BeaconGossip`] to find one. +/// +/// A block whose slot is at a fork this node does not follow (gloas) and that +/// fails to decode is `UnsupportedFork`, not `Ssz`: the container this build +/// models for that fork is the one at the pinned specification release, and +/// an honest peer's block may follow a later one. Bytes too short to name a +/// slot, and a block at a followed fork, stay `Ssz` or `Truncated`. pub fn decode_block(config: &Config, bytes: &[u8]) -> Result { let fork = fork_at_slot(config, block_slot(bytes)?); - SignedBeaconBlock::from_ssz(fork, bytes).map_err(|_| DecodeError::Ssz) + SignedBeaconBlock::from_ssz(fork, bytes).map_err(|_| { + if fork.is_followed() { + DecodeError::Ssz + } else { + DecodeError::UnsupportedFork + } + }) } /// Decode a data column sidecar off a subnet topic. @@ -460,6 +472,24 @@ mod tests { assert_eq!(decoded.topic_kind(), topics::BEACON_BLOCK); } + #[test] + fn an_undecodable_block_is_unsupported_only_at_a_fork_the_node_does_not_follow() { + let mut config = Config::mainnet(); + config.gloas_fork_epoch = config.fulu_fork_epoch + 1; + // Phase0-shaped bytes are not a valid block at either fork. + let at_fulu = phase0_block(slot_of(config.fulu_fork_epoch)).to_ssz(); + let at_gloas = phase0_block(slot_of(config.gloas_fork_epoch)).to_ssz(); + assert_eq!(decode_block(&config, &at_fulu), Err(DecodeError::Ssz)); + assert_eq!( + decode_block(&config, &at_gloas), + Err(DecodeError::UnsupportedFork) + ); + assert_eq!( + decode_block(&config, &[0xff; 3]), + Err(DecodeError::Truncated) + ); + } + #[test] fn the_slot_actually_drives_which_shape_is_decoded() { // A phase0-shaped payload whose slot lands in fulu must be refused, not diff --git a/crates/net/p2p/src/gossipsub/handler.rs b/crates/net/p2p/src/gossipsub/handler.rs index 3d6b6f5b7..631b56dd7 100644 --- a/crates/net/p2p/src/gossipsub/handler.rs +++ b/crates/net/p2p/src/gossipsub/handler.rs @@ -13,7 +13,6 @@ use ethlambda_types::{ ShortRoot, attestation::{SignedAggregatedAttestation, SignedAttestation}, beacon::containers::{SignedBeaconBlock, electra::SingleAttestation}, - beacon::fork::ForkName, block::SignedBlock, primitives::HashTreeRoot as _, time::unix_now_ms, @@ -261,6 +260,12 @@ fn triage_block(server: &P2PServer, wire: &BeaconWire, payload: &[u8]) -> Dispat const KIND: &str = beacon_topics::BEACON_BLOCK; let block = match beacon_decode::decode_block(&wire.config, payload) { Ok(block) => block, + // A block at a fork this build has no rules for, as on the aggregate + // topic: an honest peer must not be scored as a bad decoder. + Err(beacon_decode::DecodeError::UnsupportedFork) => { + metrics::inc_beacon_gossip(KIND, "unsupported_fork"); + return Dispatch::Report(Outcome::Ignore(IgnoreReason::UnsupportedFork)); + } Err(err) => { metrics::inc_beacon_gossip(KIND, "decode_failed"); debug!(kind = KIND, %err, bytes = payload.len(), "Beacon gossip decode failed"); @@ -317,7 +322,7 @@ fn triage_data_column(server: &P2PServer, payload: &[u8], subnet_id: u64) -> Dis Ok(sidecar) => sidecar, Err(err) => { let config = server.store.config(); - if beacon_decode::current_fork(&config) == ForkName::Gloas { + if !beacon_decode::current_fork(&config).is_followed() { metrics::inc_beacon_gossip(KIND, "unsupported_fork"); return Dispatch::Report(Outcome::Ignore(IgnoreReason::UnsupportedFork)); } @@ -790,6 +795,7 @@ mod tests { use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::{AttestationData, electra, phase0, shared}; use ethlambda_types::beacon::fork::ForkName; + use ethlambda_types::beacon::preset; use ethlambda_types::beacon::primitives::Slot; use super::*; @@ -873,6 +879,65 @@ mod tests { )); } + /// Phase0-shaped bytes at `slot`: not a valid block at any later fork, so + /// what `triage_block` answers depends on the fork `slot` names. + fn mismatched_block_bytes(slot: Slot) -> Vec { + phase0::SignedBeaconBlock { + message: phase0::BeaconBlock { + slot, + proposer_index: 0, + parent_root: Default::default(), + state_root: Default::default(), + body: phase0::BeaconBlockBody::default(), + }, + signature: Default::default(), + } + .to_ssz() + } + + async fn triage_block_under(config: Config, payload: &[u8]) -> Dispatch { + let server = unconnected_beacon_server(config, 0).await; + let wire = server + .wire + .beacon() + .expect("a beacon server has a beacon wire"); + triage_block(&server, wire, payload) + } + + #[tokio::test] + async fn an_undecodable_block_at_a_gloas_slot_is_ignored() { + let mut config = Config::mainnet(); + config.gloas_fork_epoch = config.fulu_fork_epoch + 1; + let slot = config.gloas_fork_epoch * preset::SLOTS_PER_EPOCH; + + assert!(matches!( + triage_block_under(config, &mismatched_block_bytes(slot)).await, + Dispatch::Report(Outcome::Ignore(IgnoreReason::UnsupportedFork)) + )); + } + + #[tokio::test] + async fn garbage_bytes_are_still_rejected_once_gloas_is_active() { + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 0); + + assert!(matches!( + triage_block_under(config, &[0xff; 3]).await, + Dispatch::Report(Outcome::Reject(RejectReason::Decode)) + )); + } + + #[tokio::test] + async fn an_undecodable_block_at_a_fulu_slot_is_still_rejected_after_the_fork() { + let mut config = Config::mainnet(); + config.gloas_fork_epoch = config.fulu_fork_epoch + 1; + let slot = config.fulu_fork_epoch * preset::SLOTS_PER_EPOCH; + + assert!(matches!( + triage_block_under(config, &mismatched_block_bytes(slot)).await, + Dispatch::Report(Outcome::Reject(RejectReason::Decode)) + )); + } + #[tokio::test] async fn garbage_bytes_on_a_data_column_subnet_are_rejected_as_undecodable() { let server = unconnected_beacon_server(Config::mainnet(), 0).await; diff --git a/crates/net/rpc/src/beacon/mod.rs b/crates/net/rpc/src/beacon/mod.rs index c18485b1d..8fa54f6c9 100644 --- a/crates/net/rpc/src/beacon/mod.rs +++ b/crates/net/rpc/src/beacon/mod.rs @@ -11,6 +11,7 @@ use axum::{ response::{IntoResponse, Response}, }; use ethlambda_storage::Store; +use ethlambda_types::beacon::{config::Config, primitives::Epoch}; use serde::Serialize; use crate::shared::block_id::IdError; @@ -59,6 +60,26 @@ pub(crate) enum ApiError { ServiceUnavailable(&'static str), } +/// Refuses `epoch` when the node's schedule places it at a fork the node does +/// not follow (gloas, see +/// [`ethlambda_types::beacon::fork::ForkName::is_followed`]). +/// +/// The head is a fulu state, so an answer for a gloas epoch would carry fulu's +/// shapes under gloas's fork (a vote a validator client would sign as valid, +/// an aggregate of the wrong shape). Refused by name, as the submit endpoints +/// refuse the gloas header. +pub(crate) fn refuse_unfollowed_epoch( + config: &Config, + epoch: Epoch, + message: &'static str, +) -> Result<(), ApiError> { + if config.fork_at_epoch(epoch).is_followed() { + Ok(()) + } else { + Err(ApiError::BadRequest(message)) + } +} + impl From for ApiError { fn from(err: IdError) -> Self { match err { diff --git a/crates/net/rpc/src/beacon/pool.rs b/crates/net/rpc/src/beacon/pool.rs index 63273e0d5..dfaffaafa 100644 --- a/crates/net/rpc/src/beacon/pool.rs +++ b/crates/net/rpc/src/beacon/pool.rs @@ -49,7 +49,7 @@ use ethlambda_types::{ use serde::{Deserialize, Serialize}; use tracing::{debug, warn}; -use crate::beacon::{ApiError, validator::head}; +use crate::beacon::{ApiError, refuse_unfollowed_epoch, validator::head}; pub(crate) fn routes() -> Router { Router::new() @@ -363,20 +363,13 @@ async fn get_aggregate_attestation( .fork_at_epoch(compute_epoch_at_slot(query.slot)); // The pool holds electra-shaped votes, and the response is labelled with // the slot's fork, so a gloas slot would be served as gloas with a body of - // the wrong shape. Refused by name, as the submit endpoints refuse the - // gloas header. - match fork { - ForkName::Gloas => { - return ApiError::BadRequest("gloas aggregates are not supported").into_response(); - } - ForkName::Phase0 - | ForkName::Altair - | ForkName::Bellatrix - | ForkName::Capella - | ForkName::Deneb - | ForkName::Electra - | ForkName::Fulu - | ForkName::Lean => {} + // the wrong shape. + if let Err(err) = refuse_unfollowed_epoch( + &store.config(), + compute_epoch_at_slot(query.slot), + "gloas aggregates are not supported", + ) { + return err.into_response(); } let aggregate = pool .lock() diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs index 5d1844ad3..fb0162b37 100644 --- a/crates/net/rpc/src/beacon/proposal.rs +++ b/crates/net/rpc/src/beacon/proposal.rs @@ -37,6 +37,7 @@ use ethlambda_state_transition::beacon::{ use ethlambda_storage::Store; use ethlambda_types::{ beacon::{ + config::Config, containers::{ self, BeaconState, deneb::Blob, @@ -45,6 +46,7 @@ use ethlambda_types::{ fork::ForkName, preset, primitives::{BlsSignature, Bytes32, ExecutionAddress, KzgProof, Slot}, + signing::compute_epoch_at_slot, }, primitives::H256, }; @@ -149,6 +151,13 @@ async fn post_block( if block.slot() <= head_state.slot() { return ApiError::BadRequest("the block is not after this node's head").into_response(); } + if let Err(err) = require_fulu_slot( + &store.config(), + block.slot(), + "blocks are accepted for fulu slots only", + ) { + return err.into_response(); + } let Ok(state) = advance_to_slot(&head_state, block.slot(), &store.config()) else { return ApiError::Internal("advancing the head state failed").into_response(); }; @@ -168,6 +177,24 @@ struct ProduceQuery { graffiti: Option, } +/// Refuses a slot the schedule does not place at fulu, before anything +/// advances a state to it: advancing a fulu head into a gloas slot would run +/// `upgrade_to_gloas` only to refuse the result. Every other fork is refused +/// too, since the containers here are fulu's. +fn require_fulu_slot(config: &Config, slot: Slot, message: &'static str) -> Result<(), ApiError> { + match config.fork_at_epoch(compute_epoch_at_slot(slot)) { + ForkName::Fulu => Ok(()), + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Gloas + | ForkName::Lean => Err(ApiError::BadRequest(message)), + } +} + async fn get_block( Path(slot): Path, Query(query): Query, @@ -180,6 +207,15 @@ async fn get_block( let Ok(slot) = slot.parse::() else { return ApiError::BadRequest("invalid slot").into_response(); }; + // Ahead of the engine check: a slot this node cannot build is a 400 on + // any node, not a 503 on one with no execution client. + if let Err(err) = require_fulu_slot( + &store.config(), + slot, + "block production is served for fulu only", + ) { + return err.into_response(); + } let Some(engine) = engine else { return ApiError::ServiceUnavailable( "no execution client configured to build a payload with", @@ -251,11 +287,6 @@ async fn produce( let state = advance_to_slot(&head_state, slot, &config) .map_err(|_| ApiError::Internal("advancing the head state failed"))?; let fork = state.fork_name(); - if fork != ForkName::Fulu { - return Err(ApiError::BadRequest( - "block production is served for fulu only", - )); - } let proposer = ethlambda_state_transition::beacon::helpers::accessors::get_beacon_proposer_index(&state) .map_err(|_| ApiError::Internal("no proposer for the slot"))?; @@ -377,3 +408,79 @@ fn decimal(value: ðlambda_types::beacon::primitives::Uint256) -> String { .map(|value| value.to_string()) .unwrap_or_else(|_| "0".to_string()) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{RecordingNetwork, beacon_store_with_config}; + use axum::{body::Body, http::Request}; + use ethlambda_state_transition::beacon::helpers::test_state::with_signing_validators_at; + use ethlambda_types::beacon::config::Config; + use http_body_util::BodyExt as _; + use std::sync::Arc; + use tower::ServiceExt as _; + + /// A store holding a fulu head, under a schedule that puts every slot at + /// gloas: the head is behind the fork, as on a node running across it. + fn gloas_scheduled_store() -> (Store, Slot) { + let state = with_signing_validators_at(ForkName::Fulu, 64); + let head_slot = state.slot(); + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 0); + let (store, _root) = beacon_store_with_config(state, config); + (store, head_slot) + } + + async fn respond(app: Router, request: Request) -> (StatusCode, serde_json::Value) { + let response = app.oneshot(request).await.unwrap(); + let status = response.status(); + let body = response.into_body().collect().await.unwrap().to_bytes(); + (status, serde_json::from_slice(&body).unwrap_or_default()) + } + + #[tokio::test] + async fn producing_a_gloas_slot_is_a_400_even_with_no_execution_client() { + let (store, head_slot) = gloas_scheduled_store(); + let app = routes() + .with_state(store) + .layer(Extension(None::)) + .layer(Extension(SharedAttestationPool::default())) + .layer(Extension(FeeRecipients::default())); + let uri = format!( + "/eth/v3/validator/blocks/{}?randao_reveal=0x{}", + head_slot + 1, + "00".repeat(96) + ); + let (status, json) = respond(app, Request::get(uri).body(Body::empty()).unwrap()).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(json["message"], "block production is served for fulu only"); + } + + #[tokio::test] + async fn publishing_a_fulu_body_at_a_gloas_slot_is_refused_before_advancing() { + let (store, head_slot) = gloas_scheduled_store(); + let contents = FuluSignedBlockContents { + signed_block: SignedBeaconBlock { + message: BeaconBlock { + slot: head_slot + 1, + proposer_index: 0, + parent_root: H256::ZERO, + state_root: H256::ZERO, + body: containers::electra::BeaconBlockBody::empty(), + }, + signature: Default::default(), + }, + kzg_proofs: Default::default(), + blobs: Default::default(), + }; + let network: RpcToP2PRef = Arc::new(RecordingNetwork::default()); + let app = routes().with_state(store).layer(Extension(network)); + let request = Request::post("/eth/v2/beacon/blocks") + .header("eth-consensus-version", "fulu") + .header(header::CONTENT_TYPE, crate::SSZ_CONTENT_TYPE) + .body(Body::from(contents.to_ssz())) + .unwrap(); + let (status, json) = respond(app, request).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(json["message"], "blocks are accepted for fulu slots only"); + } +} diff --git a/crates/net/rpc/src/beacon/validator.rs b/crates/net/rpc/src/beacon/validator.rs index abbbd95ee..bfc3514de 100644 --- a/crates/net/rpc/src/beacon/validator.rs +++ b/crates/net/rpc/src/beacon/validator.rs @@ -36,7 +36,7 @@ use ethlambda_state_transition::beacon::{ helpers::accessors::{CommitteeCacheExt as _, get_block_root_at_slot}, }; -use crate::beacon::ApiError; +use crate::beacon::{ApiError, refuse_unfollowed_epoch}; pub(crate) fn routes() -> Router { Router::new() @@ -206,6 +206,11 @@ async fn get_proposer_duties(Path(epoch): Path, State(store): State Result { let epoch = parse_epoch(epoch)?; + refuse_unfollowed_epoch( + &store.config(), + epoch, + "gloas proposer duties are not supported", + )?; let (head_root, state) = head(store)?; let BeaconState::Fulu(fulu) = state.as_ref() else { @@ -303,6 +308,11 @@ fn attester_duties( .map(|index| index.parse::()) .collect::, _>>() .map_err(|_| ApiError::BadRequest("invalid validator index"))?; + refuse_unfollowed_epoch( + &store.config(), + epoch, + "gloas attester duties are not supported", + )?; let (head_root, state) = head(store)?; let state_epoch = compute_epoch_at_slot(state.slot()); @@ -389,6 +399,11 @@ async fn get_attestation_data( } fn attestation_data(store: &Store, slot: Slot) -> Result { + refuse_unfollowed_epoch( + &store.config(), + compute_epoch_at_slot(slot), + "gloas attestation data is not supported", + )?; let (head_root, state) = head(store)?; if slot < state.slot() { return Err(ApiError::BadRequest("slot is before the head block")); @@ -799,6 +814,88 @@ mod tests { assert_eq!(*network.subscriptions.lock().unwrap(), vec![(11, 34)]); } + /// Sends the request `request` builds to a store holding a fulu head under + /// a schedule that puts gloas one epoch past the head's: the head's epoch + /// is the last fulu one, and the next is the first gloas one. `request` + /// gets the head's epoch and slot. + async fn with_gloas_next_epoch( + request: impl Fn(Epoch, Slot) -> Request, + ) -> (StatusCode, serde_json::Value) { + let state = fulu_state(); + let head_slot = state.slot(); + let head_epoch = compute_epoch_at_slot(head_slot); + let config = ethlambda_types::beacon::config::Config::mainnet() + .with_fork_epoch(ForkName::Gloas, head_epoch + 1); + let (store, _root) = crate::test_utils::beacon_store_with_config(state, config); + let response = routes() + .with_state(store) + .oneshot(request(head_epoch, head_slot)) + .await + .unwrap(); + let status = response.status(); + let body = response.into_body().collect().await.unwrap().to_bytes(); + (status, serde_json::from_slice(&body).unwrap()) + } + + fn get_request(uri: String) -> Request { + Request::get(uri).body(Body::empty()).unwrap() + } + + fn attestation_data_request(slot: Slot) -> Request { + get_request(format!( + "/eth/v1/validator/attestation_data?slot={slot}&committee_index=0" + )) + } + + #[tokio::test] + async fn attestation_data_is_answered_up_to_the_last_fulu_slot_and_refused_after() { + let (status, _) = + with_gloas_next_epoch(|_, head_slot| attestation_data_request(head_slot)).await; + assert_eq!(status, StatusCode::OK); + + let (status, json) = with_gloas_next_epoch(|head_epoch, _| { + attestation_data_request(compute_start_slot_at_epoch(head_epoch + 1)) + }) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(json["message"], "gloas attestation data is not supported"); + } + + #[tokio::test] + async fn proposer_duties_are_answered_for_the_last_fulu_epoch_and_refused_after() { + let (status, _) = with_gloas_next_epoch(|head_epoch, _| { + get_request(format!("/eth/v1/validator/duties/proposer/{head_epoch}")) + }) + .await; + assert_eq!(status, StatusCode::OK); + + let (status, json) = with_gloas_next_epoch(|head_epoch, _| { + get_request(format!( + "/eth/v1/validator/duties/proposer/{}", + head_epoch + 1 + )) + }) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(json["message"], "gloas proposer duties are not supported"); + } + + #[tokio::test] + async fn attester_duties_are_answered_for_the_last_fulu_epoch_and_refused_after() { + let attester = |epoch: Epoch| { + Request::post(format!("/eth/v1/validator/duties/attester/{epoch}")) + .header("content-type", "application/json") + .body(Body::from("[\"0\"]")) + .unwrap() + }; + let (status, _) = with_gloas_next_epoch(|head_epoch, _| attester(head_epoch)).await; + assert_eq!(status, StatusCode::OK); + + let (status, json) = with_gloas_next_epoch(|head_epoch, _| attester(head_epoch + 1)).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(json["message"], "gloas attester duties are not supported"); + } + #[tokio::test] async fn an_epoch_outside_the_lookahead_is_a_400() { let state = fulu_state(); diff --git a/docs/beacon_wire.md b/docs/beacon_wire.md index 6a7d94711..2eec3b2d8 100644 --- a/docs/beacon_wire.md +++ b/docs/beacon_wire.md @@ -152,7 +152,8 @@ chain actor to do with one. The rules above are fulu's. A message from a fork this build has no rules for, gloas today, is IGNOREd with the reason `unsupported_fork` and is never scored against the peer: the decoders answer `DecodeError::UnsupportedFork`, not `Ssz`, -for a gloas aggregate, attester slashing or subnet attestation, since an honest +for a gloas aggregate, attester slashing or subnet attestation, or for a block +at a gloas slot that fails to decode, since an honest gloas peer sends exactly those once this node's clock reaches the fork. A gloas subnet attestation has the bytes of an electra one, but its `data.index` is the payload flag, so decoding it under electra's rules would reject honest votes. diff --git a/docs/rpc.md b/docs/rpc.md index ee355cb8f..6e50f0bd8 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -259,12 +259,15 @@ the chain actor writes, so no request waits on the actor. next; attester duties cover the head's previous, current and next epoch, which is as far as its shuffling is already fixed. Anything else is a `400`. `dependent_root` follows each endpoint's v1 definition. Attester duties walk - every committee of the epoch, a full shuffle per request on mainnet. + every committee of the epoch, a full shuffle per request on mainnet. An + epoch the node's schedule places at gloas is a `400` on both duty endpoints: + the head is a fulu state, and this node does not follow gloas. - **`attestation_data`** follows phase0's `validator.md`: the head block, the epoch's boundary block as target, and as source the current justified checkpoint of the head state advanced to the slot's epoch (through fork choice's cached `checkpoint_state`, and only when the head is in an earlier - epoch). A slot before the head, or past the wall clock, is a `400`. + epoch). A slot before the head, or past the wall clock, is a `400`, as is a + slot in a gloas epoch: the answer would be a fulu-shaped vote under gloas. - **`pool/attestations`** checks each attestation against the electra `beacon_attestation_{subnet_id}` gossip conditions it can evaluate (clock window, `data.index == 0`, target epoch, the voted block known and the target @@ -313,10 +316,13 @@ the chain actor writes, so no request waits on the actor. the block through `process_block`. The answer is fulu `BlockContents`, with `Eth-Execution-Payload-Blinded: false`; there is no builder flow. It is a **`503`** without a configured execution client, or when the payload carries - blobs. + blobs. A slot the schedule does not place at fulu (gloas included) is a + `400`, checked before the execution client. - **`POST beacon/blocks`** takes SSZ `SignedBlockContents`, checks the block is after the head and its proposer signature, then gossips it on - `beacon_block` and hands it to the chain actor to import. + `beacon_block` and hands it to the chain actor to import. A slot the + schedule does not place at fulu (gloas included) is a `400`, checked before + the head state is advanced. **Blobs are not supported yet.** Publishing a blob-carrying block means computing and gossiping its data column sidecars, which this node does not do, From f31ef79efb768ed9fc86b9f99d1f640d44002487 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 12:24:43 -0300 Subject: [PATCH 030/132] refactor(beacon): share what gloas leaves unmodified Gloas modifies process_pending_deposits only by swapping the churn budget (and, like fulu, retiring the eth1-bridge gate), yet it had its own copy of the drain loop next to electra's and fulu's. The queue accessors refused gloas while the consolidation pair accepted it, which is what forced the copy. The deposit pair now accepts gloas and one drain loop serves all three forks. It takes the churn limit as its one parameter and reads the eth1-bridge gate off the state's own fork, so no caller can pass the wrong gate. The deadline helpers existed twice, a pre-gloas set taking an epoch and a gloas set the specification marks Modified. get_attestation_due_ms now dispatches on the fork of its epoch, and the reorg cutoff loses an epoch argument the specification does not take. Delete what nothing calls: the gloas checkpoint block (its reasoning moves onto gloas_get_ancestor), the gloas proposer head, which no fixture reaches and this node never needs since it proposes on no gloas slot, the latest-message epoch helper, the due-ms helpers left after the fold, and the spec-literal is_pending_validator, whose only user carries its own cached index. --- .../src/beacon/fork_choice.rs | 251 ++++-------------- .../src/beacon/helpers/electra.rs | 38 ++- .../src/beacon/helpers/gloas.rs | 35 +-- .../src/beacon/stf/epoch/electra.rs | 95 +++++-- .../src/beacon/stf/epoch/fulu.rs | 138 ++-------- .../src/beacon/stf/epoch/gloas.rs | 130 ++------- .../state_transition/src/beacon/upgrade.rs | 21 +- crates/common/types/src/beacon/config.rs | 4 +- docs/spec_deviations.md | 12 +- 9 files changed, 211 insertions(+), 513 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index a3ebca083..61593813d 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -1277,7 +1277,7 @@ pub fn get_proposer_score(store: &Store, config: &Config) -> Result { /// [`get_weight`]'s own attestation half; [`is_head_weak`] and /// [`is_parent_strong`] call it too, [`is_head_weak`] underneath the public /// [`should_apply_proposer_boost`] and [`is_parent_strong`] underneath -/// `gloas_get_proposer_head`. The head walk does not: it reads the same scores +/// [`get_proposer_head`]. The head walk does not: it reads the same scores /// from [`compute_node_weights`], which drops such a vote too. pub fn get_attestation_score( store: &Store, @@ -2240,25 +2240,28 @@ pub fn get_slot_component_duration_ms(basis_points: u64, config: &Config) -> u64 /// How far into a slot, in milliseconds, an attestation is due. /// -/// `epoch` is accepted, matching the specification's signature, but not read: -/// the deadline is a fixed fraction of the slot in every epoch this module -/// implements. -pub fn get_attestation_due_ms(_epoch: Epoch, config: &Config) -> u64 { - get_slot_component_duration_ms(config.attestation_due_bps, config) +/// Gloas modifies the function (`fork-choice.md`'s "Modified +/// `get_attestation_due_ms`"): its deadline is the earlier +/// `ATTESTATION_DUE_BPS_GLOAS`, leaving room in the same slot for the payload +/// and payload attestation deadlines that follow. +/// +/// Implementation choice, not spec text: the specification's function takes no +/// argument and each fork has its own. `epoch` is how this crate picks the +/// fork's rule, so one function serves both sides of the boundary. +pub fn get_attestation_due_ms(epoch: Epoch, config: &Config) -> u64 { + let basis_points = match ForkRules::of(config.fork_at_epoch(epoch)) { + ForkRules::PreGloas => config.attestation_due_bps, + ForkRules::Gloas => config.attestation_due_bps_gloas, + }; + get_slot_component_duration_ms(basis_points, config) } /// How far into a slot, in milliseconds, a proposer must stop attempting a -/// late-block reorg. See [`get_attestation_due_ms`] for why `epoch` is unused. -pub fn get_proposer_reorg_cutoff_ms(_epoch: Epoch, config: &Config) -> u64 { +/// late-block reorg. Gloas does not modify this deadline. +pub fn get_proposer_reorg_cutoff_ms(config: &Config) -> u64 { get_slot_component_duration_ms(config.proposer_reorg_cutoff_bps, config) } -/// How far into a slot, in milliseconds, an aggregate attestation is due. See -/// [`get_attestation_due_ms`] for why `epoch` is unused. -pub fn get_aggregate_due_ms(_epoch: Epoch, config: &Config) -> u64 { - get_slot_component_duration_ms(config.aggregate_due_bps, config) -} - // --------------------------------------------------------------------------- // Proposer head and reorg helpers // --------------------------------------------------------------------------- @@ -2319,8 +2322,7 @@ pub fn is_finalization_ok(store: &Store, slot: Slot, config: &Config) -> bool { /// building now still counts as on time. pub fn is_proposing_on_time(store: &Store, config: &Config) -> bool { let time_into_slot_ms = store.ms_since_genesis() % config.slot_duration_ms; - let epoch = get_current_store_epoch(store, config); - time_into_slot_ms <= get_proposer_reorg_cutoff_ms(epoch, config) + time_into_slot_ms <= get_proposer_reorg_cutoff_ms(config) } /// Whether `head_root` has few enough votes to be overpowered by the @@ -2468,9 +2470,9 @@ pub fn is_proposer_equivocation( /// Fulu (EIP-7917) drops the `shuffling_stable` requirement: `shuffling_stable` /// is folded into `true` rather than left out of the `&&` chain, which reads /// the same as the specification's own two near-identical copies of this -/// function without keeping two Rust copies to drift apart. Every fork this -/// module implements shares one copy for the same reason no other condition -/// here has a per-fork variant. +/// function without keeping two Rust copies to drift apart. Every fork through +/// fulu shares that one copy. Gloas modifies the function and has no version +/// here, since this node proposes on no gloas slot. pub fn get_proposer_head( store: &Store, head_root: Root, @@ -2596,24 +2598,21 @@ pub fn get_proposer_head( // // Functions here are named exactly as the specification names them, with a // `gloas_` prefix only where a pre-gloas function of the same name already -// exists in this file: `gloas_get_ancestor`, `gloas_get_checkpoint_block`, -// `gloas_get_weight`, `gloas_get_head`, and the four `*_due_ms` functions -// gloas narrows to its own basis-point fields (`gloas_get_attestation_due_ms`, -// `gloas_get_aggregate_due_ms`, `gloas_get_sync_message_due_ms`, -// `gloas_get_contribution_due_ms`; the last two rename an altair-shaped name -// no pre-gloas function of this file actually carries, since a bare -// `get_sync_message_due_ms` would otherwise misname a function that only -// ever reads gloas's own basis points). Two further functions +// exists in this file and the two cannot be one function: `gloas_get_ancestor`, +// `gloas_get_weight`, `gloas_get_head`, `gloas_verify_data_column_sidecar` and +// `gloas_verify_data_column_sidecar_kzg_proofs`. The functions the specification +// modifies in a way one function can dispatch on (`get_attestation_due_ms` +// reads the fork of its `epoch`) are not split at all. Two further functions // (`gloas_bid`, `gloas_get_attestation_score`) are private and have no name // of their own in the specification at all; they stay `gloas_`-prefixed for // the same reason, since both exist only to bind gloas's own semantics // (a block's bid, an ancestor check with a payload dimension) into either a // projection this crate needs or spec-unmodified text (`get_attestation_score`) // that the specification's own per-fork composition binds differently per -// fork. `is_head_weak` and `is_parent_strong` need no gloas copy at all: see -// their own call sites below for why the pre-gloas functions already answer -// gloas's question. Every other function below is new to this file, -// pre-gloas and gloas alike, and keeps the specification's bare name. +// fork. `is_head_weak` needs no gloas copy at all: see its call site below +// for why the pre-gloas function already answers gloas's question. Every +// other function below is new to this file, pre-gloas and gloas alike, and +// keeps the specification's bare name. /// `block`'s own execution payload bid, projected out of the fork-generic /// [`SignedBeaconBlock`] enum. @@ -2864,6 +2863,17 @@ pub fn is_parent_node_full(store: &Store, block: &SignedBeaconBlock) -> Result Result { @@ -2920,27 +2929,6 @@ pub fn is_ancestor(store: &Store, node: ForkChoiceNode, ancestor: ForkChoiceNode || ancestor.payload_status == PayloadStatus::Pending) } -/// `get_checkpoint_block` (gloas `fork-choice.md`, modified): the checkpoint -/// block for `epoch` on `root`'s chain. -/// -/// Its `.root` agrees with the pre-gloas, index-only [`get_checkpoint_block`] -/// on any chain both could answer: [`gloas_get_ancestor`]'s descent picks its -/// next step by `block.slot` alone, so the payload status threaded through -/// the walk never changes which root it lands on, only the label this -/// function then discards. [`filter_block_tree`] (the only caller of -/// [`get_checkpoint_block`] in this file; [`get_voting_source`] never calls -/// it) therefore keeps calling the pre-gloas version rather than growing a -/// gloas copy of its own, which is what lets [`get_filtered_block_tree`] -/// serve [`gloas_get_head`] with no per-block fetch. -pub fn gloas_get_checkpoint_block(store: &Store, root: Root, epoch: Epoch) -> Result { - let epoch_first_slot = compute_start_slot_at_epoch(epoch); - let node = ForkChoiceNode { - root, - payload_status: PayloadStatus::Pending, - }; - Ok(gloas_get_ancestor(store, node, epoch_first_slot)?.root) -} - /// `get_supported_node` (gloas `fork-choice.md`): the node `message` /// supports, full or empty once its slot has passed the block's own, pending /// otherwise. @@ -3467,7 +3455,7 @@ pub fn get_node_children( /// alone. /// /// Reuses the pre-gloas, index-only [`get_filtered_block_tree`] for the -/// candidate tree: see [`gloas_get_checkpoint_block`]'s own doc for why that +/// candidate tree: see [`gloas_get_ancestor`]'s own doc for why that /// is sound rather than a shortcut. /// /// **A reference, not the node's head computation.** [`get_head_node`] runs @@ -3507,146 +3495,12 @@ pub fn gloas_get_head( } } -/// `get_latest_message_epoch` (gloas `fork-choice.md`, modified): the epoch -/// `message`'s slot falls in, now that gloas keeps a [`LatestMessage`]'s slot -/// rather than its epoch directly. No pre-gloas counterpart exists in this -/// file: every pre-gloas reader of a [`LatestMessage`] reads its `epoch` -/// field directly instead. -pub fn get_latest_message_epoch(message: LatestMessage) -> Epoch { - compute_epoch_at_slot(message.slot) -} - -/// `get_attestation_due_ms` (gloas `fork-choice.md`, modified): gloas moves -/// the attestation deadline to its own `ATTESTATION_DUE_BPS_GLOAS`, earlier -/// than pre-gloas's [`get_attestation_due_ms`], to leave room in the same slot -/// for the payload and payload-attestation deadlines that follow it. -pub fn gloas_get_attestation_due_ms(config: &Config) -> u64 { - get_slot_component_duration_ms(config.attestation_due_bps_gloas, config) -} - -/// `get_aggregate_due_ms` (gloas `fork-choice.md`, modified). See -/// [`gloas_get_attestation_due_ms`]. -pub fn gloas_get_aggregate_due_ms(config: &Config) -> u64 { - get_slot_component_duration_ms(config.aggregate_due_bps_gloas, config) -} - -/// `get_sync_message_due_ms` (gloas `fork-choice.md`, modified): reads -/// `SYNC_MESSAGE_DUE_BPS_GLOAS`. `gloas_`-prefixed like its four siblings -/// above even though no altair-named `get_sync_message_due_ms` exists in -/// this file to collide with: the plain name takes altair's own, and this -/// function only ever reads gloas's basis points, never altair's, so the -/// bare name would misname what it does. -pub fn gloas_get_sync_message_due_ms(config: &Config) -> u64 { - get_slot_component_duration_ms(config.sync_message_due_bps_gloas, config) -} - -/// `get_contribution_due_ms` (gloas `fork-choice.md`, modified): reads -/// `CONTRIBUTION_DUE_BPS_GLOAS`. See [`gloas_get_sync_message_due_ms`]. -pub fn gloas_get_contribution_due_ms(config: &Config) -> u64 { - get_slot_component_duration_ms(config.contribution_due_bps_gloas, config) -} - -/// `get_payload_due_ms` (gloas `fork-choice.md`): how far into a slot, in -/// milliseconds, a builder's execution payload envelope is due. -pub fn get_payload_due_ms(config: &Config) -> u64 { - get_slot_component_duration_ms(config.payload_due_bps, config) -} - /// `get_payload_attestation_due_ms` (gloas `fork-choice.md`): how far into a /// slot, in milliseconds, the payload timeliness committee's votes are due. pub fn get_payload_attestation_due_ms(config: &Config) -> u64 { get_slot_component_duration_ms(config.payload_attestation_due_bps, config) } -/// `get_proposer_head` (gloas `fork-choice.md`, modified): the node a -/// proposer at `slot` should build on, as a full [`ForkChoiceNode`] so a -/// re-org preserves the parent's own payload status rather than collapsing -/// it back to `Pending`. -/// -/// Drops the pre-gloas `shuffling_stable` condition entirely, matching the -/// specification's own text: pre-gloas [`get_proposer_head`] still gates it -/// on `head_block.fork_name() >= ForkName::Fulu` for the forks before fulu. -/// [`get_head_node`] only ever reaches this function's gloas arm once the -/// chain has reached `GLOAS_FORK_EPOCH`, so `head_block` here is gloas, or -/// fulu right at the fulu-to-gloas boundary (this crate's decided rule for -/// a pre-gloas head; see the module documentation's "Gloas: payload-aware -/// fork choice" section), so `>= ForkName::Fulu` holds unconditionally for -/// every call this function gets, and the specification's own gloas text -/// simply omits the condition rather than spell out an always-true check. -pub fn gloas_get_proposer_head( - store: &Store, - head_node: ForkChoiceNode, - slot: Slot, - config: &Config, - committees: &CommitteeCache, -) -> Result { - let head_block = store - .get_signed_block(&head_node.root) - .expect("get") - .ok_or(Error::SpecAssert("head_node.root in store.blocks"))?; - let parent_root = head_block.parent_root(); - let (parent_slot, _) = store - .block_entry(&parent_root) - .ok_or(Error::SpecAssert("parent_root in store.blocks"))?; - let parent_payload_status = get_parent_payload_status(store, &head_block)?; - let parent_node = ForkChoiceNode { - root: parent_root, - payload_status: parent_payload_status, - }; - - // Only re-org the head block if it arrived later than the attestation - // deadline. - let head_late = is_head_late(store, head_node.root)?; - // Ensure that the FFG information of the new head will be competitive - // with the current head. - let ffg_competitive = is_ffg_competitive(store, head_node.root, parent_root)?; - // Do not re-org if the chain is not finalizing with acceptable frequency. - let finalization_ok = is_finalization_ok(store, slot, config); - // Only re-org if we are proposing on-time. - let proposing_on_time = is_proposing_on_time(store, config); - - let head_slot = head_block.slot(); - // Only re-org a single slot at most. - let parent_slot_ok = parent_slot.checked_add(1) == Some(head_slot); - let current_time_ok = head_slot.checked_add(1) == Some(slot); - let single_slot_reorg = parent_slot_ok && current_time_ok; - - // Check that the head has few enough votes to be overpowered by our - // proposer boost. - verify( - store.proposer_boost_root() != head_node.root, - "store.proposer_boost_root != head_node.root", - )?; - // Pre-gloas `is_head_weak`, not a gloas copy: see `is_ancestor`'s own - // doc for why a `Pending` node's score is the same either way. - let head_weak = is_head_weak(store, head_node.root, config, committees)?; - - // Check that the missing votes are assigned to the parent and not being - // hoarded. Pre-gloas `is_parent_strong`, for the same reason. - let parent_strong = is_parent_strong(store, head_node.root, config)?; - - // Re-org more aggressively if there is a proposer equivocation in the - // previous slot. - let index = store.block_index(); - let proposer_equivocation = is_proposer_equivocation(store, &index, head_node.root)?; - - if head_late - && ffg_competitive - && finalization_ok - && proposing_on_time - && single_slot_reorg - && head_weak - && parent_strong - { - // We can re-org the current head by building upon its parent node. - Ok(parent_node) - } else if head_weak && current_time_ok && proposer_equivocation { - Ok(parent_node) - } else { - Ok(head_node) - } -} - // --------------------------------------------------------------------------- // Merge transition helpers (bellatrix) // --------------------------------------------------------------------------- @@ -4027,8 +3881,12 @@ pub fn compute_pulled_up_tip( /// [`is_head_late`] only reads the attestation entry. Gloas modifies the /// function: a block is timely for a deadline only if it arrived in its own /// slot and before that deadline, and the two deadlines are gloas's own -/// attestation deadline ([`gloas_get_attestation_due_ms`]) and the payload +/// attestation deadline ([`get_attestation_due_ms`]) and the payload /// timeliness committee's ([`get_payload_attestation_due_ms`]). +/// +/// The attestation deadline is chosen by the clock's epoch and the PTC entry +/// by the block's own `rules`; they can disagree only when the block is not +/// from the current slot, where every entry is `false` either way. fn block_timeliness( store: &Store, block_slot: Slot, @@ -4037,15 +3895,14 @@ fn block_timeliness( ) -> [bool; 2] { let time_into_slot_ms = store.ms_since_genesis() % config.slot_duration_ms; let is_current_slot = get_current_slot(store, config) == block_slot; + let epoch = get_current_store_epoch(store, config); + let attestation_threshold_ms = get_attestation_due_ms(epoch, config); match rules { ForkRules::PreGloas => { - let epoch = get_current_store_epoch(store, config); - let attestation_threshold_ms = get_attestation_due_ms(epoch, config); let is_timely = is_current_slot && time_into_slot_ms < attestation_threshold_ms; [is_timely, is_timely] } ForkRules::Gloas => { - let attestation_threshold_ms = gloas_get_attestation_due_ms(config); let ptc_threshold_ms = get_payload_attestation_due_ms(config); [ is_current_slot && time_into_slot_ms < attestation_threshold_ms, @@ -4116,7 +3973,7 @@ fn get_ancestor_or_lowest_indexed( /// Serves gloas unchanged. Gloas's `get_shuffling_dependent_root` walks from a /// `PENDING` node with its payload-aware `get_ancestor` and returns the root of /// the node it lands on; that walk picks each step by `block.slot` alone (see -/// [`gloas_get_checkpoint_block`]), so the root is the one this index walk +/// [`gloas_get_ancestor`]), so the root is the one this index walk /// answers, and the payload status the gloas walk threads through is /// discarded. pub fn get_shuffling_dependent_root( @@ -4325,7 +4182,7 @@ fn validate_on_attestation_indexed( // LMD vote must be consistent with FFG vote target. Gloas's own // `get_checkpoint_block` answers the same root through a payload-aware walk - // (see [`gloas_get_checkpoint_block`]), so the index-only walk serves both + // (see [`gloas_get_ancestor`]), so the index-only walk serves both // forks and spares a gloas attestation one block decode per hop. let checkpoint_block = get_checkpoint_block(index, data.beacon_block_root, target.epoch)?; verify( diff --git a/crates/blockchain/state_transition/src/beacon/helpers/electra.rs b/crates/blockchain/state_transition/src/beacon/helpers/electra.rs index 77e292cfa..c3fd3bcdb 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/electra.rs @@ -957,13 +957,13 @@ impl<'a> PendingQueueFields<'a> { /// The execution-layer deposit request index at which the state switched /// from crediting deposits off `Eth1Data` votes to crediting them off /// `DepositRequest`s directly, read by - /// `crate::beacon::stf::epoch::electra::process_pending_deposits` to know + /// `crate::beacon::stf::epoch::electra::drain_pending_deposits` to know /// whether any eth1-bridge deposit is still outstanding. /// /// `pub(crate)`: fulu's own tests read this to assert a test state starts /// with it at [`crate::beacon::constants::UNSET_DEPOSIT_REQUESTS_START_INDEX`]. - /// Fulu's and gloas's own `process_pending_deposits` never read this - /// field at all. + /// Fulu's and gloas's `process_pending_deposits` retire the gate that + /// uses it, so the drain reads it only for electra. pub(crate) fn deposit_requests_start_index(&self) -> u64 { match self { PendingQueueFields::Electra(state) => state.deposit_requests_start_index, @@ -1008,31 +1008,26 @@ impl<'a> PendingQueueFields<'a> { /// back. `Vec`-level, the same shape /// [`Self::take_pending_consolidations`]/[`Self::set_pending_consolidations`] /// already use, for the same reason: - /// `crate::beacon::stf::epoch::electra::process_pending_deposits` needs + /// `crate::beacon::stf::epoch::electra::drain_pending_deposits` needs /// to drain the whole queue into a `Vec` it can freely mutate `state` /// around, not a reference still borrowing it. - /// - /// Refuses gloas: gloas modifies `process_pending_deposits` and drains - /// the queue through its own state, so a gloas state reaching electra's - /// or fulu's copy is a dispatch mistake, not a caller to serve. - pub(crate) fn take_pending_deposits(&mut self) -> Result> { + pub(crate) fn take_pending_deposits(&mut self) -> Vec { match self { PendingQueueFields::Electra(state) => { - Ok(core::mem::take(&mut state.pending_deposits).into_inner()) + core::mem::take(&mut state.pending_deposits).into_inner() } PendingQueueFields::Fulu(state) => { - Ok(core::mem::take(&mut state.pending_deposits).into_inner()) + core::mem::take(&mut state.pending_deposits).into_inner() + } + PendingQueueFields::Gloas(state) => { + core::mem::take(&mut state.pending_deposits).into_inner() } - PendingQueueFields::Gloas(_) => Err(Error::UnsupportedForFork { - function: "process_pending_deposits", - fork: ForkName::Gloas, - }), } } - /// See [`Self::take_pending_deposits`], including why gloas is refused. - /// Also fails when the queue outgrows electra's and fulu's bounded - /// `SszList`. + /// See [`Self::take_pending_deposits`]. Fallible only for electra's and + /// fulu's bounded `SszList`; gloas's progressive one never rejects a + /// length. pub(crate) fn set_pending_deposits( &mut self, deposits: Vec, @@ -1044,11 +1039,8 @@ impl<'a> PendingQueueFields<'a> { PendingQueueFields::Fulu(state) => { state.pending_deposits = electra::PendingDeposits::try_from(deposits)?; } - PendingQueueFields::Gloas(_) => { - return Err(Error::UnsupportedForFork { - function: "process_pending_deposits", - fork: ForkName::Gloas, - }); + PendingQueueFields::Gloas(state) => { + state.pending_deposits = gloas::PendingDeposits::from(deposits); } } Ok(()) diff --git a/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs b/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs index 8998e7eb3..a8f97f9d9 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs @@ -90,7 +90,7 @@ use crate::beacon::bls; use crate::beacon::config::Config; use crate::beacon::constants; use crate::beacon::containers::shared::AttestationData; -use crate::beacon::containers::{BeaconState, electra, gloas}; +use crate::beacon::containers::{BeaconState, gloas}; use crate::beacon::error::{Error, Result}; use crate::beacon::hash::hash; use crate::beacon::preset; @@ -164,8 +164,9 @@ pub fn is_valid_indexed_attestation( /// abstracts over (see its own doc), so this is a thin wrapper over that /// shared walk, the same shape electra's own `get_attesting_indices` is. /// `committee_bits` is untouched by the EIP-7688 change ([`gloas::Attestation`] -/// reuses [`electra::CommitteeBits`] outright, see `containers::gloas`'s own -/// module doc), which is what lets both forks share one walk over one type. +/// reuses [`crate::beacon::containers::electra::CommitteeBits`] outright, see +/// `containers::gloas`'s own module doc), which is what lets both forks share +/// one walk over one type. pub fn get_attesting_indices( state: &BeaconState, attestation: &gloas::Attestation, @@ -292,34 +293,6 @@ pub fn is_valid_indexed_payload_attestation( bls::fast_aggregate_verify(&pubkeys, signing_root, &attestation.signature) } -/// Whether a pending deposit with a valid signature is already queued for -/// `pubkey`. -/// -/// *Note (from the specification):* this naively reverifies a deposit -/// signature on every call; a caller iterating many pubkeys should cache the -/// verification instead of calling this in a loop. -pub fn is_pending_validator( - pending_deposits: &[electra::PendingDeposit], - pubkey: BlsPubkey, - config: &Config, -) -> bool { - for pending_deposit in pending_deposits { - if pending_deposit.pubkey != pubkey { - continue; - } - if crate::beacon::stf::electra::is_valid_deposit_signature( - pending_deposit.pubkey, - pending_deposit.withdrawal_credentials, - pending_deposit.amount, - &pending_deposit.signature, - config, - ) { - return true; - } - } - false -} - // --------------------------------------------------------------------------- // Misc // --------------------------------------------------------------------------- diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs index 701fed5ab..a28002745 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/electra.rs @@ -39,8 +39,8 @@ use crate::beacon::containers::{BeaconState, electra}; use crate::beacon::error::{Error, Result}; use crate::beacon::helpers::accessors::{get_current_epoch, get_total_active_balance}; use crate::beacon::helpers::electra::{ - get_activation_exit_churn_limit, get_max_effective_balance, initiate_validator_exit, - is_eligible_for_activation_queue, pending_queue_fields, + PendingQueueFields, get_activation_exit_churn_limit, get_max_effective_balance, + initiate_validator_exit, is_eligible_for_activation_queue, pending_queue_fields, }; use crate::beacon::helpers::misc::{ compute_activation_exit_epoch, compute_deposit_domain, compute_signing_root, @@ -305,16 +305,33 @@ pub fn process_slashings(state: &mut BeaconState, _config: &Config) -> Result<() /// described above. /// /// Electra's own version, in effect only through this fork. Fulu retires the -/// eth1-bridge-ahead-of-requests gate outright (`beacon-chain.md`'s "Modified -/// `process_pending_deposits`"), so [`super::fulu::process_pending_deposits`] -/// keeps its own copy of this loop rather than sharing this one; see that -/// function's own doc for why running this version under fulu would be -/// wrong, not merely redundant. +/// eth1-bridge-ahead-of-requests gate and gloas swaps the churn budget +/// (`beacon-chain.md`'s "Modified `process_pending_deposits`" in each); both +/// drain through `drain_pending_deposits`, which this calls with electra's +/// combined activation and exit budget. pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Result<()> { - let next_epoch = get_current_epoch(state) + 1; let churn_limit = get_activation_exit_churn_limit(state, config)?; + drain_pending_deposits(state, config, churn_limit) +} + +/// The drain loop of `process_pending_deposits`, shared by every fork that +/// has the queue. The churn budget, `churn_limit`, is the one thing a caller +/// chooses (electra and fulu: `get_activation_exit_churn_limit`; gloas: +/// `get_activation_churn_limit`). The eth1-bridge gate is a fact about the +/// state's fork, so it is derived from the state here rather than passed: +/// electra holds every deposit request back while `eth1_deposit_index` is +/// below `deposit_requests_start_index`, and fulu and gloas retire the gate +/// (fulu never writes that index, so keeping it would stall the queue for +/// good rather than merely once). The rest of the function is unmodified by +/// fulu and gloas, so it lives here once. See [`process_pending_deposits`]'s +/// doc for the drain itself. +pub(crate) fn drain_pending_deposits( + state: &mut BeaconState, + config: &Config, + churn_limit: Gwei, +) -> Result<()> { + let next_epoch = get_current_epoch(state) + 1; let finalized_slot = compute_start_slot_at_epoch(state.finalized_checkpoint().epoch); - let eth1_deposit_index = state.eth1_deposit_index(); // `pending_queue_fields` borrows the whole state, so everything read // through it below has to finish before the loop's own, ordinary @@ -323,21 +340,25 @@ pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Res // those: once the queue is a plain `Vec` of its own, reading // `state.iter_validators()` and crediting balances through `state` cannot // conflict with walking the deposits that drive those reads and writes. - let (deposit_requests_start_index, available_for_processing, deposits) = { + let (eth1_bridge_gate, available_for_processing, deposits) = { let mut fields = pending_queue_fields(state, "process_pending_deposits")?; - let deposit_requests_start_index = fields.deposit_requests_start_index(); + // `Some((eth1_deposit_index, deposit_requests_start_index))` only + // where the gate is in force; fulu and gloas read neither field. + let eth1_bridge_gate = match &fields { + PendingQueueFields::Electra(electra_state) => Some(( + electra_state.eth1_deposit_index, + fields.deposit_requests_start_index(), + )), + PendingQueueFields::Fulu(_) | PendingQueueFields::Gloas(_) => None, + }; let available_for_processing = fields .deposit_balance_to_consume() .checked_add(churn_limit) .ok_or(Error::ArithmeticOverflow( - "deposit_balance_to_consume + get_activation_exit_churn_limit", + "deposit_balance_to_consume + churn limit", ))?; - let deposits: Vec = fields.take_pending_deposits()?; - ( - deposit_requests_start_index, - available_for_processing, - deposits, - ) + let deposits: Vec = fields.take_pending_deposits(); + (eth1_bridge_gate, available_for_processing, deposits) }; let mut processed_amount: Gwei = 0; @@ -350,7 +371,8 @@ pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Res // before the first deposit *request* is: the two sources are ordered // relative to each other only by this check, since a request's own // slot says nothing about where it falls in the bridge's queue. - if deposit.slot > constants::GENESIS_SLOT + if let Some((eth1_deposit_index, deposit_requests_start_index)) = eth1_bridge_gate + && deposit.slot > constants::GENESIS_SLOT && eth1_deposit_index < deposit_requests_start_index { break; @@ -708,6 +730,41 @@ mod tests { // process_pending_deposits // ----------------------------------------------------------------------- + #[test] + fn a_deposit_request_waits_behind_the_eth1_bridge_while_the_start_index_is_unset() { + let config = Config::mainnet(); + let mut state = electra_state_with_validators(2); + // Past the deposit's own slot, so only the eth1-bridge gate, not the + // unrelated finality gate, is under test. + state.finalized_checkpoint_mut().epoch = 1; + let validator = state.validator(0).unwrap(); + let deposit = electra::PendingDeposit { + pubkey: validator.pubkey, + withdrawal_credentials: validator.withdrawal_credentials, + amount: preset::EFFECTIVE_BALANCE_INCREMENT, + signature: BlsSignature::default(), + slot: constants::GENESIS_SLOT + 1, + }; + let mut fields = pending_queue_fields(&mut state, "test setup").unwrap(); + assert_eq!( + fields.deposit_requests_start_index(), + constants::UNSET_DEPOSIT_REQUESTS_START_INDEX + ); + fields.push_pending_deposit(deposit.clone()).unwrap(); + + let balance_before = state.balance(0).unwrap(); + process_pending_deposits(&mut state, &config).unwrap(); + + // Blocked: `eth1_deposit_index < deposit_requests_start_index`, so the + // whole pass breaks before reaching this request-sourced entry. + assert_eq!(state.balance(0).unwrap(), balance_before); + let BeaconState::Electra(inner) = &state else { + unreachable!("built as Electra"); + }; + assert_eq!(inner.pending_deposits.len(), 1); + assert_eq!(inner.pending_deposits[0], deposit); + } + #[test] fn a_pending_deposit_for_an_already_withdrawn_validator_bypasses_churn() { let config = Config::mainnet(); diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/fulu.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/fulu.rs index 39ddc6061..c9fa07edc 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/fulu.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/fulu.rs @@ -53,18 +53,14 @@ //! only the registry and balance state matters for the ordering here. use crate::beacon::config::Config; -use crate::beacon::constants::FAR_FUTURE_EPOCH; use crate::beacon::containers::BeaconState; -use crate::beacon::containers::electra as electra_containers; use crate::beacon::error::{Error, Result}; use crate::beacon::fork::ForkName; use crate::beacon::helpers::accessors::get_current_epoch; -use crate::beacon::helpers::electra::{get_activation_exit_churn_limit, pending_queue_fields}; +use crate::beacon::helpers::electra::get_activation_exit_churn_limit; use crate::beacon::helpers::fulu::{get_beacon_proposer_indices, proposer_lookahead_mut}; -use crate::beacon::helpers::misc::compute_start_slot_at_epoch; use crate::beacon::lean_state_unreachable; use crate::beacon::preset; -use crate::beacon::primitives::Gwei; use super::electra; @@ -96,7 +92,7 @@ pub fn process_epoch(state: &mut BeaconState, config: &Config) -> Result<()> { // Pending deposits // --------------------------------------------------------------------------- -/// Drains a balance-churn-limited amount of [`electra_containers::PendingDeposit`]s +/// Drains a balance-churn-limited amount of [`crate::beacon::containers::electra::PendingDeposit`]s /// into the validator registry. /// /// [`electra::process_pending_deposits`]'s own doc covers the shape this @@ -123,105 +119,14 @@ pub fn process_epoch(state: &mut BeaconState, config: &Config) -> Result<()> { /// still unset or set but not yet caught up: see /// [`super::super::fulu::process_deposit_request`]'s own doc for why fulu /// never writes it, and why pairing that with electra's gate here would stall -/// the queue for good rather than merely once. This function reads neither -/// field. +/// the queue for good rather than merely once. The drain is +/// `electra::drain_pending_deposits`, which derives from the state's fork that +/// the gate is retired here and so reads neither `eth1_deposit_index` nor +/// `deposit_requests_start_index`. // [Modified in Fulu] pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Result<()> { - let next_epoch = get_current_epoch(state) + 1; let churn_limit = get_activation_exit_churn_limit(state, config)?; - let finalized_slot = compute_start_slot_at_epoch(state.finalized_checkpoint().epoch); - - // See `electra::process_pending_deposits`'s own doc for why the queue is - // taken by value here rather than iterated in place. - let (available_for_processing, deposits) = { - let mut fields = pending_queue_fields(state, "process_pending_deposits")?; - let available_for_processing = fields - .deposit_balance_to_consume() - .checked_add(churn_limit) - .ok_or(Error::ArithmeticOverflow( - "deposit_balance_to_consume + get_activation_exit_churn_limit", - ))?; - let deposits: Vec = fields.take_pending_deposits()?; - (available_for_processing, deposits) - }; - - let mut processed_amount: Gwei = 0; - let mut next_deposit_index = 0usize; - let mut deposits_to_postpone: Vec = Vec::new(); - let mut is_churn_limit_reached = false; - - for deposit in &deposits { - // A deposit whose queue position could still be reorged out must - // wait: crediting it now and reverting later is not an option, since - // nothing else in the state transition undoes a balance change. - if deposit.slot > finalized_slot { - break; - } - - if next_deposit_index >= preset::MAX_PENDING_DEPOSITS_PER_EPOCH as usize { - break; - } - - let (is_validator_exited, is_validator_withdrawn) = state - .iter_validators() - .find(|validator| validator.pubkey == deposit.pubkey) - .map(|validator| { - ( - validator.exit_epoch < FAR_FUTURE_EPOCH, - validator.withdrawable_epoch < next_epoch, - ) - }) - .unwrap_or((false, false)); - - if is_validator_withdrawn { - electra::apply_pending_deposit(state, deposit, config)?; - } else if is_validator_exited { - deposits_to_postpone.push(deposit.clone()); - } else { - match processed_amount.checked_add(deposit.amount) { - Some(sum) if sum <= available_for_processing => { - processed_amount = sum; - electra::apply_pending_deposit(state, deposit, config)?; - } - // Either the sum overflowed (certainly too much) or it fit in - // a `u64` but still exceeded the budget: both mean this - // epoch's processing stops here. - _ => { - is_churn_limit_reached = true; - break; - } - } - } - - next_deposit_index += 1; - } - - let remaining: Vec = deposits - .into_iter() - .skip(next_deposit_index) - .chain(deposits_to_postpone) - .collect(); - - // Leftover churn is only worth remembering when it was actually the - // reason processing stopped: if the queue simply ran out, or the - // finality or per-epoch-cap gate stopped it first, next epoch's budget - // starts fresh rather than inheriting room this epoch never even tried - // to spend. - let deposit_balance_to_consume = if is_churn_limit_reached { - available_for_processing - .checked_sub(processed_amount) - .ok_or(Error::ArithmeticOverflow( - "available_for_processing - processed_amount", - ))? - } else { - 0 - }; - - let mut fields = pending_queue_fields(state, "process_pending_deposits")?; - fields.set_pending_deposits(remaining)?; - *fields.deposit_balance_to_consume_mut() = deposit_balance_to_consume; - - Ok(()) + electra::drain_pending_deposits(state, config, churn_limit) } /// Shifts `proposer_lookahead` forward by one epoch. @@ -283,7 +188,9 @@ pub fn process_proposer_lookahead(state: &mut BeaconState) -> Result<()> { mod tests { use super::*; use crate::beacon::constants; + use crate::beacon::containers::electra as electra_containers; use crate::beacon::fork::ForkName; + use crate::beacon::helpers::electra::pending_queue_fields; use crate::beacon::helpers::fulu::initialize_proposer_lookahead; use crate::beacon::primitives::BlsSignature; @@ -430,7 +337,7 @@ mod tests { } #[test] - fn the_same_queue_stalls_forever_under_electras_own_gate() { + fn electras_entry_point_applies_the_state_forks_gate_to_a_fulu_state() { let config = Config::mainnet(); let mut state = fulu_state_with_validators(2); // See the previous test: past the deposit's own slot, so only the @@ -439,25 +346,20 @@ mod tests { let deposit = request_sourced_pending_deposit(&state, 0); pending_queue_fields(&mut state, "test setup") .unwrap() - .push_pending_deposit(deposit.clone()) + .push_pending_deposit(deposit) .unwrap(); let balance_before = state.balance(0).unwrap(); - // `electra::process_pending_deposits` accepts a fulu state too (its - // `PendingQueueFields` projection covers both), which is exactly what - // let the pre-fix code call it here unnoticed. + // `electra::process_pending_deposits` accepts a fulu state too. The + // gate is derived from the state's fork, not chosen by the caller, so + // it cannot stall a fulu queue whose `deposit_requests_start_index` + // fulu never writes (the same scenario stalls on an electra state: + // see `electra::tests`). electra::process_pending_deposits(&mut state, &config).unwrap(); - // Blocked: `deposit_requests_start_index` is still - // `UNSET_DEPOSIT_REQUESTS_START_INDEX`, so electra's - // `eth1_deposit_index < deposit_requests_start_index` gate holds for - // this request-sourced entry and the whole pass breaks before ever - // reaching it. - assert_eq!(state.balance(0).unwrap(), balance_before); - let BeaconState::Fulu(inner) = &state else { - unreachable!("built as Fulu"); - }; - assert_eq!(inner.pending_deposits.len(), 1); - assert_eq!(inner.pending_deposits[0], deposit); + assert_eq!( + state.balance(0).unwrap(), + balance_before + preset::EFFECTIVE_BALANCE_INCREMENT + ); } } diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs index 57e3dd7d8..abb21585b 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs @@ -48,13 +48,13 @@ //! [`super::capella::process_historical_summaries_update`], //! [`super::altair::process_sync_committee_updates`]) call straight through, //! through accessors that already list gloas among the forks they serve. -//! - **[`process_pending_deposits`] and [`process_participation_flag_updates`] -//! are gloas's own copy**, over [`gloas::BeaconState`] directly (through -//! `helpers::gloas::gloas_state`). `process_pending_deposits` is -//! spec-modified (EIP-8061: gloas's own activation-only churn budget) and -//! still calls the shared [`super::electra::apply_pending_deposit`] for the -//! "credit one dequeued deposit" sub-step. -//! `process_participation_flag_updates` is unmodified in the specification, +//! - **[`process_pending_deposits`]** is spec-modified (EIP-8061: gloas's own +//! activation-only churn budget), and that budget is all it changes: it +//! passes it to `super::electra::drain_pending_deposits`, the loop electra +//! and fulu run. +//! - **[`process_participation_flag_updates`] is gloas's own copy**, over +//! [`gloas::BeaconState`] directly (through `helpers::gloas::gloas_state`). +//! It is unmodified in the specification, //! but replaces the whole list (a new length, not an element write), which //! only the fork's own concrete container type can do: electra's and //! fulu's `EpochParticipation::try_from` is fallible where gloas's @@ -62,8 +62,7 @@ //! `dispatch_state_from!` body could give both. use crate::beacon::config::Config; -use crate::beacon::constants::FAR_FUTURE_EPOCH; -use crate::beacon::containers::{BeaconState, electra, gloas}; +use crate::beacon::containers::{BeaconState, gloas}; use crate::beacon::error::{Error, Result}; use crate::beacon::helpers::accessors::{CommitteeCache, get_current_epoch}; use crate::beacon::helpers::gloas::{ @@ -71,7 +70,6 @@ use crate::beacon::helpers::gloas::{ }; use crate::beacon::helpers::misc::compute_start_slot_at_epoch; use crate::beacon::preset; -use crate::beacon::primitives::Gwei; /// Gloas's epoch-boundary driver, in the specification's own order /// (`beacon-chain.md`'s "Modified `process_epoch`"): fulu's own step list, @@ -136,110 +134,19 @@ pub fn process_participation_flag_updates(state: &mut BeaconState) -> Result<()> /// Modified from electra's and fulu's (EIP-8061): the churn budget is /// [`get_activation_churn_limit`]'s own, independent activation-only budget, /// not electra's combined activation/exit one -/// (`crate::beacon::helpers::electra::get_activation_exit_churn_limit`); like fulu's -/// own copy, the eth1-bridge-ahead-of-requests gate is gone outright (see +/// (`crate::beacon::helpers::electra::get_activation_exit_churn_limit`); like fulu's, +/// the eth1-bridge-ahead-of-requests gate is gone outright (see /// `crate::beacon::stf::epoch::fulu::process_pending_deposits`'s own doc for why /// dropping it changes nothing observable once a chain has reached this far). -/// The queue is gloas's own progressive one (EIP-7688), read and written -/// directly through [`gloas_state`] rather than through -/// [`crate::beacon::helpers::electra::pending_queue_fields`]'s [`PendingQueueFields`](crate::beacon::helpers::electra::PendingQueueFields), -/// which refuses that field for a gloas state (see its own doc). Once a -/// deposit is dequeued and cleared to apply, though, -/// [`super::electra::apply_pending_deposit`] is the identical function electra -/// and fulu use: nothing about crediting one already-dequeued deposit differs -/// for gloas. +/// The drain itself is `super::electra::drain_pending_deposits`, the loop +/// electra and fulu run: the specification leaves everything but the churn +/// budget and the retired gate unmodified. The queue is gloas's progressive +/// one (EIP-7688), which +/// [`PendingQueueFields`](crate::beacon::helpers::electra::PendingQueueFields) +/// hands over as a `Vec` like the bounded one. pub fn process_pending_deposits(state: &mut BeaconState, config: &Config) -> Result<()> { - let next_epoch = get_current_epoch(state) + 1; let churn_limit = get_activation_churn_limit(state, config)?; - let finalized_slot = compute_start_slot_at_epoch(state.finalized_checkpoint().epoch); - - // See `crate::beacon::stf::epoch::electra::process_pending_deposits`'s own doc - // for why the queue is taken by value here rather than iterated in place. - let (available_for_processing, deposits) = { - let inner = gloas_state(state, "process_pending_deposits")?; - let available_for_processing = inner - .deposit_balance_to_consume - .checked_add(churn_limit) - .ok_or(Error::ArithmeticOverflow( - "deposit_balance_to_consume + get_activation_churn_limit", - ))?; - let deposits: Vec = - core::mem::take(&mut inner.pending_deposits).into_inner(); - (available_for_processing, deposits) - }; - - let mut processed_amount: Gwei = 0; - let mut next_deposit_index = 0usize; - let mut deposits_to_postpone: Vec = Vec::new(); - let mut is_churn_limit_reached = false; - - for deposit in &deposits { - // A deposit whose queue position could still be reorged out must - // wait: crediting it now and reverting later is not an option, since - // nothing else in the state transition undoes a balance change. - if deposit.slot > finalized_slot { - break; - } - - if next_deposit_index >= preset::MAX_PENDING_DEPOSITS_PER_EPOCH as usize { - break; - } - - let (is_validator_exited, is_validator_withdrawn) = state - .iter_validators() - .find(|validator| validator.pubkey == deposit.pubkey) - .map(|validator| { - ( - validator.exit_epoch < FAR_FUTURE_EPOCH, - validator.withdrawable_epoch < next_epoch, - ) - }) - .unwrap_or((false, false)); - - if is_validator_withdrawn { - super::electra::apply_pending_deposit(state, deposit, config)?; - } else if is_validator_exited { - deposits_to_postpone.push(deposit.clone()); - } else { - match processed_amount.checked_add(deposit.amount) { - Some(sum) if sum <= available_for_processing => { - processed_amount = sum; - super::electra::apply_pending_deposit(state, deposit, config)?; - } - // Either the sum overflowed (certainly too much) or it fit in - // a `u64` but still exceeded the budget: both mean this - // epoch's processing stops here. - _ => { - is_churn_limit_reached = true; - break; - } - } - } - - next_deposit_index += 1; - } - - let remaining: Vec = deposits - .into_iter() - .skip(next_deposit_index) - .chain(deposits_to_postpone) - .collect(); - - let deposit_balance_to_consume = if is_churn_limit_reached { - available_for_processing - .checked_sub(processed_amount) - .ok_or(Error::ArithmeticOverflow( - "available_for_processing - processed_amount", - ))? - } else { - 0 - }; - - let inner = gloas_state(state, "process_pending_deposits")?; - inner.pending_deposits = gloas::PendingDeposits::from(remaining); - inner.deposit_balance_to_consume = deposit_balance_to_consume; - - Ok(()) + super::electra::drain_pending_deposits(state, config, churn_limit) } // --------------------------------------------------------------------------- @@ -327,7 +234,8 @@ pub fn process_ptc_window(state: &mut BeaconState) -> Result<()> { #[cfg(test)] mod tests { use super::*; - use crate::beacon::constants; + use crate::beacon::constants::{self, FAR_FUTURE_EPOCH}; + use crate::beacon::containers::electra; use crate::beacon::fork::ForkName; use crate::beacon::helpers::accessors::get_total_active_balance; use crate::beacon::helpers::gloas::{get_ptc, gloas_state_ref}; diff --git a/crates/blockchain/state_transition/src/beacon/upgrade.rs b/crates/blockchain/state_transition/src/beacon/upgrade.rs index b0bf4880c..dbcaa52b0 100644 --- a/crates/blockchain/state_transition/src/beacon/upgrade.rs +++ b/crates/blockchain/state_transition/src/beacon/upgrade.rs @@ -884,15 +884,12 @@ fn initialize_ptc_window(state: &BeaconState) -> Result>, @@ -907,8 +904,8 @@ impl PendingValidatorIndex { self.verdicts.push(None); } - /// [`is_pending_validator`](crate::beacon::helpers::gloas::is_pending_validator)'s - /// exact result for `pubkey`, reached through the pubkey index instead of + /// The result the specification's `is_pending_validator` gives + /// for `pubkey`, reached through the pubkey index instead of /// a linear scan, with each candidate's signature verdict computed at /// most once no matter how many later deposits ask the same question. fn is_pending_validator( @@ -966,7 +963,7 @@ fn requeue( /// deposit whose pubkey neither registry recognizes yet is onboarded as a /// builder only if its withdrawal credential is builder-prefixed, no deposit /// already re-queued in this same pass could still turn that pubkey into a -/// validator instead ([`is_pending_validator`](crate::beacon::helpers::gloas::is_pending_validator)), +/// validator instead (the specification's `is_pending_validator`), /// and its signature checks out. A deposit that fails the credential or /// pending-validator check stays queued instead; one that fails the /// signature check is dropped outright, matching phase0's own "an invalid diff --git a/crates/common/types/src/beacon/config.rs b/crates/common/types/src/beacon/config.rs index 9732c99f5..cd3b56e48 100644 --- a/crates/common/types/src/beacon/config.rs +++ b/crates/common/types/src/beacon/config.rs @@ -363,7 +363,9 @@ pub struct Config { #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub attestation_due_bps: u64, /// Basis points of [`Self::slot_duration_ms`] by which an aggregate - /// attestation is due; read by `get_aggregate_due_ms`. + /// attestation is due; the specification's `get_aggregate_due_ms`. Nothing + /// in the state transition's fork choice calls it: the reader is the + /// validator's `SlotClock`. #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub aggregate_due_bps: u64, /// Basis points of [`Self::slot_duration_ms`] past which a proposer must diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 2ea512cc8..6a85d7372 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -178,7 +178,7 @@ that the tests compare like with like. - **Pruned votes** (`compute_node_weights`; in the references, `get_attestation_score`, and through it `is_head_weak` and - `is_parent_strong`, the latter reached from the gloas proposer head). A vote + `is_parent_strong`, the latter reached from `get_proposer_head`). A vote for a root missing from the block index contributes nothing, where `get_ancestor` would raise. Pruning removes only entries below the finalized slot, and every root scored is indexed at or above it, so such a vote cannot @@ -216,6 +216,16 @@ the root of the state's latest block header. shapes give the same root. - **Equivalence:** the same check, on both kinds of state. +## The attestation deadline takes the epoch that picks the fork's rule + +- **The specification:** `get_attestation_due_ms()` takes no argument, and + gloas replaces it with a function that reads `ATTESTATION_DUE_BPS_GLOAS`. +- **ethlambda:** one `get_attestation_due_ms(epoch, config)` + (`fork_choice.rs`) that reads the basis points of the fork `epoch` falls in + (`ForkRules::of`), so both sides of the fulu-to-gloas boundary share it. +- **Equivalence:** the same deadline for every slot, since the epoch is the + one the specification's own per-fork function would have been selected for. + ## The head is one bottom-up walk, not the specification's per-node `get_weight` The specification's gloas `get_head` calls `get_weight` for every candidate From 5102c814347a2362ff0f91b37cf71f21936ae81c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 12:26:11 -0300 Subject: [PATCH 031/132] docs(beacon): correct stale gloas comments - gloas get_execution_requests_list said electra's is uncalled; the live follower's Engine API client calls it. - LatestMessage's doc said one of epoch and slot is left neutral, but update_latest_messages fills both for every fork. - shared.rs and gloas.rs each imported a type named ProgressiveList (the merkle-tree one and libssz's); alias the tree-backed one so the two cannot be confused. - execution_block_hash named a later part of the work as its replacement; say in specification terms what a gloas caller reads instead. - committees.rs named state.validators(), an accessor the element-level registry removed. --- .../state_transition/src/beacon/stf/gloas.rs | 15 ++++++++------- crates/common/types/src/beacon/committees.rs | 2 +- crates/common/types/src/beacon/containers/mod.rs | 8 ++++++-- .../common/types/src/beacon/containers/shared.rs | 6 +++--- crates/common/types/src/beacon/fork_choice.rs | 11 ++++++----- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/stf/gloas.rs b/crates/blockchain/state_transition/src/beacon/stf/gloas.rs index 65615b528..303dd403e 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/gloas.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/gloas.rs @@ -512,13 +512,14 @@ pub fn process_withdrawals(state: &mut BeaconState) -> Result<()> { /// built from the revealed envelope's own `execution_requests`. Not from /// `state.latest_execution_payload_bid`: the bid commits to that list only /// by its hash ([`gloas::ExecutionPayloadBid::execution_requests_root`]), it -/// never carries the list itself. Nothing calls this even now that envelope -/// verification is transcribed, the same place electra's own -/// [`crate::beacon::stf::electra::get_execution_requests_list`] already is -/// (see [`crate::beacon::stf::deneb::process_execution_payload`]'s own -/// documentation for why: [`ExecutionEngine`] collapses the whole -/// `verify_and_notify_new_payload` interface to one boolean and never -/// inspects the list either implementation builds). +/// never carries the list itself. Nothing calls this: envelope verification +/// does not build the request list, since [`ExecutionEngine`] collapses the +/// whole `verify_and_notify_new_payload` interface to one boolean and never +/// inspects it (see [`crate::beacon::stf::deneb::process_execution_payload`]'s +/// own documentation). Electra's own +/// [`crate::beacon::stf::electra::get_execution_requests_list`] is called, by +/// the live follower's Engine API client (`ethlambda-blockchain`'s +/// `beacon_engine`), which sends the list to a real execution client. pub fn get_execution_requests_list(requests: &gloas::ExecutionRequests) -> Vec> { let mut list = Vec::new(); diff --git a/crates/common/types/src/beacon/committees.rs b/crates/common/types/src/beacon/committees.rs index ba6d1606f..85875c869 100644 --- a/crates/common/types/src/beacon/committees.rs +++ b/crates/common/types/src/beacon/committees.rs @@ -54,7 +54,7 @@ use crate::beacon::signing::compute_epoch_at_slot; /// # Why the active set is not memoized on `epoch` or `seed` alone /// /// `get_active_validator_indices` reads `activation_epoch` and `exit_epoch` -/// off every validator in `state.validators()`, so it is a function of the +/// off every validator in `state.iter_validators()`, so it is a function of the /// state's registry, not of `epoch` or `seed` alone. Two different states can /// share an epoch number, or even a seed (it comes from a RANDAO mix fixed /// before either state's fork point, so two sibling branches diverging diff --git a/crates/common/types/src/beacon/containers/mod.rs b/crates/common/types/src/beacon/containers/mod.rs index 882b59698..d624b1e9c 100644 --- a/crates/common/types/src/beacon/containers/mod.rs +++ b/crates/common/types/src/beacon/containers/mod.rs @@ -1291,8 +1291,12 @@ impl SignedBeaconBlock { /// (`signed_execution_payload_bid`), not the payload itself. The block's /// own EL hash depends on whether that payload is later revealed and /// attested available, which this accessor, reading only the block, - /// cannot answer. Part B replaces this once the payload envelope and - /// availability tracking exist. + /// cannot answer. The hash a gloas block commits to is the bid's + /// `block_hash` (`signed_execution_payload_bid.message.block_hash`), and it + /// names an executed payload only once `on_execution_payload_envelope` has + /// verified a matching envelope, which is store state, so a caller that + /// needs it reads the store's payload verification for that block's root + /// (`is_payload_verified(store, root)`) rather than this accessor. pub fn execution_block_hash(&self) -> Option { match self { Self::Phase0(_) | Self::Altair(_) | Self::Gloas(_) | Self::Lean(_) => None, diff --git a/crates/common/types/src/beacon/containers/shared.rs b/crates/common/types/src/beacon/containers/shared.rs index fb2e65b67..c7d857f1c 100644 --- a/crates/common/types/src/beacon/containers/shared.rs +++ b/crates/common/types/src/beacon/containers/shared.rs @@ -9,7 +9,7 @@ use std::collections::BTreeMap; -use ethlambda_ssz_tree::{List, ProgressiveList}; +use ethlambda_ssz_tree::{List, ProgressiveList as TreeProgressiveList}; use libssz_derive::{HashTreeRoot, SszDecode, SszEncode}; use libssz_types::{SszBitvector, SszList, SszVector}; @@ -102,12 +102,12 @@ pub type HistoricalSummaries = SszList>; +pub type ProgressiveValidators = TreeProgressiveList>; /// The gloas balances, positionally parallel to [`ProgressiveValidators`]. /// Tree-backed like [`Balances`], with writes buffered densely (the default /// `VecMap`), since epoch processing writes every balance. -pub type ProgressiveBalances = ProgressiveList; +pub type ProgressiveBalances = TreeProgressiveList; // --------------------------------------------------------------------------- // Misc diff --git a/crates/common/types/src/beacon/fork_choice.rs b/crates/common/types/src/beacon/fork_choice.rs index f7515cf0d..c39486a2a 100644 --- a/crates/common/types/src/beacon/fork_choice.rs +++ b/crates/common/types/src/beacon/fork_choice.rs @@ -35,11 +35,12 @@ use crate::beacon::primitives::{Epoch, ExecutionBlockHash, Root, Slot, Uint256}; /// is what lets `update_latest_messages` tell such a re-vote apart from a /// stale one). Splitting the two into per-fork types would mean every reader /// of a [`LatestMessage`] picks a variant instead of a field, for a value that -/// is otherwise identical; carrying both instead lets every fork's own -/// constructor fill in the one it has and leave the other at its fork's own -/// neutral value (pre-gloas: `slot: data.slot, payload_present: false`; gloas -/// has no use for `epoch`, so nothing in this crate reads it off a gloas -/// message). +/// is otherwise identical; carrying both instead lets one constructor, +/// `update_latest_messages`, fill in both for every fork (`epoch` from the +/// attestation's target, `slot` from its data), with only the fork's own +/// comparison reading one of them: pre-gloas orders by `epoch`, gloas by +/// `slot`. `payload_present` is the one field a fork leaves neutral +/// (`false` before gloas). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct LatestMessage { pub epoch: Epoch, From 4137aa23d1817b457a9df17e190e944273a7f5dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 12:26:38 -0300 Subject: [PATCH 032/132] test(ssz-tree): cover progressive subtrees that span leaves The progressive list model tests stopped at 342 elements, where every packed u64 subtree still fits in one leaf of the paged tree. Each subtree is paged from its own start, so its first leaf edge falls at 852 for the subtree at 340 and at 1876 for the one at 1364. Add a boundary case around the subtree starts (340, 1364, 5460) and those edges, a deterministic case that writes on both sides of each edge through get_mut and pushes across them, and property tests over 1364 to 5461 elements (plus starts just short of each boundary) comparing hash_tree_root and the encoding with libssz's ProgressiveList. run_progressive takes the write path as a parameter, so the existing tests keep their IndexMut coverage. --- crates/common/ssz-tree/tests/model.rs | 117 ++++++++++++++++++++++++-- 1 file changed, 112 insertions(+), 5 deletions(-) diff --git a/crates/common/ssz-tree/tests/model.rs b/crates/common/ssz-tree/tests/model.rs index b3367404a..f41652401 100644 --- a/crates/common/ssz-tree/tests/model.rs +++ b/crates/common/ssz-tree/tests/model.rs @@ -546,7 +546,16 @@ where assert_eq!(root(&decoded), root(&reference)); } -fn run_progressive(initial: Vec, ops: Vec>) +/// How `run_progressive` writes an element in place. +#[derive(Clone, Copy)] +enum Write { + /// `list[index] = value`. + IndexMut, + /// `*list.get_mut(index) = value`, the path the beacon state's balances take. + GetMut, +} + +fn run_progressive(initial: Vec, ops: Vec>, write: Write) where T: Value + Debug, U: UpdateMap, @@ -563,7 +572,10 @@ where } Op::Set(index, value) if !model.is_empty() => { let index = index % model.len(); - list[index] = value.clone(); + match write { + Write::IndexMut => list[index] = value.clone(), + Write::GetMut => *list.get_mut(index).expect("in bounds") = value.clone(), + } model[index] = value; } Op::Set(..) => {} @@ -577,17 +589,17 @@ proptest! { // Packed: 4 per chunk, so subtrees start at elements 0, 4, 20, 84, 340. #[test] fn progressive_u64(initial in vec(any::(), 0..400), ops in ops(any::())) { - run_progressive::>(initial, ops); + run_progressive::>(initial, ops, Write::IndexMut); } // Composite: one per chunk, subtrees start at 0, 1, 5, 21, 85. #[test] fn progressive_item(initial in vec(item(), 0..100), ops in ops(item())) { - run_progressive::>(initial, ops); + run_progressive::>(initial, ops, Write::IndexMut); } // Variable-size elements: the offset table spans every subtree. #[test] fn progressive_blob(initial in vec(blob(), 0..30), ops in ops(blob())) { - run_progressive::>(initial, ops); + run_progressive::>(initial, ops, Write::IndexMut); } } @@ -606,6 +618,101 @@ fn progressive_boundaries_hash_like_libssz() { } } +// ── ProgressiveList: subtrees that span leaves ── +// +// The tests above stop at 342 elements, where every packed `u64` subtree still +// fits in one leaf of the paged tree (a leaf holds 512 `u64`s). Subtree `k` of +// a progressive list holds `4^k` chunks of four `u64`s, so packed `u64` +// subtrees start at element 0, 4, 20, 84, 340, 1364 and 5460, and each is +// paged from its own start. The subtree that starts at 340 holds 1024 +// elements, so its second leaf starts at element 852 (340 + 512). The one +// that starts at 1364 holds 4096, so its leaves start at 1364, 1876, 2388, and +// so on in steps of 512. The cases below run from element 1364 to 5461, which +// covers the whole of the subtree that starts at 1364 and the start of the +// next. + +proptest! { + // Each case hashes thousands of elements after every operation, so far + // fewer cases than the default keep the run short. + #![proptest_config(ProptestConfig::with_cases(16))] + + /// The whole subtree that starts at element 1364 (which spans eight + /// leaves), and the start of the one at 5460. + #[test] + fn progressive_u64_spanning_leaves( + initial in vec(any::(), 1364..=5461), + ops in ops(any::()), + ) { + run_progressive::>(initial, ops, Write::GetMut); + } + + /// Starts just short of element 1364, so pushes open that subtree. + #[test] + fn progressive_u64_pushes_open_a_spanning_subtree( + initial in vec(any::(), 1350..=1364), + ops in ops(any::()), + ) { + run_progressive::>(initial, ops, Write::GetMut); + } + + /// Starts just short of element 5460, so pushes open that subtree. + #[test] + fn progressive_u64_pushes_open_the_next_spanning_subtree( + initial in vec(any::(), 5440..=5460), + ops in ops(any::()), + ) { + run_progressive::>(initial, ops, Write::GetMut); + } +} + +#[test] +fn progressive_spanning_boundaries_hash_like_libssz() { + // Around the second leaf of the subtree that starts at 340 (852), around the + // start of the subtree that starts at 1364 and its second and third leaves + // (1876, 2388), and around the start of the next subtree (5460). + for len in [ + 851usize, 852, 853, 1363, 1364, 1365, 1875, 1876, 1877, 2387, 2388, 2389, 5459, 5460, 5461, + ] { + let values: Vec = (0..len as u64).collect(); + check_progressive_matches(&ProgressiveList::::from(values.clone()), &values); + } +} + +#[test] +fn progressive_writes_and_pushes_across_the_first_spanning_boundary() { + let mut model: Vec = (0..1363).collect(); + let mut list = ProgressiveList::::from(model.clone()); + check_progressive_matches(&list, &model); + + // Writes on both sides of the boundary and at leaf edges, pending until + // the push below opens the subtree that starts at element 1364. + for index in [0usize, 339, 340, 851, 852, 1362] { + *list.get_mut(index).expect("in bounds") = 0xdead_0000 + index as u64; + model[index] = 0xdead_0000 + index as u64; + } + check_progressive_matches(&list, &model); + + for value in 0..12u64 { + list.push(value).expect("a progressive list has no limit"); + model.push(value); + check_progressive_matches(&list, &model); + } + + // Grow past the second leaf of the new subtree (1876), then write on both + // sides of each edge. + for value in 12..520u64 { + list.push(value).expect("a progressive list has no limit"); + model.push(value); + } + check_progressive_matches(&list, &model); + for index in [1363usize, 1364, 1365, 1374, 1875, 1876] { + *list.get_mut(index).expect("in bounds") = !(index as u64); + model[index] = !(index as u64); + list.apply_updates(); + check_progressive_matches(&list, &model); + } +} + // ── ProgressiveList: iter_from ── #[test] From 1d667c4b6ecf023f67bd57bbd1c50ee4af39e31d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 13:56:27 -0300 Subject: [PATCH 033/132] docs(beacon): refresh the lib test counts The status table's lib counts predate the head walk's tests, the final-review fixes and the packing tests beacon-chain-integration brought in. The fixture counts are unchanged. --- docs/beacon_stf.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/beacon_stf.md b/docs/beacon_stf.md index e2cee346c..23a006932 100644 --- a/docs/beacon_stf.md +++ b/docs/beacon_stf.md @@ -526,8 +526,8 @@ that is not ignored passes on both presets: | Preset | Fixture cases passed | Ignored | Lib tests | |--------|----------------------|---------|-----------| -| mainnet | 7175 | 1285 | 422 passed, 1 ignored | -| minimal | 49261 | 5403 | 423 passed, 1 ignored | +| mainnet | 7175 | 1285 | 433 passed, 1 ignored | +| minimal | 49261 | 5403 | 434 passed, 1 ignored | The lib counts are with `beacon-spec-tests` on, and include lean's own unit tests, since the two chains share one lib target. The one ignored lib test is From 225461a1b4628a4e9856e4e5442335671dccb660 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:17:14 -0300 Subject: [PATCH 034/132] feat(beacon): validate gloas data column sidecars Gloas sidecars carry no signed header or inclusion proof: they name their block by root and slot, and the commitments their cells are checked against live in that block's bid. The fulu rules cannot judge them, so add gloas's modified data_column_sidecar rules beside them, keyed on (block root, index) per the modified Seen, with a fork-agnostic DataColumnSidecar enum in the types crate for the later plumbing to carry. A sidecar whose block has not been seen is queued (the spec's "MAY be queued"). A block seen without a post-state is queued rather than rejected, the same deviation the fulu parent rule makes until a bad-block cache exists, so that one vector is listed in SKIPPED. The spec-test runner now keeps a per-handler list of validated forks instead of a blanket fulu filter, and reads the per-message current_time_ms gloas's column vectors use in place of offset_ms. --- .../src/beacon/gossip/column.rs | 154 +++++++++++++++++- .../state_transition/src/beacon/gossip/mod.rs | 47 ++++++ .../tests/beacon_spec/gossip.rs | 120 ++++++++++---- .../common/types/src/beacon/containers/mod.rs | 152 +++++++++++++++++ 4 files changed, 443 insertions(+), 30 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/gossip/column.rs b/crates/blockchain/state_transition/src/beacon/gossip/column.rs index 2d7130abd..a74facbe2 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/column.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/column.rs @@ -10,17 +10,20 @@ use ethlambda_storage::CacheKey; use super::{ - IgnoreReason, Outcome, QueueReason, RejectReason, SeenColumns, finalized_ancestry, - finalized_start_slot, is_future_slot, + IgnoreReason, Outcome, QueueReason, RejectReason, SeenBlockColumns, SeenColumns, + finalized_ancestry, finalized_start_slot, is_future_slot, }; use crate::beacon::bls; use crate::beacon::config::Config; use crate::beacon::constants::{DATA_COLUMN_SIDECAR_SUBNET_COUNT, DOMAIN_BEACON_PROPOSER}; use crate::beacon::containers::BeaconState; +use crate::beacon::containers::SignedBeaconBlock; use crate::beacon::containers::fulu::DataColumnSidecar; +use crate::beacon::containers::gloas; use crate::beacon::fork_choice::{self, Store}; use crate::beacon::helpers::accessors::get_beacon_proposer_index; use crate::beacon::helpers::misc::{compute_domain, compute_epoch_at_slot, compute_signing_root}; +use crate::beacon::lean_boundary::lean_block_unreachable; use crate::beacon::precheck; use crate::beacon::primitives::HashTreeRoot as _; use crate::beacon::primitives::{Slot, ValidatorIndex}; @@ -306,6 +309,96 @@ pub fn validate( stateful_checks(store, sidecar) } +/// Gloas's `validate_data_column_sidecar_gossip` (`specs/gloas/p2p-interface.md`, +/// "Modified `data_column_sidecar_{subnet_id}`"), beside fulu's [`validate`]. +/// +/// One function rather than a cheap/stateful pair: a gloas sidecar carries no +/// header, so there is no signature, proposer or parent-state rule left to +/// put on a blocking thread. What remains is a block lookup and the KZG +/// batch, in the spec's own order. The caller records the sidecar in `seen` +/// once this answers [`Outcome::Accept`]. +/// +/// A block the store has not seen is [`Outcome::Queue`], the specification's +/// "MAY be queued until block is retrieved". A block seen without a post-state +/// is queued too where the specification rejects it, the same deviation +/// fulu's parent rule makes until a bad-block cache exists. +pub fn validate_gloas( + seen: &SeenBlockColumns, + store: &Store, + sidecar: &gloas::DataColumnSidecar, + subnet_id: u64, + now_ms: u64, +) -> Outcome { + let config = store.config(); + // [IGNORE] The first sidecar seen for this block root and column index. + if seen.contains(sidecar.beacon_block_root, sidecar.index) { + return Outcome::Ignore(IgnoreReason::AlreadySeen); + } + // [REJECT] The sidecar is for the correct subnet. + if sidecar.index % DATA_COLUMN_SIDECAR_SUBNET_COUNT != subnet_id { + return Outcome::Reject(RejectReason::WrongSubnet); + } + // [IGNORE] The sidecar is not from a future slot. + if is_future_slot(&config, sidecar.slot, now_ms) { + return Outcome::Ignore(IgnoreReason::FutureSlot); + } + // [IGNORE] Already stored, fetched over req/resp before gossip delivered + // it. Not a rule of the specification: it saves the KZG batch below. + if store.has_data_column(sidecar.slot, &sidecar.beacon_block_root, sidecar.index) { + return Outcome::Ignore(IgnoreReason::AlreadyStored); + } + // [IGNORE] A block for the sidecar has been seen (MAY queue). + let block = match store.get_signed_block(&sidecar.beacon_block_root) { + Ok(Some(block)) => block, + Ok(None) => return Outcome::Queue(QueueReason::BlockUnknown), + Err(_) => return Outcome::Ignore(IgnoreReason::Internal), + }; + // [REJECT] The block for the sidecar passes validation. A block seen + // without a post-state is queued rather than rejected, see above. + match store.has_state(&sidecar.beacon_block_root) { + Ok(true) => {} + Ok(false) => return Outcome::Queue(QueueReason::BlockNotReady), + Err(_) => return Outcome::Ignore(IgnoreReason::Internal), + } + // [REJECT] The sidecar's slot matches the slot of the block. + if sidecar.slot != block.slot() { + return Outcome::Reject(RejectReason::SlotMismatch); + } + let kzg_commitments = match &block { + SignedBeaconBlock::Gloas(block) => { + &block + .message + .body + .signed_execution_payload_bid + .message + .blob_kzg_commitments + } + // A block of any other fork has no bid to read commitments from, so + // no sidecar of this shape can be valid for it. + SignedBeaconBlock::Phase0(_) + | SignedBeaconBlock::Altair(_) + | SignedBeaconBlock::Bellatrix(_) + | SignedBeaconBlock::Capella(_) + | SignedBeaconBlock::Deneb(_) + | SignedBeaconBlock::Electra(_) + | SignedBeaconBlock::Fulu(_) => return Outcome::Reject(RejectReason::Malformed), + SignedBeaconBlock::Lean(_) => lean_block_unreachable("column::validate_gloas"), + }; + // [REJECT] The sidecar passes structural validation. + if !fork_choice::gloas_verify_data_column_sidecar(sidecar, kzg_commitments) { + return Outcome::Reject(RejectReason::Malformed); + } + // [REJECT] The sidecar's column data passes KZG verification. + let kzg = { + let _timing = metrics::time_data_column_kzg_verify(); + fork_choice::gloas_verify_data_column_sidecar_kzg_proofs(sidecar, kzg_commitments) + }; + if !matches!(kzg, Ok(true)) { + return Outcome::Reject(RejectReason::Kzg); + } + Outcome::Accept +} + /// Whether `parent_state`'s proposer lookahead covers `slot` at all. /// /// Checked last in [`stateful_checks`], after every REJECT-worthy rule: the @@ -933,4 +1026,61 @@ mod tests { ChainVerdict::Drop(Outcome::Reject(RejectReason::WrongProposer)) ); } + + /// Empty column, so only the rules before the block lookup can be reached. + fn gloas_sidecar(slot: Slot, index: u64) -> gloas::DataColumnSidecar { + gloas::DataColumnSidecar { + index, + slot, + beacon_block_root: Root::from([7; 32]), + ..Default::default() + } + } + + fn block_columns() -> SeenBlockColumns { + SeenBlockColumns::new(std::num::NonZeroUsize::new(8).expect("non-zero")) + } + + #[test] + fn a_gloas_sidecar_for_another_subnet_is_rejected() { + let store = store(0); + let now = slot_start_ms(&store, 5); + assert_eq!( + validate_gloas(&block_columns(), &store, &gloas_sidecar(5, 1), 0, now), + Outcome::Reject(RejectReason::WrongSubnet) + ); + } + + #[test] + fn a_gloas_sidecar_from_a_future_slot_is_ignored() { + let store = store(0); + let now = slot_start_ms(&store, 5) - DISPARITY - 1; + assert_eq!( + validate_gloas(&block_columns(), &store, &gloas_sidecar(5, 0), 0, now), + Outcome::Ignore(IgnoreReason::FutureSlot) + ); + } + + #[test] + fn a_gloas_sidecar_for_an_unseen_block_is_queued() { + let store = store(0); + let now = slot_start_ms(&store, 5); + assert_eq!( + validate_gloas(&block_columns(), &store, &gloas_sidecar(5, 0), 0, now), + Outcome::Queue(QueueReason::BlockUnknown) + ); + } + + #[test] + fn a_gloas_sidecar_seen_for_its_block_root_and_index_is_ignored() { + let store = store(0); + let now = slot_start_ms(&store, 5); + let sidecar = gloas_sidecar(5, 0); + let mut seen = block_columns(); + seen.record(sidecar.beacon_block_root, sidecar.index); + assert_eq!( + validate_gloas(&seen, &store, &sidecar, 0, now), + Outcome::Ignore(IgnoreReason::AlreadySeen) + ); + } } diff --git a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs index bd6657079..2094a1d76 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs @@ -73,6 +73,10 @@ pub enum QueueReason { ParentNotReady, /// Its slot is outside the parent state's proposer lookahead. ShufflingUnavailable, + /// A gloas sidecar names a block that has never been seen. + BlockUnknown, + /// A gloas sidecar names a block that is stored but has no post-state yet. + BlockNotReady, } impl QueueReason { @@ -81,6 +85,8 @@ impl QueueReason { Self::ParentUnknown => "parent_unknown", Self::ParentNotReady => "parent_not_ready", Self::ShufflingUnavailable => "shuffling_unavailable", + Self::BlockUnknown => "block_unknown", + Self::BlockNotReady => "block_not_ready", } } } @@ -158,6 +164,8 @@ pub enum RejectReason { PayloadTimestamp, InclusionProof, Kzg, + /// A gloas sidecar's slot is not the slot of the block it names. + SlotMismatch, /// An attestation's target epoch is not its slot's epoch. EpochMismatch, /// An aggregate with no aggregation bit set. @@ -202,6 +210,7 @@ impl RejectReason { Self::PayloadTimestamp => "payload_timestamp", Self::InclusionProof => "inclusion_proof", Self::Kzg => "kzg", + Self::SlotMismatch => "slot_mismatch", Self::EpochMismatch => "epoch_mismatch", Self::NoParticipants => "no_participants", Self::NonZeroDataIndex => "non_zero_data_index", @@ -282,6 +291,34 @@ impl SeenColumns { } } +/// The first valid gloas sidecar per `(block root, column index)`: the +/// specification's modified `Seen.data_column_sidecar_tuples`. +/// +/// A separate type from [`SeenColumns`] since the key differs: fulu's names the +/// proposer, which a gloas sidecar does not carry, and names the block by +/// `(slot, proposer)` where gloas names it by root. Bounded the same way. +pub struct SeenBlockColumns(LruCache<(Root, u64), ()>); + +impl SeenBlockColumns { + pub fn new(capacity: NonZeroUsize) -> Self { + Self(LruCache::new(capacity)) + } + + pub fn contains(&self, block_root: Root, index: u64) -> bool { + self.0.contains(&(block_root, index)) + } + + /// Record the first valid sidecar for its key. Returns `false`, changing + /// nothing, when one is already recorded. + pub fn record(&mut self, block_root: Root, index: u64) -> bool { + if self.0.contains(&(block_root, index)) { + return false; + } + self.0.put((block_root, index), ()); + true + } +} + /// The specification's `compute_time_at_slot_ms`: the clock reading at the /// start of `slot`. pub(crate) fn slot_start_ms(config: &Config, slot: Slot) -> u64 { @@ -446,6 +483,16 @@ mod tests { assert!(seen.record(10, 3, 1)); } + #[test] + fn a_block_column_key_records_once_per_root_and_index() { + let mut seen = SeenBlockColumns::new(capacity(4)); + let root = Root::from([1; 32]); + assert!(seen.record(root, 0)); + assert!(!seen.record(root, 0)); + assert!(seen.record(root, 1)); + assert!(seen.record(Root::from([2; 32]), 0)); + } + #[test] fn the_caches_forget_their_oldest_entry_past_capacity() { let mut seen = SeenBlocks::new(capacity(2)); diff --git a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs index a3c0a81f2..85e355dec 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs @@ -16,13 +16,15 @@ use std::sync::Arc; use ethlambda_state_transition::beacon::ForkName; use ethlambda_state_transition::beacon::config::Config; use ethlambda_state_transition::beacon::containers::{ - BeaconState, Checkpoint, SignedAggregateAndProof, SignedBeaconBlock, electra, fulu, phase0, + BeaconState, Checkpoint, DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock, + electra, phase0, }; use ethlambda_state_transition::beacon::fork_choice::{ self, DataAvailability, PayloadValidity, Store, }; use ethlambda_state_transition::beacon::gossip::{ - self as rules, Outcome, SeenAggregates, SeenAttestations, SeenBlocks, SeenColumns, + self as rules, Outcome, SeenAggregates, SeenAttestations, SeenBlockColumns, SeenBlocks, + SeenColumns, }; use ethlambda_state_transition::beacon::helpers::accessors::CommitteeCache; use ethlambda_state_transition::beacon::primitives::Root; @@ -40,6 +42,26 @@ const HANDLERS: &[&str] = &[ "gossip_beacon_attestation", ]; +/// The forks each of [`HANDLERS`] validates. A case from any other fork is +/// reported as ignored rather than run, the same "known gap, not a silent one" +/// treatment `case.in_scope()` gives a fork past `HIGHEST_IMPLEMENTED_FORK`. +/// +/// `case.in_scope()` alone would pass every fork up to gloas, since the state +/// transition handles them all; the gossip rules do not. The block rules +/// (`beacon::gossip::block`) are fulu's `validate_beacon_block_gossip`, and the +/// aggregate and attestation rules (`beacon::gossip::{aggregate, attestation}`) +/// are electra's, which fulu keeps. The column rules have both fulu's and +/// gloas's (`beacon::gossip::column`). +fn validated_forks(handler: &str) -> &'static [ForkName] { + match handler { + "gossip_data_column_sidecar" => &[ForkName::Fulu, ForkName::Gloas], + "gossip_beacon_block" + | "gossip_beacon_aggregate_and_proof" + | "gossip_beacon_attestation" => &[ForkName::Fulu], + other => panic!("{other} is not in HANDLERS, so it has no validated forks"), + } +} + /// Every other `gossip_*` handler the fixture tree ships, none of which this /// node validates yet: the topics it neither subscribes to nor has a /// validator for. Reported as ignored, named individually, rather than left @@ -82,6 +104,10 @@ const SKIPPED: &[(&str, &str)] = &[ "gossip_data_column_sidecar__reject_parent_failed_validation", "a parent seen without a post-state is queued, not rejected, until a bad-block cache exists", ), + ( + "gossip_data_column_sidecar__reject_block_failed_validation", + "a block seen without a post-state is queued, not rejected, until a bad-block cache exists", + ), ( "gossip_beacon_aggregate_and_proof__reject_block_failed_validation", "a vote block seen without a post-state is ignored, not rejected, until a bad-block cache exists", @@ -124,13 +150,31 @@ struct FinalizedOverride { #[derive(serde::Deserialize)] struct GossipMessage { - offset_ms: u64, + /// Missing from gloas's column vectors, which name the message's clock + /// reading in `current_time_ms` instead; see [`GossipMessage::arrival_ms`]. + offset_ms: Option, + current_time_ms: Option, subnet_id: Option, message: String, expected: String, reason: Option, } +impl GossipMessage { + /// The clock reading, in milliseconds since genesis, the message arrives + /// at: the format's `meta.current_time_ms + offset_ms`. Gloas's column + /// vectors instead carry a per-message `current_time_ms`, which is read as + /// the absolute reading (every such vector states a time later than the + /// meta's own base and inside the slot its sidecar is for). + fn arrival_ms(&self, base_ms: u64) -> u64 { + match (self.offset_ms, self.current_time_ms) { + (Some(offset), _) => base_ms + offset, + (None, Some(absolute)) => absolute, + (None, None) => base_ms, + } + } +} + fn parse_root(hex_root: &str) -> Root { let stripped = hex_root.strip_prefix("0x").unwrap_or(hex_root); let bytes = hex::decode(stripped).expect("the fixture's root is valid hex"); @@ -205,13 +249,21 @@ fn build_store( .map_err(|err| format!("get_forkchoice_store: {err:?}"))?; // The store's clock at the case's base time, so `on_block` accepts every - // listed block. - let now_s = (config.genesis_time_ms() + meta.current_time_ms) / 1000; - fork_choice::on_tick(&mut store, now_s, config); + // listed block; a block from a slot past that time advances it to that + // slot's start, since a vector may place its base time just before the + // slot a clock-disparity case sends its sidecar for. + let mut clock_s = (config.genesis_time_ms() + meta.current_time_ms) / 1000; + fork_choice::on_tick(&mut store, clock_s, config); for entry in rest { let block = decode_block(case, &entry.block)?; let root = block.message_hash_tree_root(); + let block_start_s = + (config.genesis_time_ms() + block.slot() * config.slot_duration_ms) / 1000; + if block_start_s > clock_s { + clock_s = block_start_s; + fork_choice::on_tick(&mut store, clock_s, config); + } // Seen without a post-state. An `INVALIDATED` payload lands here too: // this store never keeps a post-state for one, since `on_block` fails // it and invalidates the branch. @@ -288,11 +340,12 @@ fn run_case(case: &Case) -> Result<(), String> { let capacity = NonZeroUsize::new(SEEN_CAPACITY).expect("non-zero"); let mut seen_blocks = SeenBlocks::new(capacity); let mut seen_columns = SeenColumns::new(capacity); + let mut seen_block_columns = SeenBlockColumns::new(capacity); let mut seen_aggregates = SeenAggregates::new(capacity, capacity); let mut seen_attestations = SeenAttestations::new(capacity); for (index, message) in meta.messages.iter().enumerate() { - let now_ms = config.genesis_time_ms() + meta.current_time_ms + message.offset_ms; + let now_ms = config.genesis_time_ms() + message.arrival_ms(meta.current_time_ms); let outcome = match meta.topic.as_str() { "beacon_block" => { let block = decode_block(case, &message.message)?; @@ -305,18 +358,40 @@ fn run_case(case: &Case) -> Result<(), String> { } "data_column_sidecar" => { let sidecar = - fulu::DataColumnSidecar::from_ssz_bytes(&case.ssz_bytes(&message.message)) + DataColumnSidecar::from_ssz(case.fork, &case.ssz_bytes(&message.message)) .map_err(|err| format!("decoding {}: {err:?}", message.message))?; let subnet_id = message .subnet_id .ok_or("a data_column_sidecar message names its subnet")?; - let outcome = - rules::column::validate(&seen_columns, &store, &sidecar, subnet_id, now_ms); - if outcome == Outcome::Accept { - let header = &sidecar.signed_block_header.message; - seen_columns.record(header.slot, header.proposer_index, sidecar.index); + match sidecar { + DataColumnSidecar::Fulu(sidecar) => { + let outcome = rules::column::validate( + &seen_columns, + &store, + &sidecar, + subnet_id, + now_ms, + ); + if outcome == Outcome::Accept { + let header = &sidecar.signed_block_header.message; + seen_columns.record(header.slot, header.proposer_index, sidecar.index); + } + outcome + } + DataColumnSidecar::Gloas(sidecar) => { + let outcome = rules::column::validate_gloas( + &seen_block_columns, + &store, + &sidecar, + subnet_id, + now_ms, + ); + if outcome == Outcome::Accept { + seen_block_columns.record(sidecar.beacon_block_root, sidecar.index); + } + outcome + } } - outcome } "beacon_aggregate_and_proof" => { let aggregate = decode_signed_aggregate(case, &message.message)?; @@ -369,21 +444,10 @@ pub fn trials() -> Vec { cases.len(), )); for case in cases { - // The block and column rules (`beacon::gossip::{block, column}`) - // are fulu's own `validate_beacon_block_gossip` and - // `validate_data_column_sidecar_gossip`. The aggregate and - // attestation rules (`beacon::gossip::{aggregate, attestation}`) - // are electra's (`p2p-interface.md`), which fulu keeps. Either - // way a case from any other fork is ignored rather than run, the - // same "known gap, not a silent one" treatment `case.in_scope()` - // already gives a fork - // past `HIGHEST_IMPLEMENTED_FORK`. `case.in_scope()` alone would - // pass every fork up to gloas, since the state transition handles - // them all; the gossip rules do not. In particular the node does - // not validate gloas gossip yet, so a gloas case stays ignored - // here even though its state transition runs. + // A fork the handler's rules do not cover is ignored rather than + // run; see [`validated_forks`]. let ignored = !case.in_scope() - || case.fork != ForkName::Fulu + || !validated_forks(handler).contains(&case.fork) || SKIPPED.iter().any(|(name, _)| *name == case.name); trials.push(super::case_trial("gossip", case, run_case).with_ignored_flag(ignored)); } diff --git a/crates/common/types/src/beacon/containers/mod.rs b/crates/common/types/src/beacon/containers/mod.rs index d624b1e9c..7dc56c216 100644 --- a/crates/common/types/src/beacon/containers/mod.rs +++ b/crates/common/types/src/beacon/containers/mod.rs @@ -1515,6 +1515,90 @@ signed_beacon_block_accessors!( ], ); +/// A data column sidecar in either shape: fulu's carries a signed header and +/// an inclusion proof, gloas's names its block by root and reads its +/// commitments from that block's bid. +/// +/// Two variants rather than one per fork, since the shape changes only at +/// gloas; [`DataColumnSidecar::fork`] answers which fork's rules apply. +// Fulu's carries a header and an inclusion proof inline, so the variants differ +// in size; a sidecar is held and passed by value, and boxing one would only make +// every reader dereference it. +#[allow(clippy::large_enum_variant)] +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DataColumnSidecar { + Fulu(fulu::DataColumnSidecar), + Gloas(gloas::DataColumnSidecar), +} + +impl DataColumnSidecar { + /// The column this sidecar carries. + pub fn index(&self) -> u64 { + match self { + Self::Fulu(sidecar) => sidecar.index, + Self::Gloas(sidecar) => sidecar.index, + } + } + + /// The slot of the block this sidecar belongs to. + pub fn slot(&self) -> Slot { + match self { + Self::Fulu(sidecar) => sidecar.signed_block_header.message.slot, + Self::Gloas(sidecar) => sidecar.slot, + } + } + + /// The root of the block this sidecar belongs to: fulu's is the header's + /// hash tree root, gloas's is named outright. + pub fn block_root(&self) -> Root { + match self { + Self::Fulu(sidecar) => sidecar.signed_block_header.message.hash_tree_root(), + Self::Gloas(sidecar) => sidecar.beacon_block_root, + } + } + + /// The fork whose rules apply to this sidecar. + pub fn fork(&self) -> ForkName { + match self { + Self::Fulu(_) => ForkName::Fulu, + Self::Gloas(_) => ForkName::Gloas, + } + } + + /// Decodes a sidecar of a known fork; the bytes carry no tag, so the fork + /// comes from context (the gossip topic's digest, a request's fork digest). + /// + /// Forks before fulu have no data columns and lean has none at all, so + /// they answer with an error rather than a panic: the fork comes off the + /// wire. + pub fn from_ssz(fork: ForkName, bytes: &[u8]) -> Result { + match fork { + ForkName::Fulu => Ok(Self::Fulu(fulu::DataColumnSidecar::from_ssz_bytes(bytes)?)), + ForkName::Gloas => Ok(Self::Gloas(gloas::DataColumnSidecar::from_ssz_bytes( + bytes, + )?)), + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Lean => Err(Error::UnsupportedForFork { + function: "DataColumnSidecar", + fork, + }), + } + } + + /// Encodes the sidecar. + pub fn to_ssz(&self) -> Vec { + match self { + Self::Fulu(sidecar) => sidecar.to_ssz(), + Self::Gloas(sidecar) => sidecar.to_ssz(), + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -2204,4 +2288,72 @@ mod tests { assert_eq!(block.blob_kzg_commitment_count(), 0); assert_eq!(block.execution_payload_timestamp(), None); } + + fn fulu_sidecar(index: u64, header: BeaconBlockHeader) -> fulu::DataColumnSidecar { + fulu::DataColumnSidecar { + index, + column: Default::default(), + kzg_commitments: Default::default(), + kzg_proofs: Default::default(), + signed_block_header: SignedBeaconBlockHeader { + message: header, + signature: Default::default(), + }, + kzg_commitments_inclusion_proof: vec![ + Root::ZERO; + preset::KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH + ] + .try_into() + .unwrap(), + } + } + + #[test] + fn a_data_column_sidecar_round_trips_per_fork() { + let header = BeaconBlockHeader { + slot: 9, + ..Default::default() + }; + let fulu = DataColumnSidecar::Fulu(fulu_sidecar(7, header)); + let gloas = DataColumnSidecar::Gloas(gloas::DataColumnSidecar { + index: 3, + slot: 11, + beacon_block_root: Root::from([5; 32]), + ..Default::default() + }); + for sidecar in [fulu, gloas] { + let decoded = DataColumnSidecar::from_ssz(sidecar.fork(), &sidecar.to_ssz()).unwrap(); + assert_eq!(decoded, sidecar); + } + } + + #[test] + fn a_data_column_sidecar_reports_its_block_root_per_variant() { + let header = BeaconBlockHeader { + slot: 9, + proposer_index: 2, + ..Default::default() + }; + let fulu = DataColumnSidecar::Fulu(fulu_sidecar(7, header.clone())); + assert_eq!(fulu.block_root(), header.hash_tree_root()); + assert_eq!((fulu.index(), fulu.slot()), (7, 9)); + assert_eq!(fulu.fork(), ForkName::Fulu); + + let root = Root::from([5; 32]); + let gloas = DataColumnSidecar::Gloas(gloas::DataColumnSidecar { + index: 3, + slot: 11, + beacon_block_root: root, + ..Default::default() + }); + assert_eq!(gloas.block_root(), root); + assert_eq!((gloas.index(), gloas.slot()), (3, 11)); + assert_eq!(gloas.fork(), ForkName::Gloas); + } + + #[test] + fn a_data_column_sidecar_does_not_decode_before_fulu_or_as_lean() { + assert!(DataColumnSidecar::from_ssz(ForkName::Electra, &[]).is_err()); + assert!(DataColumnSidecar::from_ssz(ForkName::Lean, &[]).is_err()); + } } From beb3b8368c992160e479c73d1200a43fc15163fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:28:07 -0300 Subject: [PATCH 035/132] refactor(beacon): split the gloas column rule and test its deviations Fulu's column rule is split into a cheap half and a stateful half so a duplicate is dropped without a blocking-pool permit and the KZG batch stays off the swarm loop; the p2p wiring will call the halves separately. Give the gloas rule the same shape, with validate_gloas as the combined form. The stateful half now checks for a post-state before decoding the block, so a block without one costs no read of it. A sidecar naming a block of an earlier fork gets its own reject reason rather than reusing Malformed. Tests pin the deliberate deviations: a block without a post-state is queued, an already stored sidecar is ignored, and a non-gloas block is rejected by name. The spec runner now fails a message that names no clock reading, and the pre-fulu decode test uses bytes valid for fulu so it can only fail on the fork. --- .../src/beacon/gossip/column.rs | 132 ++++++++++++++---- .../state_transition/src/beacon/gossip/mod.rs | 4 + .../tests/beacon_spec/gossip.rs | 32 +++-- .../common/types/src/beacon/containers/mod.rs | 11 +- 4 files changed, 137 insertions(+), 42 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/gossip/column.rs b/crates/blockchain/state_transition/src/beacon/gossip/column.rs index a74facbe2..b5aa7c010 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/column.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/column.rs @@ -1,5 +1,7 @@ //! `data_column_sidecar_{subnet_id}` gossip validation: fulu's -//! `validate_data_column_sidecar_gossip` (`specs/fulu/p2p-interface.md`). +//! `validate_data_column_sidecar_gossip` (`specs/fulu/p2p-interface.md`), and +//! gloas's modified one (`specs/gloas/p2p-interface.md`) in the `_gloas` +//! functions beside it. //! //! Also every check the chain relies on before it keeps a sidecar gossip did //! not accept ([`chain_checks`]): one fetched over req/resp, one gossip queued @@ -309,57 +311,70 @@ pub fn validate( stateful_checks(store, sidecar) } -/// Gloas's `validate_data_column_sidecar_gossip` (`specs/gloas/p2p-interface.md`, -/// "Modified `data_column_sidecar_{subnet_id}`"), beside fulu's [`validate`]. -/// -/// One function rather than a cheap/stateful pair: a gloas sidecar carries no -/// header, so there is no signature, proposer or parent-state rule left to -/// put on a blocking thread. What remains is a block lookup and the KZG -/// batch, in the spec's own order. The caller records the sidecar in `seen` -/// once this answers [`Outcome::Accept`]. +/// The gloas rules that read only the sidecar, the clock and the store's +/// metadata: the half [`validate_gloas`] runs first, beside fulu's +/// [`cheap_checks`]. /// -/// A block the store has not seen is [`Outcome::Queue`], the specification's -/// "MAY be queued until block is retrieved". A block seen without a post-state -/// is queued too where the specification rejects it, the same deviation -/// fulu's parent rule makes until a bad-block cache exists. -pub fn validate_gloas( +/// Split from [`stateful_checks_gloas`] for the reason fulu's is: a duplicate +/// or a misrouted sidecar is dropped without taking a blocking-pool permit, +/// and the KZG batch stays off the swarm loop. `Err` carries the verdict; `Ok` +/// sends the sidecar on to [`stateful_checks_gloas`]. The caller records the +/// sidecar in `seen` once the whole rule answers [`Outcome::Accept`]. +pub fn cheap_checks_gloas( seen: &SeenBlockColumns, store: &Store, sidecar: &gloas::DataColumnSidecar, subnet_id: u64, now_ms: u64, -) -> Outcome { +) -> Result<(), Outcome> { let config = store.config(); // [IGNORE] The first sidecar seen for this block root and column index. if seen.contains(sidecar.beacon_block_root, sidecar.index) { - return Outcome::Ignore(IgnoreReason::AlreadySeen); + return Err(Outcome::Ignore(IgnoreReason::AlreadySeen)); } // [REJECT] The sidecar is for the correct subnet. if sidecar.index % DATA_COLUMN_SIDECAR_SUBNET_COUNT != subnet_id { - return Outcome::Reject(RejectReason::WrongSubnet); + return Err(Outcome::Reject(RejectReason::WrongSubnet)); } // [IGNORE] The sidecar is not from a future slot. if is_future_slot(&config, sidecar.slot, now_ms) { - return Outcome::Ignore(IgnoreReason::FutureSlot); + return Err(Outcome::Ignore(IgnoreReason::FutureSlot)); } // [IGNORE] Already stored, fetched over req/resp before gossip delivered - // it. Not a rule of the specification: it saves the KZG batch below. + // it. Not a rule of the specification: it saves the KZG batch. if store.has_data_column(sidecar.slot, &sidecar.beacon_block_root, sidecar.index) { - return Outcome::Ignore(IgnoreReason::AlreadyStored); + return Err(Outcome::Ignore(IgnoreReason::AlreadyStored)); } + Ok(()) +} + +/// The gloas rules that need the named block and its post-state, then the KZG +/// batch. Runs on a blocking thread, beside fulu's [`stateful_checks`]. +/// +/// A block the store has not seen is [`Outcome::Queue`], the specification's +/// "MAY be queued until block is retrieved". A block seen without a post-state +/// is queued too where the specification rejects it, the same deviation +/// fulu's parent rule makes until a bad-block cache exists. +pub fn stateful_checks_gloas(store: &Store, sidecar: &gloas::DataColumnSidecar) -> Outcome { + let root = sidecar.beacon_block_root; // [IGNORE] A block for the sidecar has been seen (MAY queue). - let block = match store.get_signed_block(&sidecar.beacon_block_root) { - Ok(Some(block)) => block, - Ok(None) => return Outcome::Queue(QueueReason::BlockUnknown), - Err(_) => return Outcome::Ignore(IgnoreReason::Internal), - }; + if !store.has_block(&root) { + return Outcome::Queue(QueueReason::BlockUnknown); + } // [REJECT] The block for the sidecar passes validation. A block seen - // without a post-state is queued rather than rejected, see above. - match store.has_state(&sidecar.beacon_block_root) { + // without a post-state is queued rather than rejected, see above. Checked + // before the block is decoded, so one without a state costs no read of it. + match store.has_state(&root) { Ok(true) => {} Ok(false) => return Outcome::Queue(QueueReason::BlockNotReady), Err(_) => return Outcome::Ignore(IgnoreReason::Internal), } + let block = match store.get_signed_block(&root) { + Ok(Some(block)) => block, + // Stored a moment ago and gone now (pruned): nothing to judge against. + Ok(None) => return Outcome::Queue(QueueReason::BlockUnknown), + Err(_) => return Outcome::Ignore(IgnoreReason::Internal), + }; // [REJECT] The sidecar's slot matches the slot of the block. if sidecar.slot != block.slot() { return Outcome::Reject(RejectReason::SlotMismatch); @@ -381,8 +396,8 @@ pub fn validate_gloas( | SignedBeaconBlock::Capella(_) | SignedBeaconBlock::Deneb(_) | SignedBeaconBlock::Electra(_) - | SignedBeaconBlock::Fulu(_) => return Outcome::Reject(RejectReason::Malformed), - SignedBeaconBlock::Lean(_) => lean_block_unreachable("column::validate_gloas"), + | SignedBeaconBlock::Fulu(_) => return Outcome::Reject(RejectReason::BlockNotGloas), + SignedBeaconBlock::Lean(_) => lean_block_unreachable("column::stateful_checks_gloas"), }; // [REJECT] The sidecar passes structural validation. if !fork_choice::gloas_verify_data_column_sidecar(sidecar, kzg_commitments) { @@ -399,6 +414,23 @@ pub fn validate_gloas( Outcome::Accept } +/// [`cheap_checks_gloas`] then [`stateful_checks_gloas`], for callers with no +/// reason to split them, such as the spec vectors: gloas's +/// `validate_data_column_sidecar_gossip` (`specs/gloas/p2p-interface.md`, +/// "Modified `data_column_sidecar_{subnet_id}`"), beside fulu's [`validate`]. +pub fn validate_gloas( + seen: &SeenBlockColumns, + store: &Store, + sidecar: &gloas::DataColumnSidecar, + subnet_id: u64, + now_ms: u64, +) -> Outcome { + if let Err(outcome) = cheap_checks_gloas(seen, store, sidecar, subnet_id, now_ms) { + return outcome; + } + stateful_checks_gloas(store, sidecar) +} + /// Whether `parent_state`'s proposer lookahead covers `slot` at all. /// /// Checked last in [`stateful_checks`], after every REJECT-worthy rule: the @@ -1083,4 +1115,46 @@ mod tests { Outcome::Ignore(IgnoreReason::AlreadySeen) ); } + + #[test] + fn a_gloas_sidecar_whose_block_has_no_post_state_is_queued() { + let mut store = store(0); + let card = gloas_sidecar(5, 0); + store + .insert_pending_block(card.beacon_block_root, parent_block(5)) + .expect("insert pending block"); + assert_eq!( + stateful_checks_gloas(&store, &card), + Outcome::Queue(QueueReason::BlockNotReady) + ); + } + + #[test] + fn an_already_stored_gloas_sidecar_is_ignored() { + let store = store(0); + let card = gloas_sidecar(5, 0); + store + .put_data_column_sidecar(card.slot, &card.beacon_block_root, card.index, Vec::new()) + .expect("store the column"); + assert_eq!( + cheap_checks_gloas(&block_columns(), &store, &card, 0, slot_start_ms(&store, 5)), + Err(Outcome::Ignore(IgnoreReason::AlreadyStored)) + ); + } + + #[test] + fn a_gloas_sidecar_naming_a_non_gloas_block_is_rejected() { + let mut store = store(0); + let card = gloas_sidecar(5, 0); + store + .insert_pending_block(card.beacon_block_root, parent_block(5)) + .expect("insert the block"); + store + .insert_state(card.beacon_block_root, fulu_parent(3)) + .expect("insert the state"); + assert_eq!( + stateful_checks_gloas(&store, &card), + Outcome::Reject(RejectReason::BlockNotGloas) + ); + } } diff --git a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs index 2094a1d76..17893a082 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs @@ -166,6 +166,9 @@ pub enum RejectReason { Kzg, /// A gloas sidecar's slot is not the slot of the block it names. SlotMismatch, + /// A gloas sidecar names a block of an earlier fork, which has no bid to + /// take commitments from. + BlockNotGloas, /// An attestation's target epoch is not its slot's epoch. EpochMismatch, /// An aggregate with no aggregation bit set. @@ -211,6 +214,7 @@ impl RejectReason { Self::InclusionProof => "inclusion_proof", Self::Kzg => "kzg", Self::SlotMismatch => "slot_mismatch", + Self::BlockNotGloas => "block_not_gloas", Self::EpochMismatch => "epoch_mismatch", Self::NoParticipants => "no_participants", Self::NonZeroDataIndex => "non_zero_data_index", diff --git a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs index 85e355dec..d5e612fe1 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs @@ -150,8 +150,9 @@ struct FinalizedOverride { #[derive(serde::Deserialize)] struct GossipMessage { - /// Missing from gloas's column vectors, which name the message's clock - /// reading in `current_time_ms` instead; see [`GossipMessage::arrival_ms`]. + /// The format's offset from `meta.current_time_ms`. Some of gloas's + /// column vectors carry `current_time_ms` instead; see + /// [`GossipMessage::arrival_ms`]. offset_ms: Option, current_time_ms: Option, subnet_id: Option, @@ -162,15 +163,21 @@ struct GossipMessage { impl GossipMessage { /// The clock reading, in milliseconds since genesis, the message arrives - /// at: the format's `meta.current_time_ms + offset_ms`. Gloas's column - /// vectors instead carry a per-message `current_time_ms`, which is read as - /// the absolute reading (every such vector states a time later than the - /// meta's own base and inside the slot its sidecar is for). - fn arrival_ms(&self, base_ms: u64) -> u64 { + /// at: the format's `meta.current_time_ms + offset_ms`. + /// + /// Some of gloas's column vectors carry a per-message `current_time_ms` + /// instead: the generator (`test/gloas/networking/ + /// test_gossip_data_column_sidecar.py`) writes `compute_time_at_slot_ms` + /// plus 500 there, the same basis as `meta.current_time_ms`, so it is the + /// absolute reading. + fn arrival_ms(&self, base_ms: u64) -> Result { match (self.offset_ms, self.current_time_ms) { - (Some(offset), _) => base_ms + offset, - (None, Some(absolute)) => absolute, - (None, None) => base_ms, + (Some(offset), _) => Ok(base_ms + offset), + (None, Some(absolute)) => Ok(absolute), + (None, None) => Err(format!( + "{}: names neither offset_ms nor current_time_ms", + self.message + )), } } } @@ -345,7 +352,10 @@ fn run_case(case: &Case) -> Result<(), String> { let mut seen_attestations = SeenAttestations::new(capacity); for (index, message) in meta.messages.iter().enumerate() { - let now_ms = config.genesis_time_ms() + message.arrival_ms(meta.current_time_ms); + let now_ms = config.genesis_time_ms() + + message + .arrival_ms(meta.current_time_ms) + .map_err(|err| format!("message {index}: {err}"))?; let outcome = match meta.topic.as_str() { "beacon_block" => { let block = decode_block(case, &message.message)?; diff --git a/crates/common/types/src/beacon/containers/mod.rs b/crates/common/types/src/beacon/containers/mod.rs index 7dc56c216..89f466e73 100644 --- a/crates/common/types/src/beacon/containers/mod.rs +++ b/crates/common/types/src/beacon/containers/mod.rs @@ -2353,7 +2353,14 @@ mod tests { #[test] fn a_data_column_sidecar_does_not_decode_before_fulu_or_as_lean() { - assert!(DataColumnSidecar::from_ssz(ForkName::Electra, &[]).is_err()); - assert!(DataColumnSidecar::from_ssz(ForkName::Lean, &[]).is_err()); + let valid = DataColumnSidecar::Fulu(fulu_sidecar(7, BeaconBlockHeader::default())).to_ssz(); + // Bytes that decode fine as fulu's, so the error can only be the fork. + assert!(DataColumnSidecar::from_ssz(ForkName::Fulu, &valid).is_ok()); + for fork in [ForkName::Electra, ForkName::Lean] { + assert!(matches!( + DataColumnSidecar::from_ssz(fork, &valid), + Err(Error::UnsupportedForFork { fork: got, .. }) if got == fork + )); + } } } From d91c6e5e6e4832a6da171bcb78f4dd01d2d813ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:36:49 -0300 Subject: [PATCH 036/132] =?UTF-8?q?feat(beacon):=20add=20Plat=C3=A5berget?= =?UTF-8?q?=20as=20a=20built-in=20network?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Platåberget is the long-lived Glamsterdam testnet (glamsterdam-devnet-8), meant to run for months as the place to test post-Glamsterdam Ethereum before Sepolia and Hoodi fork. It is the one public chain already past gloas, so building it in gives the gloas follower a live target: once the follower takes gloas blocks, `--network plataberget` follows it without fetching a network directory. It has no eth-clients repo, so its config.yaml and bootstrap_nodes.yaml come byte for byte from ethpandaops/glamsterdam-devnets' network-configs/devnet-8. The genesis values are read off that directory's genesis.ssz and match what its checkpoint provider serves. The root and both fork versions are pinned offline by the digests its bootnode ENRs publish, from both sides of the gloas fork. The file says `CONFIG_NAME: 'testnet'`, a placeholder ethpandaops keeps for Prysm, so a built-in network now takes its CONFIG_NAME from its `--network` name. The eth-clients files already carry their own names, so nothing changes for mainnet, Sepolia or Hoodi. Checkpoint sync against Platåberget downloads a gloas anchor and verifies it against these genesis values, then stops at `refuse_unfollowable_fork`, since the follower does not take gloas blocks yet. --- CLAUDE.md | 19 +- README.md | 6 +- .../assets/plataberget/bootstrap_nodes.yaml | 20 ++ bin/ethlambda/assets/plataberget/config.yaml | 242 ++++++++++++++++++ bin/ethlambda/src/checkpoint_sync.rs | 7 +- bin/ethlambda/src/cli.rs | 15 +- bin/ethlambda/src/main.rs | 10 +- bin/ethlambda/src/network/built_in.rs | 100 ++++++-- bin/ethlambda/src/network/mod.rs | 4 +- docs/beacon_wire.md | 6 +- docs/checkpoint_sync.md | 4 +- docs/cli.md | 6 +- 12 files changed, 391 insertions(+), 48 deletions(-) create mode 100644 bin/ethlambda/assets/plataberget/bootstrap_nodes.yaml create mode 100644 bin/ethlambda/assets/plataberget/config.yaml diff --git a/CLAUDE.md b/CLAUDE.md index 1867d5dc5..5ef12fd3a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,8 +19,8 @@ bin/ethlambda/ # Entry point, CLI, orchestration ├─ src/beacon.rs # Beacon wire params derived from a resolved network: epoch, fork digest ├─ src/checkpoint_sync.rs # Checkpoint sync for both chains (lean's `/lean/v0/...`, beacon's Beacon API) ├─ src/network/ # --network resolution: built-in name vs. directory of published files - │ └─ built_in.rs # Built-in chains (mainnet, sepolia, hoodi) and their genesis constants - ├─ assets/{mainnet,sepolia,hoodi}/ # config.yaml + bootstrap_nodes.yaml (eth-clients/'s files) + │ └─ built_in.rs # Built-in chains (mainnet, sepolia, hoodi, plataberget) and their genesis constants + ├─ assets/{mainnet,sepolia,hoodi,plataberget}/ # config.yaml + bootstrap_nodes.yaml (each publisher's files) ├─ tests/fixtures/networks/mainnet/genesis.ssz # Mainnet genesis state, test-only └─ src/version.rs # Build-time version info (vergen-git2) crates/ @@ -429,7 +429,7 @@ lean does (it used to park on `std::future::pending()`). ### Built-in networks `beacon` takes a `--network` flag (built-in name `mainnet` (default), -`sepolia` or `hoodi`, or a path to a directory of published network files) +`sepolia`, `hoodi` or `plataberget`, or a path to a directory of published network files) and resolves it into a `NetworkSource` before doing anything else; see `bin/ethlambda/src/network/`. `genesis_time` and `genesis_validators_root`, which the fork digest keying every gossip topic, the ENR `eth2` entry and @@ -437,14 +437,15 @@ discv5 admission are computed from, come from that resolved network. Every built-in chain is a `network::built_in::EmbeddedChain`: its `eth-clients` repo's `config.yaml` and `bootstrap_nodes.yaml` byte for byte -(`bin/ethlambda/assets//`), parsed through the same +(Platåberget's come from `ethpandaops/glamsterdam-devnets`' +`network-configs/devnet-8`) (`bin/ethlambda/assets//`), parsed through the same `ConfigFile::parse`/bootnode reader a directory goes through, plus the two genesis values as constants. None carries a **genesis state**: a built-in network never anchors at genesis, and the states are 5 MB (mainnet) to 150 MB (Hoodi). The constants are checked offline (mainnet's against `tests/fixtures/networks/mainnet/genesis.ssz`, eth-clients' file, whose SHA-256 -`beacon::tests::the_fixture_state_is_eth_clients_file` pins; Sepolia's and -Hoodi's against fork digests published in their own bootnode ENRs), and at +`beacon::tests::the_fixture_state_is_eth_clients_file` pins; Sepolia's, +Hoodi's and Platåberget's against fork digests published in their own bootnode ENRs), and at runtime by checkpoint sync and resume, which both check the anchor state against them. `beacon::tests::the_built_in_network_derives_what_it_always_did` pins the parsed mainnet config to `Config::mainnet()`, so the file and the @@ -461,6 +462,12 @@ explicitly at startup, once per scheduled fork for which `ForkName::is_followed` answers no (the one place that says which forks the node follows), separately from the ignored-keys warning, which now only ever names heze's keys and `GAS_LIMIT_SCHEDULE`/`INCLUSION_LIST_DUE_BPS`. +Platåberget (the Glamsterdam testnet) is already past its `GLOAS_FORK_EPOCH`, +so its checkpoint anchor is a gloas state that `refuse_unfollowable_fork` +refuses; its entry is there for when the live follower takes gloas blocks. Its +file says `CONFIG_NAME: 'testnet'` (a placeholder for Prysm), so +`BuiltInNetwork::resolve` sets every built-in's `CONFIG_NAME` to its +`--network` name. A loaded network decodes its own `genesis.ssz` instead, at whatever fork its own schedule names for epoch 0. Every `config.yaml`, built-in or loaded, goes diff --git a/README.md b/README.md index ee14645bc..e0fb368bd 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,10 @@ provider. See [`ethlambda beacon`](#ethlambda-beacon--the-ethereum-beacon-chain) below for the remaining flags. +`--network plataberget` (the Glamsterdam testnet) is built in as well, but +Platåberget has run gloas since its `GLOAS_FORK_EPOCH` and the follower does +not take gloas blocks or a gloas anchor yet, so the node cannot follow it yet. + ### Lean consensus devnet To run a local lean devnet with ethlambda, follow the instructions on the @@ -227,7 +231,7 @@ no attestation or sync committee subnet. | Flag | Meaning | |---|---| -| `--network` | `mainnet` (default), `sepolia`, `hoodi`, or a path to a directory of network files (`config.yaml`, `genesis.ssz`, optionally `bootstrap_nodes.yaml`), the layout `eth-clients` publishes and kurtosis mounts | +| `--network` | `mainnet` (default), `sepolia`, `hoodi`, `plataberget`, or a path to a directory of network files (`config.yaml`, `genesis.ssz`, optionally `bootstrap_nodes.yaml`), the layout `eth-clients` publishes and kurtosis mounts | | `--checkpoint-sync-url` | Where the anchor comes from. Required on a fresh data directory for a built-in network, since those never start from genesis. A resumable data directory is used before it; a network loaded from a directory anchors at its own `genesis.ssz` when no URL is given | | `--node-key` | Optional, but worth persisting: the columns this node custodies are a function of its node id, so without a key file it changes identity, and custody set, on every restart | | `--execution-endpoint`, `--execution-jwt-secret` | Optional Engine API pairing, given together. Without them, blocks import without payload validation | diff --git a/bin/ethlambda/assets/plataberget/bootstrap_nodes.yaml b/bin/ethlambda/assets/plataberget/bootstrap_nodes.yaml new file mode 100644 index 000000000..9760301f4 --- /dev/null +++ b/bin/ethlambda/assets/plataberget/bootstrap_nodes.yaml @@ -0,0 +1,20 @@ +- enr:-Nm4QPXGASzUuGms8SNcTssTcr7grNKan3z7icEA4DCyBEslO6hYExCa1RTeJO1ixQZgysYNWX13AuNxJ0rNtvXzl_qGAZ_6u353h2F0dG5ldHOI__________-DY2djgYCEZXRoMpCYyRDPgHMxg___________gmlkgnY0gmlwhI5djeSDbmZkhAAAAACEcXVpY4IyyIlzZWNwMjU2azGhAt7VH1W8VHb4u-uKSNx_bkuwlG53pOYmZacvWOLy_E6viHN5bmNuZXRzAIN0Y3CCIyiDdWRwgiMo +- enr:-Nm4QEVFVQbiiXSwlJCdX5v6LkcrloL51ehKNOJPflznWDKYODvDKgoCgKMq58NwIE9VysVCvxMmt9D86ZCtWSqLbdeGAZ_60r5sh2F0dG5ldHOI__________-DY2djgYCEZXRoMpCYyRDPgHMxg___________gmlkgnY0gmlwhKiQtFiDbmZkhAAAAACEcXVpY4IyyIlzZWNwMjU2azGhAn8t3459ORe6Z7XWmK-sBrRgbIvtDjp7EiYuzrsc9iXGiHN5bmNuZXRzAIN0Y3CCIyiDdWRwgiMo +- enr:-QEXuEBiDkCfc3hB4T9PixYuIYzbiR5O5_4VntYekTaCTWtM5y3o8ak_LjsEWy5VF6qyljQz4ux7A_qfZYKzRfDbwp6dRodhdHRuZXRziP__________g2NnYwSGY2xpZW50z4hHcmFuZGluZYUyLjAuNYRldGgykJjJEM-AczGD__________-CaWSCdjSCaXCEqkCk9INpcDaQJABhgAAQAgAAAAAA28iAC4NuZmSEAAAAAIRxdWljgiMphXF1aWM2giNbiXNlY3AyNTZrMaEDf0VGB3-ABOw6d0mj2IxXi8s-fNHZSaMAHnO1SerUpZiIc3luY25ldHMPg3RjcIIjKIR0Y3A2giNag3VkcIIjKIR1ZHA2giNa +- enr:-QEZuEAIY_Gyc62AY753YbfjXvJC1bJ-iiMjuiNXQSGS9bE2UAaVN5DHoBixt_nfo7mei1nDvL9tJvO1NoM0sbIGXo4zRYdhdHRuZXRziP__________g2NnYwSGY2xpZW500YpMaWdodGhvdXNlhTguMi4xhGV0aDKQXJQ4B4BzMYMABgAAAAAAAIJpZIJ2NIJpcIQuZUmFg2lwNpAqA7DAAAEA4AAAAAGlQMALg25mZISYyRDPhHF1aWOCIymFcXVpYzaCIymJc2VjcDI1NmsxoQIDtUPCwK4eWR0IJb5l-3zwOxxrE0TTAIRwWPkCWTiWgYhzeW5jbmV0cw-DdGNwgiMohHRjcDaCIyiDdWRwgiMohHVkcDaCIyg +- enr:-QEZuEARbbwpYtSzfihstpGzZG5I_r8F7jmvn6n91Ims9cC0SncA75PmaWRxcDLPDQMrMM2wd2zPXdoVZXkq-4Qk6BL4QYdhdHRuZXRziP__________g2NnYwSGY2xpZW500YpMaWdodGhvdXNlhTguMi4yhGV0aDKQmMkQz4BzMYP__________4JpZIJ2NIJpcISokBA9g2lwNpAkAGGAAQAA0AAAAAGBCbALg25mZIQAAAAAhHF1aWOCIymFcXVpYzaCIymJc2VjcDI1NmsxoQIRNN_qOxRAVi5vowWakdp4Bes8spZtIQVWhh5tof2goYhzeW5jbmV0cwCDdGNwgiMohHRjcDaCIyiDdWRwgiMohHVkcDaCIyg +- enr:-QEAuEConRusQCS4SqkO5fqwFNQJXhbK2R86q9AqpJmLg49UAXvqKSyXAUBDPaDcN95W70VzFnyJ2ZzPy_4EDKWvMuayBodhdHRuZXRziAAAAAADAAAAg2NnYwSEZXRoMpCYyRDPgHMxg___________gmlkgnY0gmlwhKpAhXODaXA2kCQAYYAAEAIAAAAAANvIkAuDbmZkhAAAAACEcXVpY4IjKYVxdWljNoIjK4lzZWNwMjU2azGhArqsAqRydR0Okd4qw_kJPvx3uUwb_b3NIAj9JZ3DOegwiHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjKoN1ZHCCIyiEdWRwNoIjKg +- enr:-QEAuEAMEaYw8LFo5LX16MPNsQOvIHH7eAgcLz4gk2x_ZPhDDBquq2ALyWfmjnK1Qgq-Yt__H-U9DZ4Sfh5Jf_qk36l0BodhdHRuZXRziAAAAAAAAMAAg2NnYwSEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhKpA04yDaXA2kCQAYYAAEAIAAAAAANjJMAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjK4lzZWNwMjU2azGhA7tBIf5SwD2Tzmy25-uslQAsAFL-zM6DX0G78yFef4TgiHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjKoN1ZHCCIyiEdWRwNoIjKg +- enr:-Mi4QJf9GKh8dkheSoE9gOPgBZ_zSkP0zzwyXF_pwT9Gl3ZCA44FTuuna7BAbbKkdWchlOrRcDmvjhURMbAkDspKnTYDh2F0dG5ldHOI__________-DY2djBIRldGgykJjJEM-AczGD__________-CaWSCdjSCaXCEqJArnYNuZmSEAAAAAIRxdWljgiMpiXNlY3AyNTZrMaECHBw0gzzECxIZ5JxwBLbqG15dAQai3x56jGkr_lK60qeDdGNwgiMog3VkcIIjKA +- enr:-Ni4QK-MdSOl5Zam_puaMNb0VJ5QouxEn-BRM3cNNQ2L_3IdVsoCElRa1olJT5H5VyJGxz-f7oyqH9dzg2sihoqquSuGAZ_65aq1h2F0dG5ldHOI__________-DY2djBIRldGgykFyUOAeAczGDAAYAAAAAAACCaWSCdjSCaXCEQBfXTINuZmSEmMkQz4RxdWljgjLIiXNlY3AyNTZrMaEDDu_e1O_fd1uc9-gJ0WgVILH0Ix85MtY-HxsAws_LrWCIc3luY25ldHMAg3RjcIIjKIN1ZHCCIyg +- enr:-Li4QFuQCN3eqEsUwmOYpGC16gSFhQ7HtgFhWrfUwQGEO8uucqwSrf9EsGY5JCZViicQ9q7BwPTbFAId4N0S5SUcGqaBm4NjZ2MEhGV0aDKQmMkQz4BzMYP__________4JpZIJ2NIJpcIREt8lQg25mZIQAAAAAhHF1aWOCIymJc2VjcDI1NmsxoQKaWbx7I0HUqpOYKs7-goPu_myo7e8vFNG4CrSBnNqFoYN0Y3CCIyiDdWRwgiMo +- enr:-QEYuECJkYqGfhFmM8alfBJOp0VUaBXWu-uBq8O6EDCZt4IUlRWUOpq9FYXue1gjIXr1pEkPK2_QEpaIaLnob_bq4mUGAYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhJ_LBKmDaXA2kCYEqIAMrQDQAAAAAaN4cAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhAjPAcyQXBn8ISq63R_3Sg3gu6SC6IpbtiNCSMrFwRdSFiHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg +- enr:-QEYuED8Hm5EW_x3Pg9tERZtRu1syR87Bb4BGHGeDHEiDXQJHmDD5vXol6DKNh8avwqe1kAn7SsshN-Vrflqgc31eo4ZAYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhJ31aSGDaXA2kCQAYYABAADQAAAAAXl_QAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhAgvWQ3QMaNmKNx_mR1YVY1kFyKYnFJfuYBv_BBbQCZy4iHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg +- enr:-QEYuEAMENdhgbfS7XRUcl3k-O2_f9tjc0PBHnVaXiC8vaXd9zZpmSQYpYI4PY1Anr5thI1br-rNoUl1FbbhouKq4XGzAYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhIbHn9ODaXA2kCQAYYAAEAIAAAAAANjJIAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhA4EKEgHSBAmKqEjWVZGCjn1TmPdbv0VO8AbjpcocDeQ6iHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg +- enr:-QEYuEApS_2JUFBGwhRgg3mS4n-RxjevE-4XqNcSkUSfWnVAuTzqWBbjbbFgk_f9d_ZNrhDTAcf3N-ibRGleTxDbDVdbAYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhC5l_2aDaXA2kCoDsMAAAwDwAAAAAs2MAAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhA_Q9CjhlGYywa7fylxY5Qi6HMubCdDzwavcDq4ctRd2hiHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg +- enr:-QEYuEB882HfZiAfsbCvoadDpm2iCAqpy64tlbvYuYSLAQ0yGksn7tiADjU62Tw22GaXx96fz2-LVYlS38hg_5taKNS0AYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhIs7IY-DaXA2kCQAYYABAADQAAAAAXl_gAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhAoVD1N5sqhIE4Z2MO5_iEz5fgsW5HvjenNUu97i0gV5RiHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg +- enr:-QEYuEDebd1bjfQYHTgOj85z5RUwdd9KxU1tQkGzNbqSCCZri2ZdU1SFk6Tv-p2F5pq5qtS8_44keTOn6FirGEIfqytwAYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhIm4emeDaXA2kCYEqIAABAHQAAAAA0hsEAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhAi_CwnwYpvgM9GKZv-oDRkoSwC1hRmrou2BOFeN4lxzfiHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg +- enr:-QEYuECNqtrKxqKB5dF-8rYKDoogeNCgHrZ9FR2g9KEhVnYCTFuk-o-NrCDXXKWql4gBK9iMmHerLG_6OhwVJL8otgO0AYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhKiQtOyDaXA2kCQAYYABAADQAAAAAXl-8AuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhA56KySRQoku4PbUzYWuqzkTTulEt574e9qkmdEBeAzFiiHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg +- enr:-QEYuEC7v0dZOZswuKbzVXUcD3Vkx2sxCDWthxh-uExjhlsj-UXiSOY66BG4yebK0tdyBpt30gTdzwMN8kIirFRqhB5KAYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhKpAoDGDaXA2kCQAYYAAEAIAAAAAANjJEAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhAkzN2TBr3faHcjF0W-L_p5FdSvk1MbeFIFEG3NXduY7RiHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg +- enr:-QEYuEAtmWBixpnqPbdELk5e6P9ZAUK7kEs37UtkhFc-59RC6Ged9iatnPgkWt3jki9rDZln7AKZIcs3BgfAJfOTWh4MAYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhKiQdc2DaXA2kCQAYYABAADQAAAAAXl_MAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhAgFphB9XXsRdGLS3LZkjy0TTx3o0EdHIwsY5D0nI-NKliHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg +- enr:-QEYuEBpJHYFTuleF94yNA-eJXNOq4ZQuuGb4vs7uMfGxKXFjQDk5-hyqpKGDh2GSh7rYMHYdml8QPq1jj1_KDSH65btAYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhI_GaY2DaXA2kCYEqIAABAHQAAAAA0hsQAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhA7mE6wvAxEMRBo5iIclDEOMWhCudz8dSYx6N4pxKrXroiHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg \ No newline at end of file diff --git a/bin/ethlambda/assets/plataberget/config.yaml b/bin/ethlambda/assets/plataberget/config.yaml new file mode 100644 index 000000000..dba28d401 --- /dev/null +++ b/bin/ethlambda/assets/plataberget/config.yaml @@ -0,0 +1,242 @@ +# Mainnet config + +# Extends the mainnet preset +PRESET_BASE: 'mainnet' + +# Free-form short name of the network that this configuration applies to - known +# canonical network names include: +# * 'mainnet' - there can be only one +# * 'sepolia' - testnet +# * 'holesky' - testnet +# * 'hoodi' - testnet +# Must match the regex: [a-z0-9\-] +CONFIG_NAME: 'testnet' # needs to exist because of Prysm. Otherwise it conflicts with mainnet genesis + +# Transition +# --------------------------------------------------------------- +# Estimated on Sept 15, 2022 +TERMINAL_TOTAL_DIFFICULTY: 0 +# By default, don't use these params +TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 +TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 + +# Genesis +# --------------------------------------------------------------- +# `2**14` (= 16,384) +MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 84000 +# 2026-Aug-13 12:00:00 PM UTC +MIN_GENESIS_TIME: 1786622400 +# Mainnet initial fork version, recommend altering for testnets +GENESIS_FORK_VERSION: 0x10733183 +# Some seconds +GENESIS_DELAY: 0 + +# Forking +# --------------------------------------------------------------- +# Some forks are disabled for now: +# - These may be re-assigned to another fork-version later +# - Temporarily set to max uint64 value: 2**64 - 1 + +# Altair +ALTAIR_FORK_VERSION: 0x20733183 +ALTAIR_FORK_EPOCH: 0 +# Merge +BELLATRIX_FORK_VERSION: 0x30733183 +BELLATRIX_FORK_EPOCH: 0 +# Capella +CAPELLA_FORK_VERSION: 0x40733183 +CAPELLA_FORK_EPOCH: 0 +# Deneb +DENEB_FORK_VERSION: 0x50733183 +DENEB_FORK_EPOCH: 0 +# Electra +ELECTRA_FORK_VERSION: 0x60733183 +ELECTRA_FORK_EPOCH: 0 +# Fulu +FULU_FORK_VERSION: 0x70733183 +FULU_FORK_EPOCH: 0 +# Gloas +GLOAS_FORK_VERSION: 0x80733183 +GLOAS_FORK_EPOCH: 1536 +# Heze +HEZE_FORK_VERSION: 0x90000000 +HEZE_FORK_EPOCH: 18446744073709551615 + +# Time parameters +# --------------------------------------------------------------- +# 12000 milliseconds +SLOT_DURATION_MS: 12000 +# 14 (estimate from Eth1 mainnet) +SECONDS_PER_ETH1_BLOCK: 14 +# 2**8 (= 256) epochs ~27 hours +MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 +# 2**8 (= 256) epochs ~27 hours +SHARD_COMMITTEE_PERIOD: 256 +# 2**11 (= 2,048) Eth1 blocks ~8 hours +ETH1_FOLLOW_DISTANCE: 2048 +# 1667 basis points, ~17% of SLOT_DURATION_MS +PROPOSER_REORG_CUTOFF_BPS: 1667 +# 3333 basis points, ~33% of SLOT_DURATION_MS +ATTESTATION_DUE_BPS: 3333 +# 6667 basis points, ~67% of SLOT_DURATION_MS +AGGREGATE_DUE_BPS: 6667 + +# Altair +# 3333 basis points, ~33% of SLOT_DURATION_MS +SYNC_MESSAGE_DUE_BPS: 3333 +# 6667 basis points, ~67% of SLOT_DURATION_MS +CONTRIBUTION_DUE_BPS: 6667 + +# Gloas +# 2**13 (= 8,192) epochs +MIN_BUILDER_WITHDRAWABILITY_DELAY: 64 +# 2500 basis points, 25% of SLOT_DURATION_MS +ATTESTATION_DUE_BPS_GLOAS: 2500 +# 5000 basis points, 50% of SLOT_DURATION_MS +AGGREGATE_DUE_BPS_GLOAS: 5000 +# 2500 basis points, 25% of SLOT_DURATION_MS +SYNC_MESSAGE_DUE_BPS_GLOAS: 2500 +# 5000 basis points, 50% of SLOT_DURATION_MS +CONTRIBUTION_DUE_BPS_GLOAS: 5000 +# 5000 basis points, 50% of SLOT_DURATION_MS +PAYLOAD_DUE_BPS: 5000 +# 7500 basis points, 75% of SLOT_DURATION_MS +PAYLOAD_ATTESTATION_DUE_BPS: 7500 + +# Heze +# 6667 basis points, ~67% of SLOT_DURATION_MS +INCLUSION_LIST_DUE_BPS: 6667 + +# Validator cycle +# --------------------------------------------------------------- +# 2**2 (= 4) +INACTIVITY_SCORE_BIAS: 4 +# 2**4 (= 16) +INACTIVITY_SCORE_RECOVERY_RATE: 16 +# 2**4 * 10**9 (= 16,000,000,000) Gwei +EJECTION_BALANCE: 16000000000 +# 2**2 (= 4) +MIN_PER_EPOCH_CHURN_LIMIT: 4 +# 2**16 (= 65,536) +CHURN_LIMIT_QUOTIENT: 65536 + +# Deneb +# 2**3 (= 8) (*deprecated*) +MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 8 + +# Electra +# 2**7 * 10**9 (= 128,000,000,000) Gwei +MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: 128000000000 +# 2**8 * 10**9 (= 256,000,000,000) Gwei +MAX_PER_EPOCH_ACTIVATION_EXIT_CHURN_LIMIT: 256000000000 + +# Gloas +# 2**15 (= 32,768) +CHURN_LIMIT_QUOTIENT_GLOAS: 32768 +# 2**16 (= 65,536) +CONSOLIDATION_CHURN_LIMIT_QUOTIENT: 65536 +# 2**8 * 10**9 (= 256,000,000,000) Gwei +MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT_GLOAS: 256000000000 + +# Fork choice +# --------------------------------------------------------------- +# 40% +PROPOSER_SCORE_BOOST: 40 +# 20% +REORG_HEAD_WEIGHT_THRESHOLD: 20 +# 160% +REORG_PARENT_WEIGHT_THRESHOLD: 160 +# `2` epochs +REORG_MAX_EPOCHS_SINCE_FINALIZATION: 2 + +# Deposit contract +# --------------------------------------------------------------- +# Ethereum PoW Mainnet +DEPOSIT_CHAIN_ID: 7091047534 +DEPOSIT_NETWORK_ID: 7091047534 +DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa + +# Networking +# --------------------------------------------------------------- +# `10 * 2**20` (= 10485760, 10 MiB) +MAX_PAYLOAD_SIZE: 10485760 +# 2**10 (= 1,024) blocks +MAX_REQUEST_BLOCKS: 1024 +# 2**8 (= 256) epochs +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 +# 2**5 (= 32) slots +ATTESTATION_PROPAGATION_SLOT_RANGE: 32 +# 500ms +MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500 +MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000 +MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 +# 2 subnets per node +SUBNETS_PER_NODE: 2 +# 2**6 (= 64) subnets +ATTESTATION_SUBNET_COUNT: 64 +# 0 bits +ATTESTATION_SUBNET_EXTRA_BITS: 0 + +# Deneb +# 2**7 (= 128) blocks +MAX_REQUEST_BLOCKS_DENEB: 128 +# 2**12 (= 4,096) epochs +MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 +# 6 subnets +BLOB_SIDECAR_SUBNET_COUNT: 6 +# 6 blobs +MAX_BLOBS_PER_BLOCK: 6 + +# Electra +# 9 subnets +BLOB_SIDECAR_SUBNET_COUNT_ELECTRA: 9 +# 9 blobs +MAX_BLOBS_PER_BLOCK_ELECTRA: 9 + +# Fulu +# 2**7 (= 128) groups +NUMBER_OF_CUSTODY_GROUPS: 128 +# 2**7 (= 128) subnets +DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 +# 2**3 (= 8) samples +SAMPLES_PER_SLOT: 8 +# 2**2 (= 4) sidecars +CUSTODY_REQUIREMENT: 4 +# 2**3 (= 8) sidecars +VALIDATOR_CUSTODY_REQUIREMENT: 8 +# 2**5 * 10**9 (= 32,000,000,000) Gwei +BALANCE_PER_ADDITIONAL_CUSTODY_GROUP: 32000000000 +# 2**12 (= 4,096) epochs +MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096 + +# Gloas +# 2**7 (= 128) payloads +MAX_REQUEST_PAYLOADS: 128 + +# Heze +# 2**4 (= 16) inclusion lists +MAX_REQUEST_INCLUSION_LIST: 16 +# 1 slots +MIN_SLOTS_FOR_INCLUSION_LISTS_REQUESTS: 1 +# 2**13 (= 8,192) bytes +MAX_TRANSACTIONS_BYTES_PER_INCLUSION_LIST: 8192 + + +# Blob Scheduling +# --------------------------------------------------------------- + +BLOB_SCHEDULE: + - EPOCH: 0 + MAX_BLOBS_PER_BLOCK: 21 + +# Gas Limit Scheduling (EIP-8261) +# --------------------------------------------------------------- + +GAS_LIMIT_SCHEDULE: + - EPOCH: 1566 + GAS_LIMIT: 200000000 + +# Fast Confirmation Rule +# --------------------------------------------------------------- +# 25% +CONFIRMATION_BYZANTINE_THRESHOLD: 25 diff --git a/bin/ethlambda/src/checkpoint_sync.rs b/bin/ethlambda/src/checkpoint_sync.rs index 882d49be3..12af5a821 100644 --- a/bin/ethlambda/src/checkpoint_sync.rs +++ b/bin/ethlambda/src/checkpoint_sync.rs @@ -207,10 +207,9 @@ pub enum CheckpointSyncError { #[error("peer served no block at the anchor slot {slot}")] AnchorBlockMissing { slot: u64 }, // Only a built-in network reaches this: a `--network ` network - // anchors at the directory's own `genesis.ssz`. Every built-in network has - // been live for years and this follower imports nothing at startup, so - // anchoring there would park it at slot 0 while claiming to follow a live - // chain. + // anchors at the directory's own `genesis.ssz`. Every built-in network is + // a live chain and this follower imports nothing at startup, so anchoring + // there would park it at slot 0 while claiming to follow it. #[error( "a built-in network has no genesis-sync path, so a fresh data directory needs \ --checkpoint-sync-url; a network loaded with --network anchors at its own \ diff --git a/bin/ethlambda/src/cli.rs b/bin/ethlambda/src/cli.rs index 45cbe0fa6..4dbe91741 100644 --- a/bin/ethlambda/src/cli.rs +++ b/bin/ethlambda/src/cli.rs @@ -87,11 +87,11 @@ pub(crate) struct CommonOptions { /// published files) anchors at its own `genesis.ssz` when there is /// neither a resumable directory nor a URL, since a freshly started devnet /// has no checkpoint provider at slot 0 and this is the only way to join - /// one. The built-in networks (`mainnet`, `sepolia`, `hoodi`) still refuse - /// that fallback: each has been live for years, and this follower imports - /// nothing at startup, so anchoring there would park it at slot 0 while - /// claiming to follow a live chain. With neither a resumable directory, a - /// URL, nor a loaded network's genesis, startup aborts. + /// one. The built-in networks (`mainnet`, `sepolia`, `hoodi`, + /// `plataberget`) still refuse that fallback: each is a live chain, and + /// this follower imports nothing at startup, so anchoring there would park + /// it at slot 0 while claiming to follow it. With neither a resumable + /// directory, a URL, nor a loaded network's genesis, startup aborts. #[arg(long, value_delimiter = ',')] pub(crate) checkpoint_sync_url: Vec, #[command(flatten)] @@ -296,8 +296,9 @@ pub(crate) struct MainnetOptions { #[arg(long, default_value_t = SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY)] pub(crate) safe_slots_to_import_optimistically: u64, - /// Which network to follow: a built-in name (`mainnet`, `sepolia` or - /// `hoodi`), or a path to a directory of published network files. + /// Which network to follow: a built-in name (`mainnet`, `sepolia`, + /// `hoodi` or `plataberget`), or a path to a directory of published + /// network files. /// /// A value containing a slash is always read as a directory, so `mainnet` /// names the built-in network and `./mainnet` names a directory. The diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index 8ad84ce64..e25dfacfa 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -1866,8 +1866,8 @@ async fn fetch_initial_beacon_state( // A loaded network carries its own genesis state, which is a legitimate // anchor: a fresh devnet has no checkpoint provider at slot 0, so this is // the only way to join one. A built-in network still refuses, because - // every one has been live for years and this follower would sit at slot 0 - // claiming to follow a live chain. + // every one is a live chain and this follower would sit at slot 0 + // claiming to follow it. if checkpoint_urls.is_empty() { let network::NetworkSource::Loaded(loaded) = source else { return Err(checkpoint_sync::CheckpointSyncError::BeaconGenesisSync); @@ -2858,9 +2858,9 @@ validators: ); } - /// Every built-in network has been live for years, and this follower - /// imports nothing at startup, so anchoring at genesis would park it at - /// slot 0 while claiming to follow a live chain. That refusal is + /// Every built-in network is a live chain, and this follower imports + /// nothing at startup, so anchoring at genesis would park it at slot 0 + /// while claiming to follow it. That refusal is /// deliberate and must survive; no built-in network carries a genesis /// state to anchor at in the first place. #[tokio::test] diff --git a/bin/ethlambda/src/network/built_in.rs b/bin/ethlambda/src/network/built_in.rs index 1d389d5a8..d3cf05f8f 100644 --- a/bin/ethlambda/src/network/built_in.rs +++ b/bin/ethlambda/src/network/built_in.rs @@ -1,19 +1,20 @@ //! The networks compiled into the binary, by the name `--network` accepts. //! -//! Every built-in chain is an [`EmbeddedChain`]: its `eth-clients` repo's +//! Every built-in chain is an [`EmbeddedChain`]: its publisher's //! `metadata/config.yaml` and `metadata/bootstrap_nodes.yaml` byte for byte, //! read through the same parsers a `--network ` goes through, plus its two -//! genesis values. Refreshing a chain is a file copy that can be diffed against -//! upstream. +//! genesis values. The publisher is the chain's `eth-clients` repo, or for a +//! devnet the ethpandaops repo that runs it. Refreshing a chain is a file copy +//! that can be diffed against upstream. //! //! No chain carries its genesis *state*, only the two values the wire is //! derived from (`genesis_time`, `genesis_validators_root`). A built-in network //! never anchors at genesis (`fetch_initial_beacon_state` refuses, since every -//! one of them has been live for years and this follower would sit at slot 0), -//! so those two values are all the state would be read for; mainnet's is 5 MB -//! and Hoodi's 150 MB. A wrong constant still fails loudly rather than -//! silently: the anchor state checkpoint sync downloads is checked against -//! both, and so is a resumed data directory's. +//! one of them is a live chain and this follower would sit at slot 0), so those +//! two values are all the state would be read for; mainnet's is 5 MB, +//! Platåberget's 14 MB and Hoodi's 150 MB. A wrong constant still fails loudly +//! rather than silently: the anchor state checkpoint sync downloads is checked +//! against both, and so is a resumed data directory's. use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::primitives::Root; @@ -28,11 +29,13 @@ pub(crate) enum BuiltInNetwork { Mainnet, Sepolia, Hoodi, + Plataberget, } impl BuiltInNetwork { /// Every built-in network, in the order an unknown-name error lists them. - pub(crate) const ALL: [Self; 3] = [Self::Mainnet, Self::Sepolia, Self::Hoodi]; + pub(crate) const ALL: [Self; 4] = + [Self::Mainnet, Self::Sepolia, Self::Hoodi, Self::Plataberget]; /// The name `--network` accepts for this network. pub(crate) const fn name(self) -> &'static str { @@ -40,6 +43,7 @@ impl BuiltInNetwork { Self::Mainnet => "mainnet", Self::Sepolia => "sepolia", Self::Hoodi => "hoodi", + Self::Plataberget => "plataberget", } } @@ -52,6 +56,7 @@ impl BuiltInNetwork { Self::Mainnet => &MAINNET, Self::Sepolia => &SEPOLIA, Self::Hoodi => &HOODI, + Self::Plataberget => &PLATABERGET, } } @@ -74,6 +79,15 @@ impl BuiltInNetwork { let genesis = chain.genesis(); let mut config = parsed.config; super::derive_genesis_fields(&mut config, genesis.genesis_time); + // The name is the one value taken from `--network` rather than the + // file. Platåberget's file says `testnet`, a placeholder ethpandaops + // devnets carry for Prysm's sake, and a node told `--network + // plataberget` should log and report that name. The `eth-clients` + // files already carry their own, so for them this changes nothing. + config.config_name = self + .name() + .try_into() + .expect("a built-in network's name fits a ConfigName"); super::warn_if_unfollowed_fork_scheduled(self.name(), &config); Ok(BuiltIn { @@ -94,9 +108,9 @@ pub(crate) struct BuiltIn { /// A built-in chain's embedded files and genesis values. struct EmbeddedChain { - /// `metadata/config.yaml` from the chain's `eth-clients` repo. + /// `metadata/config.yaml` from the chain's publisher. config_yaml: &'static str, - /// `metadata/bootstrap_nodes.yaml` from the chain's `eth-clients` repo. + /// `metadata/bootstrap_nodes.yaml` from the chain's publisher. bootnodes_yaml: &'static str, genesis_time: u64, /// Hex, without a `0x` prefix. @@ -150,6 +164,23 @@ const HOODI: EmbeddedChain = EmbeddedChain { genesis_validators_root: "212f13fc4df078b6cb7db228f1c8307566dcecf900867401a92023d7ba99cb5f", }; +/// Platåberget, the long-lived Glamsterdam testnet, from +/// `ethpandaops/glamsterdam-devnets`' `network-configs/devnet-8` (Platåberget +/// is glamsterdam-devnet-8 under its public name). +/// +/// Both values are read off that directory's `metadata/genesis.ssz`, agree with +/// its `genesis_validators_root.txt`, and are what +/// `checkpoint-sync.plataberget.ethpandaops.io/eth/v1/beacon/genesis` returns. +/// The chain started at fulu and has run gloas since its `GLOAS_FORK_EPOCH`, +/// a fork the live follower does not take blocks or an anchor from yet. +const PLATABERGET: EmbeddedChain = EmbeddedChain { + config_yaml: include_str!("../../assets/plataberget/config.yaml"), + bootnodes_yaml: include_str!("../../assets/plataberget/bootstrap_nodes.yaml"), + // 2026-08-13 12:00:00 UTC. + genesis_time: 1_786_622_400, + genesis_validators_root: "bb4a1a9e3f7f4e10edcd734e4acc3b5ffd4f830efe0af2748fa458cfee5d2658", +}; + #[cfg(test)] mod tests { use super::*; @@ -213,7 +244,7 @@ mod tests { /// cannot be told apart from that fallback by value; it is pinned against /// `Config::mainnet()` instead (`beacon::tests`). #[test] - fn the_sepolia_and_hoodi_configs_are_their_own() { + fn the_testnet_configs_are_their_own() { let sepolia = BuiltInNetwork::Sepolia.resolve().unwrap().config; assert_eq!(sepolia.genesis_fork_version, [0x90, 0x00, 0x00, 0x69]); assert_eq!(sepolia.fulu_fork_version, [0x90, 0x00, 0x00, 0x75]); @@ -226,12 +257,31 @@ mod tests { assert_eq!(hoodi.fulu_fork_version, [0x70, 0x00, 0x09, 0x10]); assert_eq!(hoodi.fulu_fork_epoch, 50_688); assert_eq!(hoodi.deposit_chain_id, 560_048); + + let plataberget = BuiltInNetwork::Plataberget.resolve().unwrap().config; + assert_eq!(plataberget.genesis_fork_version, [0x10, 0x73, 0x31, 0x83]); + assert_eq!(plataberget.fork_at_epoch(0), ForkName::Fulu); + assert_eq!(plataberget.fulu_fork_version, [0x70, 0x73, 0x31, 0x83]); + assert_eq!(plataberget.gloas_fork_version, [0x80, 0x73, 0x31, 0x83]); + assert_eq!(plataberget.gloas_fork_epoch, 1_536); + assert_eq!(plataberget.deposit_chain_id, 7_091_047_534); } - /// Sepolia's and Hoodi's genesis roots have no other offline check, since - /// the state each is the root of is not carried even as a test fixture - /// (mainnet's is, and `beacon::tests` checks mainnet's constants against - /// it). What pins them is a digest someone else computed + /// Platåberget's file names the chain `testnet`, so its `CONFIG_NAME` + /// comes from `--network` instead; this pins why `resolve` overrides it. + #[test] + fn plataberget_is_named_for_its_flag_rather_than_its_file() { + let parsed = ConfigFile::parse(PLATABERGET.config_yaml).unwrap(); + assert_eq!(parsed.config.config_name.as_str(), "testnet"); + + let resolved = BuiltInNetwork::Plataberget.resolve().unwrap(); + assert_eq!(resolved.config.config_name.as_str(), "plataberget"); + } + + /// Sepolia's, Hoodi's and Platåberget's genesis roots have no other + /// offline check, since the state each is the root of is not carried even + /// as a test fixture (mainnet's is, and `beacon::tests` checks mainnet's + /// constants against it). What pins them is a digest someone else computed /// from it: each expected value below is the `eth2` entry of a bootnode /// ENR in the network's own `bootstrap_nodes.yaml`, published while that /// fork was current. The digest mixes the fork version with the root, so @@ -258,6 +308,24 @@ mod tests { compute_fork_digest(&hoodi.config, hoodi.genesis.genesis_validators_root, 0), [0xd2, 0xf1, 0x99, 0x7f] ); + + // Platåberget's bootnodes come from both sides of its gloas fork. + // Those signed before it name gloas as their next fork, at its + // `GLOAS_FORK_EPOCH`; genesis was fulu, so the digest also mixes in + // the blob schedule's first entry. Those signed after it repeat + // gloas's version with FAR_FUTURE_EPOCH, so they pin the gloas fork + // version as well. + let plataberget = BuiltInNetwork::Plataberget.resolve().unwrap(); + let root = plataberget.genesis.genesis_validators_root; + assert_eq!( + compute_fork_digest(&plataberget.config, root, 0), + [0x5c, 0x94, 0x38, 0x07] + ); + let gloas = plataberget.config.gloas_fork_epoch; + assert_eq!( + compute_fork_digest(&plataberget.config, root, gloas), + [0x98, 0xc9, 0x10, 0xcf] + ); } /// Sepolia's README publishes its genesis digest directly. diff --git a/bin/ethlambda/src/network/mod.rs b/bin/ethlambda/src/network/mod.rs index a1f36424e..570fdb40b 100644 --- a/bin/ethlambda/src/network/mod.rs +++ b/bin/ethlambda/src/network/mod.rs @@ -319,7 +319,8 @@ impl NetworkSource { } /// The resolved network's `CONFIG_NAME`, for logging. A built-in - /// network's is its own name, which `every_built_in_network_resolves` + /// network's is its own name, whatever its file says (see + /// [`BuiltInNetwork::resolve`]), which `every_built_in_network_resolves` /// checks. pub(crate) fn name(&self) -> &str { self.config().config_name.as_str() @@ -356,6 +357,7 @@ mod tests { ("mainnet", BuiltInNetwork::Mainnet), ("sepolia", BuiltInNetwork::Sepolia), ("hoodi", BuiltInNetwork::Hoodi), + ("plataberget", BuiltInNetwork::Plataberget), ] { assert!( matches!( diff --git a/docs/beacon_wire.md b/docs/beacon_wire.md index 2eec3b2d8..d278f389e 100644 --- a/docs/beacon_wire.md +++ b/docs/beacon_wire.md @@ -25,7 +25,7 @@ ethlambda beacon \ No flag is required, but that is only true for a built-in network: `ethlambda beacon` on its own follows mainnet, since `--network` defaults to -`mainnet`. Sepolia and Hoodi are built in too, and any other network is a +`mainnet`. Sepolia, Hoodi and Platåberget are built in too, and any other network is a `--network` naming a directory of published files (`config.yaml` and `genesis.ssz`, see [`cli.md`](./cli.md)): @@ -36,8 +36,8 @@ ethlambda beacon --network ./my-network --node-key ./node-key `genesis_time` and `genesis_validators_root`, and therefore the fork digest, are derived from the resolved network: a built-in network (`mainnet`, -`sepolia`, `hoodi`) carries the two values as constants beside its -`eth-clients` `config.yaml` and bootnode list, and no genesis state; for a +`sepolia`, `hoodi`, `plataberget`) carries the two values as constants beside +its published `config.yaml` and bootnode list, and no genesis state; for a loaded network they are read off that directory's own `genesis.ssz`. Nothing about startup touches the network to get there. discv5 is forced on and needs no flag either: published mainnet bootnodes are largely seed-only, so a crawl is how a peer is reached. diff --git a/docs/checkpoint_sync.md b/docs/checkpoint_sync.md index 48a70c5a4..5c9e34b7b 100644 --- a/docs/checkpoint_sync.md +++ b/docs/checkpoint_sync.md @@ -32,13 +32,13 @@ State already on disk takes precedence over both checkpoint sync and genesis: if ### `ethlambda beacon` -`beacon` takes no genesis config, validator registry, or bootnode file of its own: it takes `--network` instead, naming a built-in network or a directory of published network files (see [`cli.md`](cli.md)). `genesis_time`, `genesis_validators_root`, and the bootnode list all come from whichever network `--network` resolves to; for the built-in networks (`mainnet`, the default, plus `sepolia` and `hoodi`), they are built into the binary (see `CLAUDE.md`'s "Built-in networks"). +`beacon` takes no genesis config, validator registry, or bootnode file of its own: it takes `--network` instead, naming a built-in network or a directory of published network files (see [`cli.md`](cli.md)). `genesis_time`, `genesis_validators_root`, and the bootnode list all come from whichever network `--network` resolves to; for the built-in networks (`mainnet`, the default, plus `sepolia`, `hoodi` and `plataberget`), they are built into the binary (see `CLAUDE.md`'s "Built-in networks"). ```bash ethlambda beacon --network --checkpoint-sync-url ``` -The anchor precedence is, in order: a resumable data directory, then `--checkpoint-sync-url`, then, for a loaded network only, that directory's own `genesis.ssz`, then abort. The URL is therefore **required** on a fresh data directory only for a built-in network, unlike on `node`: this follower imports nothing past its anchor, so anchoring a built-in network at genesis would leave it parked at slot 0 while claiming to follow a chain that has been live for years. A loaded network's own genesis state is a legitimate anchor instead, since a freshly started devnet has no checkpoint provider at slot 0 and this is the only way to join one. With neither a resumable DB, a URL, nor (for a loaded network) a genesis fallback, startup aborts with `CheckpointSyncError::BeaconGenesisSync`. A data directory already anchored from a previous run resumes exactly as `node`'s does, without a URL, subject to the same resume window (see [Restarts and Existing State](#restarts-and-existing-state)). +The anchor precedence is, in order: a resumable data directory, then `--checkpoint-sync-url`, then, for a loaded network only, that directory's own `genesis.ssz`, then abort. The URL is therefore **required** on a fresh data directory only for a built-in network, unlike on `node`: this follower imports nothing past its anchor, so anchoring a built-in network at genesis would leave it parked at slot 0 while claiming to follow a live chain. A loaded network's own genesis state is a legitimate anchor instead, since a freshly started devnet has no checkpoint provider at slot 0 and this is the only way to join one. With neither a resumable DB, a URL, nor (for a loaded network) a genesis fallback, startup aborts with `CheckpointSyncError::BeaconGenesisSync`. A data directory already anchored from a previous run resumes exactly as `node`'s does, without a URL, subject to the same resume window (see [Restarts and Existing State](#restarts-and-existing-state)). ## Checkpoint Sources diff --git a/docs/cli.md b/docs/cli.md index 48cf91180..4661706ad 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -142,7 +142,7 @@ something specific here. | Flag | Default | Meaning | |---|---|---| -| `--network` | `mainnet` | A built-in network name (`mainnet`, `sepolia` or `hoodi`), or a path to a directory of published network files. A value containing a slash is always a path, so `mainnet` is the built-in and `./mainnet` is a directory. The directory must hold `config.yaml` and `genesis.ssz`, and may hold `bootstrap_nodes.yaml` or `bootstrap_nodes.txt` | +| `--network` | `mainnet` | A built-in network name (`mainnet`, `sepolia`, `hoodi` or `plataberget`), or a path to a directory of published network files. A value containing a slash is always a path, so `mainnet` is the built-in and `./mainnet` is a directory. The directory must hold `config.yaml` and `genesis.ssz`, and may hold `bootstrap_nodes.yaml` or `bootstrap_nodes.txt` | `genesis_validators_root` and `genesis_time`, which the fork digest that keys every gossip topic, the ENR `eth2` entry and discv5 admission is computed from, @@ -150,7 +150,7 @@ come from the resolved network: | `--network` | genesis values | config | bootnodes | |---|---|---|---| -| `mainnet` (default), `sepolia`, `hoodi` | two constants in `network::built_in`; no genesis state is carried (a built-in network never anchors at genesis, and the states run from 5 MB to 150 MB) | `bin/ethlambda/assets//config.yaml`, `eth-clients/`'s file byte for byte | `bin/ethlambda/assets//bootstrap_nodes.yaml`, from the same repo | +| `mainnet` (default), `sepolia`, `hoodi`, `plataberget` | two constants in `network::built_in`; no genesis state is carried (a built-in network never anchors at genesis, and the states run from 5 MB to 150 MB) | `bin/ethlambda/assets//config.yaml`, byte for byte from `eth-clients/` (Platåberget: `ethpandaops/glamsterdam-devnets`' `network-configs/devnet-8`) | `bin/ethlambda/assets//bootstrap_nodes.yaml`, from the same repo | | a directory | read off the directory's own `genesis.ssz` | the directory's `config.yaml` | the directory's `bootstrap_nodes.yaml`/`.txt` | The genesis constants are checked twice at runtime: checkpoint sync verifies the @@ -194,7 +194,7 @@ for a loaded network only, that directory's own `genesis.ssz`, then abort. The URL is therefore **required** on a fresh data directory only for a built-in network: this follower imports nothing past its anchor, so anchoring a built-in network at genesis would leave it parked at slot 0 while claiming to follow a -chain that has been live for years. A loaded network's own genesis +live chain. A loaded network's own genesis state is a legitimate anchor instead, since a freshly started devnet has no checkpoint provider at slot 0 and this is the only way to join one. A directory already anchored from a previous run resumes without the flag From 2dd8f5e6e91488dd39c97ca01571ef2003ed88c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:21:38 -0300 Subject: [PATCH 037/132] feat(p2p): cross fork digests without a restart The beacon wire computed its fork digest once at startup, so a follower was stranded on topics nobody publishes to the moment a fork activated. Blob-parameter-only (BPO) boundaries move the digest too, and Sepolia's fulu to gloas boundary is close, so a long-running node has to cross them on its own. ForkSchedule is a pure function of the config and genesis validators root: every digest from fork and blob-schedule epochs, digest to fork, and the spec's window ("Transitioning the gossip", altair p2p-interface.md). Post-fork topics are joined one epoch ahead of the boundary and the pre-fork topics left two epochs after it, so a node started inside a window holds both, same as one that crossed into it. The P2P actor applies the schedule idempotently on a self-rescheduling timer (and once at startup): it joins and leaves topic sets, swaps the published digest and fork, bumps the metadata sequence number, accepts peer Status at any held digest, and moves the discovery admission filter, which is now shared with the peer table's copy. Attestation decode takes its fork from the incoming topic's digest instead of the current one. The served ENR eth2 entry cannot be updated yet: ethrex's DiscoveryServer::spawn returns no handle and has no message that replaces its record. update_served_enr marks the gap. --- bin/ethlambda/src/beacon.rs | 67 ++- crates/common/types/src/beacon/fork_digest.rs | 19 + crates/net/p2p/src/beacon/fork_schedule.rs | 396 ++++++++++++++++++ crates/net/p2p/src/beacon/handler.rs | 5 + crates/net/p2p/src/beacon/mod.rs | 37 +- crates/net/p2p/src/beacon/topics.rs | 55 +++ crates/net/p2p/src/beacon/transition.rs | 334 +++++++++++++++ crates/net/p2p/src/discovery/admission.rs | 87 +++- crates/net/p2p/src/discovery/dial.rs | 11 +- crates/net/p2p/src/discovery/mod.rs | 27 ++ crates/net/p2p/src/gossipsub/handler.rs | 40 +- crates/net/p2p/src/lib.rs | 43 +- crates/net/p2p/src/req_resp/handlers.rs | 4 +- 13 files changed, 1047 insertions(+), 78 deletions(-) create mode 100644 crates/net/p2p/src/beacon/fork_schedule.rs create mode 100644 crates/net/p2p/src/beacon/transition.rs diff --git a/bin/ethlambda/src/beacon.rs b/bin/ethlambda/src/beacon.rs index 0528e5724..64ab98861 100644 --- a/bin/ethlambda/src/beacon.rs +++ b/bin/ethlambda/src/beacon.rs @@ -24,11 +24,10 @@ //! all. `node`'s checkpoint sync is untouched: a lean node fetches a //! *finalized* anchor, which genuinely has no local source. +use ethlambda_p2p::beacon::fork_schedule::ForkSchedule; use ethlambda_p2p::beacon::swarm::BeaconWireConfig; use ethlambda_types::beacon::config::Config; -use ethlambda_types::beacon::constants::FAR_FUTURE_EPOCH; use ethlambda_types::beacon::containers::BeaconState; -use ethlambda_types::beacon::fork_digest::{compute_fork_digest, next_fork_boundary}; use ethlambda_types::beacon::preset; use ethlambda_types::beacon::primitives::{Epoch, Root}; use ethlambda_types::enr::EnrForkId; @@ -47,23 +46,7 @@ pub fn time_at_epoch(config: &Config, genesis_time: u64, epoch: Epoch) -> u64 { genesis_time + epoch * config.seconds_per_slot * preset::SLOTS_PER_EPOCH } -/// The `eth2` ENR entry for this chain at this epoch. -/// -/// `next_fork_*` point at the next boundary that moves the digest, which -/// includes blob-parameter-only forks. Peers tolerate a difference here by -/// design: only `fork_digest` has to match. -pub fn enr_fork_id(config: &Config, genesis_validators_root: Root, epoch: Epoch) -> EnrForkId { - let fork_digest = compute_fork_digest(config, genesis_validators_root, epoch); - // With no boundary ahead, the spec says to repeat the current fork's own - // version and name `FAR_FUTURE_EPOCH` as the epoch it activates at. - let boundary = next_fork_boundary(config, epoch); - let named_epoch = boundary.unwrap_or(epoch); - EnrForkId { - fork_digest, - next_fork_version: config.fork_version(config.fork_at_epoch(named_epoch)), - next_fork_epoch: boundary.unwrap_or(FAR_FUTURE_EPOCH), - } -} +pub use ethlambda_types::beacon::fork_digest::enr_fork_id; /// Ethereum mainnet's genesis `BeaconState`, SSZ-encoded: a test fixture. /// @@ -187,28 +170,32 @@ pub fn wire_params( ); ethlambda_p2p::metrics::set_beacon_fork_digest(&digest_hex); - // The digest is computed once. Crossing a boundary while running strands - // this node on topic names nobody publishes to, so say when that is. + // The digest moves at every fork and blob-schedule boundary, and the p2p + // actor follows it without a restart (`ethlambda_p2p::beacon::transition`): + // next-digest topics are joined one epoch ahead, the switch happens at the + // boundary, and the old topics are left two epochs after. Say what is + // coming. // - // The gloas boundary gets its own message rather than the generic one: a - // restart there recomputes the digest and resubscribes to the right - // topics, the same as any other boundary, but it does not make this - // build any more able to import a gloas block or cross a gloas epoch - // boundary (the chain actor refuses every gloas block, since nothing - // delivers payload envelopes or payload attestations to it yet). - // Suggesting a restart "crosses" it would read as a stall a restart - // clears, which this is not. - match next_fork_boundary(&chain, epoch) { - Some(boundary) if boundary == chain.gloas_fork_epoch => warn!( - boundary_epoch = boundary, - boundary_unix_time = time_at_epoch(&chain, genesis.genesis_time, boundary), - "The fork digest changes at this boundary, but this build cannot follow past it: \ - a restart resubscribes to the right topics, not past the gloas gap itself" + // The gloas boundary keeps its own warning: crossing its digest works like + // any other, but the chain actor refuses every gloas block (nothing + // delivers payload envelopes or payload attestations to it yet), so the + // node stops tracking the chain there regardless. + let schedule = ForkSchedule::new(&chain, genesis.genesis_validators_root); + match schedule.next_boundary_after(epoch) { + Some(next) if !next.fork.is_followed() => warn!( + boundary_epoch = next.activation_epoch, + boundary_unix_time = time_at_epoch(&chain, genesis.genesis_time, next.activation_epoch), + fork = next.fork.as_str(), + fork_digest = %hex::encode(next.digest), + "The fork digest changes at this boundary and the node will switch topics, but this \ + build cannot follow the fork itself" ), - Some(boundary) => info!( - boundary_epoch = boundary, - boundary_unix_time = time_at_epoch(&chain, genesis.genesis_time, boundary), - "The fork digest changes at this boundary; restart the node to cross it" + Some(next) => info!( + boundary_epoch = next.activation_epoch, + boundary_unix_time = time_at_epoch(&chain, genesis.genesis_time, next.activation_epoch), + fork = next.fork.as_str(), + fork_digest = %hex::encode(next.digest), + "The fork digest changes at this boundary; the node will cross it without a restart" ), None => info!("No fork or blob-schedule boundary is scheduled"), } @@ -293,7 +280,7 @@ pub fn wire_params( #[cfg(test)] mod tests { use super::*; - use ethlambda_types::beacon::constants::CUSTODY_REQUIREMENT; + use ethlambda_types::beacon::constants::{CUSTODY_REQUIREMENT, FAR_FUTURE_EPOCH}; use ethlambda_types::beacon::fork::ForkName; /// Mainnet's genesis, 2020-12-01 12:00:23 UTC. diff --git a/crates/common/types/src/beacon/fork_digest.rs b/crates/common/types/src/beacon/fork_digest.rs index 69a42488a..8aaab3254 100644 --- a/crates/common/types/src/beacon/fork_digest.rs +++ b/crates/common/types/src/beacon/fork_digest.rs @@ -14,6 +14,7 @@ use crate::beacon::constants; use crate::beacon::containers::shared::ForkData; use crate::beacon::fork::ForkName; use crate::beacon::primitives::{Epoch, ForkDigest, HashTreeRoot as _, Root, Version}; +use crate::enr::EnrForkId; /// The root binding a fork version to a chain's genesis validator set. /// @@ -71,6 +72,24 @@ pub fn next_fork_boundary(config: &Config, epoch: Epoch) -> Option { .min() } +/// The `eth2` ENR entry for this chain at this epoch. +/// +/// `next_fork_*` point at the next boundary that moves the digest, which +/// includes blob-parameter-only forks. Peers tolerate a difference here by +/// design: only `fork_digest` has to match. +pub fn enr_fork_id(config: &Config, genesis_validators_root: Root, epoch: Epoch) -> EnrForkId { + let fork_digest = compute_fork_digest(config, genesis_validators_root, epoch); + // With no boundary ahead, the spec says to repeat the current fork's own + // version and name `FAR_FUTURE_EPOCH` as the epoch it activates at. + let boundary = next_fork_boundary(config, epoch); + let named_epoch = boundary.unwrap_or(epoch); + EnrForkId { + fork_digest, + next_fork_version: config.fork_version(config.fork_at_epoch(named_epoch)), + next_fork_epoch: boundary.unwrap_or(constants::FAR_FUTURE_EPOCH), + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/net/p2p/src/beacon/fork_schedule.rs b/crates/net/p2p/src/beacon/fork_schedule.rs new file mode 100644 index 000000000..d5fd3996e --- /dev/null +++ b/crates/net/p2p/src/beacon/fork_schedule.rs @@ -0,0 +1,396 @@ +//! Which fork digests the chain will use, and when a running node must move +//! between them. +//! +//! Every fork activation and every blob-schedule entry (a blob-parameter-only +//! fork, BPO) changes the fork digest, and the digest is in every gossip topic +//! name, the `Status` handshake and the ENR `eth2` entry. A node that computed +//! its digest once at startup is stranded on topics nobody publishes to the +//! moment a boundary passes, so the node follows this schedule instead. +//! +//! The transition rules are the spec's ("Transitioning the gossip" in altair's +//! `p2p-interface.md`): "In advance of the fork, a node SHOULD subscribe to the +//! post-fork variants of the topics", and the pre-fork topics are dropped "two +//! epochs after the fork". A digest is therefore held from one epoch before it +//! activates until two epochs after the next one does, and the node is +//! subscribed to whichever digests that window covers at the current epoch. +//! Everything here is a pure function of the epoch, so applying it is +//! idempotent and a node started inside a window lands in the same state as one +//! that crossed into it. +//! +//! Pure on purpose: the P2P actor owns the clock and the swarm; this owns only +//! the arithmetic, so it is testable without either. + +use ethlambda_types::beacon::config::Config; +use ethlambda_types::beacon::constants::FAR_FUTURE_EPOCH; +use ethlambda_types::beacon::fork::ForkName; +use ethlambda_types::beacon::fork_digest::{compute_fork_digest, enr_fork_id}; +use ethlambda_types::beacon::primitives::{Epoch, ForkDigest, Root}; +use ethlambda_types::enr::EnrForkId; + +/// Epochs before a digest activates at which its topics are joined. +pub const SUBSCRIBE_LEAD_EPOCHS: Epoch = 1; + +/// Epochs after the *next* digest activates at which a digest's topics are +/// left. +pub const UNSUBSCRIBE_LAG_EPOCHS: Epoch = 2; + +/// One digest and the epoch it becomes current. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ScheduledDigest { + /// The first epoch this digest is current at. + pub activation_epoch: Epoch, + pub digest: ForkDigest, + /// The fork in effect at `activation_epoch`, so the fork of everything + /// published under `digest`. + pub fork: ForkName, +} + +/// Every digest a chain will use, in activation order. +#[derive(Debug, Clone)] +pub struct ForkSchedule { + config: Config, + genesis_validators_root: Root, + /// Never empty: the first entry is the genesis digest. Consecutive equal + /// digests are merged, so each entry is a real change on the wire. + entries: Vec, +} + +impl ForkSchedule { + /// Build the schedule from every fork epoch and blob-schedule epoch in + /// `config`. Unscheduled forks carry [`FAR_FUTURE_EPOCH`] and are skipped. + pub fn new(config: &Config, genesis_validators_root: Root) -> Self { + let mut epochs: Vec = ForkName::ALL + .into_iter() + .map(|fork| config.fork_epoch(fork)) + .chain(config.blob_schedule.iter().map(|entry| entry.epoch)) + .chain([0]) + .filter(|&epoch| epoch != FAR_FUTURE_EPOCH) + .collect(); + epochs.sort_unstable(); + epochs.dedup(); + + let mut entries: Vec = Vec::with_capacity(epochs.len()); + for epoch in epochs { + let digest = compute_fork_digest(config, genesis_validators_root, epoch); + // A blob-schedule entry that restates the parameters already in + // force leaves the digest alone; nothing changes on the wire there. + if entries.last().is_some_and(|last| last.digest == digest) { + continue; + } + entries.push(ScheduledDigest { + activation_epoch: epoch, + digest, + fork: config.fork_at_epoch(epoch), + }); + } + Self { + config: config.clone(), + genesis_validators_root, + entries, + } + } + + /// Every scheduled digest, in activation order. + pub fn entries(&self) -> &[ScheduledDigest] { + &self.entries + } + + fn index_at(&self, epoch: Epoch) -> usize { + self.entries + .partition_point(|entry| entry.activation_epoch <= epoch) + .saturating_sub(1) + } + + /// The digest (and its fork) that is current at `epoch`. + pub fn current_at(&self, epoch: Epoch) -> ScheduledDigest { + self.entries[self.index_at(epoch)] + } + + /// The digest current at `epoch`. + pub fn digest_at(&self, epoch: Epoch) -> ForkDigest { + self.current_at(epoch).digest + } + + /// The fork a message published under `digest` is in, if any scheduled + /// digest is `digest`. + /// + /// This is how a gossip message's fork is taken from its own topic rather + /// than from whichever digest the node currently advertises. + pub fn fork_for_digest(&self, digest: ForkDigest) -> Option { + self.entries + .iter() + .find(|entry| entry.digest == digest) + .map(|entry| entry.fork) + } + + /// The digests whose topics the node holds at `epoch`, in activation order. + /// + /// Entry `i` is held from [`SUBSCRIBE_LEAD_EPOCHS`] before it activates + /// until [`UNSUBSCRIBE_LAG_EPOCHS`] after entry `i + 1` does; the first is + /// held from genesis and the last forever. Always contains the current + /// digest, since `epoch` lies inside its own window. + pub fn held_at(&self, epoch: Epoch) -> Vec { + self.entries + .iter() + .enumerate() + .filter(|&(index, entry)| { + let joined = index == 0 + || epoch.saturating_add(SUBSCRIBE_LEAD_EPOCHS) >= entry.activation_epoch; + let left = self.entries.get(index + 1).is_some_and(|next| { + epoch >= next.activation_epoch.saturating_add(UNSUBSCRIBE_LAG_EPOCHS) + }); + joined && !left + }) + .map(|(_, entry)| *entry) + .collect() + } + + /// The first epoch after `epoch` at which anything above changes: a join, a + /// switch or a leave. `None` once the schedule has run out. + pub fn next_change_after(&self, epoch: Epoch) -> Option { + self.entries + .iter() + .enumerate() + .flat_map(|(index, entry)| { + let join = entry.activation_epoch.saturating_sub(SUBSCRIBE_LEAD_EPOCHS); + let leave = self + .entries + .get(index + 1) + .map(|next| next.activation_epoch.saturating_add(UNSUBSCRIBE_LAG_EPOCHS)); + [Some(join), Some(entry.activation_epoch), leave] + }) + .flatten() + .filter(|&change| change > epoch) + .min() + } + + /// The next digest change after `epoch`, for the `eth2` entry's + /// `next_fork_*` and for the startup log. + pub fn next_boundary_after(&self, epoch: Epoch) -> Option { + self.entries + .iter() + .find(|entry| entry.activation_epoch > epoch) + .copied() + } + + /// The `eth2` ENR entry at `epoch`. + pub fn enr_fork_id(&self, epoch: Epoch) -> EnrForkId { + enr_fork_id(&self.config, self.genesis_validators_root, epoch) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ethlambda_types::beacon::config::BlobScheduleEntry; + use libssz_types::SszList; + + const FULU: Epoch = 1_000; + const GLOAS: Epoch = 5_000; + + fn gvr() -> Root { + Root::repeat_byte(7) + } + + /// Sepolia's shape: fulu, then gloas, with no blob-schedule entry between. + fn fulu_then_gloas() -> Config { + let mut config = Config::mainnet() + .with_fork_epoch(ForkName::Altair, 1) + .with_fork_epoch(ForkName::Bellatrix, 2) + .with_fork_epoch(ForkName::Capella, 3) + .with_fork_epoch(ForkName::Deneb, 4) + .with_fork_epoch(ForkName::Electra, 5) + .with_fork_epoch(ForkName::Fulu, FULU) + .with_fork_epoch(ForkName::Gloas, GLOAS); + config.blob_schedule = SszList::new(); + config + } + + /// Fulu, two BPOs, then gloas. + fn with_bpos() -> Config { + let mut config = fulu_then_gloas(); + config.blob_schedule = SszList::try_from(vec![ + BlobScheduleEntry { + epoch: 2_000, + max_blobs_per_block: 15, + }, + BlobScheduleEntry { + epoch: 3_000, + max_blobs_per_block: 21, + }, + ]) + .expect("within capacity"); + config + } + + fn digests_held(schedule: &ForkSchedule, epoch: Epoch) -> Vec { + schedule + .held_at(epoch) + .into_iter() + .map(|entry| entry.digest) + .collect() + } + + #[test] + fn every_scheduled_digest_resolves_to_its_fork() { + let config = with_bpos(); + let schedule = ForkSchedule::new(&config, gvr()); + // Genesis, altair..fulu, two BPOs, gloas: as many entries as distinct + // digests, each mapped back to the fork in force at its activation. + assert!(schedule.entries().len() >= 4); + for entry in schedule.entries() { + assert_eq!(schedule.fork_for_digest(entry.digest), Some(entry.fork)); + assert_eq!( + entry.digest, + compute_fork_digest(&config, gvr(), entry.activation_epoch) + ); + } + assert_eq!(schedule.fork_for_digest([0xde, 0xad, 0xbe, 0xef]), None); + } + + #[test] + fn a_bpo_moves_the_digest_but_not_the_fork() { + let schedule = ForkSchedule::new(&with_bpos(), gvr()); + let before = schedule.current_at(1_999); + let after = schedule.current_at(2_000); + assert_ne!(before.digest, after.digest); + assert_eq!(before.fork, ForkName::Fulu); + assert_eq!(after.fork, ForkName::Fulu); + assert_eq!(schedule.fork_for_digest(after.digest), Some(ForkName::Fulu)); + } + + #[test] + fn the_gloas_boundary_resolves_both_sides() { + let schedule = ForkSchedule::new(&fulu_then_gloas(), gvr()); + assert_eq!(schedule.current_at(GLOAS - 1).fork, ForkName::Fulu); + assert_eq!(schedule.current_at(GLOAS).fork, ForkName::Gloas); + assert_eq!( + schedule.digest_at(GLOAS), + compute_fork_digest(&fulu_then_gloas(), gvr(), GLOAS) + ); + } + + #[test] + fn a_blob_entry_restating_the_current_parameters_is_not_a_boundary() { + let mut config = fulu_then_gloas(); + // Electra's parameters, at fulu's own epoch: the digest cannot move. + let (_, max_blobs) = config.blob_parameters(FULU); + config.blob_schedule = SszList::try_from(vec![BlobScheduleEntry { + epoch: config.electra_fork_epoch, + max_blobs_per_block: max_blobs, + }]) + .expect("within capacity"); + let schedule = ForkSchedule::new(&config, gvr()); + for pair in schedule.entries().windows(2) { + assert_ne!(pair[0].digest, pair[1].digest); + } + } + + #[test] + fn the_next_digest_is_joined_one_epoch_early_and_the_old_left_two_late() { + let schedule = ForkSchedule::new(&fulu_then_gloas(), gvr()); + let fulu = schedule.digest_at(GLOAS - 1); + let gloas = schedule.digest_at(GLOAS); + + // Two epochs ahead: only the pre-boundary digest. + assert_eq!(digests_held(&schedule, GLOAS - 2), vec![fulu]); + // B-1: both, and the old one is still the current. + assert_eq!(digests_held(&schedule, GLOAS - 1), vec![fulu, gloas]); + assert_eq!(schedule.digest_at(GLOAS - 1), fulu); + // B: both, and the new one is current. + assert_eq!(digests_held(&schedule, GLOAS), vec![fulu, gloas]); + assert_eq!(schedule.digest_at(GLOAS), gloas); + // B+1: still both. + assert_eq!(digests_held(&schedule, GLOAS + 1), vec![fulu, gloas]); + // B+2: the old topics go. + assert_eq!(digests_held(&schedule, GLOAS + 2), vec![gloas]); + } + + #[test] + fn a_node_started_inside_the_window_holds_both() { + // Start at B+1: nothing crossed the boundary, yet the state is the one + // a node that did cross it would be in. + let schedule = ForkSchedule::new(&fulu_then_gloas(), gvr()); + let held = digests_held(&schedule, GLOAS + 1); + assert_eq!(held.len(), 2); + assert!(held.contains(&schedule.digest_at(GLOAS + 1))); + } + + #[test] + fn the_current_digest_is_always_held() { + let schedule = ForkSchedule::new(&with_bpos(), gvr()); + for epoch in (0..6_000).step_by(1) { + assert!( + digests_held(&schedule, epoch).contains(&schedule.digest_at(epoch)), + "epoch {epoch}" + ); + } + } + + #[test] + fn changes_are_announced_at_every_join_switch_and_leave() { + let schedule = ForkSchedule::new(&fulu_then_gloas(), gvr()); + assert_eq!(schedule.next_change_after(GLOAS - 10), Some(GLOAS - 1)); + assert_eq!(schedule.next_change_after(GLOAS - 1), Some(GLOAS)); + assert_eq!(schedule.next_change_after(GLOAS), Some(GLOAS + 2)); + assert_eq!(schedule.next_change_after(GLOAS + 2), None); + } + + #[test] + fn the_held_set_only_changes_at_announced_epochs() { + let schedule = ForkSchedule::new(&with_bpos(), gvr()); + let mut previous = (digests_held(&schedule, 0), schedule.digest_at(0)); + let mut epoch = 0; + while epoch < 6_000 { + let next = schedule.next_change_after(epoch); + let until = next.unwrap_or(6_000); + for probe in epoch..until { + assert_eq!( + (digests_held(&schedule, probe), schedule.digest_at(probe)), + previous, + "epoch {probe} changed without being announced" + ); + } + epoch = until; + previous = (digests_held(&schedule, epoch), schedule.digest_at(epoch)); + } + } + + #[test] + fn the_next_boundary_names_the_digest_and_fork_ahead() { + let schedule = ForkSchedule::new(&fulu_then_gloas(), gvr()); + let next = schedule.next_boundary_after(FULU + 1).expect("gloas ahead"); + assert_eq!(next.activation_epoch, GLOAS); + assert_eq!(next.fork, ForkName::Gloas); + assert_eq!(schedule.next_boundary_after(GLOAS), None); + } + + #[test] + fn the_enr_entry_points_at_the_next_boundary() { + let schedule = ForkSchedule::new(&fulu_then_gloas(), gvr()); + let before = schedule.enr_fork_id(GLOAS - 1); + assert_eq!(before.next_fork_epoch, GLOAS); + let after = schedule.enr_fork_id(GLOAS); + assert_eq!(after.fork_digest, schedule.digest_at(GLOAS)); + assert_eq!(after.next_fork_epoch, FAR_FUTURE_EPOCH); + } + + #[test] + fn closely_spaced_boundaries_hold_each_digest_in_its_own_window() { + let mut config = fulu_then_gloas(); + config.blob_schedule = SszList::try_from(vec![ + BlobScheduleEntry { + epoch: 2_000, + max_blobs_per_block: 15, + }, + BlobScheduleEntry { + epoch: 2_001, + max_blobs_per_block: 21, + }, + ]) + .expect("within capacity"); + let schedule = ForkSchedule::new(&config, gvr()); + // Three digests are live at once around 2_001; none is dropped early. + assert_eq!(schedule.held_at(2_001).len(), 3); + } +} diff --git a/crates/net/p2p/src/beacon/handler.rs b/crates/net/p2p/src/beacon/handler.rs index c593815da..b8065a37d 100644 --- a/crates/net/p2p/src/beacon/handler.rs +++ b/crates/net/p2p/src/beacon/handler.rs @@ -231,7 +231,12 @@ mod tests { BeaconWire { fork_digest: [0x8c, 0x9f, 0x62, 0xfe], fork: ForkName::Fulu, + schedule: crate::beacon::fork_schedule::ForkSchedule::new( + &Config::mainnet(), + Root::ZERO, + ), topics: topics::BeaconTopics::new([0x8c, 0x9f, 0x62, 0xfe], &[], &[]), + window_topics: Vec::new(), config: Config::mainnet(), genesis_time: 1_606_824_023, genesis_validators_root: Root::ZERO, diff --git a/crates/net/p2p/src/beacon/mod.rs b/crates/net/p2p/src/beacon/mod.rs index 3369ccba0..44903b698 100644 --- a/crates/net/p2p/src/beacon/mod.rs +++ b/crates/net/p2p/src/beacon/mod.rs @@ -16,12 +16,14 @@ pub mod column_checks; pub mod decode; pub mod encoding; +pub mod fork_schedule; pub mod handler; pub mod messages; pub mod protocols; pub mod subnets; pub mod swarm; pub mod topics; +pub mod transition; pub mod verdict; use ethlambda_types::beacon::config::Config; @@ -34,15 +36,23 @@ use ethlambda_types::beacon::primitives::{ForkDigest, Root}; /// fork a gossip payload decodes under is derived from its slot, and that /// derivation must use the same schedule the fork digest was computed from. pub struct BeaconWire { + /// The digest this node publishes under and advertises in `Status`. Moves + /// at each boundary of [`Self::schedule`]; see [`transition`]. pub fork_digest: ForkDigest, - /// The fork `fork_digest` was computed at, so the fork of everything - /// that arrives on a subscribed topic. + /// The fork `fork_digest` was computed at. /// - /// Only `beacon_attestation_{subnet_id}` reads it. Every other - /// fork-dependent topic finds its fork from the slot inside the payload, - /// which this one cannot do; see [`decode::decode_attestation`]. + /// What a message published now is in. A received message's fork comes from + /// its own topic's digest through [`Self::schedule`] instead, since while a + /// boundary's window is open this node is subscribed under two digests. pub fork: ForkName, + /// Every digest the chain will use, and when to join and leave each. + pub schedule: fork_schedule::ForkSchedule, + /// The topics under `fork_digest`. pub topics: topics::BeaconTopics, + /// The topics of every other digest the subscription window holds: the + /// next one from an epoch ahead of its boundary, the previous one until two + /// epochs after. Empty outside a window. + pub window_topics: Vec, pub config: Config, pub genesis_time: u64, /// The chain every fork digest is bound to. @@ -52,8 +62,8 @@ pub struct BeaconWire { /// epoch, so serving history means computing digests this node never runs /// on. See [`encoding`]'s module docs for the rule. pub genesis_validators_root: Root, - /// Advertised in `Ping` responses and in `MetaData`. Never bumped today: - /// nothing this node advertises changes at runtime. + /// Advertised in `Ping` responses and in `MetaData`. Bumped when the + /// digest switches, the one advertised value that changes at runtime. pub metadata_seq_number: u64, /// The columns this node custodies, carried alongside `topics` so the /// gossip handler and the request handlers can check what this node @@ -67,6 +77,19 @@ pub struct BeaconWire { } impl BeaconWire { + /// Every topic set this node is subscribed to: the current digest's, then + /// the window's. + pub fn held_topics(&self) -> impl Iterator { + std::iter::once(&self.topics).chain(self.window_topics.iter()) + } + + /// Whether this node is subscribed under `digest`, so whether a peer on it + /// is on a digest this node still speaks. + pub fn holds_digest(&self, digest: ForkDigest) -> bool { + self.held_topics() + .any(|topics| topics.fork_digest == digest) + } + /// The two values the codec needs to put a block chunk on or off the wire. pub fn codec_context(&self) -> BeaconContext { BeaconContext { diff --git a/crates/net/p2p/src/beacon/topics.rs b/crates/net/p2p/src/beacon/topics.rs index b367a0536..8f57f90a5 100644 --- a/crates/net/p2p/src/beacon/topics.rs +++ b/crates/net/p2p/src/beacon/topics.rs @@ -28,6 +28,7 @@ use std::collections::BTreeMap; +use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::primitives::ForkDigest; use libp2p::gossipsub::IdentTopic; @@ -99,6 +100,29 @@ pub fn topic_kind(topic: &str) -> Option<&str> { crate::gossipsub::topic_kind(topic) } +/// The fork digest embedded in a full topic name, or `None` if the name is not +/// shaped like a beacon topic or carries something other than four hex bytes. +/// +/// A message's fork is the fork its topic's digest names, which is not the +/// digest this node currently advertises while a fork boundary's subscription +/// window is open. +pub fn topic_digest(topic: &str) -> Option { + let digest = topic.split('/').nth(2)?; + hex::decode(digest).ok()?.try_into().ok() +} + +/// The column subnets a custody set maps onto, in custody order. +/// +/// A column's subnet is `column % DATA_COLUMN_SIDECAR_SUBNET_COUNT`, computed +/// rather than assumed so a network that ever separates the two counts still +/// subscribes to the right topic. `BeaconTopics::new` deduplicates. +pub fn column_subnets(custody_columns: &[u64]) -> Vec { + custody_columns + .iter() + .map(|column| column % ethlambda_types::beacon::constants::DATA_COLUMN_SIDECAR_SUBNET_COUNT) + .collect() +} + /// Topic family for unaggregated attestations, one topic per subnet. pub const BEACON_ATTESTATION_PREFIX: &str = "beacon_attestation_"; @@ -184,6 +208,22 @@ pub struct BeaconTopics { } impl BeaconTopics { + /// The topics to hold under `fork_digest`, for the fork that digest names. + /// + /// The one place a fork may change which topic kinds exist: a fork that adds + /// a kind extends what this returns for its own digest, and the transition + /// then joins those topics at the pre-boundary epoch the spec asks for. + /// Today every fork subscribes to the same kinds, so `fork` is read by + /// nothing yet. + pub fn for_fork( + _fork: ForkName, + fork_digest: ForkDigest, + column_subnets: &[u64], + attestation_subnets: &[u64], + ) -> Self { + Self::new(fork_digest, column_subnets, attestation_subnets) + } + pub fn new( fork_digest: ForkDigest, column_subnets: &[u64], @@ -287,6 +327,21 @@ mod tests { } } + #[test] + fn the_digest_reads_back_from_a_topic_name() { + assert_eq!( + topic_digest(&topic_name(MAINNET, BEACON_BLOCK)), + Some(MAINNET) + ); + assert_eq!( + topic_digest(&attestation_topic_name([0x0a, 0xbc, 0xde, 0xf0], 3)), + Some([0x0a, 0xbc, 0xde, 0xf0]) + ); + assert_eq!(topic_digest("/eth2/zz/beacon_block/ssz_snappy"), None); + assert_eq!(topic_digest("/eth2/8c9f62/beacon_block/ssz_snappy"), None); + assert_eq!(topic_digest("garbage"), None); + } + #[test] fn topic_kind_reads_the_name_back() { for kind in SUBSCRIBED_TOPIC_KINDS { diff --git a/crates/net/p2p/src/beacon/transition.rs b/crates/net/p2p/src/beacon/transition.rs new file mode 100644 index 000000000..6a844208f --- /dev/null +++ b/crates/net/p2p/src/beacon/transition.rs @@ -0,0 +1,334 @@ +//! Crossing fork-digest boundaries on a running node. +//! +//! [`ForkSchedule`](super::fork_schedule::ForkSchedule) says which digests the +//! node should hold at an epoch; this applies that to the swarm, the wire state +//! and discovery, and works out when to look again. Every fork activation and +//! every blob-parameter-only fork is a boundary, since both move the digest. +//! +//! [`apply`] is idempotent: it computes the target state from the epoch alone +//! and changes only what differs, so the first call at startup (which joins the +//! rest of a window the node started inside) and a call at a live crossing are +//! one code path, and a timer that fires twice does nothing the second time. + +use std::time::Duration; + +use ethlambda_types::beacon::config::Config; +use ethlambda_types::beacon::preset; +use ethlambda_types::beacon::primitives::{Epoch, ForkDigest}; +use ethlambda_types::time::unix_now_ms; +use libp2p::gossipsub::IdentTopic; +use tracing::{error, info}; + +use super::topics::{self, BeaconTopics}; +use crate::{P2PServer, Wire, discovery, metrics}; + +fn epoch_duration_ms(config: &Config) -> u64 { + config + .slot_duration_ms + .max(1) + .saturating_mul(preset::SLOTS_PER_EPOCH) +} + +/// The epoch the wall clock reads at `now_ms`; epoch 0 before genesis. +pub(crate) fn epoch_at_ms(config: &Config, genesis_time: u64, now_ms: u64) -> Epoch { + now_ms.saturating_sub(genesis_time.saturating_mul(1000)) / epoch_duration_ms(config) +} + +/// How long from `now_ms` until `epoch` begins; zero if it already has. +pub(crate) fn delay_until_epoch( + config: &Config, + genesis_time: u64, + epoch: Epoch, + now_ms: u64, +) -> Duration { + let start_ms = genesis_time + .saturating_mul(1000) + .saturating_add(epoch.saturating_mul(epoch_duration_ms(config))); + Duration::from_millis(start_ms.saturating_sub(now_ms)) +} + +/// Apply the schedule at the current wall-clock epoch, and return how long to +/// wait before the next join, switch or leave. `None` on a lean node or once the +/// schedule has run out. +pub(crate) fn advance(server: &mut P2PServer) -> Option { + let wire = server.wire.beacon()?; + let now_ms = unix_now_ms(); + let epoch = epoch_at_ms(&wire.config, wire.genesis_time, now_ms); + let next = wire.schedule.next_change_after(epoch).map(|next| { + // A timer that fires a hair early would read the same epoch and + // reschedule itself with no delay, spinning until the boundary; one + // millisecond floor keeps that to a handful of wakeups. + delay_until_epoch(&wire.config, wire.genesis_time, next, now_ms) + .max(Duration::from_millis(1)) + }); + apply(server, epoch); + next +} + +/// Move the node to the state the schedule prescribes at `epoch`. +/// +/// 1. Join the topics of every digest the window now covers, including the +/// aggregator-subnet topics already held for the current slot. +/// 2. Leave the topics of every digest the window no longer covers. +/// 3. If the current digest changed, switch what this node publishes and +/// advertises: digest, fork, metadata sequence number, metric, and the +/// `eth2` entry. +/// 4. Point the admission filter at the new entry, admitting every held digest. +pub(crate) fn apply(server: &mut P2PServer, epoch: Epoch) { + let Wire::Beacon(wire) = &mut server.wire else { + return; + }; + let held = wire.schedule.held_at(epoch); + let current = wire.schedule.current_at(epoch); + let column_subnets = topics::column_subnets(&wire.custody_columns); + + let mut existing = std::mem::take(&mut wire.window_topics); + existing.push(wire.topics.clone()); + + let mut kept: Vec = Vec::with_capacity(held.len()); + for entry in &held { + if let Some(index) = existing + .iter() + .position(|topics| topics.fork_digest == entry.digest) + { + kept.push(existing.swap_remove(index)); + continue; + } + let joined = BeaconTopics::for_fork( + entry.fork, + entry.digest, + &column_subnets, + &wire.attestation_subnets, + ); + for topic in &joined.topics { + server.swarm_handle.subscribe(topic.clone()); + } + for &subnet_id in server.aggregator_subnets.keys() { + server + .swarm_handle + .subscribe(attestation_topic(entry.digest, subnet_id)); + } + info!( + fork_digest = %hex::encode(entry.digest), + fork = entry.fork.as_str(), + activation_epoch = entry.activation_epoch, + epoch, + topics = joined.topics.len(), + "Joined the topics of an upcoming or live fork digest" + ); + kept.push(joined); + } + + for left in existing { + for topic in &left.topics { + server.swarm_handle.unsubscribe(topic.clone()); + } + for &subnet_id in server.aggregator_subnets.keys() { + server + .swarm_handle + .unsubscribe(attestation_topic(left.fork_digest, subnet_id)); + } + info!( + fork_digest = %hex::encode(left.fork_digest), + epoch, + "Left the topics of a past fork digest" + ); + } + + let Some(index) = kept + .iter() + .position(|topics| topics.fork_digest == current.digest) + else { + // `held_at` always contains the current digest, so this is a schedule + // bug; keep the node on what it had rather than publishing nowhere. + error!( + epoch, + "The fork schedule holds no topics for the current digest" + ); + return; + }; + wire.topics = kept.remove(index); + wire.window_topics = kept; + + let switched = wire.fork_digest != current.digest; + if switched { + let previous = wire.fork_digest; + wire.fork_digest = current.digest; + wire.fork = current.fork; + wire.metadata_seq_number += 1; + let digest_hex = hex::encode(current.digest); + metrics::set_beacon_fork_digest(&digest_hex); + info!( + epoch, + previous_digest = %hex::encode(previous), + fork_digest = %digest_hex, + fork = current.fork.as_str(), + metadata_seq_number = wire.metadata_seq_number, + "Crossed a fork digest boundary" + ); + } + + let fork_id = wire.schedule.enr_fork_id(epoch); + let also_admitted: Vec = wire + .window_topics + .iter() + .map(|topics| topics.fork_digest) + .collect(); + // Without discovery there is no peer table whose admission to update. + if let Some(state) = &server.discovery { + state.filter().set_fork_id(fork_id, also_admitted); + } + if switched { + discovery::update_served_enr(&fork_id); + } +} + +fn attestation_topic(digest: ForkDigest, subnet_id: u64) -> IdentTopic { + IdentTopic::new(topics::attestation_topic_name(digest, subnet_id)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::unconnected_beacon_server; + use ethlambda_types::beacon::fork::ForkName; + use libssz_types::SszList; + + use ethlambda_types::beacon::config::BlobScheduleEntry; + + const FULU: Epoch = 1_000; + const BPO: Epoch = 2_000; + const GLOAS: Epoch = 5_000; + + /// Fulu, one BPO, then gloas, on a chain whose wire the test server builds + /// with `Root::ZERO` as its genesis validators root. + fn config() -> Config { + let mut config = Config::mainnet() + .with_fork_epoch(ForkName::Altair, 1) + .with_fork_epoch(ForkName::Bellatrix, 2) + .with_fork_epoch(ForkName::Capella, 3) + .with_fork_epoch(ForkName::Deneb, 4) + .with_fork_epoch(ForkName::Electra, 5) + .with_fork_epoch(ForkName::Fulu, FULU) + .with_fork_epoch(ForkName::Gloas, GLOAS); + config.blob_schedule = SszList::try_from(vec![BlobScheduleEntry { + epoch: BPO, + max_blobs_per_block: 15, + }]) + .expect("within capacity"); + config + } + + fn digests(server: &P2PServer) -> (ForkDigest, Vec) { + let wire = server.wire.beacon().expect("a beacon wire"); + ( + wire.fork_digest, + wire.window_topics + .iter() + .map(|topics| topics.fork_digest) + .collect(), + ) + } + + #[test] + fn the_epoch_is_read_off_the_slot_duration() { + let config = Config::mainnet(); + let epoch_ms = config.slot_duration_ms * preset::SLOTS_PER_EPOCH; + let genesis = 1_000; + assert_eq!(epoch_at_ms(&config, genesis, 0), 0); + assert_eq!( + epoch_at_ms(&config, genesis, genesis * 1000 + epoch_ms - 1), + 0 + ); + assert_eq!(epoch_at_ms(&config, genesis, genesis * 1000 + epoch_ms), 1); + assert_eq!( + delay_until_epoch(&config, genesis, 3, genesis * 1000 + epoch_ms), + Duration::from_millis(2 * epoch_ms) + ); + assert_eq!( + delay_until_epoch(&config, genesis, 1, genesis * 1000 + 2 * epoch_ms), + Duration::ZERO + ); + } + + #[tokio::test] + async fn a_live_crossing_joins_switches_and_leaves_in_order() { + let config = config(); + let mut server = unconnected_beacon_server(config.clone(), 0).await; + let schedule = server.wire.beacon().expect("beacon").schedule.clone(); + let bpo = schedule.digest_at(BPO); + let before = schedule.digest_at(BPO - 1); + + // Before the window: only the current digest. + apply(&mut server, BPO - 2); + assert_eq!(digests(&server), (before, vec![])); + let seq_before = server.wire.beacon().unwrap().metadata_seq_number; + + // B-1: the next digest is joined while the old one stays current. + apply(&mut server, BPO - 1); + assert_eq!(digests(&server), (before, vec![bpo])); + assert_eq!( + server.wire.beacon().unwrap().metadata_seq_number, + seq_before + ); + + // B: the switch. Publishing and Status move, both digests stay held, + // and the sequence number is bumped exactly once. + apply(&mut server, BPO); + assert_eq!(digests(&server), (bpo, vec![before])); + let wire = server.wire.beacon().unwrap(); + assert_eq!(wire.metadata_seq_number, seq_before + 1); + assert!(wire.holds_digest(before) && wire.holds_digest(bpo)); + + // Applying again at the same epoch changes nothing. + apply(&mut server, BPO); + assert_eq!(digests(&server), (bpo, vec![before])); + assert_eq!( + server.wire.beacon().unwrap().metadata_seq_number, + seq_before + 1 + ); + + // B+2: the old topics are left. + apply(&mut server, BPO + 2); + assert_eq!(digests(&server), (bpo, vec![])); + assert!(!server.wire.beacon().unwrap().holds_digest(before)); + } + + #[tokio::test] + async fn the_gloas_crossing_switches_the_fork_the_node_publishes_in() { + let config = config(); + let mut server = unconnected_beacon_server(config, 0).await; + apply(&mut server, GLOAS - 1); + assert_eq!(server.wire.beacon().unwrap().fork, ForkName::Fulu); + apply(&mut server, GLOAS); + assert_eq!(server.wire.beacon().unwrap().fork, ForkName::Gloas); + } + + #[tokio::test] + async fn a_node_started_inside_the_window_joins_both_digests() { + let config = config(); + let mut server = unconnected_beacon_server(config, 0).await; + let schedule = server.wire.beacon().expect("beacon").schedule.clone(); + + // The startup call, at B+1: the node was built on one digest and the + // window says two. + apply(&mut server, GLOAS + 1); + let (current, window) = digests(&server); + assert_eq!(current, schedule.digest_at(GLOAS)); + assert_eq!(window, vec![schedule.digest_at(GLOAS - 1)]); + } + + #[tokio::test] + async fn a_message_takes_its_fork_from_its_topic() { + let config = config(); + let mut server = unconnected_beacon_server(config, 0).await; + apply(&mut server, GLOAS); + let wire = server.wire.beacon().unwrap(); + // Gloas is current, yet a message on the still-held fulu topic is fulu. + let fulu = wire.schedule.digest_at(GLOAS - 1); + let topic = topics::topic_name(fulu, topics::BEACON_BLOCK); + let digest = topics::topic_digest(&topic).expect("a digest"); + assert_eq!(wire.schedule.fork_for_digest(digest), Some(ForkName::Fulu)); + assert_eq!(wire.fork, ForkName::Gloas); + } +} diff --git a/crates/net/p2p/src/discovery/admission.rs b/crates/net/p2p/src/discovery/admission.rs index c4068ba60..6a1c6adf5 100644 --- a/crates/net/p2p/src/discovery/admission.rs +++ b/crates/net/p2p/src/discovery/admission.rs @@ -18,6 +18,7 @@ //! ([`rank_candidates`]). use std::collections::HashSet; +use std::sync::{Arc, RwLock}; use ethrex_p2p::peer_filter::PeerFilter; use ethrex_p2p::types::NodeRecord; @@ -27,6 +28,7 @@ use tracing::debug; use ethlambda_state_transition::beacon::das; use ethlambda_types::beacon::constants; +use ethlambda_types::beacon::primitives::ForkDigest; use super::enr::{ ATTNETS_ENR_KEY, CGC_ENR_KEY, ETH2_ENR_KEY, EnrForkId, node_id_from_peer_id, read_ip, @@ -80,23 +82,51 @@ pub(crate) enum RejectReason { /// Holds what [`admit`] needs to judge a record, so the dial loop no longer /// carries the local fork id and committee count around: the value handed to /// [`PeerTableServer::spawn_with_filter`](ethrex_p2p::peer_table::PeerTableServer::spawn_with_filter) -/// judges by the same rules the dial loop later asks for dial targets. It is +/// judges by the same rules the dial loop later asks for dial targets. +/// /// [`Clone`] because the peer table takes ownership of the filter it runs, and -/// both fields are plain data. +/// every clone shares one [`Policy`], so [`Self::set_fork_id`] at a fork +/// boundary changes what the table's copy admits as well as the dial loop's. #[derive(Clone)] pub struct LeanFilter { - fork_id: EnrForkId, + policy: Arc>, attestation_committee_count: u64, } +/// The part of admission that moves at a fork boundary. +#[derive(Debug, Clone)] +struct Policy { + /// This node's own `eth2` entry; its digest is always admitted. + fork_id: EnrForkId, + /// Further digests admitted alongside it: while a boundary's subscription + /// window is open, peers are still legitimately on the other side of it. + also_admitted: Vec, +} + impl LeanFilter { pub(crate) fn new(fork_id: EnrForkId, attestation_committee_count: u64) -> Self { Self { - fork_id, + policy: Arc::new(RwLock::new(Policy { + fork_id, + also_admitted: Vec::new(), + })), attestation_committee_count, } } + /// Replace the local `eth2` entry and the further digests admitted beside + /// it. + /// + /// The peer table re-judges a record only when its `seq` rises, so a peer + /// rejected under the old policy stays rejected until it republishes; + /// peers cross a boundary by doing exactly that. + pub(crate) fn set_fork_id(&self, fork_id: EnrForkId, also_admitted: Vec) { + *self.policy.write().expect("admission policy lock poisoned") = Policy { + fork_id, + also_admitted, + }; + } + /// What to dial for a record that has already been admitted, or `None` if it /// would not be. /// @@ -106,13 +136,23 @@ impl LeanFilter { /// unchanged, and the peer table hands out clones: a caller that reaches /// this with an arbitrary record should get nothing to dial, not a panic. pub(crate) fn dial_target(&self, record: &NodeRecord) -> Option { - admit(record, &self.fork_id, self.attestation_committee_count).ok() + self.judge(record).ok() + } + + fn judge(&self, record: &NodeRecord) -> Result { + let policy = self.policy.read().expect("admission policy lock poisoned"); + admit( + record, + &policy.fork_id, + &policy.also_admitted, + self.attestation_committee_count, + ) } } impl PeerFilter for LeanFilter { fn accepts(&self, record: &NodeRecord) -> bool { - admit(record, &self.fork_id, self.attestation_committee_count) + self.judge(record) // The only place a rejection is visible: the peer table records // that the record failed the filter but says nothing about why. .inspect_err(|reason| { @@ -136,6 +176,7 @@ impl PeerFilter for LeanFilter { fn admit( record: &NodeRecord, local: &EnrForkId, + also_admitted: &[ForkDigest], attestation_committee_count: u64, ) -> Result { let pairs = record.pairs(); @@ -144,7 +185,7 @@ fn admit( .ok_or(RejectReason::MissingForkId)?; let remote = EnrForkId::from_ssz_bytes(&raw).map_err(|_| RejectReason::MissingForkId)?; - if remote.fork_digest != local.fork_digest { + if remote.fork_digest != local.fork_digest && !also_admitted.contains(&remote.fork_digest) { return Err(RejectReason::ForkDigestMismatch); } if remote.next_fork_version != local.next_fork_version @@ -333,7 +374,7 @@ mod tests { } fn admit_record(record: &NodeRecord) -> Result { - admit(record, &EnrForkId::local(), TEST_COMMITTEE_COUNT) + admit(record, &EnrForkId::local(), &[], TEST_COMMITTEE_COUNT) } /// Build a record whose pairs are fully controlled, bypassing @@ -575,6 +616,36 @@ mod tests { assert!(filter().dial_target(&record).is_none()); } + #[test] + fn a_fork_boundary_moves_what_every_clone_admits() { + // The peer table owns a clone of the filter, so a boundary has to reach + // it through the shared policy rather than through the dial loop's copy. + let table_copy = filter(); + let dial_copy = table_copy.clone(); + + let mut next = EnrForkId::local(); + next.fork_digest = [0xde, 0xad, 0xbe, 0xef]; + let on_next = record_with(|pairs| { + set_eth2(pairs, next); + set_quic(pairs, 9001); + }); + let on_local = record_with(|pairs| { + set_eth2(pairs, EnrForkId::local()); + set_quic(pairs, 9001); + }); + assert!(!table_copy.accepts(&on_next)); + + // Window open: both sides of the boundary are admitted. + dial_copy.set_fork_id(EnrForkId::local(), vec![next.fork_digest]); + assert!(table_copy.accepts(&on_next)); + assert!(table_copy.accepts(&on_local)); + + // Switched and window closed: only the new digest. + dial_copy.set_fork_id(next, Vec::new()); + assert!(table_copy.accepts(&on_next)); + assert!(!table_copy.accepts(&on_local)); + } + #[test] fn a_missing_quic_port_is_rejected() { // Discoverable, but not over the only transport we speak. The peer can diff --git a/crates/net/p2p/src/discovery/dial.rs b/crates/net/p2p/src/discovery/dial.rs index bd1ef0c4c..229573c26 100644 --- a/crates/net/p2p/src/discovery/dial.rs +++ b/crates/net/p2p/src/discovery/dial.rs @@ -40,6 +40,9 @@ pub(crate) struct DiscoveryState { local_peer_id: PeerId, /// Connected-peer count above which the loop stops dialing. target_peers: usize, + /// A handle on the admission policy the peer table also runs, so a fork + /// boundary can move both with one call. See [`LeanFilter::set_fork_id`]. + filter: LeanFilter, } impl DiscoveryState { @@ -47,14 +50,20 @@ impl DiscoveryState { /// runtime. Every call site builds a `P2PServer` from an async context. pub(crate) fn new(handle: DiscoveryHandle, local_peer_id: PeerId) -> Self { Self { - contacts: spawn_contact_poll(handle.peer_table, handle.filter), + contacts: spawn_contact_poll(handle.peer_table, handle.filter.clone()), candidates: VecDeque::new(), peer_attnets: HashMap::new(), local_peer_id, target_peers: handle.target_peers, + filter: handle.filter, } } + /// The admission policy shared with the peer table. + pub(crate) fn filter(&self) -> &LeanFilter { + &self.filter + } + /// Connected-peer count above which the loop stops dialing. pub(crate) fn target_peers(&self) -> usize { self.target_peers diff --git a/crates/net/p2p/src/discovery/mod.rs b/crates/net/p2p/src/discovery/mod.rs index 6a6e3a09b..4020dc27b 100644 --- a/crates/net/p2p/src/discovery/mod.rs +++ b/crates/net/p2p/src/discovery/mod.rs @@ -259,6 +259,33 @@ pub async fn spawn_discovery( }) } +/// Publish a new `eth2` entry in the ENR this node serves over discv5. +/// +/// Not possible with the discovery stack this node is built on, so this only +/// says so. [`DiscoveryServer::spawn`] takes the local record by value, moves it +/// into an actor it starts itself, and returns `()`: no actor handle comes back, +/// and `DiscoveryServerProtocol` (`raw_packet`, `lookup_*`, `revalidate_*`, +/// `enr_lookup`, `prune`, `shutdown`) has no message that replaces the record. +/// The record is re-signed only internally, by discv5 IP voting +/// (`update_local_ip`), which edits the address and nothing else. What would +/// close the gap is a protocol message on `DiscoveryServer` that edits the +/// local record and bumps its `seq`, or `spawn` returning the actor's handle. +/// +/// Until then the record served to discv5 queries keeps the `eth2` entry it +/// started with. Peers already connected are unaffected (they handshake on +/// `Status`, which does follow the switch) and the admission filter does move, +/// but a peer that discovers this node after a boundary reads the stale digest +/// and rejects the record, so a long-running node becomes harder to find until +/// it is restarted. +pub(crate) fn update_served_enr(fork_id: &EnrForkId) { + warn!( + fork_digest = %hex::encode(fork_id.fork_digest), + next_fork_epoch = fork_id.next_fork_epoch, + "The served ENR keeps its startup eth2 entry: the discovery server offers no way to \ + replace its record at runtime" + ); +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/net/p2p/src/gossipsub/handler.rs b/crates/net/p2p/src/gossipsub/handler.rs index 631b56dd7..acbdaf521 100644 --- a/crates/net/p2p/src/gossipsub/handler.rs +++ b/crates/net/p2p/src/gossipsub/handler.rs @@ -13,6 +13,7 @@ use ethlambda_types::{ ShortRoot, attestation::{SignedAggregatedAttestation, SignedAttestation}, beacon::containers::{SignedBeaconBlock, electra::SingleAttestation}, + beacon::fork::ForkName, block::SignedBlock, primitives::HashTreeRoot as _, time::unix_now_ms, @@ -98,7 +99,9 @@ pub async fn handle_gossip_message( handle_lean_attestation(server, &payload, compressed_len).await } _ => match beacon_id { - Some(id) => handle_beacon_gossip(server, ctx, id, kind, &payload), + Some(id) => { + handle_beacon_gossip(server, ctx, id, message.topic.as_str(), kind, &payload) + } None => trace!(topic = %message.topic, "Gossip on an unhandled topic"), }, } @@ -222,6 +225,7 @@ fn handle_beacon_gossip( server: &P2PServer, ctx: &Context, id: GossipId, + topic: &str, kind: &str, payload: &[u8], ) { @@ -238,7 +242,13 @@ fn handle_beacon_gossip( } else if kind == beacon_topics::BEACON_AGGREGATE_AND_PROOF { triage_aggregate(server, wire, payload, id.received_at) } else if let Some(subnet_id) = beacon_topics::attestation_subnet(kind) { - triage_attestation(server, wire, payload, subnet_id) + // The fork comes from the topic's own digest: while a boundary's + // window is open the node is subscribed under two, and `wire.fork` is + // only the one it publishes under. + let fork = beacon_topics::topic_digest(topic) + .and_then(|digest| wire.schedule.fork_for_digest(digest)) + .unwrap_or(wire.fork); + triage_attestation(server, fork, payload, subnet_id) } else { triage_other(wire, kind, payload) }; @@ -427,12 +437,12 @@ fn triage_aggregate( /// already carries in full. fn triage_attestation( server: &P2PServer, - wire: &BeaconWire, + fork: ForkName, payload: &[u8], subnet_id: u64, ) -> Dispatch { const KIND: &str = beacon_topics::BEACON_ATTESTATION_KIND; - let attestation = match beacon_decode::decode_attestation(wire.fork, payload) { + let attestation = match beacon_decode::decode_attestation(fork, payload) { Ok(attestation) => attestation, // As on the aggregate topic: an honest peer on a fork this node has no // attestation rules for must not be scored as a bad decoder. @@ -740,9 +750,13 @@ pub fn join_aggregator_subnets(server: &mut P2PServer, subnets: Vec<(u64, u64)>) }); *until = (*until).max(slot); } + // Under every digest the node holds: while a boundary's window is open, + // attestations are published on either side of it. for &subnet_id in &joined { - let topic = beacon_topics::attestation_topic_name(wire.fork_digest, subnet_id); - server.swarm_handle.subscribe(IdentTopic::new(topic)); + for held in wire.held_topics() { + let topic = beacon_topics::attestation_topic_name(held.fork_digest, subnet_id); + server.swarm_handle.subscribe(IdentTopic::new(topic)); + } } if !joined.is_empty() { info!(?joined, "Joined attestation subnets for aggregation"); @@ -779,8 +793,10 @@ pub fn leave_expired_aggregator_subnets(server: &mut P2PServer) { .collect(); for subnet_id in &expired { server.aggregator_subnets.remove(subnet_id); - let topic = beacon_topics::attestation_topic_name(wire.fork_digest, *subnet_id); - server.swarm_handle.unsubscribe(IdentTopic::new(topic)); + for held in wire.held_topics() { + let topic = beacon_topics::attestation_topic_name(held.fork_digest, *subnet_id); + server.swarm_handle.unsubscribe(IdentTopic::new(topic)); + } } if !expired.is_empty() { debug!( @@ -1062,7 +1078,7 @@ mod tests { .expect("a beacon server has a beacon wire"); assert!(matches!( - triage_attestation(&server, wire, &[0xff; 3], 0), + triage_attestation(&server, wire.fork, &[0xff; 3], 0), Dispatch::Report(Outcome::Reject(RejectReason::Decode)) )); } @@ -1083,7 +1099,7 @@ mod tests { let payload = phase0_attestation(4).to_ssz(); assert!(matches!( - triage_attestation(&server, wire, &payload, 0), + triage_attestation(&server, wire.fork, &payload, 0), Dispatch::Report(Outcome::Ignore(IgnoreReason::NoConsumer)) )); } @@ -1100,7 +1116,7 @@ mod tests { let payload = electra_single_attestation(4, 1).to_ssz(); assert!(matches!( - triage_attestation(&server, wire, &payload, 0), + triage_attestation(&server, wire.fork, &payload, 0), Dispatch::Report(Outcome::Reject(RejectReason::NonZeroDataIndex)) )); } @@ -1119,7 +1135,7 @@ mod tests { let payload = electra_single_attestation(4, 1).to_ssz(); assert!(matches!( - triage_attestation(&server, wire, &payload, 0), + triage_attestation(&server, wire.fork, &payload, 0), Dispatch::Report(Outcome::Ignore(IgnoreReason::UnsupportedFork)) )); } diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index 80a8ffa09..01a7054df 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -972,14 +972,9 @@ pub fn build_swarm(config: SwarmConfig) -> Result { // `column % DATA_COLUMN_SIDECAR_SUBNET_COUNT`, computed rather than // assumed so a network that ever separates the two counts still // subscribes to the right topic. - let column_subnets: Vec = beacon - .custody_columns - .iter() - .map(|column| { - column % ethlambda_types::beacon::constants::DATA_COLUMN_SIDECAR_SUBNET_COUNT - }) - .collect(); - let topics = beacon::topics::BeaconTopics::new( + let column_subnets = beacon::topics::column_subnets(&beacon.custody_columns); + let topics = beacon::topics::BeaconTopics::for_fork( + beacon.fork, beacon.fork_digest, &column_subnets, &beacon.attestation_subnets, @@ -998,10 +993,21 @@ pub fn build_swarm(config: SwarmConfig) -> Result { "Beacon P2P node started" ); + // Only the current digest is joined here. The rest of the window + // around a boundary (the next digest's topics, or the previous + // one's if the node started just after one) is joined by the first + // `AdvanceForkSchedule`, which `P2P::spawn` sends at once, so that + // startup and a live crossing run one code path. + let schedule = beacon::fork_schedule::ForkSchedule::new( + &beacon.config, + beacon.genesis_validators_root, + ); Wire::Beacon(Box::new(beacon::BeaconWire { fork_digest: beacon.fork_digest, fork: beacon.fork, + schedule, topics, + window_topics: Vec::new(), config: beacon.config, genesis_time: beacon.genesis_time, genesis_validators_root: beacon.genesis_validators_root, @@ -1122,6 +1128,12 @@ impl P2P { p2p_protocol::DiscoverPeers, ); } + // Immediate on a beacon wire: joins the rest of a window the node + // started inside, then keeps itself scheduled. A no-op on lean. + let _ = handle + .context() + .send(p2p_protocol::AdvanceForkSchedule) + .inspect_err(|err| warn!(%err, "Could not schedule the fork schedule")); spawn_listener(handle.context(), swarm_stream.map(WrappedSwarmEvent)); Ok(P2P { handle }) } @@ -1310,6 +1322,10 @@ pub(crate) trait P2PProtocol: Send + Sync { fn leave_expired_aggregator_subnets(&self) -> Result<(), ActorError>; #[allow(dead_code)] // invoked via send_after, not called directly fn retry_beacon_range_batch(&self) -> Result<(), ActorError>; + /// Apply the fork schedule at the wall-clock epoch. Sent once at startup and + /// then by its own timer for the next join, switch or leave. + #[allow(dead_code)] // invoked via send_after, not called directly + fn advance_fork_schedule(&self) -> Result<(), ActorError>; } #[actor(protocol = P2PProtocol)] @@ -1394,6 +1410,17 @@ impl P2PServer { gossipsub::prune_attestation_pool(self); } + #[send_handler] + async fn handle_advance_fork_schedule( + &mut self, + _msg: p2p_protocol::AdvanceForkSchedule, + ctx: &Context, + ) { + if let Some(delay) = beacon::transition::advance(self) { + send_after(delay, ctx.clone(), p2p_protocol::AdvanceForkSchedule); + } + } + #[send_handler] async fn handle_discover_peers( &mut self, diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index 5cb0f09c5..b3b2ea688 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -1576,7 +1576,7 @@ async fn handle_status_request( // `refuse` above (which needs `&mut server`) is behind us. let wire = server.wire.beacon().expect("checked above"); - if peer_status.fork_digest() != wire.fork_digest { + if !wire.holds_digest(peer_status.fork_digest()) { // Not grounds for closing the stream: the peer told us who it is and we // answer honestly. Counting it is how a digest that has moved under us // becomes visible. @@ -1703,7 +1703,7 @@ async fn handle_status_response( let Some(wire) = server.wire.beacon() else { return; }; - if status.fork_digest() != wire.fork_digest { + if !wire.holds_digest(status.fork_digest()) { warn!( %peer, peer_digest = %hex::encode(status.fork_digest()), From 4f5893f377d4b5a0593803ffce44566ddd98af2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:38:24 -0300 Subject: [PATCH 038/132] fix(p2p): tighten the fork-digest switch Review follow-ups to crossing fork digests at runtime. The fork digest gauge is reset before it is set, since the digest now changes while running and the old label would otherwise keep reading 1. Range sync starts only from a peer on the current digest (or the next one within the gossip clock disparity of its boundary); a Status on any held digest is still answered. Publishing picks the digest from the message's own slot, so a slot-B message cannot leave on the old topics before the switch runs, and data column decode failures take their fork from the topic's digest instead of the wall clock. ForkSchedule derives the eth2 entry's next_fork_* from its merged entries and no longer clones the Config. Only a blob entry before fulu is merged away; a post-fulu entry always moves the digest since it hashes the entry's own epoch. BeaconTopics::for_fork matches on every fork, apply() checks the current digest is held before joining anything, and returns the topics it joined and left so the aggregator-subnet handling is tested. Stale docs about a frozen-at-startup digest are rewritten. --- bin/ethlambda/src/beacon.rs | 13 +- crates/common/types/src/beacon/fork_digest.rs | 19 --- crates/net/p2p/src/beacon/decode.rs | 29 +--- crates/net/p2p/src/beacon/fork_schedule.rs | 113 ++++++++++--- crates/net/p2p/src/beacon/mod.rs | 16 +- crates/net/p2p/src/beacon/topics.rs | 26 ++- crates/net/p2p/src/beacon/transition.rs | 160 +++++++++++++++--- crates/net/p2p/src/gossipsub/handler.rs | 68 ++++---- crates/net/p2p/src/metrics.rs | 31 +++- crates/net/p2p/src/req_resp/handlers.rs | 14 ++ docs/beacon_wire.md | 20 ++- 11 files changed, 365 insertions(+), 144 deletions(-) diff --git a/bin/ethlambda/src/beacon.rs b/bin/ethlambda/src/beacon.rs index 64ab98861..ea340a1b1 100644 --- a/bin/ethlambda/src/beacon.rs +++ b/bin/ethlambda/src/beacon.rs @@ -46,7 +46,12 @@ pub fn time_at_epoch(config: &Config, genesis_time: u64, epoch: Epoch) -> u64 { genesis_time + epoch * config.seconds_per_slot * preset::SLOTS_PER_EPOCH } -pub use ethlambda_types::beacon::fork_digest::enr_fork_id; +/// The `eth2` ENR entry for this chain at this epoch, from the same schedule +/// the running node follows, so the record it starts with and the one it would +/// switch to name the same boundaries. +pub fn enr_fork_id(config: &Config, genesis_validators_root: Root, epoch: Epoch) -> EnrForkId { + ForkSchedule::new(config, genesis_validators_root).enr_fork_id(epoch) +} /// Ethereum mainnet's genesis `BeaconState`, SSZ-encoded: a test fixture. /// @@ -172,9 +177,9 @@ pub fn wire_params( // The digest moves at every fork and blob-schedule boundary, and the p2p // actor follows it without a restart (`ethlambda_p2p::beacon::transition`): - // next-digest topics are joined one epoch ahead, the switch happens at the - // boundary, and the old topics are left two epochs after. Say what is - // coming. + // the next digest's topics are joined ahead of the boundary, the switch + // happens at it, and the old topics are left after it, per + // `SUBSCRIBE_LEAD_EPOCHS` and `UNSUBSCRIBE_LAG_EPOCHS`. Say what is coming. // // The gloas boundary keeps its own warning: crossing its digest works like // any other, but the chain actor refuses every gloas block (nothing diff --git a/crates/common/types/src/beacon/fork_digest.rs b/crates/common/types/src/beacon/fork_digest.rs index 8aaab3254..69a42488a 100644 --- a/crates/common/types/src/beacon/fork_digest.rs +++ b/crates/common/types/src/beacon/fork_digest.rs @@ -14,7 +14,6 @@ use crate::beacon::constants; use crate::beacon::containers::shared::ForkData; use crate::beacon::fork::ForkName; use crate::beacon::primitives::{Epoch, ForkDigest, HashTreeRoot as _, Root, Version}; -use crate::enr::EnrForkId; /// The root binding a fork version to a chain's genesis validator set. /// @@ -72,24 +71,6 @@ pub fn next_fork_boundary(config: &Config, epoch: Epoch) -> Option { .min() } -/// The `eth2` ENR entry for this chain at this epoch. -/// -/// `next_fork_*` point at the next boundary that moves the digest, which -/// includes blob-parameter-only forks. Peers tolerate a difference here by -/// design: only `fork_digest` has to match. -pub fn enr_fork_id(config: &Config, genesis_validators_root: Root, epoch: Epoch) -> EnrForkId { - let fork_digest = compute_fork_digest(config, genesis_validators_root, epoch); - // With no boundary ahead, the spec says to repeat the current fork's own - // version and name `FAR_FUTURE_EPOCH` as the epoch it activates at. - let boundary = next_fork_boundary(config, epoch); - let named_epoch = boundary.unwrap_or(epoch); - EnrForkId { - fork_digest, - next_fork_version: config.fork_version(config.fork_at_epoch(named_epoch)), - next_fork_epoch: boundary.unwrap_or(constants::FAR_FUTURE_EPOCH), - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/net/p2p/src/beacon/decode.rs b/crates/net/p2p/src/beacon/decode.rs index 84da94623..0c85b23b9 100644 --- a/crates/net/p2p/src/beacon/decode.rs +++ b/crates/net/p2p/src/beacon/decode.rs @@ -31,7 +31,6 @@ use ethlambda_types::beacon::containers::{ use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::preset; use ethlambda_types::beacon::primitives::Slot; -use ethlambda_types::time::unix_now_ms; use libssz::SszDecode as _; use super::topics; @@ -183,25 +182,6 @@ pub fn fork_at_slot(config: &Config, slot: Slot) -> ForkName { config.fork_at_epoch(slot / preset::SLOTS_PER_EPOCH) } -/// The fork this node's own wall clock says is active right now. -/// -/// [`decode_data_column_sidecar`] has no slot to look a fork up by the way -/// [`fork_at_slot`] does: only fulu's container is modeled, so nothing reads -/// a slot out of the bytes first. A caller on the gossip path uses this to -/// approximate, once that decode has already failed, whether the failure is -/// this build's own gap rather than the sender's fault. An approximation -/// only, since gossip topic subscriptions are frozen at startup and this -/// reads the live clock instead of whatever fork the topic was actually -/// built for; see [`decode_data_column_sidecar`]'s own doc and its caller in -/// `gossipsub::handler::triage_data_column`. -pub(crate) fn current_fork(config: &Config) -> ForkName { - let slot = unix_now_ms() - .saturating_sub(config.genesis_time_ms()) - .checked_div(config.slot_duration_ms) - .unwrap_or(0); - fork_at_slot(config, slot) -} - /// Decode a `beacon_block` payload, at the fork its own slot names. /// /// Separate from [`decode_gossip`] because the two topics worth logging in @@ -233,11 +213,10 @@ pub fn decode_block(config: &Config, bytes: &[u8]) -> Result Result { fulu::DataColumnSidecar::from_ssz_bytes(bytes).map_err(|_| DecodeError::Ssz) } diff --git a/crates/net/p2p/src/beacon/fork_schedule.rs b/crates/net/p2p/src/beacon/fork_schedule.rs index d5fd3996e..5e7a125d2 100644 --- a/crates/net/p2p/src/beacon/fork_schedule.rs +++ b/crates/net/p2p/src/beacon/fork_schedule.rs @@ -10,8 +10,9 @@ //! The transition rules are the spec's ("Transitioning the gossip" in altair's //! `p2p-interface.md`): "In advance of the fork, a node SHOULD subscribe to the //! post-fork variants of the topics", and the pre-fork topics are dropped "two -//! epochs after the fork". A digest is therefore held from one epoch before it -//! activates until two epochs after the next one does, and the node is +//! epochs after the fork". A digest is therefore held from +//! [`SUBSCRIBE_LEAD_EPOCHS`] before it activates until +//! [`UNSUBSCRIBE_LAG_EPOCHS`] after the next one does, and the node is //! subscribed to whichever digests that window covers at the current epoch. //! Everything here is a pure function of the epoch, so applying it is //! idempotent and a node started inside a window lands in the same state as one @@ -23,8 +24,8 @@ use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::constants::FAR_FUTURE_EPOCH; use ethlambda_types::beacon::fork::ForkName; -use ethlambda_types::beacon::fork_digest::{compute_fork_digest, enr_fork_id}; -use ethlambda_types::beacon::primitives::{Epoch, ForkDigest, Root}; +use ethlambda_types::beacon::fork_digest::compute_fork_digest; +use ethlambda_types::beacon::primitives::{Epoch, ForkDigest, Root, Version}; use ethlambda_types::enr::EnrForkId; /// Epochs before a digest activates at which its topics are joined. @@ -40,6 +41,9 @@ pub struct ScheduledDigest { /// The first epoch this digest is current at. pub activation_epoch: Epoch, pub digest: ForkDigest, + /// The fork version in effect at `activation_epoch`, for the `eth2` entry's + /// `next_fork_version` when this is the next boundary. + pub version: Version, /// The fork in effect at `activation_epoch`, so the fork of everything /// published under `digest`. pub fork: ForkName, @@ -48,8 +52,6 @@ pub struct ScheduledDigest { /// Every digest a chain will use, in activation order. #[derive(Debug, Clone)] pub struct ForkSchedule { - config: Config, - genesis_validators_root: Root, /// Never empty: the first entry is the genesis digest. Consecutive equal /// digests are merged, so each entry is a real change on the wire. entries: Vec, @@ -72,22 +74,22 @@ impl ForkSchedule { let mut entries: Vec = Vec::with_capacity(epochs.len()); for epoch in epochs { let digest = compute_fork_digest(config, genesis_validators_root, epoch); - // A blob-schedule entry that restates the parameters already in - // force leaves the digest alone; nothing changes on the wire there. + // Only a blob-schedule entry before fulu, or one landing on an + // epoch another entry already covers, leaves the digest alone: + // from fulu on the digest hashes the entry's own epoch as well as + // its limit, so a later entry always moves it, even one restating + // the limit already in force. if entries.last().is_some_and(|last| last.digest == digest) { continue; } entries.push(ScheduledDigest { activation_epoch: epoch, digest, + version: config.fork_version(config.fork_at_epoch(epoch)), fork: config.fork_at_epoch(epoch), }); } - Self { - config: config.clone(), - genesis_validators_root, - entries, - } + Self { entries } } /// Every scheduled digest, in activation order. @@ -174,8 +176,24 @@ impl ForkSchedule { } /// The `eth2` ENR entry at `epoch`. + /// + /// `next_fork_*` name the next entry of this schedule, so never an epoch at + /// which the digest does not change. With none ahead, the spec says to + /// repeat the current version and name the far-future epoch. pub fn enr_fork_id(&self, epoch: Epoch) -> EnrForkId { - enr_fork_id(&self.config, self.genesis_validators_root, epoch) + let current = self.current_at(epoch); + match self.next_boundary_after(epoch) { + Some(next) => EnrForkId { + fork_digest: current.digest, + next_fork_version: next.version, + next_fork_epoch: next.activation_epoch, + }, + None => EnrForkId { + fork_digest: current.digest, + next_fork_version: current.version, + next_fork_epoch: FAR_FUTURE_EPOCH, + }, + } } } @@ -235,9 +253,27 @@ mod tests { fn every_scheduled_digest_resolves_to_its_fork() { let config = with_bpos(); let schedule = ForkSchedule::new(&config, gvr()); - // Genesis, altair..fulu, two BPOs, gloas: as many entries as distinct - // digests, each mapped back to the fork in force at its activation. - assert!(schedule.entries().len() >= 4); + // Genesis, each fork through fulu, two BPOs, gloas. + let shape: Vec<(Epoch, ForkName)> = schedule + .entries() + .iter() + .map(|entry| (entry.activation_epoch, entry.fork)) + .collect(); + assert_eq!( + shape, + vec![ + (0, ForkName::Phase0), + (1, ForkName::Altair), + (2, ForkName::Bellatrix), + (3, ForkName::Capella), + (4, ForkName::Deneb), + (5, ForkName::Electra), + (FULU, ForkName::Fulu), + (2_000, ForkName::Fulu), + (3_000, ForkName::Fulu), + (GLOAS, ForkName::Gloas), + ] + ); for entry in schedule.entries() { assert_eq!(schedule.fork_for_digest(entry.digest), Some(entry.fork)); assert_eq!( @@ -271,19 +307,44 @@ mod tests { } #[test] - fn a_blob_entry_restating_the_current_parameters_is_not_a_boundary() { + fn a_blob_entry_before_fulu_is_not_a_boundary() { + // The digest only hashes blob parameters from fulu on, so an entry + // placed before it cannot move anything on the wire. + let plain = ForkSchedule::new(&fulu_then_gloas(), gvr()); let mut config = fulu_then_gloas(); - // Electra's parameters, at fulu's own epoch: the digest cannot move. - let (_, max_blobs) = config.blob_parameters(FULU); config.blob_schedule = SszList::try_from(vec![BlobScheduleEntry { - epoch: config.electra_fork_epoch, - max_blobs_per_block: max_blobs, + epoch: 4, + max_blobs_per_block: 9, }]) .expect("within capacity"); + let with_entry = ForkSchedule::new(&config, gvr()); + assert_eq!(with_entry.entries().len(), plain.entries().len()); + } + + #[test] + fn a_bpo_restating_the_previous_limit_is_still_a_boundary() { + // `compute_fork_digest` hashes the entry's epoch as well as its limit, + // so a later entry with the same limit moves the digest. + let mut config = fulu_then_gloas(); + config.blob_schedule = SszList::try_from(vec![ + BlobScheduleEntry { + epoch: 2_000, + max_blobs_per_block: 15, + }, + BlobScheduleEntry { + epoch: 3_000, + max_blobs_per_block: 15, + }, + ]) + .expect("within capacity"); let schedule = ForkSchedule::new(&config, gvr()); - for pair in schedule.entries().windows(2) { - assert_ne!(pair[0].digest, pair[1].digest); - } + assert_ne!(schedule.digest_at(2_999), schedule.digest_at(3_000)); + assert!( + schedule + .entries() + .iter() + .any(|entry| entry.activation_epoch == 3_000) + ); } #[test] @@ -319,7 +380,7 @@ mod tests { #[test] fn the_current_digest_is_always_held() { let schedule = ForkSchedule::new(&with_bpos(), gvr()); - for epoch in (0..6_000).step_by(1) { + for epoch in 0..6_000 { assert!( digests_held(&schedule, epoch).contains(&schedule.digest_at(epoch)), "epoch {epoch}" diff --git a/crates/net/p2p/src/beacon/mod.rs b/crates/net/p2p/src/beacon/mod.rs index 44903b698..292fcf089 100644 --- a/crates/net/p2p/src/beacon/mod.rs +++ b/crates/net/p2p/src/beacon/mod.rs @@ -28,7 +28,7 @@ pub mod verdict; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::fork::ForkName; -use ethlambda_types::beacon::primitives::{ForkDigest, Root}; +use ethlambda_types::beacon::primitives::{ForkDigest, Root, Slot}; /// Everything the beacon wire needs after startup has computed it. /// @@ -50,8 +50,9 @@ pub struct BeaconWire { /// The topics under `fork_digest`. pub topics: topics::BeaconTopics, /// The topics of every other digest the subscription window holds: the - /// next one from an epoch ahead of its boundary, the previous one until two - /// epochs after. Empty outside a window. + /// next one from [`fork_schedule::SUBSCRIBE_LEAD_EPOCHS`] ahead of its + /// boundary, the previous one until [`fork_schedule::UNSUBSCRIBE_LAG_EPOCHS`] + /// after. Empty outside a window. pub window_topics: Vec, pub config: Config, pub genesis_time: u64, @@ -83,6 +84,15 @@ impl BeaconWire { std::iter::once(&self.topics).chain(self.window_topics.iter()) } + /// The digest a message for `slot` is published under: the one the schedule + /// names for that slot's epoch, rather than whichever this node has + /// switched to so far, so a slot-B block goes out on the new topics even if + /// the switch has not run yet. + pub fn digest_for_slot(&self, slot: Slot) -> ForkDigest { + self.schedule + .digest_at(slot / ethlambda_types::beacon::preset::SLOTS_PER_EPOCH) + } + /// Whether this node is subscribed under `digest`, so whether a peer on it /// is on a digest this node still speaks. pub fn holds_digest(&self, digest: ForkDigest) -> bool { diff --git a/crates/net/p2p/src/beacon/topics.rs b/crates/net/p2p/src/beacon/topics.rs index 8f57f90a5..b10a0a22d 100644 --- a/crates/net/p2p/src/beacon/topics.rs +++ b/crates/net/p2p/src/beacon/topics.rs @@ -211,17 +211,31 @@ impl BeaconTopics { /// The topics to hold under `fork_digest`, for the fork that digest names. /// /// The one place a fork may change which topic kinds exist: a fork that adds - /// a kind extends what this returns for its own digest, and the transition - /// then joins those topics at the pre-boundary epoch the spec asks for. - /// Today every fork subscribes to the same kinds, so `fork` is read by - /// nothing yet. + /// a kind extends its own arm, and the transition then joins those topics + /// ahead of the boundary as the spec asks. Every arm is named so a new fork + /// forces a decision here. pub fn for_fork( - _fork: ForkName, + fork: ForkName, fork_digest: ForkDigest, column_subnets: &[u64], attestation_subnets: &[u64], ) -> Self { - Self::new(fork_digest, column_subnets, attestation_subnets) + match fork { + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Fulu => Self::new(fork_digest, column_subnets, attestation_subnets), + // Gloas adds topics of its own (payload envelopes, payload + // attestations) that the follower does not yet consume, so for now + // it holds the same set. + ForkName::Gloas => Self::new(fork_digest, column_subnets, attestation_subnets), + ForkName::Lean => { + unreachable!("a beacon topic's fork is never Lean: it is absent from ForkName::ALL") + } + } } pub fn new( diff --git a/crates/net/p2p/src/beacon/transition.rs b/crates/net/p2p/src/beacon/transition.rs index 6a844208f..7607aec26 100644 --- a/crates/net/p2p/src/beacon/transition.rs +++ b/crates/net/p2p/src/beacon/transition.rs @@ -13,15 +13,55 @@ use std::time::Duration; use ethlambda_types::beacon::config::Config; +use ethlambda_types::beacon::constants::MAXIMUM_GOSSIP_CLOCK_DISPARITY; use ethlambda_types::beacon::preset; use ethlambda_types::beacon::primitives::{Epoch, ForkDigest}; use ethlambda_types::time::unix_now_ms; use libp2p::gossipsub::IdentTopic; use tracing::{error, info}; +use super::BeaconWire; use super::topics::{self, BeaconTopics}; use crate::{P2PServer, Wire, discovery, metrics}; +/// How far before a boundary a peer may already be on the next digest and still +/// count as current: clocks disagree by up to the gossip clock disparity. +const BOUNDARY_CLOCK_SKEW: Duration = Duration::from_millis(MAXIMUM_GOSSIP_CLOCK_DISPARITY); + +/// The topics [`apply`] joined and left, which it has also handed to the swarm. +/// Returned so the aggregator-subnet handling is observable in tests without a +/// second swarm. +#[derive(Debug, Default, PartialEq, Eq)] +pub(crate) struct Changes { + pub(crate) subscribed: Vec, + pub(crate) unsubscribed: Vec, +} + +/// Whether a peer whose `Status` carries `digest` is one to range-sync from. +/// +/// Answering a `Status` at any held digest is courtesy; syncing is a choice. +/// After a boundary a peer still on the previous digest has not upgraded, so +/// its chain is not the one being followed. Only the current digest qualifies, +/// or the next one within [`BOUNDARY_CLOCK_SKEW`] of its boundary, for a peer +/// whose clock is slightly ahead of ours. +pub(crate) fn is_syncable_digest(wire: &BeaconWire, digest: ForkDigest, now_ms: u64) -> bool { + let epoch = epoch_at_ms(&wire.config, wire.genesis_time, now_ms); + if digest == wire.schedule.digest_at(epoch) { + return true; + } + wire.schedule + .next_boundary_after(epoch) + .is_some_and(|next| { + next.digest == digest + && delay_until_epoch( + &wire.config, + wire.genesis_time, + next.activation_epoch, + now_ms, + ) <= BOUNDARY_CLOCK_SKEW + }) +} + fn epoch_duration_ms(config: &Config) -> u64 { config .slot_duration_ms @@ -61,7 +101,7 @@ pub(crate) fn advance(server: &mut P2PServer) -> Option { delay_until_epoch(&wire.config, wire.genesis_time, next, now_ms) .max(Duration::from_millis(1)) }); - apply(server, epoch); + let _ = apply(server, epoch); next } @@ -74,12 +114,22 @@ pub(crate) fn advance(server: &mut P2PServer) -> Option { /// advertises: digest, fork, metadata sequence number, metric, and the /// `eth2` entry. /// 4. Point the admission filter at the new entry, admitting every held digest. -pub(crate) fn apply(server: &mut P2PServer, epoch: Epoch) { +pub(crate) fn apply(server: &mut P2PServer, epoch: Epoch) -> Changes { + let mut changes = Changes::default(); let Wire::Beacon(wire) = &mut server.wire else { - return; + return changes; }; let held = wire.schedule.held_at(epoch); let current = wire.schedule.current_at(epoch); + // Checked before anything is joined, so a schedule bug cannot leave topics + // subscribed but untracked. `held_at` always contains the current digest. + if !held.iter().any(|entry| entry.digest == current.digest) { + error!( + epoch, + "The fork schedule holds no topics for the current digest" + ); + return changes; + } let column_subnets = topics::column_subnets(&wire.custody_columns); let mut existing = std::mem::take(&mut wire.window_topics); @@ -102,11 +152,12 @@ pub(crate) fn apply(server: &mut P2PServer, epoch: Epoch) { ); for topic in &joined.topics { server.swarm_handle.subscribe(topic.clone()); + changes.subscribed.push(topic.to_string()); } for &subnet_id in server.aggregator_subnets.keys() { - server - .swarm_handle - .subscribe(attestation_topic(entry.digest, subnet_id)); + let topic = attestation_topic(entry.digest, subnet_id); + changes.subscribed.push(topic.to_string()); + server.swarm_handle.subscribe(topic); } info!( fork_digest = %hex::encode(entry.digest), @@ -122,11 +173,12 @@ pub(crate) fn apply(server: &mut P2PServer, epoch: Epoch) { for left in existing { for topic in &left.topics { server.swarm_handle.unsubscribe(topic.clone()); + changes.unsubscribed.push(topic.to_string()); } for &subnet_id in server.aggregator_subnets.keys() { - server - .swarm_handle - .unsubscribe(attestation_topic(left.fork_digest, subnet_id)); + let topic = attestation_topic(left.fork_digest, subnet_id); + changes.unsubscribed.push(topic.to_string()); + server.swarm_handle.unsubscribe(topic); } info!( fork_digest = %hex::encode(left.fork_digest), @@ -135,18 +187,10 @@ pub(crate) fn apply(server: &mut P2PServer, epoch: Epoch) { ); } - let Some(index) = kept + let index = kept .iter() .position(|topics| topics.fork_digest == current.digest) - else { - // `held_at` always contains the current digest, so this is a schedule - // bug; keep the node on what it had rather than publishing nowhere. - error!( - epoch, - "The fork schedule holds no topics for the current digest" - ); - return; - }; + .expect("`kept` holds one entry per held digest, checked above to include the current"); wire.topics = kept.remove(index); wire.window_topics = kept; @@ -181,6 +225,7 @@ pub(crate) fn apply(server: &mut P2PServer, epoch: Epoch) { if switched { discovery::update_served_enr(&fork_id); } + changes } fn attestation_topic(digest: ForkDigest, subnet_id: u64) -> IdentTopic { @@ -331,4 +376,81 @@ mod tests { assert_eq!(wire.schedule.fork_for_digest(digest), Some(ForkName::Fulu)); assert_eq!(wire.fork, ForkName::Gloas); } + + fn epoch_start_ms(wire: &BeaconWire, epoch: Epoch) -> u64 { + wire.genesis_time * 1000 + epoch * epoch_duration_ms(&wire.config) + } + + #[tokio::test] + async fn aggregator_subnets_follow_every_held_digest() { + let mut server = unconnected_beacon_server(config(), 0).await; + let schedule = server.wire.beacon().expect("beacon").schedule.clone(); + let fulu = schedule.digest_at(GLOAS - 2); + let gloas = schedule.digest_at(GLOAS); + server.aggregator_subnets.insert(9, u64::MAX); + let name = |digest| topics::attestation_topic_name(digest, 9); + + // Settle on the pre-window digest first. + apply(&mut server, GLOAS - 2); + + // Joining the next digest brings the aggregator subnet along. + let joined = apply(&mut server, GLOAS - 1); + assert!(joined.subscribed.contains(&name(gloas))); + assert!(joined.unsubscribed.is_empty()); + + // Leaving the old digest takes it away again. + let left = apply(&mut server, GLOAS + 2); + assert!(left.unsubscribed.contains(&name(fulu))); + assert!(left.subscribed.is_empty()); + + // Nothing to do the second time. + assert_eq!(apply(&mut server, GLOAS + 2), Changes::default()); + } + + #[tokio::test] + async fn publishing_follows_the_messages_own_slot() { + let mut server = unconnected_beacon_server(config(), 0).await; + apply(&mut server, GLOAS - 1); + let wire = server.wire.beacon().unwrap(); + let first_gloas_slot = GLOAS * preset::SLOTS_PER_EPOCH; + // Before the switch has run, a slot-B message still names the new digest. + assert_eq!(wire.fork, ForkName::Fulu); + assert_eq!( + wire.digest_for_slot(first_gloas_slot), + wire.schedule.digest_at(GLOAS) + ); + assert_eq!( + wire.digest_for_slot(first_gloas_slot - 1), + wire.schedule.digest_at(GLOAS - 1) + ); + } + + #[tokio::test] + async fn sync_starts_only_from_a_peer_on_a_current_digest() { + let mut server = unconnected_beacon_server(config(), 0).await; + apply(&mut server, GLOAS); + let wire = server.wire.beacon().unwrap(); + let old = wire.schedule.digest_at(GLOAS - 1); + let new = wire.schedule.digest_at(GLOAS); + + // After the boundary: the current digest, not the superseded one, even + // though the latter is still held (and so still answered). + let after = epoch_start_ms(wire, GLOAS + 1); + assert!(wire.holds_digest(old)); + assert!(is_syncable_digest(wire, new, after)); + assert!(!is_syncable_digest(wire, old, after)); + + // Just before it, within the skew: a peer whose clock is ahead is fine. + let skew = BOUNDARY_CLOCK_SKEW.as_millis() as u64; + let almost = epoch_start_ms(wire, GLOAS) - skew; + assert!(is_syncable_digest(wire, old, almost)); + assert!(is_syncable_digest(wire, new, almost)); + + // Earlier than the skew allows, the next digest is not yet current. + let early = epoch_start_ms(wire, GLOAS) - skew - 1; + assert!(!is_syncable_digest(wire, new, early)); + + // A digest on no schedule never is. + assert!(!is_syncable_digest(wire, [0xde, 0xad, 0xbe, 0xef], after)); + } } diff --git a/crates/net/p2p/src/gossipsub/handler.rs b/crates/net/p2p/src/gossipsub/handler.rs index acbdaf521..866133820 100644 --- a/crates/net/p2p/src/gossipsub/handler.rs +++ b/crates/net/p2p/src/gossipsub/handler.rs @@ -235,20 +235,20 @@ fn handle_beacon_gossip( debug!(kind, "Beacon gossip arrived on a lean node"); return; }; + // The fork comes from the topic's own digest: while a boundary's window is + // open the node is subscribed under two, and `wire.fork` is only the one it + // publishes under. + let topic_fork = beacon_topics::topic_digest(topic) + .and_then(|digest| wire.schedule.fork_for_digest(digest)) + .unwrap_or(wire.fork); let dispatch = if kind == beacon_topics::BEACON_BLOCK { triage_block(server, wire, payload) } else if let Some(subnet_id) = beacon_topics::data_column_subnet(kind) { - triage_data_column(server, payload, subnet_id) + triage_data_column(server, topic_fork, payload, subnet_id) } else if kind == beacon_topics::BEACON_AGGREGATE_AND_PROOF { triage_aggregate(server, wire, payload, id.received_at) } else if let Some(subnet_id) = beacon_topics::attestation_subnet(kind) { - // The fork comes from the topic's own digest: while a boundary's - // window is open the node is subscribed under two, and `wire.fork` is - // only the one it publishes under. - let fork = beacon_topics::topic_digest(topic) - .and_then(|digest| wire.schedule.fork_for_digest(digest)) - .unwrap_or(wire.fork); - triage_attestation(server, fork, payload, subnet_id) + triage_attestation(server, topic_fork, payload, subnet_id) } else { triage_other(wire, kind, payload) }; @@ -307,32 +307,26 @@ fn triage_block(server: &P2PServer, wire: &BeaconWire, payload: &[u8]) -> Dispat /// Decode a data column sidecar and run its cheap gossip checks. Same shape as /// [`triage_block`]. /// -/// Decodes first, unlike the clock-based gate this used to be: gossipsub -/// topic subscriptions are frozen at startup (`build_swarm` does not -/// resubscribe as a fork boundary is crossed), so a node running across the -/// gloas boundary stays on its fulu-digest topic the whole time, where a -/// late but perfectly legitimate fulu sidecar can still legally arrive. A -/// clock check ahead of the decode would drop that one too, mistaking it for -/// gloas-shaped just because the clock has moved on. Only on a decode -/// failure does the clock matter, and only as an approximation: this node's -/// *topic* fork (whichever one gossip actually subscribed under at startup) -/// is not threaded down to this handler today, so [`beacon_decode::current_fork`] -/// (the wall clock) stands in for it. That is exactly backwards for a node -/// stuck on stale fulu topics past the boundary, the same case this doc -/// opens with: a genuinely malformed fulu sidecar arriving there reads as -/// `Ignore` instead of `Reject`, since the clock alone cannot tell "stale -/// topic, bad bytes" apart from "current topic, gloas-shaped bytes". Safe -/// either way, since `Ignore` never down-scores a peer; a future change that -/// carries the topic's own fork into `BeaconWire` (or wherever else carries -/// the fork digest to this handler) can make this exact instead of merely -/// safe. -fn triage_data_column(server: &P2PServer, payload: &[u8], subnet_id: u64) -> Dispatch { +/// Decodes first rather than gating on the clock: around a fork boundary the +/// node holds topics under two digests (see `beacon::transition`), so a late +/// but perfectly legitimate sidecar can still arrive on the old digest's +/// topic, and a clock check ahead of the decode would drop it, mistaking it for +/// the new fork's shape just because the clock has moved on. Only on a decode +/// failure does the fork matter, and `fork` is the one the message's own topic +/// digest names: a failure under a fork this build does not follow is its own +/// gap, so `Ignore`; under a followed fork it is the sender's fault, so +/// `Reject`. +fn triage_data_column( + server: &P2PServer, + fork: ForkName, + payload: &[u8], + subnet_id: u64, +) -> Dispatch { const KIND: &str = beacon_topics::DATA_COLUMN_SIDECAR_KIND; let sidecar = match beacon_decode::decode_data_column_sidecar(payload) { Ok(sidecar) => sidecar, Err(err) => { - let config = server.store.config(); - if !beacon_decode::current_fork(&config).is_followed() { + if !fork.is_followed() { metrics::inc_beacon_gossip(KIND, "unsupported_fork"); return Dispatch::Report(Outcome::Ignore(IgnoreReason::UnsupportedFork)); } @@ -641,7 +635,7 @@ pub async fn publish_beacon_attestation( return; } let topic = IdentTopic::new(beacon_topics::attestation_topic_name( - beacon.fork_digest, + beacon.digest_for_slot(slot), subnet_id, )); let compressed = compress_message(&attestation.to_ssz()); @@ -671,7 +665,7 @@ pub async fn publish_beacon_aggregate( return; }; let topic = IdentTopic::new(beacon_topics::topic_name( - beacon.fork_digest, + beacon.digest_for_slot(slot), beacon_topics::BEACON_AGGREGATE_AND_PROOF, )); // Each fork's container encodes as itself on the wire; the enum is only @@ -699,7 +693,7 @@ pub async fn publish_beacon_block(server: &mut P2PServer, block: SignedBeaconBlo return; }; let topic = IdentTopic::new(beacon_topics::topic_name( - beacon.fork_digest, + beacon.digest_for_slot(slot), beacon_topics::BEACON_BLOCK, )); server @@ -959,7 +953,7 @@ mod tests { let server = unconnected_beacon_server(Config::mainnet(), 0).await; assert!(matches!( - triage_data_column(&server, &[0xff; 3], 0), + triage_data_column(&server, ForkName::Fulu, &[0xff; 3], 0), Dispatch::Report(Outcome::Reject(RejectReason::Decode)) )); } @@ -974,7 +968,7 @@ mod tests { let payload = sidecar.to_ssz(); assert!(matches!( - triage_data_column(&server, &payload, 0), + triage_data_column(&server, ForkName::Fulu, &payload, 0), Dispatch::Report(Outcome::Ignore(IgnoreReason::FutureSlot)) )); } @@ -995,7 +989,7 @@ mod tests { let payload = sidecar.to_ssz(); assert!(matches!( - triage_data_column(&server, &payload, 0), + triage_data_column(&server, ForkName::Fulu, &payload, 0), Dispatch::Validate(Validated::Column(_)) )); } @@ -1013,7 +1007,7 @@ mod tests { let payload = sidecar.to_ssz(); assert!(matches!( - triage_data_column(&server, &payload, 1), + triage_data_column(&server, ForkName::Fulu, &payload, 1), Dispatch::Report(Outcome::Reject(RejectReason::WrongSubnet)) )); } diff --git a/crates/net/p2p/src/metrics.rs b/crates/net/p2p/src/metrics.rs index c56f21bbc..88a006796 100644 --- a/crates/net/p2p/src/metrics.rs +++ b/crates/net/p2p/src/metrics.rs @@ -362,7 +362,7 @@ static LEAN_BEACON_STATUS_DIGEST_MISMATCH_TOTAL: LazyLock = LazyLock static LEAN_BEACON_FORK_DIGEST: LazyLock = LazyLock::new(|| { register_int_gauge_vec!( "lean_beacon_fork_digest", - "The fork digest this node computed at startup, as a label", + "The fork digest this node currently publishes under, as a label", &["digest"] ) .unwrap() @@ -478,9 +478,13 @@ pub fn inc_beacon_status_digest_mismatch() { LEAN_BEACON_STATUS_DIGEST_MISMATCH_TOTAL.inc(); } -/// Publish the computed fork digest as a label, so a dashboard can tell at a +/// Publish the current fork digest as a label, so a dashboard can tell at a /// glance whether a node is stranded on a boundary it failed to cross. +/// +/// Resets the vector first: the digest changes at runtime, and a label left +/// behind at 1 would read as two current digests. pub fn set_beacon_fork_digest(digest: &str) { + LEAN_BEACON_FORK_DIGEST.reset(); LEAN_BEACON_FORK_DIGEST.with_label_values(&[digest]).set(1); } @@ -615,3 +619,26 @@ pub fn observe_beacon_aggregate_decode(duration: std::time::Duration) { }); LEAN_BEACON_AGGREGATE_DECODE_SECONDS.observe(duration.as_secs_f64()); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_the_current_fork_digest_reads_one() { + set_beacon_fork_digest("aaaaaaaa"); + set_beacon_fork_digest("bbbbbbbb"); + assert_eq!( + LEAN_BEACON_FORK_DIGEST + .with_label_values(&["bbbbbbbb"]) + .get(), + 1 + ); + assert_eq!( + LEAN_BEACON_FORK_DIGEST + .with_label_values(&["aaaaaaaa"]) + .get(), + 0 + ); + } +} diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index b3b2ea688..6af186c52 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -46,6 +46,7 @@ use crate::beacon::messages::{ use crate::beacon::protocols::{ MAX_REQUEST_BLOCKS as MAX_BEACON_REQUEST_BLOCKS, MAX_REQUEST_BLOCKS_DENEB, }; +use crate::beacon::transition; use crate::discovery::enr::node_id_from_peer_id; use crate::lean::messages::{BlocksByRootRequest, RequestedBlockRoots, Status}; use crate::lean::protocols::MAX_REQUEST_BLOCKS; @@ -56,6 +57,7 @@ use crate::{ RangeSyncState, ReqRespProtocol, ReqRespRequestId, UNKNOWN_CUSTODY_RANGE_PEERS, metrics, p2p_protocol, }; +use ethlambda_types::time::unix_now_ms; use libp2p::request_response::ResponseChannel; /// `protocol` names which [`ReqResp`](super::ReqResp) field `event` came @@ -1713,6 +1715,18 @@ async fn handle_status_response( metrics::inc_beacon_status_digest_mismatch(); return; } + // Answered at any held digest, but only synced from on the current one (or + // the next, within clock skew): after a boundary, a peer still on the old + // digest has not upgraded, and its chain is not the one being followed. + if !transition::is_syncable_digest(wire, status.fork_digest(), unix_now_ms()) { + debug!( + %peer, + peer_digest = %hex::encode(status.fork_digest()), + our_digest = %hex::encode(wire.fork_digest), + "Not syncing from a peer on a superseded fork digest" + ); + return; + } let peer_head_slot = status.head_slot(); trace!( %peer, diff --git a/docs/beacon_wire.md b/docs/beacon_wire.md index d278f389e..da47284fe 100644 --- a/docs/beacon_wire.md +++ b/docs/beacon_wire.md @@ -61,9 +61,23 @@ digest = base[..4] if epoch < FULU_FOR EIP-7892's, which is why mainnet's digest is `8c9f62fe` rather than fulu's bare `82fae541`. -Startup logs the next boundary's epoch and wall-clock time. The digest is -computed once, so crossing one strands the node on topic names nobody publishes -to; restart it to pick up the new digest. +Startup logs the next boundary's epoch, wall-clock time, fork and digest. A +running node crosses it without a restart (`beacon::transition` in +`ethlambda-p2p`, driven by `beacon::fork_schedule`). Both fork activations and +blob-parameter-only forks move the digest, so both are boundaries. Following the +spec's "Transitioning the gossip" rules, the next digest's topics are joined +`SUBSCRIBE_LEAD_EPOCHS` before its boundary and the previous digest's topics +are left `UNSUBSCRIBE_LAG_EPOCHS` after it. At the boundary the node +publishes and advertises the new digest, bumps its metadata sequence number +and moves its discv5 admission filter. While the window is open a gossip +message's fork comes from its own topic's digest, and a `Status` on either +digest is answered, but range sync starts only from a peer on the current +digest (or the next one, within clock skew of the boundary). + +One gap: the ENR this node serves over discv5 keeps the `eth2` entry it started +with, because ethrex's `DiscoveryServer` offers no way to replace its record at +runtime. Peers already connected are unaffected; peers discovering the node +after a boundary read the stale digest and reject the record. ## Gossip From 29d0981f102f0e8f54c6663e1505313902bc4bae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:43:54 -0300 Subject: [PATCH 039/132] test(p2p): make the fork-digest tests prove what they claim The gauge test wrote the global fork-digest gauge while transition tests in the same binary wrote it too, so it could fail under parallelism; the reset-then-set logic is now a helper tested on its own vector. The pre-fulu blob entry test put its entry on deneb's own fork epoch, where the epoch dedup dropped it before the digest comparison ran; it now uses an epoch no fork has. Publishing now refuses a slot whose digest this node does not hold instead of fanning out to a topic nobody joined. next_fork_boundary had no callers left after the schedule took over, so it goes with its tests, and the wire docs no longer say the digest is computed once. --- crates/common/types/src/beacon/fork_digest.rs | 36 ------------------- crates/net/p2p/src/beacon/fork_schedule.rs | 4 ++- crates/net/p2p/src/beacon/mod.rs | 8 +++++ crates/net/p2p/src/beacon/transition.rs | 18 +++++++++- crates/net/p2p/src/gossipsub/handler.rs | 24 +++++++++---- crates/net/p2p/src/metrics.rs | 33 +++++++++-------- docs/beacon_wire.md | 5 +-- 7 files changed, 65 insertions(+), 63 deletions(-) diff --git a/crates/common/types/src/beacon/fork_digest.rs b/crates/common/types/src/beacon/fork_digest.rs index 69a42488a..33e40475e 100644 --- a/crates/common/types/src/beacon/fork_digest.rs +++ b/crates/common/types/src/beacon/fork_digest.rs @@ -10,7 +10,6 @@ use sha2::{Digest as _, Sha256}; use crate::beacon::config::Config; -use crate::beacon::constants; use crate::beacon::containers::shared::ForkData; use crate::beacon::fork::ForkName; use crate::beacon::primitives::{Epoch, ForkDigest, HashTreeRoot as _, Root, Version}; @@ -56,21 +55,6 @@ pub fn compute_fork_digest( core::array::from_fn(|index| base.0[index] ^ mask[index]) } -/// The next epoch at which [`compute_fork_digest`] changes, if there is one. -/// -/// Both fork activations and blob-schedule entries qualify: crossing either one -/// strands a running node on topic names no peer is publishing to. Unscheduled -/// forks carry [`constants::FAR_FUTURE_EPOCH`], a real value rather than a -/// `None`, so they are filtered out before the minimum is taken. -pub fn next_fork_boundary(config: &Config, epoch: Epoch) -> Option { - ForkName::ALL - .into_iter() - .map(|fork| config.fork_epoch(fork)) - .chain(config.blob_schedule.iter().map(|entry| entry.epoch)) - .filter(|&boundary| boundary != constants::FAR_FUTURE_EPOCH && boundary > epoch) - .min() -} - #[cfg(test)] mod tests { use super::*; @@ -130,26 +114,6 @@ mod tests { ); } - #[test] - fn next_boundary_covers_both_fork_and_blob_schedule_epochs() { - let config = Config::mainnet(); - // A plain fork boundary. - assert_eq!(next_fork_boundary(&config, 0), Some(74_240)); - // A blob-parameter-only fork is a boundary too: it moves the digest. - assert_eq!(next_fork_boundary(&config, 411_392), Some(412_672)); - assert_eq!(next_fork_boundary(&config, 412_672), Some(419_072)); - // Past the last scheduled boundary there is nothing left to warn about. - assert_eq!(next_fork_boundary(&config, 419_072), None); - } - - #[test] - fn far_future_forks_are_not_boundaries() { - // Minimal leaves every fork after phase0 at FAR_FUTURE_EPOCH, which is a - // real, enormous Epoch rather than a None; treating it as a boundary - // would schedule a warning for the heat death of the universe. - assert_eq!(next_fork_boundary(&Config::minimal(), 0), None); - } - #[test] fn fork_data_root_binds_the_version_and_the_chain() { let a = compute_fork_data_root([1, 0, 0, 0], Root::ZERO); diff --git a/crates/net/p2p/src/beacon/fork_schedule.rs b/crates/net/p2p/src/beacon/fork_schedule.rs index 5e7a125d2..252f6e9bd 100644 --- a/crates/net/p2p/src/beacon/fork_schedule.rs +++ b/crates/net/p2p/src/beacon/fork_schedule.rs @@ -313,7 +313,9 @@ mod tests { let plain = ForkSchedule::new(&fulu_then_gloas(), gvr()); let mut config = fulu_then_gloas(); config.blob_schedule = SszList::try_from(vec![BlobScheduleEntry { - epoch: 4, + // An epoch no fork uses, so the entry reaches the digest + // comparison instead of being dropped as a duplicate epoch. + epoch: 500, max_blobs_per_block: 9, }]) .expect("within capacity"); diff --git a/crates/net/p2p/src/beacon/mod.rs b/crates/net/p2p/src/beacon/mod.rs index 292fcf089..cab72149d 100644 --- a/crates/net/p2p/src/beacon/mod.rs +++ b/crates/net/p2p/src/beacon/mod.rs @@ -93,6 +93,14 @@ impl BeaconWire { .digest_at(slot / ethlambda_types::beacon::preset::SLOTS_PER_EPOCH) } + /// [`Self::digest_for_slot`], but only if this node holds that digest's + /// topics: a slot far in the past or beyond the next boundary names a digest + /// nobody is listening to, and publishing there would only fan out to peers + /// that never subscribed. + pub fn publish_digest(&self, slot: Slot) -> Option { + Some(self.digest_for_slot(slot)).filter(|&digest| self.holds_digest(digest)) + } + /// Whether this node is subscribed under `digest`, so whether a peer on it /// is on a digest this node still speaks. pub fn holds_digest(&self, digest: ForkDigest) -> bool { diff --git a/crates/net/p2p/src/beacon/transition.rs b/crates/net/p2p/src/beacon/transition.rs index 7607aec26..4447984ad 100644 --- a/crates/net/p2p/src/beacon/transition.rs +++ b/crates/net/p2p/src/beacon/transition.rs @@ -101,7 +101,7 @@ pub(crate) fn advance(server: &mut P2PServer) -> Option { delay_until_epoch(&wire.config, wire.genesis_time, next, now_ms) .max(Duration::from_millis(1)) }); - let _ = apply(server, epoch); + apply(server, epoch); next } @@ -425,6 +425,22 @@ mod tests { ); } + #[tokio::test] + async fn a_digest_that_is_not_held_is_not_published_to() { + let mut server = unconnected_beacon_server(config(), 0).await; + apply(&mut server, GLOAS - 2); + let wire = server.wire.beacon().unwrap(); + let slot_of = |epoch: Epoch| epoch * preset::SLOTS_PER_EPOCH; + // The current digest, and the next one once its window has opened. + assert!(wire.publish_digest(slot_of(GLOAS - 2)).is_some()); + assert!(wire.publish_digest(slot_of(GLOAS)).is_none()); + apply(&mut server, GLOAS - 1); + let wire = server.wire.beacon().unwrap(); + assert!(wire.publish_digest(slot_of(GLOAS)).is_some()); + // A slot long past names a digest that is no longer held. + assert!(wire.publish_digest(slot_of(FULU)).is_none()); + } + #[tokio::test] async fn sync_starts_only_from_a_peer_on_a_current_digest() { let mut server = unconnected_beacon_server(config(), 0).await; diff --git a/crates/net/p2p/src/gossipsub/handler.rs b/crates/net/p2p/src/gossipsub/handler.rs index 866133820..310088b5f 100644 --- a/crates/net/p2p/src/gossipsub/handler.rs +++ b/crates/net/p2p/src/gossipsub/handler.rs @@ -634,10 +634,11 @@ pub async fn publish_beacon_attestation( error!(%slot, validator, subnet_id, "Attestation subnet out of range; dropping it"); return; } - let topic = IdentTopic::new(beacon_topics::attestation_topic_name( - beacon.digest_for_slot(slot), - subnet_id, - )); + let Some(digest) = beacon.publish_digest(slot) else { + warn!(%slot, validator, "No held fork digest covers this attestation's slot; not publishing"); + return; + }; + let topic = IdentTopic::new(beacon_topics::attestation_topic_name(digest, subnet_id)); let compressed = compress_message(&attestation.to_ssz()); server.swarm_handle.publish(topic, compressed); debug!( @@ -664,8 +665,12 @@ pub async fn publish_beacon_aggregate( error!(%slot, aggregator, "A beacon aggregate reached a lean node; dropping it"); return; }; + let Some(digest) = beacon.publish_digest(slot) else { + warn!(%slot, aggregator, "No held fork digest covers this aggregate's slot; not publishing"); + return; + }; let topic = IdentTopic::new(beacon_topics::topic_name( - beacon.digest_for_slot(slot), + digest, beacon_topics::BEACON_AGGREGATE_AND_PROOF, )); // Each fork's container encodes as itself on the wire; the enum is only @@ -692,8 +697,15 @@ pub async fn publish_beacon_block(server: &mut P2PServer, block: SignedBeaconBlo error!(slot, "A beacon block reached a lean node; dropping it"); return; }; + let Some(digest) = beacon.publish_digest(slot) else { + warn!( + slot, + "No held fork digest covers this block's slot; not publishing" + ); + return; + }; let topic = IdentTopic::new(beacon_topics::topic_name( - beacon.digest_for_slot(slot), + digest, beacon_topics::BEACON_BLOCK, )); server diff --git a/crates/net/p2p/src/metrics.rs b/crates/net/p2p/src/metrics.rs index 88a006796..a9779c211 100644 --- a/crates/net/p2p/src/metrics.rs +++ b/crates/net/p2p/src/metrics.rs @@ -484,8 +484,13 @@ pub fn inc_beacon_status_digest_mismatch() { /// Resets the vector first: the digest changes at runtime, and a label left /// behind at 1 would read as two current digests. pub fn set_beacon_fork_digest(digest: &str) { - LEAN_BEACON_FORK_DIGEST.reset(); - LEAN_BEACON_FORK_DIGEST.with_label_values(&[digest]).set(1); + set_only_label(&LEAN_BEACON_FORK_DIGEST, digest); +} + +/// Make `label` the one series of `gauge` that reads 1. +fn set_only_label(gauge: &IntGaugeVec, label: &str) { + gauge.reset(); + gauge.with_label_values(&[label]).set(1); } static LEAN_DATA_COLUMN_FETCH_FAILURES_TOTAL: LazyLock = LazyLock::new(|| { @@ -625,20 +630,14 @@ mod tests { use super::*; #[test] - fn only_the_current_fork_digest_reads_one() { - set_beacon_fork_digest("aaaaaaaa"); - set_beacon_fork_digest("bbbbbbbb"); - assert_eq!( - LEAN_BEACON_FORK_DIGEST - .with_label_values(&["bbbbbbbb"]) - .get(), - 1 - ); - assert_eq!( - LEAN_BEACON_FORK_DIGEST - .with_label_values(&["aaaaaaaa"]) - .get(), - 0 - ); + fn only_the_newest_label_reads_one() { + // A local vector: the global gauge is also written by every transition + // test running in parallel in this binary. + let gauge = register_int_gauge_vec!("lean_test_only_label_gauge", "test", &["digest"]) + .expect("a unique name"); + set_only_label(&gauge, "aaaaaaaa"); + set_only_label(&gauge, "bbbbbbbb"); + assert_eq!(gauge.with_label_values(&["bbbbbbbb"]).get(), 1); + assert_eq!(gauge.with_label_values(&["aaaaaaaa"]).get(), 0); } } diff --git a/docs/beacon_wire.md b/docs/beacon_wire.md index da47284fe..b1e094760 100644 --- a/docs/beacon_wire.md +++ b/docs/beacon_wire.md @@ -44,8 +44,9 @@ bootnodes are largely seed-only, so a crawl is how a peer is reached. ## The fork digest -Computed once at startup from the resolved network's genesis state, never -hardcoded as a digest: +Computed from the resolved network's genesis state and fork schedule, never +hardcoded as a digest. Startup derives the first one and a running node follows +the schedule from there (see below): ``` epoch = (now - genesis_time) / (SECONDS_PER_SLOT * SLOTS_PER_EPOCH) From 539e9f6ef1ed3c2fe59c0d5464c256801a60abad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:50:45 -0300 Subject: [PATCH 040/132] fix(beacon): time fork-choice events at arrival, not at the slot tick The beacon follower ticks once per slot, so the store clock sat at the slot's start for the whole slot. on_block read it for block timeliness (attestation deadline and, from gloas, the payload timeliness committee deadline), found zero milliseconds into the slot, and called every current-slot block timely, late ones included. A late block therefore got proposer boost and was exempt from the late-head reorg rules. Advance the store clock to the block's wire receipt (decode_start, else hand-off) before on_block. The spec's clock is whole seconds and its on_tick runs whenever time > store.time, so the deadlines are judged on the second the block fell in. The advance never goes backwards and never crosses a slot boundary, which stays the tick's job since it also drains deferred aggregates and recomputes the head. Why the tick was per slot: the follower has no sub-slot duties (see ChainDuties::Beacon, introduced with the chain actor in #9). That is a statement about duties, not about fork choice needing a coarse clock, so arrival-time advances are safe. The envelope and payload-attestation handlers are not called live yet; they get the same hook when wired. Pre-gloas blocks now also get spec-correct proposer boost: a late block is no longer boosted. --- crates/blockchain/src/lib.rs | 113 +++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index abf68a754..0c000305a 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -3741,6 +3741,44 @@ impl BlockChainServer { !fork_at(current_slot).is_followed() && fork_at(head_slot).is_followed() } + /// Advances a beacon store's clock to `event_ms` (Unix milliseconds), for + /// an event whose handler reads the clock to judge a sub-slot deadline. + /// + /// The beacon tick fires once per slot, so between ticks the store sits at + /// the slot's start and [`fork_choice::on_block`] would find every + /// current-slot block inside both the attestation and the payload + /// timeliness deadlines. The specification has no such gap: its `on_tick` + /// is called "whenever `time > store.time`", so the clock is wherever the + /// last event put it. The follower's tick was per slot only because it has + /// no sub-slot duties of its own, which nothing here changes. + /// + /// Never backwards (a re-delivered block brings an old arrival), and never + /// across a slot boundary: crossing one is the tick's job, since it also + /// drains deferred aggregates and recomputes the head, which this does + /// not. An event from a later slot than the store reads is clamped to the + /// last instant of the current one. The specification's clock is whole + /// seconds, so the deadlines are judged on the second the event fell in. + fn advance_beacon_clock_to(&mut self, event_ms: u64) { + if self.store.chain() != Chain::Beacon { + return; + } + let config = self.store.config(); + let current_slot = fork_choice::get_current_slot(&self.store, &config); + let slot_end_ms = config + .genesis_time_ms() + .saturating_add( + current_slot + .saturating_add(1) + .saturating_mul(config.slot_duration_ms), + ) + .saturating_sub(1); + let time = event_ms.min(slot_end_ms) / 1000; + let store_time_ms = self.store.time_ms().expect("store time exists"); + if fork_choice::seconds_to_milliseconds(time) > store_time_ms { + fork_choice::on_tick(&mut self.store, time, &config); + } + } + /// Milliseconds until this actor's next tick, dispatched by chain: lean's /// interval grid via [`ms_until_next_interval`], beacon's once-per-slot /// cadence via [`ms_until_next_beacon_slot`]. @@ -3965,6 +4003,15 @@ impl Handler for BlockChainServer { // the import, the catch-up tick included, so it is its own section // rather than the head of the first pass's guards. timings.admit_end = Some(Instant::now()); + // Time the import at the block's arrival on this node, not at the + // slot's start where the tick left the clock: `on_block` reads it for + // the attestation and payload timeliness deadlines. The arrival is the + // wire receipt (`decode_start`), which predates validation and the + // mailbox wait; `arrival_ms` was sampled at pickup. + let received = msg.arrival.decode_start.unwrap_or(msg.arrival.handed_off); + let in_flight_ms = u64::try_from(picked_up.saturating_duration_since(received).as_millis()) + .unwrap_or(u64::MAX); + self.advance_beacon_clock_to(arrival_ms.saturating_sub(in_flight_ms)); self.on_block(msg.block, timings).await; } } @@ -4703,6 +4750,72 @@ mod tests { store } + /// How far into its slot the store clock reads, in milliseconds. + fn ms_into_slot(server: &BlockChainServer) -> u64 { + server.store.ms_since_genesis() % server.store.config().slot_duration_ms + } + + /// A fulu-era block arriving after the attestation deadline used to be + /// judged against the slot-start clock the once-per-slot tick leaves the + /// store on, so `block_timeliness` saw zero milliseconds into the slot and + /// called every current-slot block timely. Advancing the clock to the + /// arrival puts the store past the deadline that verdict is read against. + #[test] + fn a_late_arrival_moves_the_store_clock_past_the_attestation_deadline() { + let mut server = beacon_server(beacon_store_at_slot_10()); + let config = server.store.config(); + let slot_start_ms = config.genesis_time_ms() + 10 * config.slot_duration_ms; + let deadline_ms = fork_choice::get_attestation_due_ms(10 / 32, &config); + assert_eq!( + ms_into_slot(&server), + 0, + "the tick leaves the clock at slot start" + ); + + server.advance_beacon_clock_to(slot_start_ms + deadline_ms + 500); + + assert!(ms_into_slot(&server) >= deadline_ms); + } + + #[test] + fn an_early_arrival_keeps_the_store_clock_inside_the_attestation_window() { + let mut server = beacon_server(beacon_store_at_slot_10()); + let config = server.store.config(); + let slot_start_ms = config.genesis_time_ms() + 10 * config.slot_duration_ms; + let deadline_ms = fork_choice::get_attestation_due_ms(10 / 32, &config); + + server.advance_beacon_clock_to(slot_start_ms + 1_000); + + assert!(ms_into_slot(&server) < deadline_ms); + } + + #[test] + fn the_store_clock_never_moves_backwards_on_an_arrival() { + let mut server = beacon_server(beacon_store_at_slot_10()); + let config = server.store.config(); + let slot_start_ms = config.genesis_time_ms() + 10 * config.slot_duration_ms; + server.advance_beacon_clock_to(slot_start_ms + 6_000); + let before = server.store.time_ms().unwrap(); + + // A held block re-delivered with its original, earlier arrival. + server.advance_beacon_clock_to(slot_start_ms + 2_000); + + assert_eq!(server.store.time_ms().unwrap(), before); + } + + #[test] + fn an_arrival_past_the_slot_does_not_cross_the_boundary_ahead_of_the_tick() { + // Crossing it is the tick's job: it also drains deferred aggregates + // and recomputes the head, which this advance does not. + let mut server = beacon_server(beacon_store_at_slot_10()); + let config = server.store.config(); + let slot_start_ms = config.genesis_time_ms() + 10 * config.slot_duration_ms; + + server.advance_beacon_clock_to(slot_start_ms + 3 * config.slot_duration_ms); + + assert_eq!(fork_choice::get_current_slot(&server.store, &config), 10); + } + #[tokio::test] async fn a_checked_sidecar_is_stored_as_it_is() { // Nothing on this path judges the sidecar (that is the p2p layer's From 02eba80f338759e4ef6b3a2b9fac921c0a05fe9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:35:29 -0300 Subject: [PATCH 041/132] feat(beacon): persist verified envelopes and block timeliness A restarted gloas follower lost `verified_payloads`, which silently dropped every FULL fork-choice node, and `block_timeliness`, which made the head walk fail with `Error::SpecAssert("head_root in store.block_timeliness")`. Both were in-memory only. Two new tables hold them: `ExecutionPayloadEnvelopes` (slot || root, the verified envelope, so a later change can serve envelopes by slot range) and `BlockTimeliness` (root, one byte per deadline). Writes go through to the table and the in-memory set; `from_db_state` rebuilds both for every LiveChain block at or above the finalized block's slot, the finalized block included since its payload status decides which branch its children build on. The PTC vote vectors stay in memory on purpose: after a restart only the current and previous slot's blocks read them. `insert_verified_payload` now takes the envelope and errors on an unknown block, since the row key needs the block's slot. DB_VERSION is not bumped: new column families change no existing layout, and no v5 directory holds gloas data since the previous binary refused gloas. --- .../src/beacon/fork_choice.rs | 70 +++- crates/storage/src/api/tables.rs | 25 +- crates/storage/src/store.rs | 330 ++++++++++++++++-- 3 files changed, 375 insertions(+), 50 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 61593813d..c66fd5ea7 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -4863,9 +4863,11 @@ pub fn on_execution_payload_envelope( // Verify the execution payload envelope. stf::gloas::verify_execution_payload_envelope(&state, signed_envelope, config, engine)?; - // Add execution payload envelope to the store. Only the fact that it is - // verified is kept: nothing in fork choice reads the payload itself. - store.insert_verified_payload(block_root); + // Add execution payload envelope to the store. Persisted, so a restarted + // follower keeps the full branch of this block. + store + .insert_verified_payload(signed_envelope) + .expect("insert"); Ok(()) } @@ -5156,6 +5158,50 @@ mod tests { }) } + /// Marks `root`'s payload verified by storing a default envelope for it. + /// The block must already be in the store: the envelope's row is keyed by + /// the block's slot. + fn verify_payload(store: &mut Store, root: Root) { + let payload = gloas::ExecutionPayload { + parent_hash: Default::default(), + fee_recipient: Default::default(), + state_root: Default::default(), + receipts_root: Default::default(), + logs_bloom: crate::beacon::containers::bellatrix::LogsBloom::try_from(vec![ + 0u8; + preset::BYTES_PER_LOGS_BLOOM + ]) + .expect("built at exactly BYTES_PER_LOGS_BLOOM"), + prev_randao: Default::default(), + block_number: 0, + gas_limit: 0, + gas_used: 0, + timestamp: 0, + extra_data: Default::default(), + base_fee_per_gas: Default::default(), + block_hash: Default::default(), + transactions: Default::default(), + withdrawals: Default::default(), + blob_gas_used: 0, + excess_blob_gas: 0, + block_access_list: Default::default(), + slot_number: 0, + }; + let envelope = gloas::SignedExecutionPayloadEnvelope { + message: gloas::ExecutionPayloadEnvelope { + payload, + execution_requests: Default::default(), + builder_index: 0, + beacon_block_root: root, + parent_beacon_block_root: Root::ZERO, + }, + signature: Default::default(), + }; + store + .insert_verified_payload(&envelope) + .expect("the block is in the store"); + } + /// A fulu signed block with an empty body and a zero signature, for the /// fulu-to-gloas boundary tests below, which need a pre-gloas block /// genuinely of fulu's own fork rather than the phase0-shaped [`block`] @@ -6232,7 +6278,7 @@ mod tests { "no verified payload yet: only the empty branch exists" ); - store.insert_verified_payload(a_root); + verify_payload(&mut store, a_root); assert_eq!( get_node_children(&store, &blocks, node).unwrap(), vec![ @@ -6431,7 +6477,7 @@ mod tests { "a payload nobody verified must not be extended, so full loses the tiebreak" ); - store.insert_verified_payload(a_root); + verify_payload(&mut store, a_root); let strong_votes = vec![Some(true); preset::PTC_SIZE]; store.set_payload_timeliness_vote(a_root, strong_votes.clone()); store.set_payload_data_availability_vote(a_root, strong_votes); @@ -6460,7 +6506,7 @@ mod tests { ), ) .unwrap(); - store.insert_verified_payload(a_root); + verify_payload(&mut store, a_root); let exactly_threshold = vec![Some(true); preset::PAYLOAD_TIMELY_THRESHOLD as usize]; store.set_payload_timeliness_vote(a_root, exactly_threshold.clone()); @@ -6529,7 +6575,7 @@ mod tests { "a payload nobody verified must not be extended" ); - store.insert_verified_payload(a_root); + verify_payload(&mut store, a_root); let weak_votes = vec![Some(false); preset::PTC_SIZE]; store.set_payload_timeliness_vote(a_root, weak_votes.clone()); store.set_payload_data_availability_vote(a_root, weak_votes); @@ -6589,7 +6635,7 @@ mod tests { ), ) .unwrap(); - store.insert_verified_payload(a_root); + verify_payload(&mut store, a_root); let state = test_state::with_validators_at(ForkName::Gloas, 4); store.insert_state(anchor_root, state.clone()).unwrap(); @@ -6965,7 +7011,7 @@ mod tests { ), ) .unwrap(); - store.insert_verified_payload(a_root); + verify_payload(&mut store, a_root); let state = test_state::with_validators_at(ForkName::Gloas, 4); store.insert_state(anchor_root, state.clone()).unwrap(); @@ -7469,7 +7515,7 @@ mod tests { assert_ne!(failed_assertion(rejected), Some(UNVERIFIED_PAYLOAD_PARENT)); // Once the envelope is verified the full child clears the check too. - store.insert_verified_payload(anchor_root); + verify_payload(&mut store, anchor_root); let rejected = import(&mut store, full_child); assert!(rejected.is_err()); assert_ne!(failed_assertion(rejected), Some(UNVERIFIED_PAYLOAD_PARENT)); @@ -7545,7 +7591,7 @@ mod tests { assert!(validate(&store, vote(2, 1), ForkRules::PreGloas).is_ok()); assert!(validate(&store, vote(2, 2), ForkRules::PreGloas).is_ok()); - store.insert_verified_payload(b_root); + verify_payload(&mut store, b_root); assert!(validate(&store, vote(2, 1), ForkRules::Gloas).is_ok()); } @@ -7877,7 +7923,7 @@ mod tests { .unwrap(); store.insert_state(root, gloas_state.clone()).unwrap(); if rng.chance(1, 2) { - store.insert_verified_payload(root); + verify_payload(&mut store, root); } for timely_votes in [true, false] { let votes = match rng.below(4) { diff --git a/crates/storage/src/api/tables.rs b/crates/storage/src/api/tables.rs index f2d2f0db6..87b935638 100644 --- a/crates/storage/src/api/tables.rs +++ b/crates/storage/src/api/tables.rs @@ -61,10 +61,29 @@ pub enum Table { /// re-deriving anything. Emptied by the replay that verifies a row and by /// the finality eviction that gives up on one. PendingDataColumns, + /// Verified gloas execution payload envelopes: + /// (slot || beacon_block_root) -> `SignedExecutionPayloadEnvelope` + /// + /// A row exists only for an envelope that passed verification, so the + /// table is the durable form of the store's `verified_payloads` set: a + /// restart rebuilds that set from it. Keyed slot-first, with the slot of + /// the block the envelope fulfills (the envelope carries none itself), so + /// a by-range handler can scan a slot window, like `DataColumns`. + /// + /// Never pruned, as beacon block rows are not either. + ExecutionPayloadEnvelopes, + /// Block timeliness: beacon_block_root -> one byte per deadline (0 or 1) + /// + /// The durable form of the store's `block_timeliness` map. Fork choice + /// reads it on the ordinary head-computation path, and a block whose entry + /// is lost fails the whole head walk, so it cannot be left to memory. + /// + /// Never pruned; a row is a few bytes. + BlockTimeliness, } /// All table variants. -pub const ALL_TABLES: [Table; 10] = [ +pub const ALL_TABLES: [Table; 12] = [ Table::BlockHeaders, Table::BlockBodies, Table::BlockProof, @@ -75,6 +94,8 @@ pub const ALL_TABLES: [Table; 10] = [ Table::LiveChain, Table::DataColumns, Table::PendingDataColumns, + Table::ExecutionPayloadEnvelopes, + Table::BlockTimeliness, ]; impl Table { @@ -91,6 +112,8 @@ impl Table { Table::LiveChain => "live_chain", Table::DataColumns => "data_columns", Table::PendingDataColumns => "pending_data_columns", + Table::ExecutionPayloadEnvelopes => "execution_payload_envelopes", + Table::BlockTimeliness => "block_timeliness", } } } diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 87e89df11..020fd72f0 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -18,7 +18,7 @@ use ethlambda_types::{ beacon::{ config::Config, constants::NUM_BLOCK_TIMELINESS_DEADLINES, - containers::{BeaconState, Checkpoint as BeaconCheckpoint, SignedBeaconBlock}, + containers::{BeaconState, Checkpoint as BeaconCheckpoint, SignedBeaconBlock, gloas}, fork::ForkName, fork_choice::{BlockPayloadLink, LatestMessage, PayloadStatusV1, PowBlock}, preset::{Preset, SLOTS_PER_EPOCH}, @@ -657,25 +657,21 @@ impl GossipSignatureBuffer { /// for again, and `el_block_hashes` and `payload_links` are caches over data /// already decodable from the block itself. /// -/// Gloas's own four (`verified_payloads`, `payload_timeliness_vote`, -/// `payload_data_availability_vote`, and `block_timeliness`'s widened shape) -/// do not all fit the "a restarted node just gets it again" pattern above, -/// even though they live in this same scratch. `payload_timeliness_vote` and -/// `payload_data_availability_vote` do: they are the payload timeliness -/// committee's per-slot ballot, gone the moment the slot it was cast for is -/// no longer being decided, the same kind of throwaway `pow_blocks` is. -/// `verified_payloads` and `block_timeliness` do not, and both must be -/// persisted before this runs against a live, restarting node: losing -/// `verified_payloads` on restart does not error, it silently drops every -/// full payload branch from fork choice (`is_payload_verified` answers -/// `false` for a block this store genuinely verified before restarting), -/// and losing a `block_timeliness` entry the gloas fork choice's -/// `should_apply_proposer_boost` needs makes that call fail outright with -/// `Error::SpecAssert`. Unlike pre-gloas, where `block_timeliness` is read -/// only by the same-slot reorg helpers, gloas's own weight computation -/// reads it on the ordinary head-computation path, so that failure aborts -/// the whole `gloas_get_weight`/`gloas_get_head` walk, not just an optional -/// reorg decision. Nothing in this crate persists either yet. +/// Gloas's `verified_payloads` and `block_timeliness` are the exceptions: they +/// live in this scratch for reading, but are written through to +/// `Table::ExecutionPayloadEnvelopes` and `Table::BlockTimeliness`, and +/// [`Store::from_db_state`] rebuilds both for every block at or above the +/// finalized one. Losing them would not error for `verified_payloads`, it +/// would silently drop every full payload branch from fork choice; losing a +/// `block_timeliness` entry makes `should_apply_proposer_boost` fail with +/// `Error::SpecAssert`, which aborts the whole head walk since gloas reads it +/// on the ordinary path. +/// +/// `payload_timeliness_vote` and `payload_data_availability_vote` stay in +/// memory on purpose. They are the payload timeliness committee's per-slot +/// ballot, read only while the slot they were cast for is being decided, so +/// after a restart only the current and previous slot's blocks would read +/// them and the loss lasts about a slot. /// /// Most of this is uncapped: the per-validator maps are bounded by the /// validator set, and the per-block ones (`block_timeliness`, @@ -703,13 +699,9 @@ pub(crate) struct BeaconScratch { pub(crate) unrealized_justifications: HashMap, /// Gloas: beacon block roots whose execution payload envelope has been /// locally delivered and verified (`on_execution_payload_envelope`'s own - /// `store.payloads[root] = envelope`), standing in for the specification's - /// `store.payloads` for exactly the question this crate asks of it today, - /// `is_payload_verified`'s membership test. The envelope itself is not - /// kept: nothing here reads one back out yet, and it is not clear this - /// scratch is even where a kept envelope should live, since (see this - /// struct's own doc) `verified_payloads` needs to survive a restart and - /// nothing else here does. + /// `store.payloads[root] = envelope`), the in-memory index of + /// `Table::ExecutionPayloadEnvelopes` for `is_payload_verified`'s + /// membership test. The envelopes themselves are read from the table. pub(crate) verified_payloads: HashSet, /// Each imported block's payload link (which rules it is handled under, /// and which of its parent's payload branches it builds on), so the head @@ -1105,12 +1097,12 @@ impl Store { }; info!(?chain, anchor_slot, "Loaded store from persisted DB state"); - Ok(Some(Self::from_parts( - backend, - Arc::new(config), - chain, - anchor_slot, - ))) + let store = Self::from_parts(backend, Arc::new(config), chain, anchor_slot); + match chain { + Chain::Lean => {} + Chain::Beacon => store.restore_beacon_scratch(), + } + Ok(Some(store)) } /// Checks that both the justified and finalized checkpoints have a @@ -3259,12 +3251,19 @@ impl Store { .copied() } - /// Records `root`'s two timeliness deadlines. See [`Store::block_timeliness`]. + /// Records `root`'s two timeliness deadlines, in memory and in + /// `Table::BlockTimeliness`. See [`Store::block_timeliness`]. pub fn set_block_timeliness( &mut self, root: H256, timely: [bool; NUM_BLOCK_TIMELINESS_DEADLINES], ) { + let entries = vec![(root.to_ssz(), timely.map(u8::from).to_vec())]; + let mut batch = self.backend.begin_write().expect("write batch"); + batch + .put_batch(Table::BlockTimeliness, entries) + .expect("put block timeliness"); + batch.commit().expect("commit"); self.beacon .lock() .unwrap() @@ -3297,10 +3296,101 @@ impl Store { self.beacon.lock().unwrap().verified_payloads.contains(root) } - /// Gloas: records that `root`'s execution payload envelope has been - /// verified. - pub fn insert_verified_payload(&mut self, root: H256) { + /// Gloas: stores a verified execution payload envelope and records that + /// its block's payload is verified. + /// + /// Callers verify the envelope against its block first, so the block is + /// known here; its slot is the row's key, since the envelope has none. + /// + /// # Errors + /// + /// [`Error::UnexpectedMissingBlockHeader`] when the envelope's block is + /// not in the store; nothing is written. + pub fn insert_verified_payload( + &mut self, + envelope: &gloas::SignedExecutionPayloadEnvelope, + ) -> Result<(), Error> { + let root = envelope.message.beacon_block_root; + let slot = beacon_block_slot(self.backend.as_ref(), &root) + .ok_or(Error::UnexpectedMissingBlockHeader(root))?; + let entries = vec![(encode_slot_root_key(slot, &root), envelope.to_ssz())]; + let mut batch = self.backend.begin_write().expect("write batch"); + batch + .put_batch(Table::ExecutionPayloadEnvelopes, entries) + .expect("put execution payload envelope"); + batch.commit().expect("commit"); self.beacon.lock().unwrap().verified_payloads.insert(root); + Ok(()) + } + + /// Gloas: the verified envelope stored for `root`, if any. + pub fn get_execution_payload_envelope( + &self, + root: &H256, + ) -> Result, Error> { + let Some(slot) = beacon_block_slot(self.backend.as_ref(), root) else { + return Ok(None); + }; + let view = self.backend.begin_read().expect("read view"); + let bytes = view + .get( + Table::ExecutionPayloadEnvelopes, + &encode_slot_root_key(slot, root), + ) + .expect("get"); + Ok(bytes.map(|bytes| { + gloas::SignedExecutionPayloadEnvelope::from_ssz_bytes(&bytes) + .expect("a stored envelope decodes") + })) + } + + /// Rebuilds the in-memory `verified_payloads` and `block_timeliness` from + /// their tables, for every block fork choice can still reach. + /// + /// Walks `LiveChain` from the finalized block's slot up: the finalized + /// block is included because its payload status decides which branch its + /// children build on. Older rows stay on disk unread, so a restart costs + /// the unfinalized window rather than the chain's history. + fn restore_beacon_scratch(&self) { + let finalized = self.beacon_finalized_checkpoint(); + let floor = beacon_block_slot(self.backend.as_ref(), &finalized.root) + .unwrap_or(finalized.epoch * SLOTS_PER_EPOCH); + + let view = self.backend.begin_read().expect("read view"); + let mut timeliness = HashMap::new(); + let mut payloads = HashSet::new(); + for (key, _) in view + .prefix_iterator(Table::LiveChain, &[]) + .expect("iterator") + .filter_map(Result::ok) + { + let (slot, root) = decode_slot_root_key(&key); + if slot < floor { + continue; + } + if let Some(bytes) = view + .get(Table::BlockTimeliness, &root.to_ssz()) + .expect("get") + { + let bytes: [u8; NUM_BLOCK_TIMELINESS_DEADLINES] = bytes + .as_slice() + .try_into() + .expect("a stored timeliness has one byte per deadline"); + timeliness.insert(root, bytes.map(|byte| byte != 0)); + } + if view + .get(Table::ExecutionPayloadEnvelopes, &key) + .expect("get") + .is_some() + { + payloads.insert(root); + } + } + drop(view); + + let mut scratch = self.beacon.lock().unwrap(); + scratch.block_timeliness.extend(timeliness); + scratch.verified_payloads.extend(payloads); } /// `root`'s payload link, if one has been recorded. See @@ -7056,6 +7146,172 @@ mod tests { assert_eq!(store.block_timeliness(&H256::from([2u8; 32])), None); } + /// An envelope for `root` with every field at its zero value. + fn test_envelope(root: H256) -> gloas::SignedExecutionPayloadEnvelope { + let payload = gloas::ExecutionPayload { + parent_hash: Default::default(), + fee_recipient: Default::default(), + state_root: Default::default(), + receipts_root: Default::default(), + logs_bloom: vec![0u8; ethlambda_types::beacon::preset::BYTES_PER_LOGS_BLOOM] + .try_into() + .expect("built at exactly BYTES_PER_LOGS_BLOOM"), + prev_randao: Default::default(), + block_number: 7, + gas_limit: 0, + gas_used: 0, + timestamp: 0, + extra_data: Default::default(), + base_fee_per_gas: Default::default(), + block_hash: Default::default(), + transactions: Default::default(), + withdrawals: Default::default(), + blob_gas_used: 0, + excess_blob_gas: 0, + block_access_list: Default::default(), + slot_number: 0, + }; + gloas::SignedExecutionPayloadEnvelope { + message: gloas::ExecutionPayloadEnvelope { + payload, + execution_requests: Default::default(), + builder_index: 3, + beacon_block_root: root, + parent_beacon_block_root: H256::ZERO, + }, + signature: Default::default(), + } + } + + /// A beacon directory whose finalized checkpoint is the epoch-1 boundary + /// checkpoint naming `finalized_root`, so the finalized slot is the first + /// slot of that epoch. + fn beacon_backend_finalized_at(finalized_root: H256) -> Arc { + let backend = Arc::new(InMemoryBackend::new()); + let finalized = BeaconCheckpoint { + epoch: 1, + root: finalized_root, + }; + Store::init_beacon( + backend.clone(), + 0, + Config::mainnet(), + finalized_root, + Store::beacon_checkpoint_as_stored(finalized), + 0, + ); + backend + } + + fn reopen(backend: Arc) -> Store { + Store::from_db_state(backend) + .expect("reopen") + .expect("populated directory") + } + + #[test] + fn a_verified_payload_survives_reopen() { + let root = H256::from([1u8; 32]); + let backend = beacon_backend_finalized_at(root); + let mut store = reopen(backend.clone()); + store + .insert_signed_block(root, beacon_test_block(SLOTS_PER_EPOCH, H256::ZERO)) + .expect("insert block"); + let envelope = test_envelope(root); + store + .insert_verified_payload(&envelope) + .expect("the block is known"); + assert!(store.has_verified_payload(&root)); + + let reopened = reopen(backend); + + assert!(reopened.has_verified_payload(&root)); + assert_eq!( + reopened + .get_execution_payload_envelope(&root) + .expect("read"), + Some(envelope) + ); + } + + #[test] + fn an_envelope_for_an_unknown_block_is_refused() { + let mut store = Store::init_beacon( + Arc::new(InMemoryBackend::new()), + 0, + Config::mainnet(), + H256::ZERO, + Checkpoint::default(), + 0, + ); + let root = H256::from([9u8; 32]); + + let result = store.insert_verified_payload(&test_envelope(root)); + + assert!(matches!( + result, + Err(Error::UnexpectedMissingBlockHeader(missing)) if missing == root + )); + assert!(!store.has_verified_payload(&root)); + assert_eq!(store.get_execution_payload_envelope(&root).unwrap(), None); + } + + #[test] + fn block_timeliness_survives_reopen() { + let root = H256::from([1u8; 32]); + let backend = beacon_backend_finalized_at(root); + let mut store = reopen(backend.clone()); + store + .insert_signed_block(root, beacon_test_block(SLOTS_PER_EPOCH, H256::ZERO)) + .expect("insert block"); + store.set_block_timeliness(root, [true, false]); + + let reopened = reopen(backend); + + assert_eq!(reopened.block_timeliness(&root), Some([true, false])); + } + + #[test] + fn resume_skips_payloads_below_finality() { + // The finalized block sits one slot past its epoch's first slot, so + // the floor is the block's own slot, not the checkpoint's. + let old = H256::from([1u8; 32]); + let finalized = H256::from([2u8; 32]); + let child = H256::from([3u8; 32]); + let backend = beacon_backend_finalized_at(finalized); + let mut store = reopen(backend.clone()); + let slots = [ + (old, SLOTS_PER_EPOCH - 1, H256::ZERO), + (finalized, SLOTS_PER_EPOCH + 1, old), + (child, SLOTS_PER_EPOCH + 2, finalized), + ]; + for (root, slot, parent) in slots { + store + .insert_signed_block(root, beacon_test_block(slot, parent)) + .expect("insert block"); + store + .insert_verified_payload(&test_envelope(root)) + .expect("the block is known"); + store.set_block_timeliness(root, [true, true]); + } + + let reopened = reopen(backend); + + assert!(!reopened.has_verified_payload(&old)); + assert_eq!(reopened.block_timeliness(&old), None); + for root in [finalized, child] { + assert!(reopened.has_verified_payload(&root)); + assert_eq!(reopened.block_timeliness(&root), Some([true, true])); + } + // Skipped in memory only: the row stays on disk. + assert!( + reopened + .get_execution_payload_envelope(&old) + .unwrap() + .is_some() + ); + } + #[test] fn latest_messages_skip_equivocators() { let mut store = Store::test_store(); From d7cc62bafb3e06b0e24ca34bd1f11c378db545cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:51:22 -0300 Subject: [PATCH 042/132] fix(beacon): seed the PTC vote vectors a resumed store expects Restoring verified_payloads on resume let the head walk reach payload_timeliness, which reads the payload committee's vote vectors. Those are never persisted, so the lookup failed with a SpecAssert and get_head (and a timely on_block) errored; the first child of the pre-restart tip failed the same way in notify_ptc_messages. Resume now seeds both vectors empty for every gloas block in the restored window, which is what a freshly imported block holds and is true of a process that has seen no votes. Also: persist block timeliness only for gloas blocks (nothing reads pre-gloas entries after a restart, so the rows would only grow); fall back to the anchor slot rather than the epoch start when the finalized block has no row, since a finalized block can sit below its epoch start; pass the block's slot to insert_verified_payload instead of re-decoding the block; and correct the docs that called a missing timeliness entry fatal or the scratch in-memory only. --- .../src/beacon/fork_choice.rs | 115 +++++++- crates/storage/src/store.rs | 251 ++++++++++++------ docs/data_storage.md | 36 ++- 3 files changed, 302 insertions(+), 100 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index c66fd5ea7..b41be4c27 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -929,7 +929,9 @@ pub fn is_optimistic_candidate_block( /// unrealized justifications in their own table, and the per-slot/per-epoch /// scratch (`proposer_boost_root`, `block_timeliness`, `equivocating_indices`, /// `latest_messages`, `pow_blocks`) in an in-memory struct cheap enough to -/// rebuild after a restart rather than worth persisting. See +/// rebuild after a restart rather than worth persisting; the exceptions are +/// a gloas block's timeliness and verified payload, which are also stored +/// and reloaded on resume. See /// [`ethlambda_storage::Store`]'s own documentation for the full /// field-by-field accounting; this module reads and writes it exclusively /// through its public accessors. @@ -1134,7 +1136,7 @@ pub fn get_forkchoice_store( store.set_payload_link(anchor_root, anchor_slot, anchor_rules.payload_link(None)); match anchor_rules { ForkRules::Gloas => { - store.set_block_timeliness(anchor_root, [true, true]); + store.set_gloas_block_timeliness(anchor_root, [true, true]); store.set_payload_timeliness_vote(anchor_root, vec![None; preset::PTC_SIZE]); store.set_payload_data_availability_vote(anchor_root, vec![None; preset::PTC_SIZE]); } @@ -3165,9 +3167,9 @@ fn payload_status_tiebreaker_with( /// **Implementation choice, not spec text**: a candidate with no recorded /// [`Store::block_timeliness`](ethlambda_storage::Store::block_timeliness) /// entry reads as not timely by either deadline, rather than raising -/// `Error::SpecAssert`. That scratch is in-memory only, so every entry is -/// gone after a restart; reading a gap as "not an early equivocation" -/// rather than aborting the whole weight computation over it is the +/// `Error::SpecAssert`. A gloas block's entry is stored and reloaded on +/// resume, so a gap is a block with no entry at all; reading it as "not an +/// early equivocation" rather than aborting the whole weight computation is the /// conservative answer (it can only ever miss withholding a boost, never /// wrongly withhold one), the same shape of tolerance /// [`gloas_get_attestation_score`]'s own doc gives for a pruned vote. @@ -4640,7 +4642,10 @@ pub fn on_block( // first, and shares the pre-import head's proposer shuffling. Both calls // are infallible: nothing from here to the end of this function can turn // into an `Err`, and the store has already been mutated above. - store.set_block_timeliness(block_root, timeliness); + match rules { + ForkRules::Gloas => store.set_gloas_block_timeliness(block_root, timeliness), + ForkRules::PreGloas => store.set_block_timeliness(block_root, timeliness), + } if let Some(pre_block_head) = pre_block_head { update_proposer_boost_root(store, &index, pre_block_head, block_root, config); } @@ -4865,9 +4870,7 @@ pub fn on_execution_payload_envelope( // Add execution payload envelope to the store. Persisted, so a restarted // follower keeps the full branch of this block. - store - .insert_verified_payload(signed_envelope) - .expect("insert"); + store.insert_verified_payload(block.slot(), signed_envelope); Ok(()) } @@ -5197,9 +5200,12 @@ mod tests { }, signature: Default::default(), }; - store - .insert_verified_payload(&envelope) - .expect("the block is in the store"); + let slot = store + .get_signed_block(&root) + .expect("get") + .expect("the block is in the store") + .slot(); + store.insert_verified_payload(slot, &envelope); } /// A fulu signed block with an empty body and a zero signature, for the @@ -7309,8 +7315,7 @@ mod tests { // A same-slot, same-proposer sibling of the parent that would be an // early equivocation if its timeliness were known, but whose - // `block_timeliness` entry was never recorded: the scratch is - // in-memory only and does not survive a restart. Deliberately no + // `block_timeliness` entry was never recorded. Deliberately no // `store.set_block_timeliness(twin_root, ...)` call. let twin_root = Root::repeat_byte(0xb1); store @@ -7706,6 +7711,88 @@ mod tests { ); } + /// A restarted follower holds a verified payload (restored from its table) + /// but no payload-committee votes, which are never persisted. Both the + /// head walk (`payload_timeliness` on the tiebreaker path) and the first + /// child's payload attestations (`notify_ptc_messages` ends in this same + /// vote update) read those vectors, so resume must reseed them empty. + #[test] + fn a_resumed_store_walks_the_head_and_takes_payload_attestations() { + let config = Config::active().with_fork_epoch(ForkName::Gloas, 0); + let anchor_root = Root::repeat_byte(0xb0); + let root = Root::repeat_byte(0xb1); + let backend = Arc::new(InMemoryBackend::new()); + let mut store = store_anchored_at_on(backend.clone(), anchor_root); + store + .insert_signed_block( + anchor_root, + gloas_block( + 0, + Root::ZERO, + ExecutionBlockHash::ZERO, + ExecutionBlockHash::ZERO, + ), + ) + .unwrap(); + store + .insert_signed_block( + root, + gloas_block( + 1, + anchor_root, + ExecutionBlockHash::repeat_byte(0xff), + ExecutionBlockHash::repeat_byte(0x22), + ), + ) + .unwrap(); + let mut state = test_state::with_validators_at(ForkName::Gloas, 4); + store.insert_state(anchor_root, state.clone()).unwrap(); + let BeaconState::Gloas(inner) = &mut state else { + unreachable!("with_validators_at(Gloas) builds a gloas state"); + }; + inner.slot = 1; + let mut committee = vec![0; preset::PTC_SIZE]; + committee[2] = 3; + let window_index = (preset::SLOTS_PER_EPOCH + 1) as usize; + inner.ptc_window[window_index] = committee.try_into().unwrap(); + store.insert_state(root, state).unwrap(); + // What `on_block` records for each gloas block, then the envelope. + for block_root in [anchor_root, root] { + store.set_gloas_block_timeliness(block_root, [true, true]); + store.set_payload_timeliness_vote(block_root, vec![None; preset::PTC_SIZE]); + store.set_payload_data_availability_vote(block_root, vec![None; preset::PTC_SIZE]); + } + verify_payload(&mut store, root); + // Dropping the last handle lets the state writer finish its queue. + drop(store); + + let mut store = Store::from_db_state(backend) + .expect("reopen") + .expect("populated directory"); + assert!(store.has_verified_payload(&root)); + + // The slot after the block's own: the head walk asks whether the + // verified payload is timely, which reads the votes. + store.set_time_ms(2 * config.slot_duration_ms).unwrap(); + let committees = CommitteeCache::default(); + gloas_get_head(&store, &config, &committees) + .expect("a resumed store must walk the head over a verified payload"); + + let message = gloas::PayloadAttestationMessage { + validator_index: 3, + data: gloas::PayloadAttestationData { + beacon_block_root: root, + slot: 1, + payload_present: true, + blob_data_available: true, + }, + signature: Default::default(), + }; + on_payload_attestation_message(&mut store, &message, true, &config) + .expect("the first child's attestations must find the vote vectors"); + assert_eq!(store.payload_timeliness_vote(&root).unwrap()[2], Some(true)); + } + #[test] fn get_forkchoice_store_seeds_a_gloas_anchor_and_leaves_an_earlier_one_alone() { // An exact anchor pair, built as `anchor_pair_with` builds a phase0 one. diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 020fd72f0..3229bf502 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -21,7 +21,7 @@ use ethlambda_types::{ containers::{BeaconState, Checkpoint as BeaconCheckpoint, SignedBeaconBlock, gloas}, fork::ForkName, fork_choice::{BlockPayloadLink, LatestMessage, PayloadStatusV1, PowBlock}, - preset::{Preset, SLOTS_PER_EPOCH}, + preset::{PTC_SIZE, Preset, SLOTS_PER_EPOCH}, primitives::ExecutionBlockHash, }, block::{ @@ -642,14 +642,15 @@ impl GossipSignatureBuffer { } /// Beacon fork-choice state that is per-slot or per-epoch scratch rather than -/// chain history: nothing here survives a restart, and nothing here is worth -/// the write amplification of persisting. +/// chain history: apart from the gloas entries named below, nothing here +/// survives a restart, and nothing here is worth the write amplification of +/// persisting. /// -/// `proposer_boost_root` resets every slot, `block_timeliness` is read only by -/// the same-slot reorg helpers, `equivocating_indices` is rebuilt by replaying -/// attester slashings on sync, `latest_messages` is rebuilt by the first epoch -/// of attestations, `pow_blocks` stands in for a call to an execution client -/// that a restarted node would simply make again, and +/// `proposer_boost_root` resets every slot, pre-gloas `block_timeliness` is +/// read only by the same-slot reorg helpers, `equivocating_indices` is rebuilt +/// by replaying attester slashings on sync, `latest_messages` is rebuilt by +/// the first epoch of attestations, `pow_blocks` stands in for a call to an +/// execution client that a restarted node would simply make again, and /// `unrealized_justifications` is recomputed by replaying epoch processing on a /// copy of a block's post-state, which a node resuming from an anchor does /// anyway as it re-imports the unfinalized window. `optimistic_roots` and @@ -658,20 +659,22 @@ impl GossipSignatureBuffer { /// already decodable from the block itself. /// /// Gloas's `verified_payloads` and `block_timeliness` are the exceptions: they -/// live in this scratch for reading, but are written through to -/// `Table::ExecutionPayloadEnvelopes` and `Table::BlockTimeliness`, and -/// [`Store::from_db_state`] rebuilds both for every block at or above the -/// finalized one. Losing them would not error for `verified_payloads`, it -/// would silently drop every full payload branch from fork choice; losing a -/// `block_timeliness` entry makes `should_apply_proposer_boost` fail with -/// `Error::SpecAssert`, which aborts the whole head walk since gloas reads it -/// on the ordinary path. +/// live in this scratch for reading, but a gloas block's entries are written +/// through to `Table::ExecutionPayloadEnvelopes` and `Table::BlockTimeliness`, +/// and [`Store::from_db_state`] rebuilds both for every block at or above the +/// finalized one. Losing `verified_payloads` would not error, it would +/// silently drop every full payload branch from fork choice. Losing a +/// `block_timeliness` entry does not error either: a missing entry reads as +/// not timely by either deadline, which quietly changes what +/// `should_apply_proposer_boost` withholds. Pre-gloas timeliness is never +/// persisted: nothing reads it after a restart. /// /// `payload_timeliness_vote` and `payload_data_availability_vote` stay in /// memory on purpose. They are the payload timeliness committee's per-slot -/// ballot, read only while the slot they were cast for is being decided, so -/// after a restart only the current and previous slot's blocks would read -/// them and the loss lasts about a slot. +/// ballot, read only while the slot they were cast for is being decided. A +/// resumed store reseeds them empty (every member `None`) for each restored +/// gloas block, which is what a block just imported holds, and the next +/// block's payload attestations refill them: the loss lasts about a slot. /// /// Most of this is uncapped: the per-validator maps are bounded by the /// validator set, and the per-block ones (`block_timeliness`, @@ -829,9 +832,10 @@ fn decode_beacon_block_value(bytes: &[u8]) -> SignedBeaconBlock { /// Beacon directories only: it decodes the row as a tagged /// [`SignedBeaconBlock`], which is the wrong shape for a lean directory's bare /// [`BlockHeader`] (unlike [`block_fields`](Store::block_fields), which -/// dispatches on `self.chain`). Nothing does today: [`StateWriter::write`](crate::state_writer)'s -/// non-`Lean` match arm is this function's only caller, so that invariant is -/// enforced by having exactly one caller rather than by a runtime check. +/// dispatches on `self.chain`). Its callers are [`StateWriter::write`](crate::state_writer)'s +/// non-`Lean` match arm and the beacon-only envelope and resume methods on +/// [`Store`], so that invariant is enforced by every caller being a beacon +/// path rather than by a runtime check. pub(crate) fn beacon_block_slot(backend: &dyn StorageBackend, root: &H256) -> Option { let view = backend.begin_read().expect("read view"); view.read_with(Table::BlockHeaders, &root.to_ssz(), |bytes| { @@ -3218,9 +3222,11 @@ impl Store { // ============ Beacon Fork-Choice Scratch ============ // - // None of these carry a `Result`: the beacon fork choice's error type - // lives in `ethlambda-types` and cannot name a storage error, so these - // accessors take and return plain values like the rest of this scratch. + // The accessors on in-memory scratch carry no `Result`: the beacon fork + // choice's error type lives in `ethlambda-types` and cannot name a storage + // error, so they take and return plain values. The gloas write-through + // ones (`set_gloas_block_timeliness`, `insert_verified_payload`) write a + // row and so `expect` on the backend like the other infallible writers. /// The block root proposer boost currently applies to. Resets every slot. pub fn proposer_boost_root(&self) -> H256 { @@ -3251,12 +3257,28 @@ impl Store { .copied() } - /// Records `root`'s two timeliness deadlines, in memory and in - /// `Table::BlockTimeliness`. See [`Store::block_timeliness`]. + /// Records `root`'s two timeliness deadlines in memory only. See + /// [`Store::block_timeliness`]. A gloas block uses + /// [`Store::set_gloas_block_timeliness`] instead, since only gloas reads + /// the entry back after a restart. pub fn set_block_timeliness( &mut self, root: H256, timely: [bool; NUM_BLOCK_TIMELINESS_DEADLINES], + ) { + self.beacon + .lock() + .unwrap() + .block_timeliness + .insert(root, timely); + } + + /// Records a gloas block's timeliness in memory and in + /// `Table::BlockTimeliness`, so a resumed store still has it. + pub fn set_gloas_block_timeliness( + &mut self, + root: H256, + timely: [bool; NUM_BLOCK_TIMELINESS_DEADLINES], ) { let entries = vec![(root.to_ssz(), timely.map(u8::from).to_vec())]; let mut batch = self.backend.begin_write().expect("write batch"); @@ -3264,11 +3286,7 @@ impl Store { .put_batch(Table::BlockTimeliness, entries) .expect("put block timeliness"); batch.commit().expect("commit"); - self.beacon - .lock() - .unwrap() - .block_timeliness - .insert(root, timely); + self.set_block_timeliness(root, timely); } /// Whether `index` has been observed equivocating (via a processed @@ -3299,20 +3317,15 @@ impl Store { /// Gloas: stores a verified execution payload envelope and records that /// its block's payload is verified. /// - /// Callers verify the envelope against its block first, so the block is - /// known here; its slot is the row's key, since the envelope has none. - /// - /// # Errors - /// - /// [`Error::UnexpectedMissingBlockHeader`] when the envelope's block is - /// not in the store; nothing is written. + /// `slot` is the slot of the block the envelope fulfills, which the + /// caller has already read while verifying the envelope against it; the + /// envelope carries none and the row is keyed by it. pub fn insert_verified_payload( &mut self, + slot: u64, envelope: &gloas::SignedExecutionPayloadEnvelope, - ) -> Result<(), Error> { + ) { let root = envelope.message.beacon_block_root; - let slot = beacon_block_slot(self.backend.as_ref(), &root) - .ok_or(Error::UnexpectedMissingBlockHeader(root))?; let entries = vec![(encode_slot_root_key(slot, &root), envelope.to_ssz())]; let mut batch = self.backend.begin_write().expect("write batch"); batch @@ -3320,7 +3333,6 @@ impl Store { .expect("put execution payload envelope"); batch.commit().expect("commit"); self.beacon.lock().unwrap().verified_payloads.insert(root); - Ok(()) } /// Gloas: the verified envelope stored for `root`, if any. @@ -3351,14 +3363,25 @@ impl Store { /// block is included because its payload status decides which branch its /// children build on. Older rows stay on disk unread, so a restart costs /// the unfinalized window rather than the chain's history. + /// + /// Also seeds both payload-committee vote vectors, empty, for every gloas + /// block in the window. Nothing persists the votes, and the head walk and + /// `notify_ptc_messages` read a gloas block's vectors, so an absent entry + /// would fail them with a `SpecAssert`. Empty is what `on_block` gives a + /// block it just imported, and is true here: this process has seen no + /// votes for it. fn restore_beacon_scratch(&self) { let finalized = self.beacon_finalized_checkpoint(); - let floor = beacon_block_slot(self.backend.as_ref(), &finalized.root) - .unwrap_or(finalized.epoch * SLOTS_PER_EPOCH); + // Without the finalized block the epoch start would be wrong, since a + // finalized block can sit below its checkpoint's epoch start; the + // anchor is the lowest slot this directory holds. + let floor = + beacon_block_slot(self.backend.as_ref(), &finalized.root).unwrap_or(self.anchor_slot); let view = self.backend.begin_read().expect("read view"); let mut timeliness = HashMap::new(); let mut payloads = HashSet::new(); + let mut gloas_roots = Vec::new(); for (key, _) in view .prefix_iterator(Table::LiveChain, &[]) .expect("iterator") @@ -3378,6 +3401,15 @@ impl Store { .expect("a stored timeliness has one byte per deadline"); timeliness.insert(root, bytes.map(|byte| byte != 0)); } + let is_gloas = view + .get(Table::BlockHeaders, &root.to_ssz()) + .expect("get") + .and_then(|bytes| bytes.first().copied()) + .and_then(ForkName::from_selector) + == Some(ForkName::Gloas); + if is_gloas { + gloas_roots.push(root); + } if view .get(Table::ExecutionPayloadEnvelopes, &key) .expect("get") @@ -3391,6 +3423,14 @@ impl Store { let mut scratch = self.beacon.lock().unwrap(); scratch.block_timeliness.extend(timeliness); scratch.verified_payloads.extend(payloads); + for root in gloas_roots { + scratch + .payload_timeliness_vote + .insert(root, vec![None; PTC_SIZE]); + scratch + .payload_data_availability_vote + .insert(root, vec![None; PTC_SIZE]); + } } /// `root`'s payload link, if one has been recorded. See @@ -7209,18 +7249,30 @@ mod tests { .expect("populated directory") } + /// A gloas signed block with an empty body and a zero signature. + fn gloas_test_block(slot: u64, parent_root: H256) -> SignedBeaconBlock { + SignedBeaconBlock::Gloas(gloas::SignedBeaconBlock { + message: gloas::BeaconBlock { + slot, + proposer_index: 0, + parent_root, + state_root: H256::ZERO, + body: Default::default(), + }, + signature: Default::default(), + }) + } + #[test] fn a_verified_payload_survives_reopen() { let root = H256::from([1u8; 32]); let backend = beacon_backend_finalized_at(root); let mut store = reopen(backend.clone()); store - .insert_signed_block(root, beacon_test_block(SLOTS_PER_EPOCH, H256::ZERO)) + .insert_signed_block(root, gloas_test_block(SLOTS_PER_EPOCH, H256::ZERO)) .expect("insert block"); let envelope = test_envelope(root); - store - .insert_verified_payload(&envelope) - .expect("the block is known"); + store.insert_verified_payload(SLOTS_PER_EPOCH, &envelope); assert!(store.has_verified_payload(&root)); let reopened = reopen(backend); @@ -7235,70 +7287,93 @@ mod tests { } #[test] - fn an_envelope_for_an_unknown_block_is_refused() { - let mut store = Store::init_beacon( - Arc::new(InMemoryBackend::new()), - 0, - Config::mainnet(), - H256::ZERO, - Checkpoint::default(), - 0, - ); - let root = H256::from([9u8; 32]); + fn block_timeliness_survives_reopen() { + let root = H256::from([1u8; 32]); + let backend = beacon_backend_finalized_at(root); + let mut store = reopen(backend.clone()); + store + .insert_signed_block(root, gloas_test_block(SLOTS_PER_EPOCH, H256::ZERO)) + .expect("insert block"); + store.set_gloas_block_timeliness(root, [true, false]); - let result = store.insert_verified_payload(&test_envelope(root)); + let reopened = reopen(backend); - assert!(matches!( - result, - Err(Error::UnexpectedMissingBlockHeader(missing)) if missing == root - )); - assert!(!store.has_verified_payload(&root)); - assert_eq!(store.get_execution_payload_envelope(&root).unwrap(), None); + assert_eq!(reopened.block_timeliness(&root), Some([true, false])); } #[test] - fn block_timeliness_survives_reopen() { + fn pre_gloas_timeliness_is_not_persisted() { let root = H256::from([1u8; 32]); let backend = beacon_backend_finalized_at(root); let mut store = reopen(backend.clone()); store .insert_signed_block(root, beacon_test_block(SLOTS_PER_EPOCH, H256::ZERO)) .expect("insert block"); - store.set_block_timeliness(root, [true, false]); + store.set_block_timeliness(root, [true, true]); let reopened = reopen(backend); - assert_eq!(reopened.block_timeliness(&root), Some([true, false])); + assert_eq!(reopened.block_timeliness(&root), None); + } + + #[test] + fn resume_reseeds_empty_vote_vectors_for_gloas_blocks_only() { + let gloas_root = H256::from([1u8; 32]); + let phase0_root = H256::from([2u8; 32]); + let backend = beacon_backend_finalized_at(gloas_root); + let mut store = reopen(backend.clone()); + store + .insert_signed_block(gloas_root, gloas_test_block(SLOTS_PER_EPOCH, H256::ZERO)) + .expect("insert block"); + store + .insert_signed_block( + phase0_root, + beacon_test_block(SLOTS_PER_EPOCH + 1, gloas_root), + ) + .expect("insert block"); + + let reopened = reopen(backend); + + let empty = vec![None; PTC_SIZE]; + assert_eq!( + reopened.payload_timeliness_vote(&gloas_root), + Some(empty.clone()) + ); + assert_eq!( + reopened.payload_data_availability_vote(&gloas_root), + Some(empty) + ); + assert_eq!(reopened.payload_timeliness_vote(&phase0_root), None); } #[test] fn resume_skips_payloads_below_finality() { - // The finalized block sits one slot past its epoch's first slot, so - // the floor is the block's own slot, not the checkpoint's. + // The finalized block sits below its checkpoint's epoch start, and its + // child sits between the two: a floor at the epoch start would skip + // both, so the child loading shows the floor is the block's own slot. let old = H256::from([1u8; 32]); let finalized = H256::from([2u8; 32]); let child = H256::from([3u8; 32]); let backend = beacon_backend_finalized_at(finalized); let mut store = reopen(backend.clone()); let slots = [ - (old, SLOTS_PER_EPOCH - 1, H256::ZERO), - (finalized, SLOTS_PER_EPOCH + 1, old), - (child, SLOTS_PER_EPOCH + 2, finalized), + (old, SLOTS_PER_EPOCH - 3, H256::ZERO), + (finalized, SLOTS_PER_EPOCH - 2, old), + (child, SLOTS_PER_EPOCH - 1, finalized), ]; for (root, slot, parent) in slots { store - .insert_signed_block(root, beacon_test_block(slot, parent)) + .insert_signed_block(root, gloas_test_block(slot, parent)) .expect("insert block"); - store - .insert_verified_payload(&test_envelope(root)) - .expect("the block is known"); - store.set_block_timeliness(root, [true, true]); + store.insert_verified_payload(slot, &test_envelope(root)); + store.set_gloas_block_timeliness(root, [true, true]); } let reopened = reopen(backend); assert!(!reopened.has_verified_payload(&old)); assert_eq!(reopened.block_timeliness(&old), None); + assert_eq!(reopened.payload_timeliness_vote(&old), None); for root in [finalized, child] { assert!(reopened.has_verified_payload(&root)); assert_eq!(reopened.block_timeliness(&root), Some([true, true])); @@ -7312,6 +7387,24 @@ mod tests { ); } + #[test] + fn resume_without_the_finalized_block_floors_at_the_anchor_slot() { + // The finalized block has no row, so the floor falls back; a block + // below the checkpoint's epoch start must still load. + let finalized = H256::from([2u8; 32]); + let block = H256::from([3u8; 32]); + let backend = beacon_backend_finalized_at(finalized); + let mut store = reopen(backend.clone()); + store + .insert_signed_block(block, gloas_test_block(5, H256::ZERO)) + .expect("insert block"); + store.set_gloas_block_timeliness(block, [true, true]); + + let reopened = reopen(backend); + + assert_eq!(reopened.block_timeliness(&block), Some([true, true])); + } + #[test] fn latest_messages_skip_equivocators() { let mut store = Store::test_store(); diff --git a/docs/data_storage.md b/docs/data_storage.md index cc238c836..51079b266 100644 --- a/docs/data_storage.md +++ b/docs/data_storage.md @@ -2,7 +2,7 @@ This doc explains how ethlambda saves data. Especially, the split between the fork choice `Store` and the `StorageBackend` trait, -what each of the ten tables holds, and which data is in-memory only. +what each of the twelve tables holds, and which data is in-memory only. ## Overview @@ -108,8 +108,10 @@ is built from it, and clones are handed to the BlockChain and P2P actors. │ │ LiveChain │ │ (raw XMSS sigs │ │ │ │ DataColumns │ │ awaiting │ │ │ │ PendingDataColumns │ │ aggregation) │ │ - │ └─────────────────────┘ │ state_cache (LRU) │ │ - │ └──────────────────────┘ │ + │ │ ExecutionPayload- │ │ state_cache (LRU) │ │ + │ │ Envelopes │ └──────────────────────┘ │ + │ │ BlockTimeliness │ │ + │ └─────────────────────┘ │ │ Survives restarts, except │ │ PendingDataColumns, which Lost on restart. │ │ is cleared at startup. │ @@ -118,7 +120,7 @@ is built from it, and clones are handed to the BlockChain and P2P actors. ## The Tables -The ten variants of the `Table` enum (`crates/storage/src/api/tables.rs`): +The twelve variants of the `Table` enum (`crates/storage/src/api/tables.rs`): | Table | Key | Value | Pruned? | | ------------------ | --------------------------- | ----------------------------------------- | --------------------------------- | @@ -132,6 +134,8 @@ The ten variants of the `Table` enum (`crates/storage/src/api/tables.rs`): | `LiveChain` | slot ‖ root | `parent_root` | yes: below finalized | | `DataColumns` | slot ‖ root ‖ column_index | `DataColumnSidecar` (SSZ-encoded) | no (see [DataColumns](#datacolumns) below) | | `PendingDataColumns` | slot ‖ root ‖ column_index | `DataColumnSidecar` (SSZ-encoded), unverified | yes: on replay, below finalized, and wholly at startup | +| `ExecutionPayloadEnvelopes` | slot ‖ root | gloas `SignedExecutionPayloadEnvelope` (SSZ), verified only | no: beacon block rows are kept too | +| `BlockTimeliness` | root | one byte per timeliness deadline (0 or 1), gloas blocks only | no | ### Key encoding @@ -440,6 +444,23 @@ counts. `lean_table_bytes{table="data_columns"}` (see [metrics.md](./metrics.md) is that growth made visible; watch it, and size the disk against however long the node runs unattended before a pruner exists. +### ExecutionPayloadEnvelopes and BlockTimeliness + +Gloas only. A row in `ExecutionPayloadEnvelopes` exists for an envelope that +passed verification, keyed by the slot of the block it fulfills (the envelope +has no slot of its own) so a by-range handler can scan a slot window. The +table is the durable form of the store's `verified_payloads` set. Losing that +set on restart would silently drop every full payload branch from fork choice. +`BlockTimeliness` is the durable form of the gloas part of `block_timeliness`. +Pre-gloas blocks write no row, since nothing reads their timeliness after a +restart. + +`Store::from_db_state` reloads both for every `LiveChain` block at or above the +finalized block's slot, and reseeds the in-memory payload-committee vote +vectors empty for each gloas block in that window. The votes themselves are not +persisted: the next block's payload attestations refill them. Neither table is +pruned; rows outside the window are simply not read. + ### PendingDataColumns Same key and same encoding as `DataColumns`, holding sidecars that have **not @@ -657,7 +678,8 @@ where a future pruner is meant to land ## In-Memory Only (Lost on Restart) -Five `Store` fields never touch the backend. All are bounded buffers (or, for +Five `Store` fields never touch the backend (the gloas parts of `beacon` +excepted, see above). All are bounded buffers (or, for `beacon`, bounded in practice by the validator set and the unfinalized window) shared across `Store` clones: @@ -667,7 +689,7 @@ shared across `Store` clones: | `known_payloads` | 512 messages | Fork-choice-active aggregated proofs | | `gossip_signatures` | 2048 signatures | Raw per-validator XMSS signatures awaiting aggregation (each ~3 KB, so ~6 MB worst case) | | `state_cache` | 32 states | LRU memoization of block *and* checkpoint post-states (either chain), each held behind an `Arc` so a hit is not a copy; one bound covers both kinds, keyed apart by a small enum, and a miss is just a reconstruction rather than an error | -| `beacon` | unbounded | Beacon fork-choice scratch: proposer boost root, block timeliness, equivocating validator indices, latest messages, PoW blocks, and unrealized justifications. None of it is persisted: proposer boost resets every slot, timeliness is read only by the same-slot reorg helpers, equivocators come back from replaying attester slashings on sync, latest messages from one epoch of attestations, PoW blocks stand in for an execution-client call a restarted node would simply make again, and unrealized justifications are refilled as a node re-imports the unfinalized window from its anchor | +| `beacon` | unbounded | Beacon fork-choice scratch: proposer boost root, block timeliness, equivocating validator indices, latest messages, PoW blocks, and unrealized justifications. Apart from a gloas block's timeliness and verified payload (see [ExecutionPayloadEnvelopes and BlockTimeliness](#executionpayloadenvelopes-and-blocktimeliness)), none of it is persisted: proposer boost resets every slot, pre-gloas timeliness is read only by the same-slot reorg helpers, equivocators come back from replaying attester slashings on sync, latest messages from one epoch of attestations, PoW blocks stand in for an execution-client call a restarted node would simply make again, and unrealized justifications are refilled as a node re-imports the unfinalized window from its anchor | The payload buffers evict FIFO when full, and redundant proofs (whose participants are a subset of an existing proof for the same attestation data) @@ -681,7 +703,7 @@ pools. After a restart these buffers start empty: pending attestations and un-aggregated gossip signatures are lost and must be re-collected from the -network. Everything persisted in the ten tables survives, except `PendingDataColumns`, which is cleared outright: its only index is in memory. +network. Everything persisted in the twelve tables survives, except `PendingDataColumns`, which is cleared outright: its only index is in memory. ## Startup and Restore From 1ed3ec4e063ebc732ac84f95e2899f1de251c000 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:55:31 -0300 Subject: [PATCH 043/132] test(beacon): cover the live head walk on a resumed store The resume test only exercised the spec-literal gloas_get_head. The actor uses get_head_node, whose walk_head re-derives payload links that do not survive a restart and reads the boost and weight helpers, so assert it succeeds and agrees with gloas_get_head. Also make the resume fork check an exhaustive match so a new fork forces a decision, state the real cost of the LiveChain scan in its doc, and debug-assert that insert_verified_payload is given its block's own slot. --- .../src/beacon/fork_choice.rs | 8 ++++- crates/storage/src/store.rs | 32 ++++++++++++++++--- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index b41be4c27..8680a54d2 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -7775,8 +7775,14 @@ mod tests { // verified payload is timely, which reads the votes. store.set_time_ms(2 * config.slot_duration_ms).unwrap(); let committees = CommitteeCache::default(); - gloas_get_head(&store, &config, &committees) + let walked = gloas_get_head(&store, &config, &committees) .expect("a resumed store must walk the head over a verified payload"); + // The actor's own path: `walk_head` with the payload links re-derived + // (none survive a restart) and the boost and weight helpers. + let head = get_head_node(&store, &config) + .expect("the live head walk must succeed on a resumed store"); + assert_eq!(head.root, walked.root); + assert_eq!(get_head(&mut store, &config).unwrap(), walked.root); let message = gloas::PayloadAttestationMessage { validator_index: 3, diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 3229bf502..6d8c34d7e 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -3326,6 +3326,11 @@ impl Store { envelope: &gloas::SignedExecutionPayloadEnvelope, ) { let root = envelope.message.beacon_block_root; + debug_assert_eq!( + beacon_block_slot(self.backend.as_ref(), &root), + Some(slot), + "the slot must be the envelope's block's own" + ); let entries = vec![(encode_slot_root_key(slot, &root), envelope.to_ssz())]; let mut batch = self.backend.begin_write().expect("write batch"); batch @@ -3361,8 +3366,10 @@ impl Store { /// /// Walks `LiveChain` from the finalized block's slot up: the finalized /// block is included because its payload status decides which branch its - /// children build on. Older rows stay on disk unread, so a restart costs - /// the unfinalized window rather than the chain's history. + /// children build on. Older rows stay on disk unread, but the walk still + /// iterates every `LiveChain` key to find the window (the index is not + /// pruned on a beacon directory), so the scan grows with chain length; + /// only the point reads are limited to the window. /// /// Also seeds both payload-committee vote vectors, empty, for every gloas /// block in the window. Nothing persists the votes, and the head walk and @@ -3401,12 +3408,27 @@ impl Store { .expect("a stored timeliness has one byte per deadline"); timeliness.insert(root, bytes.map(|byte| byte != 0)); } - let is_gloas = view + let fork = view .get(Table::BlockHeaders, &root.to_ssz()) .expect("get") .and_then(|bytes| bytes.first().copied()) - .and_then(ForkName::from_selector) - == Some(ForkName::Gloas); + .and_then(ForkName::from_selector); + let is_gloas = match fork { + Some(ForkName::Gloas) => true, + Some( + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Fulu, + ) => false, + // A beacon directory never holds a lean block row. + Some(ForkName::Lean) => unreachable!("lean block row in a beacon directory"), + // A LiveChain row with no block: nothing to seed. + None => false, + }; if is_gloas { gloas_roots.push(root); } From f1f62da726fc0f053cde9ebd327d33a79f4cc36b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:59:54 -0300 Subject: [PATCH 044/132] feat(beacon): gloas beacon_block gossip rules Every gloas block was ignored as an unsupported fork, so the node could not relay one even once the actor follows gloas. Gloas now runs its own modified `validate_beacon_block_gossip`: the operation and parent-request limits (gloas lists are progressive, so decoding enforces none of them), bid checks in place of execution-payload checks, and the parent payload status read through the existing `is_parent_node_full` and `is_payload_verified`. Earlier forks keep their rules unchanged. `fixed_proposer` answers from a gloas parent's lookahead, which gloas keeps unchanged from fulu. The spec runner now validates gloas `gossip_beacon_block` and delivers a listed block's payload envelope before the case runs. --- .../src/beacon/gossip/block.rs | 214 ++++++++++++++++-- .../state_transition/src/beacon/gossip/mod.rs | 16 ++ .../state_transition/src/beacon/precheck.rs | 35 ++- .../tests/beacon_spec/gossip.rs | 51 ++++- 4 files changed, 277 insertions(+), 39 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/gossip/block.rs b/crates/blockchain/state_transition/src/beacon/gossip/block.rs index 6600e2ae6..c42ab6807 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/block.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/block.rs @@ -1,5 +1,6 @@ //! `beacon_block` gossip validation: fulu's `validate_beacon_block_gossip` -//! (`specs/fulu/p2p-interface.md`), with the deviations the design spec lists. +//! (`specs/fulu/p2p-interface.md`) and gloas's modified one +//! (`specs/gloas/p2p-interface.md`), with the deviations the design spec lists. use ethlambda_storage::CacheKey; @@ -7,37 +8,30 @@ use super::{ IgnoreReason, Outcome, QueueReason, RejectReason, SeenBlocks, finalized_ancestry, finalized_start_slot, is_future_slot, }; -use crate::beacon::containers::SignedBeaconBlock; -use crate::beacon::fork_choice::Store; +use crate::beacon::containers::{BeaconState, SignedBeaconBlock, gloas}; +use crate::beacon::fork_choice::{self, Store}; use crate::beacon::helpers::misc::compute_epoch_at_slot; +use crate::beacon::lean_boundary::{lean_block_unreachable, lean_state_unreachable}; use crate::beacon::precheck::{self, PrecheckError, Reference, precheck_block}; -use crate::beacon::primitives::Root; +use crate::beacon::preset; +use crate::beacon::primitives::{ExecutionBlockHash, Root}; use crate::beacon::stf::bellatrix::compute_timestamp_at_slot; /// The rules that read only the block, the clock and the store's metadata. /// /// `Err` carries the verdict; `Ok` sends the block on to [`stateful_checks`]. +/// A gloas block runs [`cheap_checks_gloas`] instead. pub fn cheap_checks( seen: &SeenBlocks, store: &Store, block: &SignedBeaconBlock, now_ms: u64, ) -> Result<(), Outcome> { + if let Some(gloas_block) = as_gloas(block, "block::cheap_checks") { + return cheap_checks_gloas(seen, store, gloas_block, now_ms); + } let config = store.config(); let slot = block.slot(); - // [IGNORE] This build does not validate a gloas-shaped block against - // these (fulu) rules: EIP-7732 changes what a block commits to enough - // that accepting one here would propagate something this node cannot - // itself follow (its node wiring cannot deliver a gloas block's payload - // envelope yet, though `fork_choice::on_block` accepts the block). Checked - // first, so a gloas block never reaches a rule below whose fields it - // happens to still answer (its `blob_kzg_commitment_count` reads the - // builder's bid) but whose verdict would misdescribe it. `Ignore`, not - // `Reject`: the sender did nothing wrong, and every honest peer sends - // exactly this once this node's own clock reaches gloas. - if !block.fork_name().is_followed() { - return Err(Outcome::Ignore(IgnoreReason::UnsupportedFork)); - } // [IGNORE] The block is not from a future slot. if is_future_slot(&config, slot, now_ms) { return Err(Outcome::Ignore(IgnoreReason::FutureSlot)); @@ -59,12 +53,108 @@ pub fn cheap_checks( Ok(()) } +/// `block` as a gloas block, or `None` for every earlier fork. +/// +/// Spelled out arm by arm so that a new fork breaks this match rather than +/// silently taking fulu's rules. +fn as_gloas<'a>( + block: &'a SignedBeaconBlock, + function: &str, +) -> Option<&'a gloas::SignedBeaconBlock> { + match block { + SignedBeaconBlock::Gloas(block) => Some(block), + SignedBeaconBlock::Phase0(_) + | SignedBeaconBlock::Altair(_) + | SignedBeaconBlock::Bellatrix(_) + | SignedBeaconBlock::Capella(_) + | SignedBeaconBlock::Deneb(_) + | SignedBeaconBlock::Electra(_) + | SignedBeaconBlock::Fulu(_) => None, + SignedBeaconBlock::Lean(_) => lean_block_unreachable(function), + } +} + +/// The gloas rules that read only the block, the clock and the store's +/// metadata: the first half of gloas's modified `validate_beacon_block_gossip` +/// (`specs/gloas/p2p-interface.md`), in the specification's order. +/// +/// The blob-count rule sits later in the specification, after the parent +/// lookups, but reads only the bid; it runs here for the reason fulu's does, +/// so a block with too many commitments is refused before any state is read. +fn cheap_checks_gloas( + seen: &SeenBlocks, + store: &Store, + signed_block: &gloas::SignedBeaconBlock, + now_ms: u64, +) -> Result<(), Outcome> { + let config = store.config(); + let block = &signed_block.message; + // [IGNORE] The block is the first with a valid signature for its slot and + // proposer. + if seen.contains(block.slot, block.proposer_index) { + return Err(Outcome::Ignore(IgnoreReason::AlreadySeen)); + } + // [IGNORE] The block is not from a future slot. + if is_future_slot(&config, block.slot, now_ms) { + return Err(Outcome::Ignore(IgnoreReason::FutureSlot)); + } + // [IGNORE] The block is from a slot greater than the latest finalized slot. + if block.slot <= finalized_start_slot(store) { + return Err(Outcome::Ignore(IgnoreReason::Finalized)); + } + // [REJECT] The body's operation counts, and the parent execution + // requests' counts, are within their limits. + if !body_operations_within_limits(&block.body) + || !execution_requests_within_limits(&block.body.parent_execution_requests) + { + return Err(Outcome::Reject(RejectReason::OperationLimit)); + } + // [REJECT] The bid's blob KZG commitment count is within the per-epoch + // limit. + let bid = &block.body.signed_execution_payload_bid.message; + let max_blobs = config.max_blobs_per_block(compute_epoch_at_slot(block.slot)); + if bid.blob_kzg_commitments.len() as u64 > max_blobs { + return Err(Outcome::Reject(RejectReason::TooManyBlobs)); + } + Ok(()) +} + +/// `verify_block_body_operation_limits` (`specs/gloas/p2p-interface.md`). +/// +/// Gloas's operation lists are progressive, so unlike every earlier fork's +/// they carry no bound of their own and decoding does not enforce it. +fn body_operations_within_limits(body: &gloas::BeaconBlockBody) -> bool { + body.proposer_slashings.len() <= preset::MAX_PROPOSER_SLASHINGS + && body.attester_slashings.len() <= preset::MAX_ATTESTER_SLASHINGS_ELECTRA + && body.attestations.len() <= preset::MAX_ATTESTATIONS_ELECTRA + && body.deposits.is_empty() + && body.voluntary_exits.len() <= preset::MAX_VOLUNTARY_EXITS + && body.bls_to_execution_changes.len() <= preset::MAX_BLS_TO_EXECUTION_CHANGES + && body.payload_attestations.len() as u64 <= preset::MAX_PAYLOAD_ATTESTATIONS +} + +/// `verify_execution_requests_limits` (`specs/gloas/p2p-interface.md`). +/// +/// The specification's text names the withdrawal and consolidation lists; the +/// conformance vectors also reject builder deposit and exit requests over +/// their bounds, and a deposit list over its own, which every list's type +/// bound implies. +fn execution_requests_within_limits(requests: &gloas::ExecutionRequests) -> bool { + requests.deposits.len() <= preset::MAX_DEPOSIT_REQUESTS_PER_PAYLOAD + && requests.withdrawals.len() <= preset::MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD + && requests.consolidations.len() <= preset::MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD + && requests.builder_deposits.len() as u64 + <= preset::MAX_BUILDER_DEPOSIT_REQUESTS_PER_PAYLOAD + && requests.builder_exits.len() as u64 <= preset::MAX_BUILDER_EXIT_REQUESTS_PER_PAYLOAD +} + /// The rules that need the parent's post-state. Runs on a blocking thread. /// /// Reads states only through [`Store::cached_state`]: a miss would otherwise /// rebuild the state from diffs, which is far too slow for a verdict that /// gossipsub waits on. A miss queues the block instead. pub fn stateful_checks(store: &Store, block: &SignedBeaconBlock, block_root: Root) -> Outcome { + let gloas_block = as_gloas(block, "block::stateful_checks"); let config = store.config(); let parent_root = block.parent_root(); let parent_known = store.has_block(&parent_root); @@ -90,12 +180,45 @@ pub fn stateful_checks(store: &Store, block: &SignedBeaconBlock, block_root: Roo if let Err(err) = precheck_block(block, block_root, Reference::Parent(&parent_state), &config) { return Outcome::Reject(err.into()); } + // [IGNORE] (gloas) If the parent block is full, the parent payload is + // valid. The specification checks this between the signature and the + // slot-order rule, both inside `precheck_block` here, so a block that + // fails either is rejected even with an unverified parent payload. + if gloas_block.is_some() { + match fork_choice::is_parent_node_full(store, block) { + Ok(true) if !fork_choice::is_payload_verified(store, parent_root) => { + return Outcome::Ignore(IgnoreReason::ParentPayloadUnverified); + } + Ok(_) => {} + Err(_) => return Outcome::Queue(QueueReason::ParentNotReady), + } + } // [REJECT] The finalized checkpoint is an ancestor of the block. if let Err(outcome) = finalized_ancestry(store, parent_root).verdict() { return outcome; } - // [REJECT] The execution payload's timestamp is the slot's. - if let Some(timestamp) = block.execution_payload_timestamp() + if let Some(gloas_block) = gloas_block { + // [REJECT] The bid's parent equals the block's parent. + let bid = &gloas_block + .message + .body + .signed_execution_payload_bid + .message; + if bid.parent_block_root != gloas_block.message.parent_root { + return Outcome::Reject(RejectReason::BidParentMismatch); + } + // [REJECT] If the parent is not full, the bid builds on the parent's + // execution head. `process_slots` leaves `latest_block_hash` alone, + // so the parent's post-state answers for the advanced one. + match fork_choice::is_parent_node_full(store, block) { + Ok(false) if latest_block_hash(&parent_state) != Some(bid.parent_block_hash) => { + return Outcome::Reject(RejectReason::BidNotOnParentHead); + } + Ok(_) => {} + Err(_) => return Outcome::Queue(QueueReason::ParentNotReady), + } + } else if let Some(timestamp) = block.execution_payload_timestamp() + // [REJECT] The execution payload's timestamp is the slot's. && timestamp != compute_timestamp_at_slot(&parent_state, block.slot(), &config) { return Outcome::Reject(RejectReason::PayloadTimestamp); @@ -110,6 +233,22 @@ pub fn stateful_checks(store: &Store, block: &SignedBeaconBlock, block_root: Roo Outcome::Accept } +/// `state.latest_block_hash`, for a gloas state. `None` for an earlier fork, +/// which has none: its parent counts as full, so no caller asks. +fn latest_block_hash(state: &BeaconState) -> Option { + match state { + BeaconState::Gloas(state) => Some(state.latest_block_hash), + BeaconState::Phase0(_) + | BeaconState::Altair(_) + | BeaconState::Bellatrix(_) + | BeaconState::Capella(_) + | BeaconState::Deneb(_) + | BeaconState::Electra(_) + | BeaconState::Fulu(_) => None, + BeaconState::Lean(_) => lean_state_unreachable("block::latest_block_hash"), + } +} + /// `cheap_checks` then `stateful_checks`, the order the p2p actor runs them in. /// For callers that have no reason to split them, such as the spec vectors. pub fn validate( @@ -228,6 +367,43 @@ mod tests { block } + #[test] + fn a_gloas_body_is_limited_per_operation_and_carries_no_deposits() { + let body = gloas::BeaconBlockBody::default(); + assert!(body_operations_within_limits(&body)); + + let mut with_deposit = body.clone(); + let deposit = crate::beacon::containers::Deposit { + proof: vec![Root::ZERO; 33] + .try_into() + .expect("a deposit proof is thirty-three roots"), + data: Default::default(), + }; + with_deposit.deposits = vec![deposit] + .try_into() + .expect("progressive lists are unbounded"); + assert!(!body_operations_within_limits(&with_deposit)); + + let mut too_many_exits = body; + too_many_exits.voluntary_exits = vec![Default::default(); preset::MAX_VOLUNTARY_EXITS + 1] + .try_into() + .expect("progressive lists are unbounded"); + assert!(!body_operations_within_limits(&too_many_exits)); + } + + #[test] + fn gloas_execution_requests_are_limited_per_list() { + let requests = gloas::ExecutionRequests::default(); + assert!(execution_requests_within_limits(&requests)); + + let mut too_many = requests; + too_many.builder_exits = + vec![Default::default(); preset::MAX_BUILDER_EXIT_REQUESTS_PER_PAYLOAD as usize + 1] + .try_into() + .expect("progressive lists are unbounded"); + assert!(!execution_requests_within_limits(&too_many)); + } + #[test] fn a_block_within_the_clock_disparity_passes() { let store = store(0); diff --git a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs index 17893a082..d7f32bdd6 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs @@ -125,6 +125,10 @@ pub enum IgnoreReason { FinalizedNotAncestor, /// An ancestor lies outside what the state's `block_roots` can answer. AncestryUnknown, + /// A gloas block builds on its parent's full payload branch, but the + /// parent's envelope has not been seen and verified (the specification + /// lets it be queued until it is). + ParentPayloadUnverified, } impl IgnoreReason { @@ -144,6 +148,7 @@ impl IgnoreReason { Self::StateUnavailable => "state_unavailable", Self::FinalizedNotAncestor => "finalized_not_ancestor", Self::AncestryUnknown => "ancestry_unknown", + Self::ParentPayloadUnverified => "parent_payload_unverified", } } } @@ -195,6 +200,14 @@ pub enum RejectReason { AggregateSignature, /// The target is not the voted block's ancestor at the target epoch. TargetNotAncestor, + /// A gloas block body (or its parent execution requests) carries more of + /// an operation than its limit, or any deposit. + OperationLimit, + /// A gloas bid's `parent_block_root` is not the block's `parent_root`. + BidParentMismatch, + /// A gloas block builds on its parent's empty branch, but its bid's + /// `parent_block_hash` is not the parent state's `latest_block_hash`. + BidNotOnParentHead, } impl RejectReason { @@ -228,6 +241,9 @@ impl RejectReason { Self::AggregatorSignature => "aggregator_signature", Self::AggregateSignature => "aggregate_signature", Self::TargetNotAncestor => "target_not_ancestor", + Self::OperationLimit => "operation_limit", + Self::BidParentMismatch => "bid_parent_mismatch", + Self::BidNotOnParentHead => "bid_not_on_parent_head", } } } diff --git a/crates/blockchain/state_transition/src/beacon/precheck.rs b/crates/blockchain/state_transition/src/beacon/precheck.rs index 25d66d032..6731f243a 100644 --- a/crates/blockchain/state_transition/src/beacon/precheck.rs +++ b/crates/blockchain/state_transition/src/beacon/precheck.rs @@ -133,8 +133,8 @@ pub fn precheck_block( /// The proposer `parent_state` has already fixed for `slot`, if it has. /// -/// Fulu's `proposer_lookahead` (EIP-7917) holds the proposers of the state's -/// current epoch and of the `MIN_SEED_LOOKAHEAD` epochs after it. Each entry +/// Fulu's (and gloas's) `proposer_lookahead` (EIP-7917) holds the proposers of +/// the state's current epoch and of the `MIN_SEED_LOOKAHEAD` epochs after it. Each entry /// is fixed when it enters the window: epoch processing only shifts the /// window along and appends a new last epoch. A slot inside the window is /// therefore answered exactly as the state advanced to that slot would answer @@ -142,24 +142,21 @@ pub fn precheck_block( /// state, which is the import's job; the specification lets such a block /// through rather than rejecting it, and so does this. pub(crate) fn fixed_proposer(parent_state: &BeaconState, slot: Slot) -> Option { - let state = match parent_state { - BeaconState::Fulu(state) => state, - // Gloas keeps the lookahead, but its gossip rules are not applied - // here yet (block and column gossip refuse it before this is asked), - // so no window is read from a gloas parent and the caller lets the - // block through as it does before fulu. + // Gloas keeps fulu's lookahead unchanged, so both answer the same way. + let (state_slot, lookahead) = match parent_state { + BeaconState::Fulu(state) => (state.slot, &state.proposer_lookahead), + BeaconState::Gloas(state) => (state.slot, &state.proposer_lookahead), BeaconState::Phase0(_) | BeaconState::Altair(_) | BeaconState::Bellatrix(_) | BeaconState::Capella(_) | BeaconState::Deneb(_) | BeaconState::Electra(_) - | BeaconState::Gloas(_) | BeaconState::Lean(_) => return None, }; - let window_start = compute_start_slot_at_epoch(compute_epoch_at_slot(state.slot)); + let window_start = compute_start_slot_at_epoch(compute_epoch_at_slot(state_slot)); let offset = usize::try_from(slot.checked_sub(window_start)?).ok()?; - state.proposer_lookahead.get(offset).copied() + lookahead.get(offset).copied() } #[cfg(test)] @@ -261,6 +258,22 @@ mod tests { precheck_block(block, block.message_hash_tree_root(), reference, config) } + #[test] + fn a_gloas_parents_lookahead_fixes_the_proposer_like_a_fulu_one() { + let mut parent = with_validators_at(ForkName::Gloas, 8); + let BeaconState::Gloas(inner) = &mut parent else { + unreachable!("with_validators_at(Gloas) builds a gloas state"); + }; + for entry in inner.proposer_lookahead.iter_mut() { + *entry = 6; + } + let slot = parent.slot() + 1; + assert_eq!(fixed_proposer(&parent, slot), Some(6)); + let window_start = compute_start_slot_at_epoch(compute_epoch_at_slot(parent.slot())); + let beyond = window_start + preset::PROPOSER_LOOKAHEAD_LENGTH as Slot; + assert_eq!(fixed_proposer(&parent, beyond), None); + } + #[test] fn a_block_signed_by_its_expected_proposer_passes() { let config = Config::mainnet(); diff --git a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs index d5e612fe1..471690e29 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs @@ -17,7 +17,7 @@ use ethlambda_state_transition::beacon::ForkName; use ethlambda_state_transition::beacon::config::Config; use ethlambda_state_transition::beacon::containers::{ BeaconState, Checkpoint, DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock, - electra, phase0, + electra, gloas, phase0, }; use ethlambda_state_transition::beacon::fork_choice::{ self, DataAvailability, PayloadValidity, Store, @@ -28,6 +28,7 @@ use ethlambda_state_transition::beacon::gossip::{ }; use ethlambda_state_transition::beacon::helpers::accessors::CommitteeCache; use ethlambda_state_transition::beacon::primitives::Root; +use ethlambda_state_transition::beacon::stf::ExecutionEngine; use ethlambda_storage::ForkCheckpoints; use libssz::SszDecode; use libtest_mimic::{Failed, Trial}; @@ -48,16 +49,14 @@ const HANDLERS: &[&str] = &[ /// /// `case.in_scope()` alone would pass every fork up to gloas, since the state /// transition handles them all; the gossip rules do not. The block rules -/// (`beacon::gossip::block`) are fulu's `validate_beacon_block_gossip`, and the -/// aggregate and attestation rules (`beacon::gossip::{aggregate, attestation}`) -/// are electra's, which fulu keeps. The column rules have both fulu's and -/// gloas's (`beacon::gossip::column`). +/// (`beacon::gossip::block`) and the column rules (`beacon::gossip::column`) +/// have both fulu's and gloas's, and the aggregate and attestation rules +/// (`beacon::gossip::{aggregate, attestation}`) are electra's, which fulu +/// keeps. fn validated_forks(handler: &str) -> &'static [ForkName] { match handler { - "gossip_data_column_sidecar" => &[ForkName::Fulu, ForkName::Gloas], - "gossip_beacon_block" - | "gossip_beacon_aggregate_and_proof" - | "gossip_beacon_attestation" => &[ForkName::Fulu], + "gossip_data_column_sidecar" | "gossip_beacon_block" => &[ForkName::Fulu, ForkName::Gloas], + "gossip_beacon_aggregate_and_proof" | "gossip_beacon_attestation" => &[ForkName::Fulu], other => panic!("{other} is not in HANDLERS, so it has no validated forks"), } } @@ -100,6 +99,10 @@ const SKIPPED: &[(&str, &str)] = &[ "gossip_beacon_block__reject_parent_consensus_failed_execution_not_verified", "a parent seen without a post-state is queued, not rejected, until a bad-block cache exists", ), + ( + "gossip_beacon_block__reject_parent_failed_validation", + "a parent seen without a post-state is queued, not rejected, until a bad-block cache exists", + ), ( "gossip_data_column_sidecar__reject_parent_failed_validation", "a parent seen without a post-state is queued, not rejected, until a bad-block cache exists", @@ -139,6 +142,9 @@ struct StoreBlock { #[serde(default)] pending: bool, payload_status: Option, + /// The block's execution payload envelope, delivered once the block is + /// imported so `is_payload_verified` answers true for its root. + payload: Option, } #[derive(serde::Deserialize)] @@ -239,6 +245,31 @@ fn case_config(case: &Case, state: &BeaconState) -> Config { config } +/// Delivers `entry`'s execution payload envelope, if it lists one, so +/// `is_payload_verified` answers true for its block. +fn deliver_payload( + store: &mut Store, + case: &Case, + entry: &StoreBlock, + config: &Config, +) -> Result<(), String> { + let Some(name) = &entry.payload else { + return Ok(()); + }; + let envelope = gloas::SignedExecutionPayloadEnvelope::from_ssz_bytes(&case.ssz_bytes(name)) + .map_err(|err| format!("decoding {name}: {err:?}"))?; + // No sampled columns are named, so the empty retrieval reads as available, + // as in the fork-choice runner's envelope step. + fork_choice::on_execution_payload_envelope( + store, + &envelope, + config, + &[], + &ExecutionEngine::valid(), + ) + .map_err(|err| format!("delivering {name}: {err:?}")) +} + /// The store the case describes: its anchor, then each listed block. fn build_store( case: &Case, @@ -261,6 +292,7 @@ fn build_store( // slot a clock-disparity case sends its sidecar for. let mut clock_s = (config.genesis_time_ms() + meta.current_time_ms) / 1000; fork_choice::on_tick(&mut store, clock_s, config); + deliver_payload(&mut store, case, anchor, config)?; for entry in rest { let block = decode_block(case, &entry.block)?; @@ -295,6 +327,7 @@ fn build_store( &CommitteeCache::default(), ) .map_err(|err| format!("importing {}: {err:?}", entry.block))?; + deliver_payload(&mut store, case, entry, config)?; } if let Some(finalized) = &meta.finalized_checkpoint { From 3b661ca73515c650fd4c0923431b2dcd4d11fbef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:09:36 -0300 Subject: [PATCH 045/132] fix(beacon): match gloas block gossip limits to the spec Gossip rejected a block with more deposit requests than the preset bound, but gloas deposit requests are a progressive list that neither the spec nor the state transition bounds, so import would have accepted what gossip refused. The parent payload status is also read once rather than decoding the parent block twice on the validation pool, and a failed read is now an Ignore(Internal) with a log instead of a mislabelled queue. Docs no longer claim every gloas block is ignored as an unsupported fork. --- .../src/beacon/gossip/block.rs | 72 +++++++++++++------ .../state_transition/src/beacon/precheck.rs | 11 +-- docs/beacon_stf.md | 8 +-- docs/beacon_wire.md | 5 +- docs/metrics.md | 6 +- 5 files changed, 66 insertions(+), 36 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/gossip/block.rs b/crates/blockchain/state_transition/src/beacon/gossip/block.rs index c42ab6807..0fb31c2ab 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/block.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/block.rs @@ -3,6 +3,7 @@ //! (`specs/gloas/p2p-interface.md`), with the deviations the design spec lists. use ethlambda_storage::CacheKey; +use tracing::warn; use super::{ IgnoreReason, Outcome, QueueReason, RejectReason, SeenBlocks, finalized_ancestry, @@ -135,13 +136,13 @@ fn body_operations_within_limits(body: &gloas::BeaconBlockBody) -> bool { /// `verify_execution_requests_limits` (`specs/gloas/p2p-interface.md`). /// -/// The specification's text names the withdrawal and consolidation lists; the -/// conformance vectors also reject builder deposit and exit requests over -/// their bounds, and a deposit list over its own, which every list's type -/// bound implies. +/// The four lists the specification names: withdrawals, consolidations, +/// builder deposits and builder exits. Deposit requests are deliberately not +/// checked: gloas's `DepositRequests` is progressive and neither the +/// specification nor the state transition bounds it, so a limit here would +/// reject a block that import accepts. fn execution_requests_within_limits(requests: &gloas::ExecutionRequests) -> bool { - requests.deposits.len() <= preset::MAX_DEPOSIT_REQUESTS_PER_PAYLOAD - && requests.withdrawals.len() <= preset::MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD + requests.withdrawals.len() <= preset::MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD && requests.consolidations.len() <= preset::MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD && requests.builder_deposits.len() as u64 <= preset::MAX_BUILDER_DEPOSIT_REQUESTS_PER_PAYLOAD @@ -182,21 +183,32 @@ pub fn stateful_checks(store: &Store, block: &SignedBeaconBlock, block_root: Roo } // [IGNORE] (gloas) If the parent block is full, the parent payload is // valid. The specification checks this between the signature and the - // slot-order rule, both inside `precheck_block` here, so a block that - // fails either is rejected even with an unverified parent payload. - if gloas_block.is_some() { + // slot-order rule, but the slot-order and expected-proposer rules are both + // inside `precheck_block` here, so a block that fails either is rejected + // even with an unverified parent payload. The parent's fullness is read + // once: it decodes the parent block, which the bounded validation pool + // should not do twice. + let parent_full = if gloas_block.is_some() { match fork_choice::is_parent_node_full(store, block) { - Ok(true) if !fork_choice::is_payload_verified(store, parent_root) => { - return Outcome::Ignore(IgnoreReason::ParentPayloadUnverified); + Ok(full) => full, + Err(err) => { + warn!(%err, "Could not read a gloas block's parent payload status"); + return Outcome::Ignore(IgnoreReason::Internal); } - Ok(_) => {} - Err(_) => return Outcome::Queue(QueueReason::ParentNotReady), } + } else { + true + }; + if gloas_block.is_some() && parent_full && !fork_choice::is_payload_verified(store, parent_root) + { + return Outcome::Ignore(IgnoreReason::ParentPayloadUnverified); } // [REJECT] The finalized checkpoint is an ancestor of the block. if let Err(outcome) = finalized_ancestry(store, parent_root).verdict() { return outcome; } + // A gloas block has no payload to time; earlier forks: [REJECT] the + // execution payload's timestamp is the slot's. if let Some(gloas_block) = gloas_block { // [REJECT] The bid's parent equals the block's parent. let bid = &gloas_block @@ -210,15 +222,10 @@ pub fn stateful_checks(store: &Store, block: &SignedBeaconBlock, block_root: Roo // [REJECT] If the parent is not full, the bid builds on the parent's // execution head. `process_slots` leaves `latest_block_hash` alone, // so the parent's post-state answers for the advanced one. - match fork_choice::is_parent_node_full(store, block) { - Ok(false) if latest_block_hash(&parent_state) != Some(bid.parent_block_hash) => { - return Outcome::Reject(RejectReason::BidNotOnParentHead); - } - Ok(_) => {} - Err(_) => return Outcome::Queue(QueueReason::ParentNotReady), + if !parent_full && latest_block_hash(&parent_state) != Some(bid.parent_block_hash) { + return Outcome::Reject(RejectReason::BidNotOnParentHead); } } else if let Some(timestamp) = block.execution_payload_timestamp() - // [REJECT] The execution payload's timestamp is the slot's. && timestamp != compute_timestamp_at_slot(&parent_state, block.slot(), &config) { return Outcome::Reject(RejectReason::PayloadTimestamp); @@ -301,7 +308,8 @@ mod tests { use crate::beacon::bls; use crate::beacon::config::Config; use crate::beacon::constants::{ - DOMAIN_BEACON_PROPOSER, MAXIMUM_GOSSIP_CLOCK_DISPARITY as DISPARITY, + DEPOSIT_CONTRACT_TREE_DEPTH, DOMAIN_BEACON_PROPOSER, + MAXIMUM_GOSSIP_CLOCK_DISPARITY as DISPARITY, }; use crate::beacon::containers::{BeaconState, electra}; use crate::beacon::gossip::test_support::{fulu_parent, seen_blocks, slot_start_ms, store}; @@ -374,9 +382,9 @@ mod tests { let mut with_deposit = body.clone(); let deposit = crate::beacon::containers::Deposit { - proof: vec![Root::ZERO; 33] + proof: vec![Root::ZERO; DEPOSIT_CONTRACT_TREE_DEPTH + 1] .try_into() - .expect("a deposit proof is thirty-three roots"), + .expect("a deposit proof has the contract tree's depth plus one roots"), data: Default::default(), }; with_deposit.deposits = vec![deposit] @@ -391,6 +399,24 @@ mod tests { assert!(!body_operations_within_limits(&too_many_exits)); } + #[test] + fn deposit_requests_are_not_limited_by_gossip() { + let request = crate::beacon::containers::electra::DepositRequest { + pubkey: Default::default(), + withdrawal_credentials: Default::default(), + amount: 0, + signature: Default::default(), + index: 0, + }; + let requests = gloas::ExecutionRequests { + deposits: vec![request; preset::MAX_DEPOSIT_REQUESTS_PER_PAYLOAD + 1] + .try_into() + .expect("progressive lists are unbounded"), + ..Default::default() + }; + assert!(execution_requests_within_limits(&requests)); + } + #[test] fn gloas_execution_requests_are_limited_per_list() { let requests = gloas::ExecutionRequests::default(); diff --git a/crates/blockchain/state_transition/src/beacon/precheck.rs b/crates/blockchain/state_transition/src/beacon/precheck.rs index 6731f243a..c34511a17 100644 --- a/crates/blockchain/state_transition/src/beacon/precheck.rs +++ b/crates/blockchain/state_transition/src/beacon/precheck.rs @@ -264,12 +264,15 @@ mod tests { let BeaconState::Gloas(inner) = &mut parent else { unreachable!("with_validators_at(Gloas) builds a gloas state"); }; - for entry in inner.proposer_lookahead.iter_mut() { - *entry = 6; + for (offset, entry) in inner.proposer_lookahead.iter_mut().enumerate() { + *entry = offset as ValidatorIndex; } - let slot = parent.slot() + 1; - assert_eq!(fixed_proposer(&parent, slot), Some(6)); let window_start = compute_start_slot_at_epoch(compute_epoch_at_slot(parent.slot())); + let slot = parent.slot() + 1; + assert_eq!( + fixed_proposer(&parent, slot), + Some((slot - window_start) as ValidatorIndex) + ); let beyond = window_start + preset::PROPOSER_LOOKAHEAD_LENGTH as Slot; assert_eq!(fixed_proposer(&parent, beyond), None); } diff --git a/docs/beacon_stf.md b/docs/beacon_stf.md index 23a006932..2e44d0931 100644 --- a/docs/beacon_stf.md +++ b/docs/beacon_stf.md @@ -319,7 +319,8 @@ section of `fork_choice.rs` (fork choice), and `containers/gloas.rs` in The live follower does not follow gloas yet. `process_or_pend_block` refuses a gloas block and `refuse_unfollowable_fork` refuses a gloas anchor at startup, because nothing delivers payload envelopes or payload attestations to the chain -actor. Gloas gossip is answered with `Ignore(UnsupportedFork)` rather than +actor. Gloas blocks and data columns are validated by their own gossip rules; +the other gloas topics are answered with `Ignore(UnsupportedFork)` rather than scored, so an honest peer past the fork epoch is not penalized. ### The deferred payload @@ -551,9 +552,8 @@ ignored runs. The ignored cases are deliberate exclusions: state transition and fork choice, and is not in this module's scope. - `networking/gossip_*` cases outside what the node validates: every fork's cases for topics it has no validator for (`IGNORED_HANDLERS`), non-fulu cases - of the four it validates, and the fulu vectors in `SKIPPED`, which assume a - bad-block cache. A gloas message is refused with `Ignore(UnsupportedFork)` - before any rule runs. + of the aggregate and attestation topics (block and data column validate fulu + and gloas), and the vectors in `SKIPPED`, which assume a bad-block cache. - The `heze` fixture tree, one ignored entry. See "Accounting for every fork directory" below. diff --git a/docs/beacon_wire.md b/docs/beacon_wire.md index b1e094760..a770211d2 100644 --- a/docs/beacon_wire.md +++ b/docs/beacon_wire.md @@ -164,8 +164,9 @@ reaches it at all, on any outcome: verifying and relaying it is the whole of what this node owes the topic (see above), so there is nothing further for the chain actor to do with one. -The rules above are fulu's. A message from a fork this build has no rules for, -gloas today, is IGNOREd with the reason `unsupported_fork` and is never scored +The rules above are fulu's, except that blocks and data columns also have +gloas rules. An aggregate, attester slashing or attestation from a fork this +build has no rules for, gloas today, is IGNOREd with the reason `unsupported_fork` and is never scored against the peer: the decoders answer `DecodeError::UnsupportedFork`, not `Ssz`, for a gloas aggregate, attester slashing or subnet attestation, or for a block at a gloas slot that fails to decode, since an honest diff --git a/docs/metrics.md b/docs/metrics.md index 83d0627f2..b8447663b 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -386,9 +386,9 @@ other topics: `unsupported_fork` is an ignore reason for a message of a fork thi build has no gossip rules for (gloas today), on the block, data column, aggregate, attestation and attester-slashing topics, so an honest peer past the fork epoch is not scored as a bad decoder. The same label value appears as -`result` on `lean_beacon_gossip_messages_total` for every one of those but the -block topic: a gloas block decodes, so it counts as `decoded` there and -`unsupported_fork` shows only as a verdict reason. +`result` on `lean_beacon_gossip_messages_total` for the topics without gloas +rules. Gloas blocks and data columns are validated, so they carry their own +verdict reasons instead. Two permit pools bound the blocking-thread half of validation: `gossip_validation_permits` for blocks and columns, From 484947f37747d7e99b132a4d84286d20395b3797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:00:44 -0300 Subject: [PATCH 046/132] feat(beacon): receive, store and replay gloas data column sidecars Gloas redefines DataColumnSidecar (no header or inclusion proof; a slot and a block root instead), and the bytes carry no fork tag. Thread the fork-agnostic DataColumnSidecar enum through p2p, the actor API, storage reads and the chain actor so a gloas column takes the same routes a fulu one does. The fork comes from context: the gossip topic's digest, a req/resp chunk's context bytes (then checked against the digest at the sidecar's own slot), and for stored rows the fork of the slot in the key, so the table layout and DB_VERSION are unchanged. A gloas column is verified against its own block's bid, so the chain actor parks it keyed on its block root (a fulu one stays keyed on its parent) and replays it through the p2p checks once that block has a post-state. Tests that stored a fulu sidecar at a slot mainnet still schedules as phase0 now use a store with fulu active from genesis, since rows decode by their slot's fork. --- crates/blockchain/src/lib.rs | 260 +++++++++++++----- .../src/beacon/gossip/column.rs | 56 +++- crates/net/api/src/lib.rs | 13 +- crates/net/p2p/src/beacon/column_checks.rs | 53 +++- crates/net/p2p/src/beacon/decode.rs | 61 ++-- crates/net/p2p/src/beacon/encoding.rs | 41 ++- crates/net/p2p/src/beacon/verdict.rs | 30 +- crates/net/p2p/src/gossipsub/handler.rs | 61 +++- crates/net/p2p/src/lib.rs | 10 +- crates/net/p2p/src/req_resp/codec.rs | 133 +++++++-- crates/net/p2p/src/req_resp/handlers.rs | 50 ++-- crates/net/p2p/src/req_resp/messages.rs | 3 +- crates/storage/src/error.rs | 9 + crates/storage/src/store.rs | 178 +++++++++++- 14 files changed, 762 insertions(+), 196 deletions(-) diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 0c000305a..70513c2a2 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -15,7 +15,7 @@ use ethlambda_types::{ beacon::{ config::Config, constants, - containers::{SignedAggregateAndProof, SignedBeaconBlock, fulu}, + containers::{DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock}, preset, primitives::ValidatorIndex, signing::compute_epoch_at_slot, @@ -25,7 +25,6 @@ use ethlambda_types::{ primitives::{H256, HashTreeRoot as _}, time::unix_now_ms, }; -use libssz::{SszDecode as _, SszEncode as _}; use std::collections::{HashMap, HashSet, VecDeque}; use std::time::{Duration, Instant, SystemTime}; @@ -147,6 +146,18 @@ struct ParkedColumn { index: u64, } +/// The root whose post-state `sidecar` cannot be checked without. +/// +/// A fulu sidecar is verified against its parent's state (its proposer and +/// signature live there). A gloas sidecar carries neither, but reads its +/// commitments from its own block's bid, so it waits for that block instead. +fn awaited_root(sidecar: &DataColumnSidecar) -> H256 { + match sidecar { + DataColumnSidecar::Fulu(sidecar) => sidecar.signed_block_header.message.parent_root, + DataColumnSidecar::Gloas(sidecar) => sidecar.beacon_block_root, + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum SlotInterval { BlockPublication, @@ -508,7 +519,11 @@ pub struct BlockChainServer { /// never have can therefore grow it until finality reclaims the slots; /// see [`Self::queue_sidecar_awaiting_parent`]. Always empty on lean. /// - /// A set per parent, so "the same column is never parked twice" is the + /// Keyed by the root whose post-state the sidecar is judged against: its + /// parent for a fulu sidecar, its own block for a gloas one, whose + /// commitments are in that block's bid ([`awaited_root`]). + /// + /// A set per root, so "the same column is never parked twice" is the /// container's own rule rather than a scan every arrival pays for. Order /// is not one: a replay checks and stores each sidecar on its own, and a /// held block is released by its last column arriving, whichever that is. @@ -826,12 +841,18 @@ fn data_availability_for( let sidecars = custody_columns .iter() .map(|&index| { - let encoded = store - .get_data_column_sidecar(slot, &block_root, index) - .expect("DB read should succeed") - .expect("presence just confirmed above"); - fulu::DataColumnSidecar::from_ssz_bytes(&encoded) + let sidecar = store + .get_data_column(slot, &block_root, index) .expect("a sidecar this node verified before storing decodes") + .expect("presence just confirmed above"); + match sidecar { + DataColumnSidecar::Fulu(sidecar) => sidecar, + // The row decodes by its slot's fork, and this arm is + // the fulu block at that slot. + DataColumnSidecar::Gloas(_) => { + unreachable!("a fulu block's slot decodes its rows as fulu") + } + } }) .collect(); @@ -3261,15 +3282,15 @@ impl BlockChainServer { /// /// Beacon-only: a lean node subscribes to no column subnet, so nothing /// ever delivers this message there. - async fn on_checked_data_columns(&mut self, sidecars: Vec) { + async fn on_checked_data_columns(&mut self, sidecars: Vec) { for sidecar in sidecars { #[cfg(debug_assertions)] { use ethlambda_state_transition::beacon::gossip::{ Outcome, - column::{ChainVerdict, chain_checks}, + column::{ChainVerdict, chain_checks_for}, }; - let verdict = chain_checks(&self.store, &sidecar, unix_now_ms()); + let verdict = chain_checks_for(&self.store, &sidecar, unix_now_ms()); assert!( matches!( verdict, @@ -3283,10 +3304,10 @@ impl BlockChainServer { } /// Store a checked sidecar and release the held block it may complete. - async fn keep_data_column(&mut self, sidecar: fulu::DataColumnSidecar) { - let header = &sidecar.signed_block_header.message; - let slot = header.slot; - let block_root = header.hash_tree_root(); + async fn keep_data_column(&mut self, sidecar: DataColumnSidecar) { + let slot = sidecar.slot(); + let block_root = sidecar.block_root(); + let index = sidecar.index(); // Two copies of one column can pass the checks at once (from gossip // and from a fetch, say). The second write would store the same row @@ -3295,14 +3316,11 @@ impl BlockChainServer { .store .data_column_indices_for(slot, &block_root) .expect("DB read should succeed"); - if stored.contains(&sidecar.index) { + if stored.contains(&index) { return; } - if let Err(err) = - self.store - .put_data_column_sidecar(slot, &block_root, sidecar.index, sidecar.to_ssz()) - { + if let Err(err) = self.store.put_data_column(&sidecar) { error!(%err, "Failed to store a data column sidecar"); return; } @@ -3322,23 +3340,22 @@ impl BlockChainServer { /// run again, so a sidecar parked now would wait for nothing until /// finality evicts it. Asked with the same `get_state` the checks use, so /// a sidecar sent back is one they will find a parent state for. - fn park_data_columns(&mut self, sidecars: Vec) { + fn park_data_columns(&mut self, sidecars: Vec) { let mut ready = Vec::new(); for sidecar in sidecars { - let parent_root = sidecar.signed_block_header.message.parent_root; - if matches!(self.store.get_state(&parent_root), Ok(Some(_))) { + let awaited = awaited_root(&sidecar); + if matches!(self.store.get_state(&awaited), Ok(Some(_))) { ready.push(sidecar); continue; } - let block_root = sidecar.signed_block_header.message.hash_tree_root(); - self.queue_sidecar_awaiting_parent(block_root, sidecar); + self.queue_sidecar_awaiting_parent(awaited, sidecar); } self.send_data_columns_for_checks(ready); } /// Hand sidecars to the p2p layer's chain checks, which send back the /// ones that pass through `new_data_column_sidecars`. - fn send_data_columns_for_checks(&self, sidecars: Vec) { + fn send_data_columns_for_checks(&self, sidecars: Vec) { if sidecars.is_empty() { return; } @@ -3384,17 +3401,11 @@ impl BlockChainServer { /// header still reaches here and parks a row. A peer exploiting either /// gap can still park rows as fast as it can invent a slot, proposer and /// index, until finality catches up. - fn queue_sidecar_awaiting_parent( - &mut self, - block_root: H256, - sidecar: fulu::DataColumnSidecar, - ) { - let header = &sidecar.signed_block_header.message; - let parent_root = header.parent_root; + fn queue_sidecar_awaiting_parent(&mut self, awaited: H256, sidecar: DataColumnSidecar) { let parked = ParkedColumn { - slot: header.slot, - block_root, - index: sidecar.index, + slot: sidecar.slot(), + block_root: sidecar.block_root(), + index: sidecar.index(), }; // A re-delivery of something already parked. The by-root and by-range @@ -3407,7 +3418,7 @@ impl BlockChainServer { // because the write is what costs. if self .sidecars_awaiting_parent - .get(&parent_root) + .get(&awaited) .is_some_and(|parked_columns| parked_columns.contains(&parked)) { return; @@ -3415,12 +3426,7 @@ impl BlockChainServer { // The bytes go to disk before the key goes in the map, so a failed // write leaves no key pointing at a row that is not there. - if let Err(err) = self.store.put_pending_data_column_sidecar( - parked.slot, - &parked.block_root, - parked.index, - sidecar.to_ssz(), - ) { + if let Err(err) = self.store.put_pending_data_column(&sidecar) { error!(%err, "Failed to park a data column sidecar"); return; } @@ -3428,11 +3434,11 @@ impl BlockChainServer { trace!( slot = parked.slot, column = parked.index, - parent_root = %ShortRoot(&parent_root.0), - "Queueing a data column sidecar until its parent has a post-state" + awaited = %ShortRoot(&awaited.0), + "Queueing a data column sidecar until the block it is judged against has a post-state" ); self.sidecars_awaiting_parent - .entry(parent_root) + .entry(awaited) .or_default() .insert(parked); self.publish_sidecars_awaiting_parent(); @@ -3471,12 +3477,12 @@ impl BlockChainServer { // replay that passes is written to `DataColumns`, and one that // fails a check has been judged, so neither leaves anything worth // keeping here. - let encoded = match self.store.take_pending_data_column_sidecar( + let sidecar = match self.store.take_pending_data_column( parked.slot, &parked.block_root, parked.index, ) { - Ok(Some(encoded)) => encoded, + Ok(Some(sidecar)) => sidecar, Ok(None) => { error!( slot = parked.slot, @@ -3491,14 +3497,6 @@ impl BlockChainServer { continue; } }; - let Ok(sidecar) = fulu::DataColumnSidecar::from_ssz_bytes(&encoded) else { - error!( - slot = parked.slot, - column = parked.index, - "A parked data column sidecar did not decode" - ); - continue; - }; sidecars.push(sidecar); } self.send_data_columns_for_checks(sidecars); @@ -4214,11 +4212,14 @@ mod tests { use ethlambda_state_transition::beacon::fork_choice::seconds_to_milliseconds; use ethlambda_storage::backend::InMemoryBackend; use ethlambda_types::beacon::config::Config; - use ethlambda_types::beacon::containers::{BeaconState, deneb, electra, phase0, shared}; + use ethlambda_types::beacon::containers::{ + BeaconState, deneb, electra, fulu, gloas, phase0, shared, + }; use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::preset; use ethlambda_types::checkpoint::Checkpoint; use ethlambda_types::state::State; + use libssz::SszEncode as _; const GENESIS_TIME: u64 = 1_000; @@ -4691,7 +4692,7 @@ mod tests { /// recorded: nothing under test here sends the others. #[derive(Default)] struct RecordingP2P { - checks: std::sync::Mutex>>, + checks: std::sync::Mutex>>, fetches: std::sync::Mutex>, } @@ -4723,7 +4724,7 @@ mod tests { } fn check_data_column_sidecars( &self, - sidecars: Vec, + sidecars: Vec, ) -> Result<(), spawned_concurrency::error::ActorError> { self.checks.lock().unwrap().push(sidecars); Ok(()) @@ -4739,9 +4740,10 @@ mod tests { } /// A beacon store whose clock reads slot 10, so a sidecar at slot 10 is - /// neither future nor finalized. + /// neither future nor finalized. Fulu is active from genesis, since a + /// stored sidecar is decoded as the fork its slot names. fn beacon_store_at_slot_10() -> Store { - let mut store = beacon_store(GENESIS_TIME, 0); + let mut store = beacon_store_fulu_at_genesis(GENESIS_TIME, 0); store .set_time_ms(seconds_to_milliseconds( GENESIS_TIME + 10 * Config::mainnet().seconds_per_slot, @@ -4826,7 +4828,9 @@ mod tests { let sidecar = sidecar_at(10, H256::repeat_byte(9)); let block_root = sidecar.signed_block_header.message.hash_tree_root(); - server.keep_data_column(sidecar).await; + server + .keep_data_column(DataColumnSidecar::Fulu(sidecar)) + .await; assert_eq!( server @@ -4850,7 +4854,9 @@ mod tests { // keep it. let sidecar = sidecar_at(10, H256::repeat_byte(9)); - server.on_checked_data_columns(vec![sidecar]).await; + server + .on_checked_data_columns(vec![DataColumnSidecar::Fulu(sidecar)]) + .await; } #[test] @@ -4862,7 +4868,7 @@ mod tests { let sidecar = sidecar_at(10, parent_root); let block_root = sidecar.signed_block_header.message.hash_tree_root(); - server.park_data_columns(vec![sidecar]); + server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar)]); // Not stored: it has not been checked, so it has not been accepted. assert_eq!( @@ -4900,10 +4906,13 @@ mod tests { .expect("insert"); let sidecar = sidecar_at(10, parent_root); - server.park_data_columns(vec![sidecar.clone()]); + server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar.clone())]); assert!(server.sidecars_awaiting_parent.is_empty()); - assert_eq!(*p2p.checks.lock().unwrap(), vec![vec![sidecar]]); + assert_eq!( + *p2p.checks.lock().unwrap(), + vec![vec![DataColumnSidecar::Fulu(sidecar)]] + ); } #[test] @@ -4918,7 +4927,7 @@ mod tests { let sidecar = sidecar_at(10, parent_root); let block_root = sidecar.signed_block_header.message.hash_tree_root(); - server.park_data_columns(vec![sidecar.clone()]); + server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar.clone())]); assert!(server.sidecars_awaiting_parent.contains_key(&parent_root)); server.drain_sidecars_awaiting_parent(parent_root); @@ -4933,9 +4942,115 @@ mod tests { .expect("DB read should succeed") .is_none() ); + assert_eq!( + *p2p.checks.lock().unwrap(), + vec![vec![DataColumnSidecar::Fulu(sidecar)]] + ); + } + + /// [`beacon_store_at_slot_10`] with gloas active from genesis too, so a + /// slot-10 sidecar decodes as gloas's shape. + fn gloas_store_at_slot_10() -> Store { + let mut store = beacon_store_with_config( + GENESIS_TIME, + 0, + Config::mainnet() + .with_fork_epoch(ForkName::Fulu, 0) + .with_fork_epoch(ForkName::Gloas, 0), + ); + store + .set_time_ms(seconds_to_milliseconds( + GENESIS_TIME + 10 * Config::mainnet().seconds_per_slot, + )) + .unwrap(); + store + } + + fn gloas_sidecar_at(slot: u64, block_root: H256, index: u64) -> DataColumnSidecar { + DataColumnSidecar::Gloas(gloas::DataColumnSidecar { + index, + slot, + beacon_block_root: block_root, + ..Default::default() + }) + } + + #[test] + fn a_gloas_sidecar_for_an_unimported_block_parks_on_its_own_block_and_replays_when_it_imports() + { + // A gloas sidecar is judged against its own block (the commitments are + // in its bid), so it waits on that block's post-state where a fulu one + // waits on its parent's. + let (mut server, p2p) = beacon_server_recording(gloas_store_at_slot_10()); + let block_root = H256::repeat_byte(5); + let sidecar = gloas_sidecar_at(10, block_root, 3); + + server.park_data_columns(vec![sidecar.clone()]); + + assert_eq!( + server.sidecars_awaiting_parent.get(&block_root), + Some(&HashSet::from([ParkedColumn { + slot: 10, + block_root, + index: 3, + }])) + ); + // Parked and not custodied. + assert!(!server.store.has_data_column(10, &block_root, 3)); + assert!(p2p.checks.lock().unwrap().is_empty()); + + // The block imports: what was waiting on it goes back for checks, as + // it was parked. + server.drain_sidecars_awaiting_parent(block_root); + + assert!(server.sidecars_awaiting_parent.is_empty()); + assert!( + server + .store + .take_pending_data_column(10, &block_root, 3) + .expect("DB read should succeed") + .is_none() + ); assert_eq!(*p2p.checks.lock().unwrap(), vec![vec![sidecar]]); } + #[test] + fn a_gloas_sidecar_whose_block_has_a_state_goes_straight_back_for_checks() { + let (mut server, p2p) = beacon_server_recording(gloas_store_at_slot_10()); + let block_root = H256::repeat_byte(5); + server + .store + .insert_state(block_root, bare_state()) + .expect("insert"); + let sidecar = gloas_sidecar_at(10, block_root, 3); + + server.park_data_columns(vec![sidecar.clone()]); + + assert!(server.sidecars_awaiting_parent.is_empty()); + assert_eq!(*p2p.checks.lock().unwrap(), vec![vec![sidecar]]); + } + + #[tokio::test] + async fn a_checked_gloas_sidecar_is_stored_as_it_is() { + let mut server = beacon_server(gloas_store_at_slot_10()); + let block_root = H256::repeat_byte(5); + let sidecar = gloas_sidecar_at(10, block_root, 3); + + server.keep_data_column(sidecar.clone()).await; + + assert_eq!( + server + .store + .data_column_indices_for(10, &block_root) + .expect("DB read should succeed"), + vec![3] + ); + assert_eq!( + server.store.get_data_column(10, &block_root, 3).unwrap(), + Some(sidecar) + ); + } + #[test] fn a_parked_sidecar_holds_its_bytes_on_disk_and_not_in_the_queue() { // The queue's size is chosen by whoever is gossiping, so what it holds @@ -4946,7 +5061,7 @@ mod tests { let sidecar = sidecar_at(10, parent_root); let block_root = sidecar.signed_block_header.message.hash_tree_root(); - server.park_data_columns(vec![sidecar]); + server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar)]); assert_eq!( server.sidecars_awaiting_parent.get(&parent_root), @@ -4976,7 +5091,7 @@ mod tests { let sidecar = sidecar_at(10, H256::repeat_byte(9)); let block_root = sidecar.signed_block_header.message.hash_tree_root(); - server.park_data_columns(vec![sidecar]); + server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar)]); assert_eq!( server @@ -4998,8 +5113,8 @@ mod tests { let mut server = beacon_server(beacon_store_at_slot_10()); let parent_root = H256::repeat_byte(9); - server.park_data_columns(vec![sidecar_at(10, parent_root)]); - server.park_data_columns(vec![sidecar_at(10, parent_root)]); + server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar_at(10, parent_root))]); + server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar_at(10, parent_root))]); assert_eq!( server @@ -5183,7 +5298,8 @@ mod tests { #[test] fn a_block_with_every_custody_column_is_available() { - let store = beacon_store(0, 0); + // Fulu at genesis: stored rows decode by the fork their slot names. + let store = beacon_store_fulu_at_genesis(0, 0); let block = fulu_block_with_commitments(&store, 2); let root = block.message_hash_tree_root(); for index in CUSTODY { diff --git a/crates/blockchain/state_transition/src/beacon/gossip/column.rs b/crates/blockchain/state_transition/src/beacon/gossip/column.rs index b5aa7c010..852e46771 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/column.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/column.rs @@ -19,6 +19,7 @@ use crate::beacon::bls; use crate::beacon::config::Config; use crate::beacon::constants::{DATA_COLUMN_SIDECAR_SUBNET_COUNT, DOMAIN_BEACON_PROPOSER}; use crate::beacon::containers::BeaconState; +use crate::beacon::containers::DataColumnSidecar as AnyDataColumnSidecar; use crate::beacon::containers::SignedBeaconBlock; use crate::beacon::containers::fulu::DataColumnSidecar; use crate::beacon::containers::gloas; @@ -105,8 +106,10 @@ pub fn stateful_checks(store: &Store, sidecar: &DataColumnSidecar) -> Outcome { pub enum ChainVerdict { /// Passed every rule: the chain stores it as it is. Keep, - /// Its parent has no post-state yet. The chain parks it and sends it back - /// through [`chain_checks`] once the parent imports. + /// The block it is judged against has no post-state yet: its parent for a + /// fulu sidecar, its own block for a gloas one (whose commitments are in + /// that block's bid). The chain parks it and sends it back through + /// [`chain_checks_for`] once that block imports. AwaitParent, /// Not kept. The outcome says why: an `Ignore` or a `Reject`, or /// `Queue(ParentNotReady)` when the parent has a post-state but its @@ -170,6 +173,55 @@ pub fn chain_checks(store: &Store, sidecar: &DataColumnSidecar, now_ms: u64) -> } } +/// [`chain_checks`] or [`chain_checks_gloas`], by the sidecar's shape. +pub fn chain_checks_for( + store: &Store, + sidecar: &AnyDataColumnSidecar, + now_ms: u64, +) -> ChainVerdict { + match sidecar { + AnyDataColumnSidecar::Fulu(sidecar) => chain_checks(store, sidecar, now_ms), + AnyDataColumnSidecar::Gloas(sidecar) => chain_checks_gloas(store, sidecar, now_ms), + } +} + +/// [`chain_checks`] for a gloas sidecar: the gloas gossip rules minus the two +/// that only mean something on a gossip topic (the subnet match and the seen +/// cache). +/// +/// What the stateful half cannot yet judge is [`ChainVerdict::AwaitParent`]: +/// a block not seen, or seen without a post-state. Unlike a fulu header, a +/// gloas sidecar carries no signature, so nothing here can refuse a forged one +/// before it is parked; the future-slot and finalized rules bound how long a +/// made-up key can sit there. +pub fn chain_checks_gloas( + store: &Store, + sidecar: &gloas::DataColumnSidecar, + now_ms: u64, +) -> ChainVerdict { + let config = store.config(); + // [IGNORE] Not from a future slot. Also what bounds how far ahead a + // parked row can name. + if is_future_slot(&config, sidecar.slot, now_ms) { + return ChainVerdict::Drop(Outcome::Ignore(IgnoreReason::FutureSlot)); + } + // [IGNORE] From a slot greater than the latest finalized slot. + if sidecar.slot <= finalized_start_slot(store) { + return ChainVerdict::Drop(Outcome::Ignore(IgnoreReason::Finalized)); + } + // [IGNORE] Already stored, as in [`chain_checks`]. + if store.has_data_column(sidecar.slot, &sidecar.beacon_block_root, sidecar.index) { + return ChainVerdict::Drop(Outcome::Ignore(IgnoreReason::AlreadyStored)); + } + match stateful_checks_gloas(store, sidecar) { + Outcome::Accept => ChainVerdict::Keep, + Outcome::Queue(QueueReason::BlockUnknown | QueueReason::BlockNotReady) => { + ChainVerdict::AwaitParent + } + outcome => ChainVerdict::Drop(outcome), + } +} + /// What [`judge_against_parent`] does when the parent state's proposer /// lookahead has no answer for the sidecar's slot. #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/net/api/src/lib.rs b/crates/net/api/src/lib.rs index ae39a8a02..f42786280 100644 --- a/crates/net/api/src/lib.rs +++ b/crates/net/api/src/lib.rs @@ -3,8 +3,7 @@ use std::time::Instant; use ethlambda_types::{ attestation::{SignedAggregatedAttestation, SignedAttestation}, beacon::containers::{ - SignedAggregateAndProof, SignedBeaconBlock, electra::SingleAttestation, - fulu::DataColumnSidecar, + DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock, electra::SingleAttestation, }, beacon::primitives::ValidatorIndex, block::SignedBlock, @@ -27,7 +26,8 @@ pub trait BlockChainToP2P: Send + Sync { /// Ask peers for whatever of one block this node is missing. fn fetch_block(&self, request: FetchRequest) -> Result<(), ActorError>; /// Run the chain checks on sidecars the chain actor had parked, now that - /// their parent has a post-state. + /// the block they are checked against has a post-state: their parent for a + /// fulu sidecar, their own block for a gloas one. /// /// The chain actor keeps a sidecar without checking it, so it hands these /// back rather than judging them itself: the p2p layer runs every column @@ -214,9 +214,10 @@ pub trait P2PToBlockChain: Send + Sync { /// again. fn new_data_column_sidecars(&self, sidecars: Vec) -> Result<(), ActorError>; /// Data column sidecars the chain checks could not judge yet, because - /// their parent has no post-state: the chain actor parks them and sends - /// them back through [`BlockChainToP2P::check_data_column_sidecars`] once - /// the parent imports. + /// the block they are judged against has no post-state (a fulu sidecar's + /// parent, a gloas sidecar's own block): the chain actor parks them and + /// sends them back through [`BlockChainToP2P::check_data_column_sidecars`] + /// once that block imports. fn data_column_sidecars_awaiting_parent( &self, sidecars: Vec, diff --git a/crates/net/p2p/src/beacon/column_checks.rs b/crates/net/p2p/src/beacon/column_checks.rs index 9d92c0985..e161a131e 100644 --- a/crates/net/p2p/src/beacon/column_checks.rs +++ b/crates/net/p2p/src/beacon/column_checks.rs @@ -19,7 +19,7 @@ use ethlambda_state_transition::beacon::gossip::column::{self, ChainVerdict}; use ethlambda_state_transition::beacon::gossip::{IgnoreReason, Outcome}; -use ethlambda_types::beacon::containers::fulu::DataColumnSidecar; +use ethlambda_types::beacon::containers::DataColumnSidecar; use ethlambda_types::time::unix_now_ms; use tracing::{error, warn}; @@ -51,7 +51,7 @@ pub(crate) fn check_and_forward(server: &P2PServer, sidecars: Vec Result Result { - fulu::DataColumnSidecar::from_ssz_bytes(bytes).map_err(|_| DecodeError::Ssz) +/// Fulu and gloas define different containers under this name (gloas drops the +/// header and inclusion proof for a `slot` and a block root; see +/// `containers::gloas::DataColumnSidecar`'s doc), and the bytes alone do not say +/// which one they are. The fork therefore comes from context: a gossip topic's +/// digest, or a req/resp chunk's context bytes. A fork with no data columns +/// fails like a malformed payload. +pub fn decode_data_column_sidecar( + fork: ForkName, + bytes: &[u8], +) -> Result { + DataColumnSidecar::from_ssz(fork, bytes).map_err(|_| DecodeError::Ssz) } /// Decode a `beacon_aggregate_and_proof` payload, at the fork its slot names. @@ -369,6 +367,7 @@ pub fn decode_gossip( #[cfg(test)] mod tests { use super::*; + use ethlambda_types::beacon::containers::fulu; use ethlambda_types::beacon::primitives::{BlsSignature, Bytes32, Root}; use libssz::SszEncode as _; @@ -519,8 +518,28 @@ mod tests { .expect("exactly the required depth"), }; let bytes = sidecar.to_ssz(); - assert_eq!(decode_data_column_sidecar(&bytes).unwrap().index, 3); - assert!(decode_data_column_sidecar(&bytes[..bytes.len() - 1]).is_err()); + assert_eq!( + decode_data_column_sidecar(ForkName::Fulu, &bytes) + .unwrap() + .index(), + 3 + ); + assert!(decode_data_column_sidecar(ForkName::Fulu, &bytes[..bytes.len() - 1]).is_err()); + } + + #[test] + fn a_gloas_sidecar_decodes_by_the_topics_fork_and_not_as_fulu() { + let sidecar = ethlambda_types::beacon::containers::gloas::DataColumnSidecar { + index: 5, + slot: 77, + ..Default::default() + }; + let bytes = sidecar.to_ssz(); + let decoded = decode_data_column_sidecar(ForkName::Gloas, &bytes).unwrap(); + assert_eq!((decoded.index(), decoded.slot()), (5, 77)); + assert_eq!(decoded, DataColumnSidecar::Gloas(sidecar)); + assert!(decode_data_column_sidecar(ForkName::Fulu, &bytes).is_err()); + assert!(decode_data_column_sidecar(ForkName::Electra, &bytes).is_err()); } /// One attester's vote at `slot`, in the shape electra puts on a subnet. diff --git a/crates/net/p2p/src/beacon/encoding.rs b/crates/net/p2p/src/beacon/encoding.rs index d5791b445..933b33df8 100644 --- a/crates/net/p2p/src/beacon/encoding.rs +++ b/crates/net/p2p/src/beacon/encoding.rs @@ -34,8 +34,8 @@ use std::io; use ethlambda_types::beacon::config::Config; +use ethlambda_types::beacon::containers::DataColumnSidecar; use ethlambda_types::beacon::containers::SignedBeaconBlock; -use ethlambda_types::beacon::containers::fulu::DataColumnSidecar; use ethlambda_types::beacon::fork_digest::compute_fork_digest; use ethlambda_types::beacon::preset; use ethlambda_types::beacon::primitives::Root; @@ -44,6 +44,7 @@ use libssz::{SszDecode, SszEncode}; use tracing::warn; use super::decode; +use super::fork_schedule::ForkSchedule; use super::messages::{ BeaconBlocksByRangeRequest, BeaconMetaData, BeaconStatus, MetaDataV1, MetaDataV2, MetaDataV3, StatusV1, StatusV2, @@ -253,16 +254,15 @@ where let encoded = sidecar.to_ssz(); if encoded.len() > MAX_PAYLOAD_SIZE - 1024 { warn!( - index = sidecar.index, + index = sidecar.index(), size = encoded.len(), "Skipping oversized data column sidecar in response" ); continue; } - // The sidecar's own epoch, taken from the header it carries rather - // than the one this node runs on, so a backfill labels each chunk - // with its own fork. - let epoch = sidecar.signed_block_header.message.slot / preset::SLOTS_PER_EPOCH; + // The sidecar's own epoch rather than the one this node runs on, so a + // backfill labels each chunk with its own fork. + let epoch = sidecar.slot() / preset::SLOTS_PER_EPOCH; let digest = compute_fork_digest(config, genesis_validators_root, epoch); write_success_chunk(io, label, &digest, encoded).await?; } @@ -272,12 +272,13 @@ where /// Read a data column sidecar response: one `DataColumnSidecar` per chunk, /// until the peer closes. /// -/// The counterpart of [`decode_blocks_response`]. Only fulu defines this -/// container, so there is no fork ladder to select a decoder from the way a -/// block chunk's slot selects one; [`super::decode::decode_data_column_sidecar`] -/// decodes unconditionally. The context bytes are still checked against the -/// digest the sidecar's own slot implies, for the same reason a block chunk's -/// are: it catches a peer whose `genesis_validators_root` or fork schedule +/// The counterpart of [`decode_blocks_response`]. A gloas sidecar carries no +/// header to read a slot from ahead of decoding, so the fork comes from the +/// chunk's context bytes instead ([`ForkSchedule::fork_for_digest`]), and +/// [`super::decode::decode_data_column_sidecar`] decodes by it. A digest no +/// scheduled fork uses ends the stream like any other mismatch. The context +/// bytes are then checked against the digest the sidecar's own slot implies, +/// for the same reason a block chunk's are: it catches a peer whose `genesis_validators_root` or fork schedule /// differs from ours, which a signature failure would otherwise be the only /// way to notice. /// @@ -300,16 +301,26 @@ where // column protocol; see `protocols::max_request_data_column_sidecars`. max_chunks: protocols::max_request_data_column_sidecars() as usize, }; + let schedule = ForkSchedule::new(config, genesis_validators_root); read_chunked_response(io, protocol_label, limits, |context, payload| { - let sidecar = decode::decode_data_column_sidecar(payload) + let Some(fork) = <[u8; 4]>::try_from(context) + .ok() + .and_then(|digest| schedule.fork_for_digest(digest)) + else { + return Err(invalid(format!( + "data column sidecar chunk context {} is no scheduled fork digest", + hex::encode(context), + ))); + }; + let sidecar = decode::decode_data_column_sidecar(fork, payload) .map_err(|err| invalid(format!("data column sidecar chunk: {err}")))?; - let slot = sidecar.signed_block_header.message.slot; + let slot = sidecar.slot(); let epoch = slot / preset::SLOTS_PER_EPOCH; let expected = compute_fork_digest(config, genesis_validators_root, epoch); if context != expected { warn!( slot, - index = sidecar.index, + index = sidecar.index(), peer_context = %hex::encode(context), our_context = %hex::encode(expected), "Data column sidecar chunk names another fork digest" diff --git a/crates/net/p2p/src/beacon/verdict.rs b/crates/net/p2p/src/beacon/verdict.rs index 6d7c31364..c5d178400 100644 --- a/crates/net/p2p/src/beacon/verdict.rs +++ b/crates/net/p2p/src/beacon/verdict.rs @@ -20,7 +20,7 @@ use ethlambda_state_transition::beacon::helpers::accessors::CommitteeCacheExt as use ethlambda_storage::{CacheKey, Store}; use ethlambda_types::beacon::containers::electra::SingleAttestation; use ethlambda_types::beacon::containers::{ - SignedAggregateAndProof, SignedBeaconBlock, fulu::DataColumnSidecar, + DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock, }; use ethlambda_types::beacon::primitives::{Root, ValidatorIndex}; use libp2p::PeerId; @@ -86,7 +86,12 @@ impl Validated { Self::Block { block, block_root } => { gossip::block::stateful_checks(store, block, *block_root) } - Self::Column(sidecar) => gossip::column::stateful_checks(store, sidecar), + Self::Column(sidecar) => match &**sidecar { + DataColumnSidecar::Fulu(sidecar) => gossip::column::stateful_checks(store, sidecar), + DataColumnSidecar::Gloas(sidecar) => { + gossip::column::stateful_checks_gloas(store, sidecar) + } + }, Self::Aggregate { aggregate, attesting_indices, @@ -113,12 +118,17 @@ impl Validated { .seen_blocks .record(block.slot(), block.proposer_index(), *block_root) } - Self::Column(sidecar) => { - let header = &sidecar.signed_block_header.message; - server - .seen_columns - .record(header.slot, header.proposer_index, sidecar.index) - } + Self::Column(sidecar) => match &**sidecar { + DataColumnSidecar::Fulu(sidecar) => { + let header = &sidecar.signed_block_header.message; + server + .seen_columns + .record(header.slot, header.proposer_index, sidecar.index) + } + DataColumnSidecar::Gloas(sidecar) => server + .seen_block_columns + .record(sidecar.beacon_block_root, sidecar.index), + }, Self::Aggregate { aggregate, .. } => server.seen_aggregates.record(aggregate), Self::Attestation { attestation, .. } => server.seen_attestations.record(attestation), } @@ -637,7 +647,9 @@ mod tests { #[tokio::test] async fn the_first_accept_for_a_column_key_stands_and_the_second_is_marked_seen() { let mut server = unconnected_beacon_server(Config::mainnet(), 0).await; - let object = Validated::Column(Box::new(valid_shaped_sidecar(5, 0))); + let object = Validated::Column(Box::new(DataColumnSidecar::Fulu(valid_shaped_sidecar( + 5, 0, + )))); assert_eq!( settle(&mut server, Outcome::Accept, &object), diff --git a/crates/net/p2p/src/gossipsub/handler.rs b/crates/net/p2p/src/gossipsub/handler.rs index 310088b5f..681d4d349 100644 --- a/crates/net/p2p/src/gossipsub/handler.rs +++ b/crates/net/p2p/src/gossipsub/handler.rs @@ -12,7 +12,7 @@ use ethlambda_state_transition::beacon::gossip::{self, IgnoreReason, Outcome, Re use ethlambda_types::{ ShortRoot, attestation::{SignedAggregatedAttestation, SignedAttestation}, - beacon::containers::{SignedBeaconBlock, electra::SingleAttestation}, + beacon::containers::{DataColumnSidecar, SignedBeaconBlock, electra::SingleAttestation}, beacon::fork::ForkName, block::SignedBlock, primitives::HashTreeRoot as _, @@ -323,7 +323,7 @@ fn triage_data_column( subnet_id: u64, ) -> Dispatch { const KIND: &str = beacon_topics::DATA_COLUMN_SIDECAR_KIND; - let sidecar = match beacon_decode::decode_data_column_sidecar(payload) { + let sidecar = match beacon_decode::decode_data_column_sidecar(fork, payload) { Ok(sidecar) => sidecar, Err(err) => { if !fork.is_followed() { @@ -337,13 +337,23 @@ fn triage_data_column( }; metrics::inc_beacon_gossip(KIND, "decoded"); let now_ms = unix_now_ms(); - if let Err(outcome) = gossip::column::cheap_checks( - &server.seen_columns, - &server.store, - &sidecar, - subnet_id, - now_ms, - ) { + let cheap = match &sidecar { + DataColumnSidecar::Fulu(sidecar) => gossip::column::cheap_checks( + &server.seen_columns, + &server.store, + sidecar, + subnet_id, + now_ms, + ), + DataColumnSidecar::Gloas(sidecar) => gossip::column::cheap_checks_gloas( + &server.seen_block_columns, + &server.store, + sidecar, + subnet_id, + now_ms, + ), + }; + if let Err(outcome) = cheap { return Dispatch::Report(outcome); } Dispatch::Validate(Validated::Column(Box::new(sidecar))) @@ -1006,6 +1016,39 @@ mod tests { )); } + #[tokio::test] + async fn a_gloas_sidecar_decodes_by_its_topics_fork_and_is_judged_by_gloas_rules() { + let now_ms = unix_now_ms(); + let config = Config { + genesis_time: now_ms / 1_000 - 5, + slot_duration_ms: 1_000, + ..Config::mainnet() + }; + let server = unconnected_beacon_server(config, 0).await; + let sidecar = ethlambda_types::beacon::containers::gloas::DataColumnSidecar { + index: 0, + slot: 4, + ..Default::default() + }; + let payload = sidecar.to_ssz(); + + assert!(matches!( + triage_data_column(&server, ForkName::Gloas, &payload, 0), + Dispatch::Validate(Validated::Column(_)) + )); + // Gloas's rule has no header to read a proposer from, but it does have + // a subnet rule. + assert!(matches!( + triage_data_column(&server, ForkName::Gloas, &payload, 1), + Dispatch::Report(Outcome::Reject(RejectReason::WrongSubnet)) + )); + // The same bytes under fulu's topic are not a fulu sidecar. + assert!(matches!( + triage_data_column(&server, ForkName::Fulu, &payload, 0), + Dispatch::Report(Outcome::Reject(RejectReason::Decode)) + )); + } + #[tokio::test] async fn the_same_sidecar_asked_for_on_the_wrong_subnet_is_rejected() { let now_ms = unix_now_ms(); diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index 01a7054df..06a7b2187 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -51,7 +51,8 @@ use ethlambda_network_api::{ use ethlambda_state_transition::beacon::aggregate::MAX_AGGREGATES_PER_SLOT; use ethlambda_state_transition::beacon::attestation_pool::SharedAttestationPool; use ethlambda_state_transition::beacon::gossip::{ - SeenBlocks, SeenColumns, aggregate::SeenAggregates, attestation::SeenAttestations, + SeenBlockColumns, SeenBlocks, SeenColumns, aggregate::SeenAggregates, + attestation::SeenAttestations, }; use ethlambda_storage::{Chain, Store}; use ethlambda_types::beacon::preset::{MAX_VALIDATORS_PER_COMMITTEE, SLOTS_PER_EPOCH}; @@ -1095,6 +1096,7 @@ impl P2P { discovery: discovery.map(|handle| DiscoveryState::new(handle, built.local_peer_id)), seen_blocks: SeenBlocks::new(SEEN_BLOCKS_CAPACITY), seen_columns: SeenColumns::new(SEEN_COLUMNS_CAPACITY), + seen_block_columns: SeenBlockColumns::new(SEEN_COLUMNS_CAPACITY), seen_aggregates: SeenAggregates::new( SEEN_AGGREGATES_CAPACITY, SEEN_AGGREGATES_CAPACITY, @@ -1209,6 +1211,9 @@ pub struct P2PServer { /// The first valid sidecar per `(slot, proposer, index)` accepted from /// gossip. Bounded by capacity, so a fabricated slot cannot grow it. pub(crate) seen_columns: SeenColumns, + /// Gloas's counterpart of [`Self::seen_columns`]: the first valid sidecar + /// per `(block root, index)`, since a gloas sidecar names no proposer. + pub(crate) seen_block_columns: SeenBlockColumns, /// Accepted `beacon_aggregate_and_proof`s, by `(target_epoch, /// aggregator_index)` and by `(hash_tree_root(data), committee_index)`. pub(crate) seen_aggregates: SeenAggregates, @@ -2562,6 +2567,9 @@ pub(crate) mod test_support { seen_columns: ethlambda_state_transition::beacon::gossip::SeenColumns::new( crate::SEEN_COLUMNS_CAPACITY, ), + seen_block_columns: ethlambda_state_transition::beacon::gossip::SeenBlockColumns::new( + crate::SEEN_COLUMNS_CAPACITY, + ), seen_aggregates: ethlambda_state_transition::beacon::gossip::aggregate::SeenAggregates::new( crate::SEEN_AGGREGATES_CAPACITY, diff --git a/crates/net/p2p/src/req_resp/codec.rs b/crates/net/p2p/src/req_resp/codec.rs index 5aea1084e..044f39754 100644 --- a/crates/net/p2p/src/req_resp/codec.rs +++ b/crates/net/p2p/src/req_resp/codec.rs @@ -470,6 +470,8 @@ mod tests { self, ColumnIndices, DataColumnsByRootIdentifier, }; use ethlambda_types::beacon::containers::shared; + use ethlambda_types::beacon::containers::{DataColumnSidecar, gloas}; + use ethlambda_types::beacon::fork_digest::compute_fork_digest; use ethlambda_types::beacon::preset; use ethlambda_types::beacon::primitives::Root; use futures::io::Cursor; @@ -724,34 +726,75 @@ mod tests { } } + /// A config scheduling gloas shortly after mainnet's fulu, so fulu and gloas + /// sidecars get different digests. + fn gloas_config() -> Config { + let base = Config::mainnet(); + Config { + gloas_fork_epoch: base.fulu_fork_epoch + 64, + ..base + } + } + + fn codec_for(config: Config) -> Codec { + Codec::beacon(BeaconContext { + config, + genesis_validators_root: mainnet_gvr(), + }) + } + + fn fulu_slot() -> u64 { + Config::mainnet().fulu_fork_epoch * preset::SLOTS_PER_EPOCH + 1 + } + + fn gloas_slot() -> u64 { + gloas_config().gloas_fork_epoch * preset::SLOTS_PER_EPOCH + 1 + } + + fn gloas_data_column_sidecar(slot: u64, index: u64) -> gloas::DataColumnSidecar { + gloas::DataColumnSidecar { + index, + slot, + ..Default::default() + } + } + /// Exercises `write_data_column_sidecars_response` into /// `decode_data_column_sidecars_response` directly, unlike /// [`a_by_range_column_request_round_trips`] and /// [`a_by_root_column_request_round_trips`] above, which only round-trip /// the thin SSZ-derive request wrappers and never touch this pair. /// - /// The two sidecars straddle mainnet's altair fork boundary on purpose, - /// so their digests actually differ: each chunk's `` is - /// derived from *that sidecar's own* slot - /// (`write_data_column_sidecars_response`'s doc explains why — a - /// backfill answer labels each chunk with its own fork), so an - /// implementation that computed one digest for the whole response - /// (from, say, the first sidecar's epoch) would still round-trip a - /// batch that never crosses a fork boundary but fail this one. + /// The two sidecars straddle the fulu and gloas boundary on purpose, so + /// their digests actually differ and so do their shapes: each chunk's + /// `` is derived from *that sidecar's own* slot + /// (`write_data_column_sidecars_response`'s doc explains why: a backfill + /// answer labels each chunk with its own fork), and the context is what + /// selects the decoder. An implementation that computed one digest for the + /// whole response would fail this batch. #[tokio::test] async fn a_data_column_sidecars_response_round_trips_with_a_per_item_context() { - let sidecar_a = data_column_sidecar(3, 0); - let post_altair_slot = Config::mainnet().altair_fork_epoch * preset::SLOTS_PER_EPOCH + 1; - let sidecar_b = data_column_sidecar(post_altair_slot, 7); + let sidecar_a = DataColumnSidecar::Fulu(data_column_sidecar(fulu_slot(), 0)); + let sidecar_b = DataColumnSidecar::Gloas(gloas_data_column_sidecar(gloas_slot(), 7)); + let stream_protocol = StreamProtocol::new(protocols::DATA_COLUMN_SIDECARS_BY_RANGE_V1); - let decoded = response_round_trip( - protocols::DATA_COLUMN_SIDECARS_BY_RANGE_V1, - Response::success(ResponsePayload::DataColumnSidecars(vec![ - sidecar_a.clone(), - sidecar_b.clone(), - ])), - ) - .await; + let mut buffer = Cursor::new(Vec::new()); + codec_for(gloas_config()) + .write_response( + &stream_protocol, + &mut buffer, + Response::success(ResponsePayload::DataColumnSidecars(vec![ + sidecar_a.clone(), + sidecar_b.clone(), + ])), + ) + .await + .expect("writes"); + let mut buffer = Cursor::new(buffer.into_inner()); + let decoded = codec_for(gloas_config()) + .read_response(&stream_protocol, &mut buffer) + .await + .expect("reads"); match decoded { Response::Success { @@ -763,17 +806,55 @@ mod tests { } } + /// A gloas sidecar under a fulu chunk's context bytes is a shape the + /// context does not name, so the stream aborts rather than handing back a + /// sidecar decoded as the wrong fork. + #[tokio::test] + async fn a_gloas_sidecar_under_a_fulu_context_aborts_the_stream() { + let config = gloas_config(); + let root = mainnet_gvr(); + let fulu_digest = compute_fork_digest(&config, root, config.fulu_fork_epoch); + let payload = gloas_data_column_sidecar(gloas_slot(), 1).to_ssz(); + let stream_protocol = StreamProtocol::new(protocols::DATA_COLUMN_SIDECARS_BY_RANGE_V1); + + let mut buffer = Cursor::new(Vec::new()); + write_success_chunk(&mut buffer, "test", &fulu_digest, payload) + .await + .expect("writes"); + let mut buffer = Cursor::new(buffer.into_inner()); + let result = codec_for(config) + .read_response(&stream_protocol, &mut buffer) + .await; + + assert!(result.is_err()); + } + + /// A digest no scheduled fork uses names no shape at all. + #[tokio::test] + async fn a_data_column_chunk_under_an_unscheduled_digest_aborts_the_stream() { + let payload = data_column_sidecar(fulu_slot(), 1).to_ssz(); + let stream_protocol = StreamProtocol::new(protocols::DATA_COLUMN_SIDECARS_BY_RANGE_V1); + + let mut buffer = Cursor::new(Vec::new()); + write_success_chunk(&mut buffer, "test", &[0xde, 0xad, 0xbe, 0xef], payload) + .await + .expect("writes"); + let mut buffer = Cursor::new(buffer.into_inner()); + let result = codec().read_response(&stream_protocol, &mut buffer).await; + + assert!(result.is_err()); + } + /// A peer's `genesis_validators_root` differing from ours means every /// digest it labels a chunk with is for the wrong chain, even though the - /// chunk decodes cleanly on its own. `decode_data_column_sidecars_response` - /// checks the digest against what *this* node's own root implies, so - /// reading the same bytes back through a codec built with a different - /// root must abort the stream rather than hand back a sidecar under the - /// wrong context — mirroring the block response's own fork-mismatch - /// check, which nothing here exercised before. + /// chunk decodes cleanly on its own. The digest matches no scheduled one + /// under our root, so reading the same bytes back through a codec built + /// with a different root must abort the stream rather than hand back a + /// sidecar under the wrong context, mirroring the block response's own + /// fork-mismatch check. #[tokio::test] async fn a_data_column_sidecars_response_aborts_on_a_fork_digest_mismatch() { - let sidecar = data_column_sidecar(3, 0); + let sidecar = DataColumnSidecar::Fulu(data_column_sidecar(fulu_slot(), 0)); let stream_protocol = StreamProtocol::new(protocols::DATA_COLUMN_SIDECARS_BY_RANGE_V1); let mut buffer = Cursor::new(Vec::new()); diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index 6af186c52..3912077c5 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -23,10 +23,8 @@ use tracing::{debug, error, info, trace, warn}; use ethlambda_state_transition::beacon::das; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::constants; -use ethlambda_types::beacon::containers::SignedBeaconBlock; -use ethlambda_types::beacon::containers::fulu::{ - ColumnIndices, DataColumnSidecar, DataColumnsByRootIdentifier, -}; +use ethlambda_types::beacon::containers::fulu::{ColumnIndices, DataColumnsByRootIdentifier}; +use ethlambda_types::beacon::containers::{DataColumnSidecar, SignedBeaconBlock}; use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::checkpoint::Checkpoint; use ethlambda_types::primitives::HashTreeRoot as _; @@ -38,7 +36,7 @@ use super::{ }; use crate::beacon::BeaconWire; use crate::beacon::column_checks; -use crate::beacon::decode::{decode_data_column_sidecar, fork_at_slot}; +use crate::beacon::decode::fork_at_slot; use crate::beacon::handler::{self as beacon_handler, StatusVersion}; use crate::beacon::messages::{ BeaconMetaData, BeaconStatus, DataColumnsByRangeRequest, Goodbye, Ping, @@ -1321,13 +1319,11 @@ fn range_batch_needs_columns(server: &P2PServer, batch: &std::ops::Range) - /// range sync skips its custody-peer check before PeerDAS for the same reason: /// a batch with no columns to fetch has no custodian to wait for. /// -/// Gloas answers yes as fulu does, though its columns have a different shape. -/// A batch ending in gloas can begin in fulu, whose blocks need their columns. -/// A batch lying entirely in gloas also reaches this, since -/// `beacon_fetched_through` advances past the gloas blocks the chain actor -/// refuses. It is then held for custody for nothing, and its column prefetch -/// fails to decode against fulu's shape; that failure logs at debug and -/// penalizes no peer. No gloas column is ever consumed from here. +/// Gloas answers yes as fulu does, though its columns have a different shape +/// and decode as gloas sidecars by the chunk's context bytes. A batch ending in +/// gloas can begin in fulu, whose blocks need their columns; a batch lying +/// entirely in gloas still needs them too, since the chain actor will gate a +/// block's payload envelope on its columns. fn range_needs_columns( config: &Config, custody_columns: &[u64], @@ -1961,21 +1957,18 @@ async fn handle_data_column_sidecars_by_root_request( continue; }; for &column in identifier.columns.iter() { - let Ok(Some(encoded)) = - server - .store - .get_data_column_sidecar(slot, &identifier.block_root, column) - else { - continue; - }; - match decode_data_column_sidecar(&encoded) { - Ok(sidecar) => sidecars.push(sidecar), + match server + .store + .get_data_column(slot, &identifier.block_root, column) + { + Ok(Some(sidecar)) => sidecars.push(sidecar), + Ok(None) => {} Err(err) => error!( %peer, slot, column, %err, - "Stored data column sidecar failed to decode" + "Stored data column sidecar failed to read" ), } } @@ -2059,7 +2052,7 @@ async fn handle_data_column_sidecars_by_range_request( let columns = request.columns.to_vec(); let sidecars: Vec<_> = server .store - .data_column_sidecars_in_range(request.start_slot, end_slot, &columns) + .data_column_rows_in_range(request.start_slot, end_slot, &columns) .inspect_err(|err| { warn!( start_slot = request.start_slot, @@ -2070,8 +2063,10 @@ async fn handle_data_column_sidecars_by_range_request( }) .unwrap_or_default() .into_iter() - .filter_map(|encoded| { - decode_data_column_sidecar(&encoded) + .filter_map(|(slot, encoded)| { + server + .store + .decode_data_column_sidecar(slot, &encoded) .inspect_err( |err| error!(%peer, %err, "Stored data column sidecar failed to decode"), ) @@ -2348,7 +2343,7 @@ async fn handle_data_column_sidecars_range_response( let in_range: Vec = sidecars .into_iter() .filter(|sidecar| { - let slot = sidecar.signed_block_header.message.slot; + let slot = sidecar.slot(); let keep = slot >= start_slot && slot <= end_slot; if !keep { debug!(%peer, slot, start_slot, end_slot, "Dropping an out-of-range data column sidecar"); @@ -2578,6 +2573,9 @@ mod tests { seen_columns: ethlambda_state_transition::beacon::gossip::SeenColumns::new( crate::SEEN_COLUMNS_CAPACITY, ), + seen_block_columns: ethlambda_state_transition::beacon::gossip::SeenBlockColumns::new( + crate::SEEN_COLUMNS_CAPACITY, + ), seen_aggregates: ethlambda_state_transition::beacon::gossip::aggregate::SeenAggregates::new( crate::SEEN_AGGREGATES_CAPACITY, diff --git a/crates/net/p2p/src/req_resp/messages.rs b/crates/net/p2p/src/req_resp/messages.rs index 54f49bc18..f0a8df344 100644 --- a/crates/net/p2p/src/req_resp/messages.rs +++ b/crates/net/p2p/src/req_resp/messages.rs @@ -1,6 +1,7 @@ use ethlambda_types::ShortRoot; +use ethlambda_types::beacon::containers::DataColumnSidecar; use ethlambda_types::beacon::containers::SignedBeaconBlock; -use ethlambda_types::beacon::containers::fulu::{DataColumnSidecar, DataColumnsByRootIdentifier}; +use ethlambda_types::beacon::containers::fulu::DataColumnsByRootIdentifier; use libssz_types::SszList; use crate::beacon::messages::{ diff --git a/crates/storage/src/error.rs b/crates/storage/src/error.rs index 432c1582f..7e35c2709 100644 --- a/crates/storage/src/error.rs +++ b/crates/storage/src/error.rs @@ -81,4 +81,13 @@ pub enum Error { .checkpoint.slot, .checkpoint.root )] AnchorStateLost { checkpoint: Checkpoint }, + /// A stored data column sidecar did not decode as the fork its slot + /// names. The table keeps no tag beside the bytes, so the fork is read off + /// the row's slot and the config's schedule; a disagreement means the + /// row was written under another schedule or corrupted at rest. + #[error("data column sidecar at slot {slot} does not decode as a {fork:?} sidecar")] + UndecodableDataColumn { + slot: u64, + fork: ethlambda_types::beacon::fork::ForkName, + }, } diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 6d8c34d7e..dfa012fae 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -18,7 +18,10 @@ use ethlambda_types::{ beacon::{ config::Config, constants::NUM_BLOCK_TIMELINESS_DEADLINES, - containers::{BeaconState, Checkpoint as BeaconCheckpoint, SignedBeaconBlock, gloas}, + containers::{ + BeaconState, Checkpoint as BeaconCheckpoint, DataColumnSidecar, SignedBeaconBlock, + gloas, + }, fork::ForkName, fork_choice::{BlockPayloadLink, LatestMessage, PayloadStatusV1, PowBlock}, preset::{PTC_SIZE, Preset, SLOTS_PER_EPOCH}, @@ -3839,6 +3842,66 @@ impl Store { Ok(()) } + /// Decode a stored sidecar row, as the fork its slot's epoch names. + /// + /// The rows carry no fork tag (and gaining one would need a `DB_VERSION` + /// bump), but a fork is a function of the slot and the config, both known + /// here, so the schedule tells a fulu row from a gloas one. + pub fn decode_data_column_sidecar( + &self, + slot: u64, + encoded: &[u8], + ) -> Result { + let fork = self.config.fork_at_epoch(slot / SLOTS_PER_EPOCH); + DataColumnSidecar::from_ssz(fork, encoded) + .map_err(|_| Error::UndecodableDataColumn { slot, fork }) + } + + /// [`Self::put_data_column_sidecar`], for a sidecar in hand. + pub fn put_data_column(&self, sidecar: &DataColumnSidecar) -> Result<(), Error> { + self.put_data_column_sidecar( + sidecar.slot(), + &sidecar.block_root(), + sidecar.index(), + sidecar.to_ssz(), + ) + } + + /// [`Self::put_pending_data_column_sidecar`], for a sidecar in hand. + pub fn put_pending_data_column(&self, sidecar: &DataColumnSidecar) -> Result<(), Error> { + self.put_pending_data_column_sidecar( + sidecar.slot(), + &sidecar.block_root(), + sidecar.index(), + sidecar.to_ssz(), + ) + } + + /// [`Self::take_pending_data_column_sidecar`], decoded. A row that does not + /// decode is still taken, so it cannot be replayed into the same failure. + pub fn take_pending_data_column( + &self, + slot: u64, + block_root: &H256, + column_index: u64, + ) -> Result, Error> { + self.take_pending_data_column_sidecar(slot, block_root, column_index)? + .map(|encoded| self.decode_data_column_sidecar(slot, &encoded)) + .transpose() + } + + /// [`Self::get_data_column_sidecar`], decoded. + pub fn get_data_column( + &self, + slot: u64, + block_root: &H256, + column_index: u64, + ) -> Result, Error> { + self.get_data_column_sidecar(slot, block_root, column_index)? + .map(|encoded| self.decode_data_column_sidecar(slot, &encoded)) + .transpose() + } + /// Drop every parked row this directory holds. /// /// Called once at startup. The only index into `PendingDataColumns` is the @@ -4004,6 +4067,22 @@ impl Store { end_slot: u64, columns: &[u64], ) -> Result>, Error> { + Ok(self + .data_column_rows_in_range(start_slot, end_slot, columns)? + .into_iter() + .map(|(_, encoded)| encoded) + .collect()) + } + + /// [`Self::data_column_sidecars_in_range`] with each row's slot beside its + /// bytes, which is what [`Self::decode_data_column_sidecar`] needs to pick + /// the fork a range spanning one boundary decodes each row as. + pub fn data_column_rows_in_range( + &self, + start_slot: u64, + end_slot: u64, + columns: &[u64], + ) -> Result)>, Error> { let view = self.backend.begin_read().expect("read view"); let mut found = Vec::new(); for slot in start_slot..end_slot { @@ -4025,7 +4104,7 @@ impl Store { .try_into() .expect("a column key ends in an eight-byte index"); if columns.contains(&u64::from_be_bytes(index_bytes)) { - found.push(value.to_vec()); + found.push((slot, value.to_vec())); } } } @@ -7805,6 +7884,101 @@ mod tests { vec![marker; 16] } + /// A beacon store whose config puts gloas at epoch 2, so slots below + /// `2 * SLOTS_PER_EPOCH` are fulu and the rest gloas. + fn gloas_scheduled_store() -> Store { + let config = Config { + fulu_fork_epoch: 0, + gloas_fork_epoch: 2, + ..Config::mainnet() + }; + Store::init_beacon( + Arc::new(InMemoryBackend::new()), + 0, + config, + H256::ZERO, + Checkpoint::default(), + 0, + ) + } + + fn fulu_sidecar_at(slot: u64, index: u64) -> DataColumnSidecar { + let mut sidecar = ethlambda_types::beacon::containers::fulu::DataColumnSidecar { + index, + column: Default::default(), + kzg_commitments: Default::default(), + kzg_proofs: Default::default(), + signed_block_header: Default::default(), + kzg_commitments_inclusion_proof: vec![ + H256::ZERO; + ethlambda_types::beacon::preset::KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH + ] + .try_into() + .expect("exactly the required depth"), + }; + sidecar.signed_block_header.message.slot = slot; + DataColumnSidecar::Fulu(sidecar) + } + + fn gloas_sidecar_at(slot: u64, index: u64, root: H256) -> DataColumnSidecar { + DataColumnSidecar::Gloas( + ethlambda_types::beacon::containers::gloas::DataColumnSidecar { + index, + slot, + beacon_block_root: root, + ..Default::default() + }, + ) + } + + #[test] + fn a_gloas_sidecar_round_trips_through_both_column_tables() { + let store = gloas_scheduled_store(); + let root = H256::repeat_byte(7); + let slot = 2 * SLOTS_PER_EPOCH + 3; + let sidecar = gloas_sidecar_at(slot, 4, root); + + store.put_data_column(&sidecar).unwrap(); + assert!(store.has_data_column(slot, &root, 4)); + assert_eq!( + store.get_data_column(slot, &root, 4).unwrap(), + Some(sidecar.clone()) + ); + + let parked = gloas_sidecar_at(slot, 5, root); + store.put_pending_data_column(&parked).unwrap(); + // Parked rows are not custodied ones. + assert!(!store.has_data_column(slot, &root, 5)); + assert_eq!( + store.take_pending_data_column(slot, &root, 5).unwrap(), + Some(parked) + ); + assert_eq!( + store.take_pending_data_column(slot, &root, 5).unwrap(), + None + ); + } + + #[test] + fn a_row_decodes_as_the_fork_its_slot_names() { + let store = gloas_scheduled_store(); + let fulu = fulu_sidecar_at(SLOTS_PER_EPOCH, 1); + let root = fulu.block_root(); + store.put_data_column(&fulu).unwrap(); + assert_eq!( + store.get_data_column(SLOTS_PER_EPOCH, &root, 1).unwrap(), + Some(fulu) + ); + + // The same bytes filed under a gloas slot are not a gloas sidecar. + let gloas_slot = 2 * SLOTS_PER_EPOCH; + let encoded = fulu_sidecar_at(gloas_slot, 1).to_ssz(); + assert!(matches!( + store.decode_data_column_sidecar(gloas_slot, &encoded), + Err(Error::UndecodableDataColumn { .. }) + )); + } + #[test] fn a_sidecar_round_trips_by_slot_root_and_index() { let store = beacon_test_store(Arc::new(InMemoryBackend::new())); From 07e4b607f96d68c0f86ef34aa99ab26e4c36ba44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:12:54 -0300 Subject: [PATCH 047/132] refactor(beacon): move column parking into its own module The parking lot for data column sidecars the p2p checks could not judge yet is about to grow bounds of its own, so give it a module beside beacon_aggregates instead of growing lib.rs. Pure move: no behavior change. --- crates/blockchain/src/beacon_columns.rs | 525 ++++++++++++++++++++++++ crates/blockchain/src/lib.rs | 517 +---------------------- 2 files changed, 538 insertions(+), 504 deletions(-) create mode 100644 crates/blockchain/src/beacon_columns.rs diff --git a/crates/blockchain/src/beacon_columns.rs b/crates/blockchain/src/beacon_columns.rs new file mode 100644 index 000000000..75bd728e9 --- /dev/null +++ b/crates/blockchain/src/beacon_columns.rs @@ -0,0 +1,525 @@ +//! The chain actor's parking lot for data column sidecars the p2p layer's +//! checks could not judge yet. +//! +//! A sidecar is judged against a block's post-state: its parent's for a fulu +//! sidecar, its own block's for a gloas one. Until that state exists the +//! sidecar's bytes wait in `Table::PendingDataColumns`, and only their keys are +//! held in `BlockChainServer::sidecars_awaiting_parent`. + +use std::collections::HashSet; + +use ethlambda_types::ShortRoot; +use ethlambda_types::beacon::containers::DataColumnSidecar; +use ethlambda_types::primitives::H256; +use tracing::{debug, error, info, trace}; + +use crate::{BlockChainServer, metrics}; + +/// Where a parked data column sidecar was put. +/// +/// The three fields are exactly `Table::PendingDataColumns`'s key, so reading +/// the sidecar back needs nothing else. `slot` is also what the finality sweep +/// compares against. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub(crate) struct ParkedColumn { + pub(crate) slot: u64, + pub(crate) block_root: H256, + pub(crate) index: u64, +} + +/// The root whose post-state `sidecar` cannot be checked without. +/// +/// A fulu sidecar is verified against its parent's state (its proposer and +/// signature live there). A gloas sidecar carries neither, but reads its +/// commitments from its own block's bid, so it waits for that block instead. +pub(crate) fn awaited_root(sidecar: &DataColumnSidecar) -> H256 { + match sidecar { + DataColumnSidecar::Fulu(sidecar) => sidecar.signed_block_header.message.parent_root, + DataColumnSidecar::Gloas(sidecar) => sidecar.beacon_block_root, + } +} + +impl BlockChainServer { + /// Park sidecars the chain checks found no parent post-state for, or send + /// them straight back to be checked if the parent has one by now. + /// + /// The second case is a race this actor has to close, because the checks + /// run elsewhere: the p2p layer looked for the parent's post-state, found + /// none and sent these, and if the parent imported in between, its + /// [`Self::drain_sidecars_awaiting_parent`] has already run and will not + /// run again, so a sidecar parked now would wait for nothing until + /// finality evicts it. Asked with the same `get_state` the checks use, so + /// a sidecar sent back is one they will find a parent state for. + pub(crate) fn park_data_columns(&mut self, sidecars: Vec) { + let mut ready = Vec::new(); + for sidecar in sidecars { + let awaited = awaited_root(&sidecar); + if matches!(self.store.get_state(&awaited), Ok(Some(_))) { + ready.push(sidecar); + continue; + } + self.queue_sidecar_awaiting_parent(awaited, sidecar); + } + self.send_data_columns_for_checks(ready); + } + + /// Hand sidecars to the p2p layer's chain checks, which send back the + /// ones that pass through `new_data_column_sidecars`. + pub(crate) fn send_data_columns_for_checks(&self, sidecars: Vec) { + if sidecars.is_empty() { + return; + } + let Some(ref p2p) = self.p2p else { + return; + }; + let _ = p2p.check_data_column_sidecars(sidecars).inspect_err( + |err| error!(%err, "Failed to send data column sidecars to the p2p layer for checks"), + ); + } + + /// Park `sidecar` against the parent root it could not be checked against. + /// + /// Counted as `queued_for_parent` rather than as a rejection: nothing about + /// the sidecar has been judged yet, and conflating the two is what made the + /// deadlock invisible in the metrics (every column read as + /// `rejected{reason="unknown_parent"}` while the real fault was upstream). + /// + /// Nothing is refused here. The queue used to hold whole sidecars and so + /// carried a count cap, which a follower behind the tip hit constantly: + /// it receives gossip for the tip continuously, so the queue filled with + /// sidecars for blocks it would not reach for minutes and then refused + /// the ones for the block it was about to import. Measured on the eth-4 + /// follower a hundred slots behind, the queue sat pinned at its cap and + /// dropped 2,561 sidecars in ten minutes while the chain ground through + /// by-root lookups for slots whose columns gossip had already delivered + /// and this function had thrown away. Evicting the furthest-ahead entry + /// instead of the newest fixed which sidecar was lost, not that one was. + /// + /// The cap is gone now that the sidecars live in + /// `Table::PendingDataColumns` and only their keys are held here, so what + /// grows is disk rather than this actor's memory. + /// [`Self::evict_sidecars_awaiting_parent_at_or_below_finality`] is what + /// bounds it, which bounds how *long* an entry lives but not how fast + /// they arrive: the chain checks do not require `parent_root` to name a + /// block this node knows. Every sidecar reaching here, gossiped or + /// fetched, has had its header's signature checked against the head state + /// by those checks (`queue_unless_forged`, in + /// `ethlambda_state_transition::beacon::gossip::column`), but only when a + /// head state is already cached *and* the header's `proposer_index` names + /// a validator in it: with no cached head state, or a proposer index that + /// names none (`u64::MAX`, say), that check is skipped and a made-up + /// header still reaches here and parks a row. A peer exploiting either + /// gap can still park rows as fast as it can invent a slot, proposer and + /// index, until finality catches up. + pub(crate) fn queue_sidecar_awaiting_parent( + &mut self, + awaited: H256, + sidecar: DataColumnSidecar, + ) { + let parked = ParkedColumn { + slot: sidecar.slot(), + block_root: sidecar.block_root(), + index: sidecar.index(), + }; + + // A re-delivery of something already parked. The by-root and by-range + // fetch paths skip gossip's seen cache entirely, so they never touch + // the p2p actor's `SeenColumns` (which in any case only records an + // Accept, never a park); a re-delivery reaching here is ordinary, and + // without this check the same column would take a second slot in the + // queue and leave a stale key behind after the first replay took its + // row. Asked before the write rather than left to the set below, + // because the write is what costs. + if self + .sidecars_awaiting_parent + .get(&awaited) + .is_some_and(|parked_columns| parked_columns.contains(&parked)) + { + return; + } + + // The bytes go to disk before the key goes in the map, so a failed + // write leaves no key pointing at a row that is not there. + if let Err(err) = self.store.put_pending_data_column(&sidecar) { + error!(%err, "Failed to park a data column sidecar"); + return; + } + + trace!( + slot = parked.slot, + column = parked.index, + awaited = %ShortRoot(&awaited.0), + "Queueing a data column sidecar until the block it is judged against has a post-state" + ); + self.sidecars_awaiting_parent + .entry(awaited) + .or_default() + .insert(parked); + self.publish_sidecars_awaiting_parent(); + } + + /// Republish how many sidecars are parked, from the map that decides it. + pub(crate) fn publish_sidecars_awaiting_parent(&self) { + let total: usize = self + .sidecars_awaiting_parent + .values() + .map(HashSet::len) + .sum(); + metrics::set_sidecars_awaiting_parent(total as u64); + } + + /// Send every sidecar parked against `block_root` back to the p2p layer's + /// chain checks, now that it has a post-state to be checked against. + /// + /// Called from the one arm that means "this root now has a post-state". + /// The ones that pass come back through `new_data_column_sidecars` as a + /// new message, so nothing here re-enters the import path. + pub(crate) fn drain_sidecars_awaiting_parent(&mut self, block_root: H256) { + let Some(parked_columns) = self.sidecars_awaiting_parent.remove(&block_root) else { + return; + }; + debug!( + parent_root = %ShortRoot(&block_root.0), + count = parked_columns.len(), + "Replaying data column sidecars whose parent just imported" + ); + self.publish_sidecars_awaiting_parent(); + + let mut sidecars = Vec::with_capacity(parked_columns.len()); + for parked in parked_columns { + // Taken, not read: the row has served its purpose either way. A + // replay that passes is written to `DataColumns`, and one that + // fails a check has been judged, so neither leaves anything worth + // keeping here. + let sidecar = match self.store.take_pending_data_column( + parked.slot, + &parked.block_root, + parked.index, + ) { + Ok(Some(sidecar)) => sidecar, + Ok(None) => { + error!( + slot = parked.slot, + column = parked.index, + block_root = %ShortRoot(&parked.block_root.0), + "A parked data column sidecar has no row to replay from" + ); + continue; + } + Err(err) => { + error!(%err, "Failed to read back a parked data column sidecar"); + continue; + } + }; + sidecars.push(sidecar); + } + self.send_data_columns_for_checks(sidecars); + } + + /// Drop parked sidecars whose block finality has superseded. + /// + /// The counterpart of [`Self::evict_held_blocks_at_or_below_finality`] and + /// run beside it, for the same reason: a parent root that never arrives + /// would otherwise pin its children's sidecars for this node's whole + /// uptime. A sidecar at or below the finalized slot can never be needed + /// again, since the chain checks would drop it outright now. + pub(crate) fn evict_sidecars_awaiting_parent_at_or_below_finality(&mut self) { + if self.sidecars_awaiting_parent.is_empty() { + return; + } + let finalized_slot = self + .store + .latest_finalized() + .expect("finalized checkpoint exists") + .slot; + + let mut dropped: Vec = Vec::new(); + self.sidecars_awaiting_parent.retain(|_, parked_columns| { + parked_columns.retain(|parked| { + let keep = parked.slot > finalized_slot; + if !keep { + dropped.push(*parked); + } + keep + }); + !parked_columns.is_empty() + }); + + if !dropped.is_empty() { + info!( + finalized_slot, + count = dropped.len(), + "Evicting parked data column sidecars that finality has superseded" + ); + // The rows go with the keys, so a key dropped from the map never + // leaves its bytes on disk with nothing left to read them. + let keys = dropped + .iter() + .map(|parked| (parked.slot, parked.block_root, parked.index)); + let _ = self + .store + .delete_pending_data_column_sidecars(keys) + .inspect_err(|err| error!(%err, "Failed to drop parked data column sidecars")); + self.publish_sidecars_awaiting_parent(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{ + GENESIS_TIME, bare_state, beacon_server, beacon_server_recording, beacon_store, + beacon_store_at_slot_10, gloas_sidecar_at, gloas_store_at_slot_10, sidecar_at, + }; + use ethlambda_types::primitives::HashTreeRoot as _; + + #[test] + fn a_data_column_sidecar_naming_a_parent_with_no_state_is_parked_not_stored() { + // `beacon_store` writes no anchor state (see its own doc comment), so + // any parent root at all is unknown here, including the anchor's own. + let (mut server, p2p) = beacon_server_recording(beacon_store_at_slot_10()); + let parent_root = H256::repeat_byte(9); + let sidecar = sidecar_at(10, parent_root); + let block_root = sidecar.signed_block_header.message.hash_tree_root(); + + server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar)]); + + // Not stored: it has not been checked, so it has not been accepted. + assert_eq!( + server + .store + .data_column_indices_for(10, &block_root) + .unwrap(), + Vec::::new() + ); + // But kept, under the parent it is waiting on. Dropping it here is + // what deadlocks a follower running the availability gate: a held + // block writes no post-state, so every sidecar of every child of it + // lands in exactly this branch. + assert_eq!( + server + .sidecars_awaiting_parent + .get(&parent_root) + .map(HashSet::len), + Some(1) + ); + assert!(p2p.checks.lock().unwrap().is_empty()); + } + + #[test] + fn a_sidecar_whose_parent_imported_meanwhile_goes_back_for_checks_not_into_the_queue() { + // The race the p2p layer's checks open: they found no parent state, + // then the parent imported and drained its (still empty) queue before + // this message arrived. Parking it now would strand it until + // finality, since that parent never drains again. + let (mut server, p2p) = beacon_server_recording(beacon_store_at_slot_10()); + let parent_root = H256::repeat_byte(9); + server + .store + .insert_state(parent_root, bare_state()) + .expect("insert"); + let sidecar = sidecar_at(10, parent_root); + + server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar.clone())]); + + assert!(server.sidecars_awaiting_parent.is_empty()); + assert_eq!( + *p2p.checks.lock().unwrap(), + vec![vec![DataColumnSidecar::Fulu(sidecar)]] + ); + } + + #[test] + fn a_parked_sidecar_goes_back_for_checks_once_its_parent_gains_a_post_state() { + // The deadlock this closes, in miniature: while the parent has no + // post-state every sidecar under it parks, and if parking were the end + // of the story the queue would only ever grow. What breaks the cycle + // is that gaining a post-state releases them, to the p2p layer's + // checks, since this actor no longer judges a sidecar itself. + let (mut server, p2p) = beacon_server_recording(beacon_store_at_slot_10()); + let parent_root = H256::repeat_byte(9); + let sidecar = sidecar_at(10, parent_root); + let block_root = sidecar.signed_block_header.message.hash_tree_root(); + + server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar.clone())]); + assert!(server.sidecars_awaiting_parent.contains_key(&parent_root)); + + server.drain_sidecars_awaiting_parent(parent_root); + + // Gone from the queue and from `PendingDataColumns`, and handed to + // the checks exactly as it was parked. + assert!(!server.sidecars_awaiting_parent.contains_key(&parent_root)); + assert!( + server + .store + .take_pending_data_column_sidecar(10, &block_root, 0) + .expect("DB read should succeed") + .is_none() + ); + assert_eq!( + *p2p.checks.lock().unwrap(), + vec![vec![DataColumnSidecar::Fulu(sidecar)]] + ); + } + + #[test] + fn a_gloas_sidecar_for_an_unimported_block_parks_on_its_own_block_and_replays_when_it_imports() + { + // A gloas sidecar is judged against its own block (the commitments are + // in its bid), so it waits on that block's post-state where a fulu one + // waits on its parent's. + let (mut server, p2p) = beacon_server_recording(gloas_store_at_slot_10()); + let block_root = H256::repeat_byte(5); + let sidecar = gloas_sidecar_at(10, block_root, 3); + + server.park_data_columns(vec![sidecar.clone()]); + + assert_eq!( + server.sidecars_awaiting_parent.get(&block_root), + Some(&HashSet::from([ParkedColumn { + slot: 10, + block_root, + index: 3, + }])) + ); + // Parked and not custodied. + assert!(!server.store.has_data_column(10, &block_root, 3)); + assert!(p2p.checks.lock().unwrap().is_empty()); + + // The block imports: what was waiting on it goes back for checks, as + // it was parked. + server.drain_sidecars_awaiting_parent(block_root); + + assert!(server.sidecars_awaiting_parent.is_empty()); + assert!( + server + .store + .take_pending_data_column(10, &block_root, 3) + .expect("DB read should succeed") + .is_none() + ); + assert_eq!(*p2p.checks.lock().unwrap(), vec![vec![sidecar]]); + } + + #[test] + fn a_gloas_sidecar_whose_block_has_a_state_goes_straight_back_for_checks() { + let (mut server, p2p) = beacon_server_recording(gloas_store_at_slot_10()); + let block_root = H256::repeat_byte(5); + server + .store + .insert_state(block_root, bare_state()) + .expect("insert"); + let sidecar = gloas_sidecar_at(10, block_root, 3); + + server.park_data_columns(vec![sidecar.clone()]); + + assert!(server.sidecars_awaiting_parent.is_empty()); + assert_eq!(*p2p.checks.lock().unwrap(), vec![vec![sidecar]]); + } + + #[test] + fn a_parked_sidecar_holds_its_bytes_on_disk_and_not_in_the_queue() { + // The queue's size is chosen by whoever is gossiping, so what it holds + // per entry is the thing that has to stay small: a key, not a cell per + // blob. + let mut server = beacon_server(beacon_store_at_slot_10()); + let parent_root = H256::repeat_byte(9); + let sidecar = sidecar_at(10, parent_root); + let block_root = sidecar.signed_block_header.message.hash_tree_root(); + + server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar)]); + + assert_eq!( + server.sidecars_awaiting_parent.get(&parent_root), + Some(&HashSet::from([ParkedColumn { + slot: 10, + block_root, + index: 0, + }])) + ); + assert!( + server + .store + .take_pending_data_column_sidecar(10, &block_root, 0) + .expect("DB read should succeed") + .is_some(), + "the sidecar's bytes belong in PendingDataColumns" + ); + } + + #[test] + fn a_parked_sidecar_does_not_satisfy_the_availability_gate() { + // Why the parked rows get a table of their own. Nothing has judged a + // parked sidecar's inclusion proof, its KZG batch or its proposer + // signature, so a peer that could get one counted as custodied would + // be able to release a held block with a column it invented. + let mut server = beacon_server(beacon_store_at_slot_10()); + let sidecar = sidecar_at(10, H256::repeat_byte(9)); + let block_root = sidecar.signed_block_header.message.hash_tree_root(); + + server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar)]); + + assert_eq!( + server + .store + .data_column_indices_for(10, &block_root) + .expect("DB read should succeed"), + Vec::::new(), + "an unverified sidecar must be invisible to data_column_indices_for" + ); + } + + #[test] + fn a_sidecar_parked_twice_takes_one_slot_in_the_queue() { + // The by-root and by-range fetch paths skip gossip's seen cache + // entirely, so nothing between them and this actor dedups a + // re-delivery while the parent is still stateless; it is ordinary. A + // second entry would leave a key with no row behind it once the + // first replay took it. + let mut server = beacon_server(beacon_store_at_slot_10()); + let parent_root = H256::repeat_byte(9); + + server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar_at(10, parent_root))]); + server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar_at(10, parent_root))]); + + assert_eq!( + server + .sidecars_awaiting_parent + .get(&parent_root) + .map(HashSet::len), + Some(1) + ); + } + + #[test] + fn parked_sidecars_are_dropped_once_finality_passes_their_slot() { + // Populated directly rather than through `park_data_columns`: the + // chain checks refuse a sidecar at or below the finalized slot before + // it could ever be parked, so the only way to observe the sweep is to + // park one behind their back. The finalized slot is fixed at init, so + // the store carries it rather than the test moving it. + let mut server = beacon_server(beacon_store(GENESIS_TIME, 10)); + let superseded = H256::repeat_byte(1); + let still_wanted = H256::repeat_byte(2); + let parked_at = |slot: u64| ParkedColumn { + slot, + block_root: H256::repeat_byte(9), + index: 0, + }; + server + .sidecars_awaiting_parent + .insert(superseded, HashSet::from([parked_at(10)])); + server + .sidecars_awaiting_parent + .insert(still_wanted, HashSet::from([parked_at(20)])); + + server.evict_sidecars_awaiting_parent_at_or_below_finality(); + + // A parent root that never arrives would otherwise pin its children's + // sidecars for this node's whole uptime; one still above finality is + // a parent that may yet show up. + assert!(!server.sidecars_awaiting_parent.contains_key(&superseded)); + assert!(server.sidecars_awaiting_parent.contains_key(&still_wanted)); + } +} diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 70513c2a2..4184fa5ec 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -44,6 +44,7 @@ use spawned_concurrency::tasks::{ use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, trace, warn}; +use crate::beacon_columns::ParkedColumn; use crate::block_builder::ProposerConfig; use crate::events::ChainEventSnapshot; use crate::import_timing::{BlockImportReport, CascadeTimings, HeadTimings, ImportTimings}; @@ -53,6 +54,7 @@ pub use events::{ChainEvent, EventBus, Topic, UnknownTopic}; pub mod aggregation; mod beacon_aggregates; +mod beacon_columns; pub mod beacon_engine; pub mod block_builder; pub(crate) mod coverage; @@ -134,30 +136,6 @@ pub const GOSSIP_DISPARITY_INTERVALS: u64 = 1; pub const MAXIMUM_GOSSIP_CLOCK_DISPARITY: Duration = Duration::from_millis(ethlambda_types::beacon::constants::MAXIMUM_GOSSIP_CLOCK_DISPARITY); -/// Where a parked data column sidecar was put. -/// -/// The three fields are exactly `Table::PendingDataColumns`'s key, so reading -/// the sidecar back needs nothing else. `slot` is also what the finality sweep -/// compares against. -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -struct ParkedColumn { - slot: u64, - block_root: H256, - index: u64, -} - -/// The root whose post-state `sidecar` cannot be checked without. -/// -/// A fulu sidecar is verified against its parent's state (its proposer and -/// signature live there). A gloas sidecar carries neither, but reads its -/// commitments from its own block's bid, so it waits for that block instead. -fn awaited_root(sidecar: &DataColumnSidecar) -> H256 { - match sidecar { - DataColumnSidecar::Fulu(sidecar) => sidecar.signed_block_header.message.parent_root, - DataColumnSidecar::Gloas(sidecar) => sidecar.beacon_block_root, - } -} - #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum SlotInterval { BlockPublication, @@ -3330,226 +3308,6 @@ impl BlockChainServer { self.release_block_if_columns_complete(block_root).await; } - /// Park sidecars the chain checks found no parent post-state for, or send - /// them straight back to be checked if the parent has one by now. - /// - /// The second case is a race this actor has to close, because the checks - /// run elsewhere: the p2p layer looked for the parent's post-state, found - /// none and sent these, and if the parent imported in between, its - /// [`Self::drain_sidecars_awaiting_parent`] has already run and will not - /// run again, so a sidecar parked now would wait for nothing until - /// finality evicts it. Asked with the same `get_state` the checks use, so - /// a sidecar sent back is one they will find a parent state for. - fn park_data_columns(&mut self, sidecars: Vec) { - let mut ready = Vec::new(); - for sidecar in sidecars { - let awaited = awaited_root(&sidecar); - if matches!(self.store.get_state(&awaited), Ok(Some(_))) { - ready.push(sidecar); - continue; - } - self.queue_sidecar_awaiting_parent(awaited, sidecar); - } - self.send_data_columns_for_checks(ready); - } - - /// Hand sidecars to the p2p layer's chain checks, which send back the - /// ones that pass through `new_data_column_sidecars`. - fn send_data_columns_for_checks(&self, sidecars: Vec) { - if sidecars.is_empty() { - return; - } - let Some(ref p2p) = self.p2p else { - return; - }; - let _ = p2p.check_data_column_sidecars(sidecars).inspect_err( - |err| error!(%err, "Failed to send data column sidecars to the p2p layer for checks"), - ); - } - - /// Park `sidecar` against the parent root it could not be checked against. - /// - /// Counted as `queued_for_parent` rather than as a rejection: nothing about - /// the sidecar has been judged yet, and conflating the two is what made the - /// deadlock invisible in the metrics (every column read as - /// `rejected{reason="unknown_parent"}` while the real fault was upstream). - /// - /// Nothing is refused here. The queue used to hold whole sidecars and so - /// carried a count cap, which a follower behind the tip hit constantly: - /// it receives gossip for the tip continuously, so the queue filled with - /// sidecars for blocks it would not reach for minutes and then refused - /// the ones for the block it was about to import. Measured on the eth-4 - /// follower a hundred slots behind, the queue sat pinned at its cap and - /// dropped 2,561 sidecars in ten minutes while the chain ground through - /// by-root lookups for slots whose columns gossip had already delivered - /// and this function had thrown away. Evicting the furthest-ahead entry - /// instead of the newest fixed which sidecar was lost, not that one was. - /// - /// The cap is gone now that the sidecars live in - /// `Table::PendingDataColumns` and only their keys are held here, so what - /// grows is disk rather than this actor's memory. - /// [`Self::evict_sidecars_awaiting_parent_at_or_below_finality`] is what - /// bounds it, which bounds how *long* an entry lives but not how fast - /// they arrive: the chain checks do not require `parent_root` to name a - /// block this node knows. Every sidecar reaching here, gossiped or - /// fetched, has had its header's signature checked against the head state - /// by those checks (`queue_unless_forged`, in - /// `ethlambda_state_transition::beacon::gossip::column`), but only when a - /// head state is already cached *and* the header's `proposer_index` names - /// a validator in it: with no cached head state, or a proposer index that - /// names none (`u64::MAX`, say), that check is skipped and a made-up - /// header still reaches here and parks a row. A peer exploiting either - /// gap can still park rows as fast as it can invent a slot, proposer and - /// index, until finality catches up. - fn queue_sidecar_awaiting_parent(&mut self, awaited: H256, sidecar: DataColumnSidecar) { - let parked = ParkedColumn { - slot: sidecar.slot(), - block_root: sidecar.block_root(), - index: sidecar.index(), - }; - - // A re-delivery of something already parked. The by-root and by-range - // fetch paths skip gossip's seen cache entirely, so they never touch - // the p2p actor's `SeenColumns` (which in any case only records an - // Accept, never a park); a re-delivery reaching here is ordinary, and - // without this check the same column would take a second slot in the - // queue and leave a stale key behind after the first replay took its - // row. Asked before the write rather than left to the set below, - // because the write is what costs. - if self - .sidecars_awaiting_parent - .get(&awaited) - .is_some_and(|parked_columns| parked_columns.contains(&parked)) - { - return; - } - - // The bytes go to disk before the key goes in the map, so a failed - // write leaves no key pointing at a row that is not there. - if let Err(err) = self.store.put_pending_data_column(&sidecar) { - error!(%err, "Failed to park a data column sidecar"); - return; - } - - trace!( - slot = parked.slot, - column = parked.index, - awaited = %ShortRoot(&awaited.0), - "Queueing a data column sidecar until the block it is judged against has a post-state" - ); - self.sidecars_awaiting_parent - .entry(awaited) - .or_default() - .insert(parked); - self.publish_sidecars_awaiting_parent(); - } - - /// Republish how many sidecars are parked, from the map that decides it. - fn publish_sidecars_awaiting_parent(&self) { - let total: usize = self - .sidecars_awaiting_parent - .values() - .map(HashSet::len) - .sum(); - metrics::set_sidecars_awaiting_parent(total as u64); - } - - /// Send every sidecar parked against `block_root` back to the p2p layer's - /// chain checks, now that it has a post-state to be checked against. - /// - /// Called from the one arm that means "this root now has a post-state". - /// The ones that pass come back through `new_data_column_sidecars` as a - /// new message, so nothing here re-enters the import path. - fn drain_sidecars_awaiting_parent(&mut self, block_root: H256) { - let Some(parked_columns) = self.sidecars_awaiting_parent.remove(&block_root) else { - return; - }; - debug!( - parent_root = %ShortRoot(&block_root.0), - count = parked_columns.len(), - "Replaying data column sidecars whose parent just imported" - ); - self.publish_sidecars_awaiting_parent(); - - let mut sidecars = Vec::with_capacity(parked_columns.len()); - for parked in parked_columns { - // Taken, not read: the row has served its purpose either way. A - // replay that passes is written to `DataColumns`, and one that - // fails a check has been judged, so neither leaves anything worth - // keeping here. - let sidecar = match self.store.take_pending_data_column( - parked.slot, - &parked.block_root, - parked.index, - ) { - Ok(Some(sidecar)) => sidecar, - Ok(None) => { - error!( - slot = parked.slot, - column = parked.index, - block_root = %ShortRoot(&parked.block_root.0), - "A parked data column sidecar has no row to replay from" - ); - continue; - } - Err(err) => { - error!(%err, "Failed to read back a parked data column sidecar"); - continue; - } - }; - sidecars.push(sidecar); - } - self.send_data_columns_for_checks(sidecars); - } - - /// Drop parked sidecars whose block finality has superseded. - /// - /// The counterpart of [`Self::evict_held_blocks_at_or_below_finality`] and - /// run beside it, for the same reason: a parent root that never arrives - /// would otherwise pin its children's sidecars for this node's whole - /// uptime. A sidecar at or below the finalized slot can never be needed - /// again, since the chain checks would drop it outright now. - fn evict_sidecars_awaiting_parent_at_or_below_finality(&mut self) { - if self.sidecars_awaiting_parent.is_empty() { - return; - } - let finalized_slot = self - .store - .latest_finalized() - .expect("finalized checkpoint exists") - .slot; - - let mut dropped: Vec = Vec::new(); - self.sidecars_awaiting_parent.retain(|_, parked_columns| { - parked_columns.retain(|parked| { - let keep = parked.slot > finalized_slot; - if !keep { - dropped.push(*parked); - } - keep - }); - !parked_columns.is_empty() - }); - - if !dropped.is_empty() { - info!( - finalized_slot, - count = dropped.len(), - "Evicting parked data column sidecars that finality has superseded" - ); - // The rows go with the keys, so a key dropped from the map never - // leaves its bytes on disk with nothing left to read them. - let keys = dropped - .iter() - .map(|parked| (parked.slot, parked.block_root, parked.index)); - let _ = self - .store - .delete_pending_data_column_sidecars(keys) - .inspect_err(|err| error!(%err, "Failed to drop parked data column sidecars")); - self.publish_sidecars_awaiting_parent(); - } - } - /// Once a slot, revisit every held block: release the ones whose columns /// have quietly completed, and ask for whatever the rest are still /// missing. @@ -4221,7 +3979,7 @@ mod tests { use ethlambda_types::state::State; use libssz::SszEncode as _; - const GENESIS_TIME: u64 = 1_000; + pub(crate) const GENESIS_TIME: u64 = 1_000; fn config(milliseconds_per_slot: u64) -> ChainConfig { ChainConfig::new(GENESIS_TIME, milliseconds_per_slot) @@ -4403,7 +4161,7 @@ mod tests { /// /// No anchor state is written: its only caller asserts on the chain tag /// `Store::init_beacon` sets, which is seeded before any state is. - fn beacon_store(genesis_time: u64, finalized_slot: u64) -> Store { + pub(crate) fn beacon_store(genesis_time: u64, finalized_slot: u64) -> Store { beacon_store_with_config(genesis_time, finalized_slot, Config::mainnet()) } @@ -4503,7 +4261,7 @@ mod tests { // with `beacon::gossip::column`. // ----------------------------------------------------------------- - fn beacon_server(store: Store) -> BlockChainServer { + pub(crate) fn beacon_server(store: Store) -> BlockChainServer { BlockChainServer { store, p2p: None, @@ -4529,7 +4287,7 @@ mod tests { /// column cell, all the same length), so the chain checks judge it on its /// parent rather than its shape; every other field is its type's default, /// since nothing under test reads past the header. - fn sidecar_at(slot: u64, parent_root: H256) -> fulu::DataColumnSidecar { + pub(crate) fn sidecar_at(slot: u64, parent_root: H256) -> fulu::DataColumnSidecar { let cell: fulu::Cell = libssz_types::SszVector::try_from(vec![0u8; preset::BYTES_PER_CELL]) .expect("exact cell size"); fulu::DataColumnSidecar { @@ -4596,7 +4354,7 @@ mod tests { /// `insert_state` takes that as "first state on record" and stores a /// plain snapshot, rather than trying to diff against a parent state /// this helper never creates. - fn bare_state() -> BeaconState { + pub(crate) fn bare_state() -> BeaconState { let block_roots = vec![H256::ZERO; preset::SLOTS_PER_HISTORICAL_ROOT] .try_into() .expect("exactly the preset length"); @@ -4691,8 +4449,8 @@ mod tests { /// to read back. Only `check_data_column_sidecars` and `fetch_block` are /// recorded: nothing under test here sends the others. #[derive(Default)] - struct RecordingP2P { - checks: std::sync::Mutex>>, + pub(crate) struct RecordingP2P { + pub(crate) checks: std::sync::Mutex>>, fetches: std::sync::Mutex>, } @@ -4732,7 +4490,7 @@ mod tests { } /// `beacon_server(store)` with a [`RecordingP2P`] wired in as its p2p ref. - fn beacon_server_recording(store: Store) -> (BlockChainServer, Arc) { + pub(crate) fn beacon_server_recording(store: Store) -> (BlockChainServer, Arc) { let p2p = Arc::new(RecordingP2P::default()); let mut server = beacon_server(store); server.p2p = Some(p2p.clone()); @@ -4742,7 +4500,7 @@ mod tests { /// A beacon store whose clock reads slot 10, so a sidecar at slot 10 is /// neither future nor finalized. Fulu is active from genesis, since a /// stored sidecar is decoded as the fork its slot names. - fn beacon_store_at_slot_10() -> Store { + pub(crate) fn beacon_store_at_slot_10() -> Store { let mut store = beacon_store_fulu_at_genesis(GENESIS_TIME, 0); store .set_time_ms(seconds_to_milliseconds( @@ -4859,98 +4617,9 @@ mod tests { .await; } - #[test] - fn a_data_column_sidecar_naming_a_parent_with_no_state_is_parked_not_stored() { - // `beacon_store` writes no anchor state (see its own doc comment), so - // any parent root at all is unknown here, including the anchor's own. - let (mut server, p2p) = beacon_server_recording(beacon_store_at_slot_10()); - let parent_root = H256::repeat_byte(9); - let sidecar = sidecar_at(10, parent_root); - let block_root = sidecar.signed_block_header.message.hash_tree_root(); - - server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar)]); - - // Not stored: it has not been checked, so it has not been accepted. - assert_eq!( - server - .store - .data_column_indices_for(10, &block_root) - .unwrap(), - Vec::::new() - ); - // But kept, under the parent it is waiting on. Dropping it here is - // what deadlocks a follower running the availability gate: a held - // block writes no post-state, so every sidecar of every child of it - // lands in exactly this branch. - assert_eq!( - server - .sidecars_awaiting_parent - .get(&parent_root) - .map(HashSet::len), - Some(1) - ); - assert!(p2p.checks.lock().unwrap().is_empty()); - } - - #[test] - fn a_sidecar_whose_parent_imported_meanwhile_goes_back_for_checks_not_into_the_queue() { - // The race the p2p layer's checks open: they found no parent state, - // then the parent imported and drained its (still empty) queue before - // this message arrived. Parking it now would strand it until - // finality, since that parent never drains again. - let (mut server, p2p) = beacon_server_recording(beacon_store_at_slot_10()); - let parent_root = H256::repeat_byte(9); - server - .store - .insert_state(parent_root, bare_state()) - .expect("insert"); - let sidecar = sidecar_at(10, parent_root); - - server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar.clone())]); - - assert!(server.sidecars_awaiting_parent.is_empty()); - assert_eq!( - *p2p.checks.lock().unwrap(), - vec![vec![DataColumnSidecar::Fulu(sidecar)]] - ); - } - - #[test] - fn a_parked_sidecar_goes_back_for_checks_once_its_parent_gains_a_post_state() { - // The deadlock this closes, in miniature: while the parent has no - // post-state every sidecar under it parks, and if parking were the end - // of the story the queue would only ever grow. What breaks the cycle - // is that gaining a post-state releases them, to the p2p layer's - // checks, since this actor no longer judges a sidecar itself. - let (mut server, p2p) = beacon_server_recording(beacon_store_at_slot_10()); - let parent_root = H256::repeat_byte(9); - let sidecar = sidecar_at(10, parent_root); - let block_root = sidecar.signed_block_header.message.hash_tree_root(); - - server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar.clone())]); - assert!(server.sidecars_awaiting_parent.contains_key(&parent_root)); - - server.drain_sidecars_awaiting_parent(parent_root); - - // Gone from the queue and from `PendingDataColumns`, and handed to - // the checks exactly as it was parked. - assert!(!server.sidecars_awaiting_parent.contains_key(&parent_root)); - assert!( - server - .store - .take_pending_data_column_sidecar(10, &block_root, 0) - .expect("DB read should succeed") - .is_none() - ); - assert_eq!( - *p2p.checks.lock().unwrap(), - vec![vec![DataColumnSidecar::Fulu(sidecar)]] - ); - } - /// [`beacon_store_at_slot_10`] with gloas active from genesis too, so a /// slot-10 sidecar decodes as gloas's shape. - fn gloas_store_at_slot_10() -> Store { + pub(crate) fn gloas_store_at_slot_10() -> Store { let mut store = beacon_store_with_config( GENESIS_TIME, 0, @@ -4966,7 +4635,7 @@ mod tests { store } - fn gloas_sidecar_at(slot: u64, block_root: H256, index: u64) -> DataColumnSidecar { + pub(crate) fn gloas_sidecar_at(slot: u64, block_root: H256, index: u64) -> DataColumnSidecar { DataColumnSidecar::Gloas(gloas::DataColumnSidecar { index, slot, @@ -4975,61 +4644,6 @@ mod tests { }) } - #[test] - fn a_gloas_sidecar_for_an_unimported_block_parks_on_its_own_block_and_replays_when_it_imports() - { - // A gloas sidecar is judged against its own block (the commitments are - // in its bid), so it waits on that block's post-state where a fulu one - // waits on its parent's. - let (mut server, p2p) = beacon_server_recording(gloas_store_at_slot_10()); - let block_root = H256::repeat_byte(5); - let sidecar = gloas_sidecar_at(10, block_root, 3); - - server.park_data_columns(vec![sidecar.clone()]); - - assert_eq!( - server.sidecars_awaiting_parent.get(&block_root), - Some(&HashSet::from([ParkedColumn { - slot: 10, - block_root, - index: 3, - }])) - ); - // Parked and not custodied. - assert!(!server.store.has_data_column(10, &block_root, 3)); - assert!(p2p.checks.lock().unwrap().is_empty()); - - // The block imports: what was waiting on it goes back for checks, as - // it was parked. - server.drain_sidecars_awaiting_parent(block_root); - - assert!(server.sidecars_awaiting_parent.is_empty()); - assert!( - server - .store - .take_pending_data_column(10, &block_root, 3) - .expect("DB read should succeed") - .is_none() - ); - assert_eq!(*p2p.checks.lock().unwrap(), vec![vec![sidecar]]); - } - - #[test] - fn a_gloas_sidecar_whose_block_has_a_state_goes_straight_back_for_checks() { - let (mut server, p2p) = beacon_server_recording(gloas_store_at_slot_10()); - let block_root = H256::repeat_byte(5); - server - .store - .insert_state(block_root, bare_state()) - .expect("insert"); - let sidecar = gloas_sidecar_at(10, block_root, 3); - - server.park_data_columns(vec![sidecar.clone()]); - - assert!(server.sidecars_awaiting_parent.is_empty()); - assert_eq!(*p2p.checks.lock().unwrap(), vec![vec![sidecar]]); - } - #[tokio::test] async fn a_checked_gloas_sidecar_is_stored_as_it_is() { let mut server = beacon_server(gloas_store_at_slot_10()); @@ -5051,111 +4665,6 @@ mod tests { ); } - #[test] - fn a_parked_sidecar_holds_its_bytes_on_disk_and_not_in_the_queue() { - // The queue's size is chosen by whoever is gossiping, so what it holds - // per entry is the thing that has to stay small: a key, not a cell per - // blob. - let mut server = beacon_server(beacon_store_at_slot_10()); - let parent_root = H256::repeat_byte(9); - let sidecar = sidecar_at(10, parent_root); - let block_root = sidecar.signed_block_header.message.hash_tree_root(); - - server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar)]); - - assert_eq!( - server.sidecars_awaiting_parent.get(&parent_root), - Some(&HashSet::from([ParkedColumn { - slot: 10, - block_root, - index: 0, - }])) - ); - assert!( - server - .store - .take_pending_data_column_sidecar(10, &block_root, 0) - .expect("DB read should succeed") - .is_some(), - "the sidecar's bytes belong in PendingDataColumns" - ); - } - - #[test] - fn a_parked_sidecar_does_not_satisfy_the_availability_gate() { - // Why the parked rows get a table of their own. Nothing has judged a - // parked sidecar's inclusion proof, its KZG batch or its proposer - // signature, so a peer that could get one counted as custodied would - // be able to release a held block with a column it invented. - let mut server = beacon_server(beacon_store_at_slot_10()); - let sidecar = sidecar_at(10, H256::repeat_byte(9)); - let block_root = sidecar.signed_block_header.message.hash_tree_root(); - - server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar)]); - - assert_eq!( - server - .store - .data_column_indices_for(10, &block_root) - .expect("DB read should succeed"), - Vec::::new(), - "an unverified sidecar must be invisible to data_column_indices_for" - ); - } - - #[test] - fn a_sidecar_parked_twice_takes_one_slot_in_the_queue() { - // The by-root and by-range fetch paths skip gossip's seen cache - // entirely, so nothing between them and this actor dedups a - // re-delivery while the parent is still stateless; it is ordinary. A - // second entry would leave a key with no row behind it once the - // first replay took it. - let mut server = beacon_server(beacon_store_at_slot_10()); - let parent_root = H256::repeat_byte(9); - - server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar_at(10, parent_root))]); - server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar_at(10, parent_root))]); - - assert_eq!( - server - .sidecars_awaiting_parent - .get(&parent_root) - .map(HashSet::len), - Some(1) - ); - } - - #[test] - fn parked_sidecars_are_dropped_once_finality_passes_their_slot() { - // Populated directly rather than through `park_data_columns`: the - // chain checks refuse a sidecar at or below the finalized slot before - // it could ever be parked, so the only way to observe the sweep is to - // park one behind their back. The finalized slot is fixed at init, so - // the store carries it rather than the test moving it. - let mut server = beacon_server(beacon_store(GENESIS_TIME, 10)); - let superseded = H256::repeat_byte(1); - let still_wanted = H256::repeat_byte(2); - let parked_at = |slot: u64| ParkedColumn { - slot, - block_root: H256::repeat_byte(9), - index: 0, - }; - server - .sidecars_awaiting_parent - .insert(superseded, HashSet::from([parked_at(10)])); - server - .sidecars_awaiting_parent - .insert(still_wanted, HashSet::from([parked_at(20)])); - - server.evict_sidecars_awaiting_parent_at_or_below_finality(); - - // A parent root that never arrives would otherwise pin its children's - // sidecars for this node's whole uptime; one still above finality is - // a parent that may yet show up. - assert!(!server.sidecars_awaiting_parent.contains_key(&superseded)); - assert!(server.sidecars_awaiting_parent.contains_key(&still_wanted)); - } - // ----------------------------------------------------------------- // data_availability_for / hold_block_for_columns / // release_block_if_columns_complete From 7144c0a833f33c7e7c2b804fe004ebad3deeefdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:18:33 -0300 Subject: [PATCH 048/132] fix(beacon): bound gloas column parking A gloas sidecar carries no signature, so a peer can invent (slot, root, index) triples as fast as it can send them and each one parked costs a disk row. - Park only while the node follows the fork; gloas blocks cannot import yet, so nothing would ever drain a parked row. - Check shape before anything is written: slot in gloas, index in range, column and proofs agree within the blob limit, index one this node samples. - Cap distinct parked gloas roots per slot and pin each root to one slot. - Keep the parked count as a running counter instead of summing every root. - Share one "can be judged now" predicate between the actor and the p2p check, so a root with a state but no block cannot bounce between them. - Drop by-root column answers that were not asked for. Also refreshes docs that still said a column waits for its parent. --- crates/blockchain/src/beacon_columns.rs | 372 ++++++++++++++++-- crates/blockchain/src/lib.rs | 17 +- .../src/beacon/gossip/column.rs | 158 +++++++- crates/net/p2p/src/beacon/column_checks.rs | 7 +- crates/net/p2p/src/beacon/encoding.rs | 10 +- crates/net/p2p/src/gossipsub/handler.rs | 11 +- crates/net/p2p/src/lib.rs | 10 +- crates/net/p2p/src/req_resp/codec.rs | 23 ++ crates/net/p2p/src/req_resp/handlers.rs | 83 +++- 9 files changed, 614 insertions(+), 77 deletions(-) diff --git a/crates/blockchain/src/beacon_columns.rs b/crates/blockchain/src/beacon_columns.rs index 75bd728e9..f6ee4bfd6 100644 --- a/crates/blockchain/src/beacon_columns.rs +++ b/crates/blockchain/src/beacon_columns.rs @@ -5,9 +5,22 @@ //! sidecar, its own block's for a gloas one. Until that state exists the //! sidecar's bytes wait in `Table::PendingDataColumns`, and only their keys are //! held in `BlockChainServer::sidecars_awaiting_parent`. - -use std::collections::HashSet; - +//! +//! # What bounds this +//! +//! A fulu sidecar carries a signed header, which the p2p checks verify against +//! the head state before it is ever sent here. A gloas sidecar carries no +//! signature at all, so a peer can invent `(slot, root, index)` triples as fast +//! as it can send them. Gloas sidecars are therefore parked only while the node +//! follows gloas, only when their shape passes +//! [`is_parkable_gloas`](ethlambda_state_transition::beacon::gossip::column::is_parkable_gloas), +//! and at most [`MAX_PARKED_GLOAS_ROOTS_PER_SLOT`] distinct roots per slot. + +use std::collections::{HashMap, HashSet}; + +use ethlambda_state_transition::beacon::gossip::column::{ + is_parkable_gloas, judgeable_block_gloas, +}; use ethlambda_types::ShortRoot; use ethlambda_types::beacon::containers::DataColumnSidecar; use ethlambda_types::primitives::H256; @@ -27,6 +40,26 @@ pub(crate) struct ParkedColumn { pub(crate) index: u64, } +/// How many distinct gloas block roots may have sidecars parked at one slot. +/// +/// A slot has one canonical block, and a short fork or an equivocating +/// proposer adds a few more, so a handful covers every honest case. A gloas +/// sidecar names its block by an unsigned root, so past this cap the extra +/// roots at a slot are ones nobody can vouch for, and the node stops keeping +/// rows for them. +pub(crate) const MAX_PARKED_GLOAS_ROOTS_PER_SLOT: usize = 4; + +/// The bookkeeping that bounds the parking lot, beside the map it bounds. +#[derive(Default)] +pub(crate) struct ColumnParking { + /// How many sidecars are parked. Kept as a running count, updated where + /// one is parked, drained or evicted, so publishing it is not a walk over + /// every parked root. + pub(crate) count: usize, + /// The gloas roots with sidecars parked, by the slot they were parked at. + pub(crate) gloas_roots: HashMap>, +} + /// The root whose post-state `sidecar` cannot be checked without. /// /// A fulu sidecar is verified against its parent's state (its proposer and @@ -40,21 +73,50 @@ pub(crate) fn awaited_root(sidecar: &DataColumnSidecar) -> H256 { } impl BlockChainServer { - /// Park sidecars the chain checks found no parent post-state for, or send - /// them straight back to be checked if the parent has one by now. + /// Park sidecars the chain checks found no post-state for, or send them + /// straight back to be checked if the block they wait on has one by now. /// /// The second case is a race this actor has to close, because the checks - /// run elsewhere: the p2p layer looked for the parent's post-state, found - /// none and sent these, and if the parent imported in between, its + /// run elsewhere: the p2p layer looked for the post-state, found none and + /// sent these, and if the block imported in between, its /// [`Self::drain_sidecars_awaiting_parent`] has already run and will not /// run again, so a sidecar parked now would wait for nothing until - /// finality evicts it. Asked with the same `get_state` the checks use, so - /// a sidecar sent back is one they will find a parent state for. + /// finality evicts it. Asked with the same predicate the checks use + /// ([`Self::is_ready_to_judge`]), so a sidecar sent back is one they will + /// find everything for. + /// + /// A sidecar of a fork the node does not follow is dropped: no block of + /// that fork will import, so nothing would ever drain it. pub(crate) fn park_data_columns(&mut self, sidecars: Vec) { + let (followed, unfollowed): (Vec<_>, Vec<_>) = sidecars + .into_iter() + .partition(|sidecar| sidecar.fork().is_followed()); + if !unfollowed.is_empty() { + trace!( + count = unfollowed.len(), + "Dropping data column sidecars of a fork this node does not follow" + ); + } + self.park_followed_columns(followed); + } + + /// [`Self::park_data_columns`] for sidecars of a followed fork. + pub(crate) fn park_followed_columns(&mut self, sidecars: Vec) { + let config = self.store.config(); let mut ready = Vec::new(); for sidecar in sidecars { + if let DataColumnSidecar::Gloas(gloas) = &sidecar + && !is_parkable_gloas(&config, &self.custody_columns, gloas) + { + trace!( + slot = gloas.slot, + column = gloas.index, + "Dropping a gloas data column sidecar not shaped to be parked" + ); + continue; + } let awaited = awaited_root(&sidecar); - if matches!(self.store.get_state(&awaited), Ok(Some(_))) { + if self.is_ready_to_judge(&sidecar) { ready.push(sidecar); continue; } @@ -63,6 +125,24 @@ impl BlockChainServer { self.send_data_columns_for_checks(ready); } + /// Whether the p2p checks can judge `sidecar` now: a fulu sidecar needs its + /// parent's post-state, a gloas one its block and that block's post-state. + /// + /// The gloas half is [`judgeable_block_gloas`], the same call the p2p + /// check makes first, so this actor and that one can never disagree about + /// a root that has a state but no block, which would bounce its sidecars + /// between them for ever. + fn is_ready_to_judge(&self, sidecar: &DataColumnSidecar) -> bool { + match sidecar { + DataColumnSidecar::Fulu(_) => { + matches!(self.store.get_state(&awaited_root(sidecar)), Ok(Some(_))) + } + DataColumnSidecar::Gloas(gloas) => { + judgeable_block_gloas(&self.store, &gloas.beacon_block_root).is_ok() + } + } + } + /// Hand sidecars to the p2p layer's chain checks, which send back the /// ones that pass through `new_data_column_sidecars`. pub(crate) fn send_data_columns_for_checks(&self, sidecars: Vec) { @@ -111,6 +191,10 @@ impl BlockChainServer { /// header still reaches here and parks a row. A peer exploiting either /// gap can still park rows as fast as it can invent a slot, proposer and /// index, until finality catches up. + /// + /// A gloas sidecar has no header to check at all, so it is bounded + /// instead: every sidecar parked under a root must share that root's slot, + /// and a slot keeps at most [`MAX_PARKED_GLOAS_ROOTS_PER_SLOT`] roots. pub(crate) fn queue_sidecar_awaiting_parent( &mut self, awaited: H256, @@ -138,6 +222,17 @@ impl BlockChainServer { return; } + if matches!(sidecar, DataColumnSidecar::Gloas(_)) + && !self.gloas_root_fits(awaited, parked.slot) + { + trace!( + slot = parked.slot, + block_root = %ShortRoot(&awaited.0), + "Dropping a gloas data column sidecar: its root does not fit the slot's bound" + ); + return; + } + // The bytes go to disk before the key goes in the map, so a failed // write leaves no key pointing at a row that is not there. if let Err(err) = self.store.put_pending_data_column(&sidecar) { @@ -151,25 +246,51 @@ impl BlockChainServer { awaited = %ShortRoot(&awaited.0), "Queueing a data column sidecar until the block it is judged against has a post-state" ); - self.sidecars_awaiting_parent + if matches!(sidecar, DataColumnSidecar::Gloas(_)) { + self.column_parking + .gloas_roots + .entry(parked.slot) + .or_default() + .insert(awaited); + } + let inserted = self + .sidecars_awaiting_parent .entry(awaited) .or_default() .insert(parked); + if inserted { + self.column_parking.count += 1; + } self.publish_sidecars_awaiting_parent(); } - /// Republish how many sidecars are parked, from the map that decides it. - pub(crate) fn publish_sidecars_awaiting_parent(&self) { - let total: usize = self + /// Whether a gloas sidecar for `root` at `slot` may be parked: the root is + /// not already parked at another slot, and this slot has room for it. + fn gloas_root_fits(&self, root: H256, slot: u64) -> bool { + let parked_slot = self .sidecars_awaiting_parent - .values() - .map(HashSet::len) - .sum(); - metrics::set_sidecars_awaiting_parent(total as u64); + .get(&root) + .and_then(|parked| parked.iter().next()) + .map(|parked| parked.slot); + if parked_slot.is_some_and(|parked_slot| parked_slot != slot) { + return false; + } + self.column_parking + .gloas_roots + .get(&slot) + .is_none_or(|roots| { + roots.contains(&root) || roots.len() < MAX_PARKED_GLOAS_ROOTS_PER_SLOT + }) + } + + /// Republish how many sidecars are parked, from the running count. + pub(crate) fn publish_sidecars_awaiting_parent(&self) { + metrics::set_sidecars_awaiting_parent(self.column_parking.count as u64); } /// Send every sidecar parked against `block_root` back to the p2p layer's - /// chain checks, now that it has a post-state to be checked against. + /// chain checks, now that it has a post-state to be checked against (for a + /// gloas sidecar, now that it is imported itself). /// /// Called from the one arm that means "this root now has a post-state". /// The ones that pass come back through `new_data_column_sidecars` as a @@ -179,10 +300,22 @@ impl BlockChainServer { return; }; debug!( - parent_root = %ShortRoot(&block_root.0), + awaited = %ShortRoot(&block_root.0), count = parked_columns.len(), - "Replaying data column sidecars whose parent just imported" + "Replaying data column sidecars whose awaited block just imported" ); + self.column_parking.count = self + .column_parking + .count + .saturating_sub(parked_columns.len()); + if let Some(slot) = parked_columns.iter().next().map(|parked| parked.slot) + && let Some(roots) = self.column_parking.gloas_roots.get_mut(&slot) + { + roots.remove(&block_root); + if roots.is_empty() { + self.column_parking.gloas_roots.remove(&slot); + } + } self.publish_sidecars_awaiting_parent(); let mut sidecars = Vec::with_capacity(parked_columns.len()); @@ -246,6 +379,10 @@ impl BlockChainServer { }); if !dropped.is_empty() { + self.column_parking.count = self.column_parking.count.saturating_sub(dropped.len()); + self.column_parking + .gloas_roots + .retain(|slot, _| *slot > finalized_slot); info!( finalized_slot, count = dropped.len(), @@ -269,7 +406,7 @@ impl BlockChainServer { mod tests { use super::*; use crate::tests::{ - GENESIS_TIME, bare_state, beacon_server, beacon_server_recording, beacon_store, + GENESIS_TIME, bare_block, bare_state, beacon_server, beacon_server_recording, beacon_store, beacon_store_at_slot_10, gloas_sidecar_at, gloas_store_at_slot_10, sidecar_at, }; use ethlambda_types::primitives::HashTreeRoot as _; @@ -363,17 +500,34 @@ mod tests { ); } + /// A gloas-active server with a recording p2p ref, sampling columns 0..8. + fn gloas_server() -> (BlockChainServer, std::sync::Arc) { + let (mut server, p2p) = beacon_server_recording(gloas_store_at_slot_10()); + server.custody_columns = (0..8).collect(); + (server, p2p) + } + + /// The count the parking lot publishes is the sum of what it holds. + fn assert_count_matches(server: &BlockChainServer) { + let held: usize = server + .sidecars_awaiting_parent + .values() + .map(HashSet::len) + .sum(); + assert_eq!(server.column_parking.count, held); + } + #[test] fn a_gloas_sidecar_for_an_unimported_block_parks_on_its_own_block_and_replays_when_it_imports() { // A gloas sidecar is judged against its own block (the commitments are // in its bid), so it waits on that block's post-state where a fulu one // waits on its parent's. - let (mut server, p2p) = beacon_server_recording(gloas_store_at_slot_10()); + let (mut server, p2p) = gloas_server(); let block_root = H256::repeat_byte(5); let sidecar = gloas_sidecar_at(10, block_root, 3); - server.park_data_columns(vec![sidecar.clone()]); + server.park_followed_columns(vec![sidecar.clone()]); assert_eq!( server.sidecars_awaiting_parent.get(&block_root), @@ -383,6 +537,7 @@ mod tests { index: 3, }])) ); + assert_eq!(server.column_parking.count, 1); // Parked and not custodied. assert!(!server.store.has_data_column(10, &block_root, 3)); assert!(p2p.checks.lock().unwrap().is_empty()); @@ -392,6 +547,8 @@ mod tests { server.drain_sidecars_awaiting_parent(block_root); assert!(server.sidecars_awaiting_parent.is_empty()); + assert_eq!(server.column_parking.count, 0); + assert!(server.column_parking.gloas_roots.is_empty()); assert!( server .store @@ -404,20 +561,183 @@ mod tests { #[test] fn a_gloas_sidecar_whose_block_has_a_state_goes_straight_back_for_checks() { - let (mut server, p2p) = beacon_server_recording(gloas_store_at_slot_10()); + let (mut server, p2p) = gloas_server(); let block_root = H256::repeat_byte(5); + server + .store + .insert_signed_block(block_root, bare_block(10, H256::ZERO)) + .expect("insert block"); server .store .insert_state(block_root, bare_state()) .expect("insert"); let sidecar = gloas_sidecar_at(10, block_root, 3); - server.park_data_columns(vec![sidecar.clone()]); + server.park_followed_columns(vec![sidecar.clone()]); assert!(server.sidecars_awaiting_parent.is_empty()); assert_eq!(*p2p.checks.lock().unwrap(), vec![vec![sidecar]]); } + /// The p2p check will not judge a sidecar whose block is missing, so this + /// actor must not call it ready either: a state with no block would send it + /// back and forth between the two for ever. + #[test] + fn a_gloas_sidecar_whose_root_has_a_state_but_no_block_is_parked_not_bounced() { + let (mut server, p2p) = gloas_server(); + let block_root = H256::repeat_byte(5); + server + .store + .insert_state(block_root, bare_state()) + .expect("insert"); + + server.park_followed_columns(vec![gloas_sidecar_at(10, block_root, 3)]); + + assert!(server.sidecars_awaiting_parent.contains_key(&block_root)); + assert!(p2p.checks.lock().unwrap().is_empty()); + } + + #[test] + fn a_gloas_sidecar_is_dropped_while_the_node_does_not_follow_gloas() { + // `ForkName::is_followed` is false for gloas today: no gloas block + // imports, so nothing would ever drain a parked row. + let (mut server, p2p) = gloas_server(); + let block_root = H256::repeat_byte(5); + + server.park_data_columns(vec![gloas_sidecar_at(10, block_root, 3)]); + + assert!(server.sidecars_awaiting_parent.is_empty()); + assert_eq!(server.column_parking.count, 0); + assert!( + server + .store + .take_pending_data_column(10, &block_root, 3) + .expect("DB read should succeed") + .is_none() + ); + assert!(p2p.checks.lock().unwrap().is_empty()); + } + + #[test] + fn a_misshapen_or_uncustodied_gloas_sidecar_is_dropped_before_anything_is_written() { + let (mut server, _p2p) = gloas_server(); + let block_root = H256::repeat_byte(5); + // Not a column this node samples. + let uncustodied = gloas_sidecar_at(10, block_root, 9); + // No cells at all. + let mut empty = gloas_sidecar_at(10, block_root, 3); + if let DataColumnSidecar::Gloas(sidecar) = &mut empty { + sidecar.column = Default::default(); + sidecar.kzg_proofs = Default::default(); + } + + server.park_followed_columns(vec![uncustodied, empty]); + + assert!(server.sidecars_awaiting_parent.is_empty()); + assert_eq!(server.column_parking.count, 0); + for index in [3, 9] { + assert!( + server + .store + .take_pending_data_column(10, &block_root, index) + .expect("DB read should succeed") + .is_none() + ); + } + } + + #[test] + fn a_slot_parks_no_more_gloas_roots_than_the_cap() { + let (mut server, _p2p) = gloas_server(); + let roots: Vec = (1..=MAX_PARKED_GLOAS_ROOTS_PER_SLOT as u8 + 1) + .map(H256::repeat_byte) + .collect(); + for &root in &roots { + server.park_followed_columns(vec![gloas_sidecar_at(10, root, 0)]); + } + + let (admitted, refused) = roots.split_at(MAX_PARKED_GLOAS_ROOTS_PER_SLOT); + for root in admitted { + assert!(server.sidecars_awaiting_parent.contains_key(root)); + } + for root in refused { + assert!(!server.sidecars_awaiting_parent.contains_key(root)); + assert!( + server + .store + .take_pending_data_column(10, root, 0) + .expect("DB read should succeed") + .is_none(), + "a refused root leaves no row on disk" + ); + } + assert_eq!(server.column_parking.count, MAX_PARKED_GLOAS_ROOTS_PER_SLOT); + + // A root already parked keeps taking its other columns, and another + // slot has a cap of its own. + server.park_followed_columns(vec![ + gloas_sidecar_at(10, admitted[0], 1), + gloas_sidecar_at(9, refused[0], 0), + ]); + assert_eq!(server.sidecars_awaiting_parent[&admitted[0]].len(), 2); + assert!(server.sidecars_awaiting_parent.contains_key(&refused[0])); + assert_count_matches(&server); + } + + #[test] + fn every_sidecar_parked_for_a_gloas_root_shares_its_slot() { + let (mut server, _p2p) = gloas_server(); + let root = H256::repeat_byte(5); + server.park_followed_columns(vec![ + gloas_sidecar_at(10, root, 0), + gloas_sidecar_at(9, root, 1), + ]); + + assert_eq!( + server.sidecars_awaiting_parent.get(&root), + Some(&HashSet::from([ParkedColumn { + slot: 10, + block_root: root, + index: 0, + }])) + ); + assert!( + server + .store + .take_pending_data_column(9, &root, 1) + .expect("DB read should succeed") + .is_none() + ); + assert_count_matches(&server); + } + + #[test] + fn eviction_frees_the_gloas_roots_and_the_count_of_a_finalized_slot() { + // Populated directly, as `parked_sidecars_are_dropped_once_finality_ + // passes_their_slot` is: the finalized slot is fixed at init. + let mut server = beacon_server(beacon_store(GENESIS_TIME, 10)); + let root = H256::repeat_byte(5); + server.sidecars_awaiting_parent.insert( + root, + HashSet::from([ParkedColumn { + slot: 10, + block_root: root, + index: 0, + }]), + ); + server.column_parking.count = 1; + server + .column_parking + .gloas_roots + .insert(10, HashSet::from([root])); + + server.evict_sidecars_awaiting_parent_at_or_below_finality(); + + assert!(server.sidecars_awaiting_parent.is_empty()); + assert_eq!(server.column_parking.count, 0); + assert!(server.column_parking.gloas_roots.is_empty()); + } + #[test] fn a_parked_sidecar_holds_its_bytes_on_disk_and_not_in_the_queue() { // The queue's size is chosen by whoever is gossiping, so what it holds diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 4184fa5ec..4af44423b 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -44,7 +44,7 @@ use spawned_concurrency::tasks::{ use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, trace, warn}; -use crate::beacon_columns::ParkedColumn; +use crate::beacon_columns::{ColumnParking, ParkedColumn}; use crate::block_builder::ProposerConfig; use crate::events::ChainEventSnapshot; use crate::import_timing::{BlockImportReport, CascadeTimings, HeadTimings, ImportTimings}; @@ -403,6 +403,7 @@ impl BlockChain { blocks_awaiting_columns: HashMap::new(), held_timings: HashMap::new(), sidecars_awaiting_parent: HashMap::new(), + column_parking: ColumnParking::default(), beacon_aggregates: Default::default(), custody_columns, engine, @@ -506,6 +507,9 @@ pub struct BlockChainServer { /// is not one: a replay checks and stores each sidecar on its own, and a /// held block is released by its last column arriving, whichever that is. sidecars_awaiting_parent: HashMap>, + /// What bounds [`Self::sidecars_awaiting_parent`]: its running size, and + /// which gloas roots are parked at each slot. + column_parking: ColumnParking, /// The columns this node samples, computed once at startup from its node /// id (see `das::custody_columns`). Empty on lean. @@ -1957,6 +1961,7 @@ impl BlockChainServer { blocks_awaiting_columns: HashMap::new(), held_timings: HashMap::new(), sidecars_awaiting_parent: HashMap::new(), + column_parking: ColumnParking::default(), custody_columns: Vec::new(), engine, safe_slots_to_import_optimistically, @@ -4270,6 +4275,7 @@ mod tests { blocks_awaiting_columns: HashMap::new(), held_timings: HashMap::new(), sidecars_awaiting_parent: HashMap::new(), + column_parking: ColumnParking::default(), beacon_aggregates: Default::default(), custody_columns: Vec::new(), engine: None, @@ -4322,7 +4328,7 @@ mod tests { /// what `Store::insert_signed_block` writes into `LiveChain`, which is /// all `get_checkpoint_block`'s ancestry walk ever reads. The fork is /// irrelevant to that walk, so the cheapest shape to build stands in. - fn bare_block(slot: u64, parent_root: H256) -> SignedBeaconBlock { + pub(crate) fn bare_block(slot: u64, parent_root: H256) -> SignedBeaconBlock { SignedBeaconBlock::Phase0(phase0::SignedBeaconBlock { message: phase0::BeaconBlock { slot, @@ -4635,12 +4641,17 @@ mod tests { store } + /// One blob's worth of column and proof, the smallest shape the parking + /// lot accepts. pub(crate) fn gloas_sidecar_at(slot: u64, block_root: H256, index: u64) -> DataColumnSidecar { + let cell: fulu::Cell = libssz_types::SszVector::try_from(vec![0u8; preset::BYTES_PER_CELL]) + .expect("exact cell size"); DataColumnSidecar::Gloas(gloas::DataColumnSidecar { index, slot, beacon_block_root: block_root, - ..Default::default() + column: vec![cell].try_into().expect("one cell"), + kzg_proofs: vec![Default::default()].try_into().expect("one proof"), }) } diff --git a/crates/blockchain/state_transition/src/beacon/gossip/column.rs b/crates/blockchain/state_transition/src/beacon/gossip/column.rs index 852e46771..27d6c442f 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/column.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/column.rs @@ -5,8 +5,9 @@ //! //! Also every check the chain relies on before it keeps a sidecar gossip did //! not accept ([`chain_checks`]): one fetched over req/resp, one gossip queued -//! or had no free permit for, or a parked one whose parent has since -//! imported. The chain actor stores whatever reaches it without checking it +//! or had no free permit for, or a parked one whose block has since imported +//! (its parent, for a fulu sidecar; its own block, for a gloas one). The chain +//! actor stores whatever reaches it without checking it //! again, so these are the only checks such a sidecar gets. use ethlambda_storage::CacheKey; @@ -23,13 +24,15 @@ use crate::beacon::containers::DataColumnSidecar as AnyDataColumnSidecar; use crate::beacon::containers::SignedBeaconBlock; use crate::beacon::containers::fulu::DataColumnSidecar; use crate::beacon::containers::gloas; +use crate::beacon::fork::ForkName; use crate::beacon::fork_choice::{self, Store}; use crate::beacon::helpers::accessors::get_beacon_proposer_index; use crate::beacon::helpers::misc::{compute_domain, compute_epoch_at_slot, compute_signing_root}; use crate::beacon::lean_boundary::lean_block_unreachable; use crate::beacon::precheck; +use crate::beacon::preset; use crate::beacon::primitives::HashTreeRoot as _; -use crate::beacon::primitives::{Slot, ValidatorIndex}; +use crate::beacon::primitives::{Root, Slot, ValidatorIndex}; use crate::beacon::stf; use crate::metrics; @@ -408,24 +411,9 @@ pub fn cheap_checks_gloas( /// is queued too where the specification rejects it, the same deviation /// fulu's parent rule makes until a bad-block cache exists. pub fn stateful_checks_gloas(store: &Store, sidecar: &gloas::DataColumnSidecar) -> Outcome { - let root = sidecar.beacon_block_root; - // [IGNORE] A block for the sidecar has been seen (MAY queue). - if !store.has_block(&root) { - return Outcome::Queue(QueueReason::BlockUnknown); - } - // [REJECT] The block for the sidecar passes validation. A block seen - // without a post-state is queued rather than rejected, see above. Checked - // before the block is decoded, so one without a state costs no read of it. - match store.has_state(&root) { - Ok(true) => {} - Ok(false) => return Outcome::Queue(QueueReason::BlockNotReady), - Err(_) => return Outcome::Ignore(IgnoreReason::Internal), - } - let block = match store.get_signed_block(&root) { - Ok(Some(block)) => block, - // Stored a moment ago and gone now (pruned): nothing to judge against. - Ok(None) => return Outcome::Queue(QueueReason::BlockUnknown), - Err(_) => return Outcome::Ignore(IgnoreReason::Internal), + let block = match judgeable_block_gloas(store, &sidecar.beacon_block_root) { + Ok(block) => block, + Err(outcome) => return outcome, }; // [REJECT] The sidecar's slot matches the slot of the block. if sidecar.slot != block.slot() { @@ -466,6 +454,58 @@ pub fn stateful_checks_gloas(store: &Store, sidecar: &gloas::DataColumnSidecar) Outcome::Accept } +/// The block a gloas sidecar is judged against, or the verdict that says why +/// there is none yet. +/// +/// The one predicate behind "this sidecar can be judged now": the chain actor +/// asks it before sending a parked sidecar back for checks, and +/// [`stateful_checks_gloas`] asks it first thing. Two predicates that differed +/// (a post-state without the block, say) would bounce a sidecar between the two +/// for ever, each side sure the other had what it needed. +pub fn judgeable_block_gloas(store: &Store, root: &Root) -> Result { + // [IGNORE] A block for the sidecar has been seen (MAY queue). + if !store.has_block(root) { + return Err(Outcome::Queue(QueueReason::BlockUnknown)); + } + // [REJECT] The block for the sidecar passes validation. A block seen + // without a post-state is queued rather than rejected, see above. Checked + // before the block is decoded, so one without a state costs no read of it. + match store.has_state(root) { + Ok(true) => {} + Ok(false) => return Err(Outcome::Queue(QueueReason::BlockNotReady)), + Err(_) => return Err(Outcome::Ignore(IgnoreReason::Internal)), + } + match store.get_signed_block(root) { + Ok(Some(block)) => Ok(block), + // Stored a moment ago and gone now (pruned): nothing to judge against. + Ok(None) => Err(Outcome::Queue(QueueReason::BlockUnknown)), + Err(_) => Err(Outcome::Ignore(IgnoreReason::Internal)), + } +} + +/// Whether a gloas sidecar is shaped well enough to be worth parking for its +/// block: the structural rules that need no block, checked before anything is +/// written to disk on its behalf. +/// +/// A gloas sidecar carries no signature, so nothing ties a parked one to a +/// real block; these are the only bounds on what a peer can make the node keep. +/// The slot must fall in gloas's epochs, the column index and the blob count +/// must be in range, the column and proof lists must agree, and the index must +/// be one this node custodies, since no other is ever stored. +pub fn is_parkable_gloas( + config: &Config, + custody_columns: &[u64], + sidecar: &gloas::DataColumnSidecar, +) -> bool { + let epoch = compute_epoch_at_slot(sidecar.slot); + config.fork_at_epoch(epoch) == ForkName::Gloas + && (sidecar.index as usize) < preset::NUMBER_OF_COLUMNS + && !sidecar.column.is_empty() + && sidecar.column.len() == sidecar.kzg_proofs.len() + && sidecar.column.len() as u64 <= config.max_blobs_per_block(epoch) + && custody_columns.contains(&sidecar.index) +} + /// [`cheap_checks_gloas`] then [`stateful_checks_gloas`], for callers with no /// reason to split them, such as the spec vectors: gloas's /// `validate_data_column_sidecar_gossip` (`specs/gloas/p2p-interface.md`, @@ -1209,4 +1249,80 @@ mod tests { Outcome::Reject(RejectReason::BlockNotGloas) ); } + + fn parkable_config() -> Config { + Config::mainnet() + .with_fork_epoch(ForkName::Fulu, 0) + .with_fork_epoch(ForkName::Gloas, 1) + } + + /// One blob's worth of column and proof, at `slot` and `index`. + fn shaped_gloas_sidecar(slot: Slot, index: u64) -> gloas::DataColumnSidecar { + let cell: fulu::Cell = + libssz_types::SszVector::try_from(vec![0u8; preset::BYTES_PER_CELL]).expect("cell"); + gloas::DataColumnSidecar { + column: vec![cell].try_into().expect("one cell"), + kzg_proofs: vec![Default::default()].try_into().expect("one proof"), + ..gloas_sidecar(slot, index) + } + } + + #[test] + fn a_well_shaped_custodied_gloas_sidecar_is_parkable() { + let config = parkable_config(); + let slot = preset::SLOTS_PER_EPOCH; + assert!(is_parkable_gloas( + &config, + &[3], + &shaped_gloas_sidecar(slot, 3) + )); + } + + #[test] + fn a_gloas_sidecar_is_not_parkable_outside_gloas_or_off_the_custody_set_or_misshapen() { + let config = parkable_config(); + let slot = preset::SLOTS_PER_EPOCH; + let custody = [3u64]; + // A slot before gloas names another fork's shape. + assert!(!is_parkable_gloas( + &config, + &custody, + &shaped_gloas_sidecar(1, 3) + )); + // Not a column this node custodies. + assert!(!is_parkable_gloas( + &config, + &custody, + &shaped_gloas_sidecar(slot, 4) + )); + // Out of range, even if a custody set were to name it. + let beyond = preset::NUMBER_OF_COLUMNS as u64; + assert!(!is_parkable_gloas( + &config, + &[beyond], + &shaped_gloas_sidecar(slot, beyond) + )); + // No blobs. + assert!(!is_parkable_gloas( + &config, + &custody, + &gloas_sidecar(slot, 3) + )); + // Column and proofs disagree. + let mut uneven = shaped_gloas_sidecar(slot, 3); + uneven.kzg_proofs = Default::default(); + assert!(!is_parkable_gloas(&config, &custody, &uneven)); + // More blobs than the schedule allows at that epoch. + let limit = config.max_blobs_per_block(1) as usize; + let cell: fulu::Cell = + libssz_types::SszVector::try_from(vec![0u8; preset::BYTES_PER_CELL]).expect("cell"); + let mut crowded = shaped_gloas_sidecar(slot, 3); + crowded.column = vec![cell; limit + 1] + .try_into() + .expect("within the list bound"); + crowded.kzg_proofs = vec![Default::default(); limit + 1] + .try_into() + .expect("within the list bound"); + assert!(!is_parkable_gloas(&config, &custody, &crowded)); + } } diff --git a/crates/net/p2p/src/beacon/column_checks.rs b/crates/net/p2p/src/beacon/column_checks.rs index e161a131e..945b6c21f 100644 --- a/crates/net/p2p/src/beacon/column_checks.rs +++ b/crates/net/p2p/src/beacon/column_checks.rs @@ -9,8 +9,9 @@ //! - a gossiped sidecar reported `Queue` or `Ignore(Overloaded)`, whose //! stateful checks stopped early or never ran; //! - every `DataColumnsByRoot` and `DataColumnsByRange` answer; -//! - sidecars the chain actor had parked, handed back once their parent -//! imported (`BlockChainToP2P::check_data_column_sidecars`). +//! - sidecars the chain actor had parked, handed back once the block they +//! wait on imported (`BlockChainToP2P::check_data_column_sidecars`): a fulu +//! sidecar's parent, a gloas sidecar's own block. //! //! Each sidecar is checked on a blocking thread, bounded by //! [`P2PServer::column_check_permits`]. Unlike a gossip verdict, nothing here @@ -26,7 +27,7 @@ use tracing::{error, warn}; use crate::{P2PServer, metrics}; /// Run the chain checks on `sidecars`, then send the chain actor the ones that -/// passed as one batch and the ones waiting on a parent as another. +/// passed as one batch and the ones waiting on a block as another. /// /// Returns at once: the checks run on a task of their own, so the p2p actor /// goes on handling swarm events while a range batch's KZG proofs are diff --git a/crates/net/p2p/src/beacon/encoding.rs b/crates/net/p2p/src/beacon/encoding.rs index 933b33df8..575e36d7d 100644 --- a/crates/net/p2p/src/beacon/encoding.rs +++ b/crates/net/p2p/src/beacon/encoding.rs @@ -278,9 +278,10 @@ where /// [`super::decode::decode_data_column_sidecar`] decodes by it. A digest no /// scheduled fork uses ends the stream like any other mismatch. The context /// bytes are then checked against the digest the sidecar's own slot implies, -/// for the same reason a block chunk's are: it catches a peer whose `genesis_validators_root` or fork schedule -/// differs from ours, which a signature failure would otherwise be the only -/// way to notice. +/// for the same reason a block chunk's are: it catches a peer whose +/// `genesis_validators_root` or fork schedule differs from ours, which a +/// signature failure would otherwise be the only way to notice. It also +/// catches a sidecar of one fork's shape sent under another's context. /// /// A mismatch ends the stream rather than skipping the chunk, matching /// [`decode_blocks_response`]: a peer that disagrees about a historical digest @@ -315,8 +316,7 @@ where let sidecar = decode::decode_data_column_sidecar(fork, payload) .map_err(|err| invalid(format!("data column sidecar chunk: {err}")))?; let slot = sidecar.slot(); - let epoch = slot / preset::SLOTS_PER_EPOCH; - let expected = compute_fork_digest(config, genesis_validators_root, epoch); + let expected = schedule.digest_at(slot / preset::SLOTS_PER_EPOCH); if context != expected { warn!( slot, diff --git a/crates/net/p2p/src/gossipsub/handler.rs b/crates/net/p2p/src/gossipsub/handler.rs index 681d4d349..f2cc0b163 100644 --- a/crates/net/p2p/src/gossipsub/handler.rs +++ b/crates/net/p2p/src/gossipsub/handler.rs @@ -311,11 +311,12 @@ fn triage_block(server: &P2PServer, wire: &BeaconWire, payload: &[u8]) -> Dispat /// node holds topics under two digests (see `beacon::transition`), so a late /// but perfectly legitimate sidecar can still arrive on the old digest's /// topic, and a clock check ahead of the decode would drop it, mistaking it for -/// the new fork's shape just because the clock has moved on. Only on a decode -/// failure does the fork matter, and `fork` is the one the message's own topic -/// digest names: a failure under a fork this build does not follow is its own -/// gap, so `Ignore`; under a followed fork it is the sender's fault, so -/// `Reject`. +/// the new fork's shape just because the clock has moved on. `fork` is the one +/// the message's own topic digest names, and it picks the container to decode +/// (fulu's and gloas's differ) and the cheap rules to run. On a decode failure +/// it also decides the verdict: a failure under a fork this build does not +/// follow is its own gap, so `Ignore`; under a followed fork it is the +/// sender's fault, so `Reject`. fn triage_data_column( server: &P2PServer, fork: ForkName, diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index 06a7b2187..92fee32a1 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -336,10 +336,12 @@ pub(crate) enum PendingRequestKind { end_slot: u64, }, /// A `DataColumnsByRoot` lookup for this block's missing columns. Carries - /// only the root: the columns, attempts and failed-peer set live in - /// `pending_column_requests`, keyed the same way, so this is enough to - /// route the response and nothing this map needs to duplicate. - Columns(H256), + /// the root and the columns this one request asked its peer for: the + /// attempts and failed-peer set live in `pending_column_requests`, keyed + /// by the root, but one lookup fans out across peers with a subset each and + /// retires its entry on the first answer, so what a later answer may + /// contain has to travel with the request itself. + Columns(H256, Vec), /// A `DataColumnsByRange` sweep for a range sync batch, covering the same /// slots the matching `BlocksByRange` asked for. /// diff --git a/crates/net/p2p/src/req_resp/codec.rs b/crates/net/p2p/src/req_resp/codec.rs index 044f39754..b77359d2e 100644 --- a/crates/net/p2p/src/req_resp/codec.rs +++ b/crates/net/p2p/src/req_resp/codec.rs @@ -829,6 +829,29 @@ mod tests { assert!(result.is_err()); } + /// A chunk's context must also be the digest of its sidecar's own slot: a + /// fulu-shaped sidecar carrying a gloas slot, sent under the fulu digest, + /// decodes cleanly as fulu and is still refused. + #[tokio::test] + async fn a_fulu_shaped_sidecar_at_a_gloas_slot_under_the_fulu_digest_aborts_the_stream() { + let config = gloas_config(); + let root = mainnet_gvr(); + let fulu_digest = compute_fork_digest(&config, root, config.fulu_fork_epoch); + let payload = data_column_sidecar(gloas_slot(), 1).to_ssz(); + let stream_protocol = StreamProtocol::new(protocols::DATA_COLUMN_SIDECARS_BY_RANGE_V1); + + let mut buffer = Cursor::new(Vec::new()); + write_success_chunk(&mut buffer, "test", &fulu_digest, payload) + .await + .expect("writes"); + let mut buffer = Cursor::new(buffer.into_inner()); + let result = codec_for(config) + .read_response(&stream_protocol, &mut buffer) + .await; + + assert!(result.is_err()); + } + /// A digest no scheduled fork uses names no shape at all. #[tokio::test] async fn a_data_column_chunk_under_an_unscheduled_digest_aborts_the_stream() { diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index 3912077c5..5c938e76a 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -222,9 +222,9 @@ pub async fn handle_req_resp_message( // event: this response is the terminal outcome for // the id either way. match server.outbound_requests.remove(&request_id) { - Some(PendingRequestKind::Columns(block_root)) => { + Some(PendingRequestKind::Columns(block_root, requested)) => { handle_data_column_sidecars_response( - server, peer, block_root, sidecars, ctx, + server, peer, block_root, &requested, sidecars, ctx, ) .await; } @@ -305,7 +305,7 @@ pub async fn handle_req_resp_message( // data column protocol, which answers with // `DataColumnSidecars`, never with `Blocks`. Some( - PendingRequestKind::Columns(_) + PendingRequestKind::Columns(..) | PendingRequestKind::ColumnRange { .. }, ) => { error!( @@ -334,7 +334,7 @@ pub async fn handle_req_resp_message( // forever and deduplicates every later fetch. handle_fetch_failure(server, root, peer, ctx).await; } - Some(PendingRequestKind::Columns(block_root)) => { + Some(PendingRequestKind::Columns(block_root, _)) => { // Same reasoning as the `Root` arm above: an // error response is the whole exchange, so // this is the only place that can retire it. @@ -383,7 +383,7 @@ pub async fn handle_req_resp_message( "BlocksByRange request failed; retry is disabled" ); } - Some(PendingRequestKind::Columns(block_root)) => { + Some(PendingRequestKind::Columns(block_root, _)) => { handle_column_fetch_failure(server, block_root, peer, ctx).await; } // Nothing waits on a range prefetch, so a failure is only @@ -1050,6 +1050,7 @@ pub async fn fetch_data_columns_from_peer( let mut sent_count = 0usize; for (peer, columns) in by_peer { let count = columns.len(); + let requested = columns.clone(); let column_indices = match ColumnIndices::try_from(columns) { Ok(indices) => indices, Err(err) => { @@ -1078,9 +1079,10 @@ pub async fn fetch_data_columns_from_peer( debug!(%block_root, %peer, "Failed to send DataColumnsByRoot request (swarm adapter closed)"); continue; }; - server - .outbound_requests - .insert(request_id, PendingRequestKind::Columns(block_root)); + server.outbound_requests.insert( + request_id, + PendingRequestKind::Columns(block_root, requested), + ); sent_count += 1; } @@ -1322,8 +1324,8 @@ fn range_batch_needs_columns(server: &P2PServer, batch: &std::ops::Range) - /// Gloas answers yes as fulu does, though its columns have a different shape /// and decode as gloas sidecars by the chunk's context bytes. A batch ending in /// gloas can begin in fulu, whose blocks need their columns; a batch lying -/// entirely in gloas still needs them too, since the chain actor will gate a -/// block's payload envelope on its columns. +/// entirely in gloas still needs them too, since a later change makes the chain +/// actor gate a block's payload envelope on its columns. fn range_needs_columns( config: &Config, custody_columns: &[u64], @@ -2363,16 +2365,49 @@ async fn handle_data_column_sidecars_range_response( column_checks::check_and_forward(server, in_range); } +/// Keep the sidecars of `sidecars` that answer the request: the requested +/// block's, at a requested column. Returns them with how many were dropped. +/// +/// Mirrors [`handle_blocks_by_root_response`], which keeps only the block the +/// request named. Nothing downstream re-asks whether a sidecar was wanted, and +/// an unsolicited one would otherwise be parked or stored on a peer's say-so. +fn retain_requested_columns( + sidecars: Vec, + block_root: H256, + requested: &[u64], +) -> (Vec, usize) { + let received = sidecars.len(); + let kept: Vec<_> = sidecars + .into_iter() + .filter(|sidecar| { + sidecar.block_root() == block_root && requested.contains(&sidecar.index()) + }) + .collect(); + let dropped = received - kept.len(); + (kept, dropped) +} + async fn handle_data_column_sidecars_response( server: &mut P2PServer, peer: PeerId, block_root: H256, + requested: &[u64], sidecars: Vec, ctx: &Context, ) { let received = sidecars.len(); trace!(%peer, %block_root, received, "Received DataColumnsByRoot response"); + let (sidecars, unrequested) = retain_requested_columns(sidecars, block_root, requested); + if unrequested > 0 { + debug!( + %peer, + %block_root, + unrequested, + "Dropping data column sidecars the DataColumnsByRoot request did not ask for" + ); + } + if sidecars.is_empty() { debug!(%peer, %block_root, "DataColumnsByRoot response carried no sidecars"); handle_column_fetch_failure(server, block_root, peer, ctx).await; @@ -2478,6 +2513,7 @@ mod tests { use super::*; use crate::ConnectionDirection; use ethlambda_storage::{ForkCheckpoints, backend::InMemoryBackend}; + use ethlambda_types::beacon::containers::gloas; use ethlambda_types::constants::DEFAULT_MILLISECONDS_PER_SLOT; use ethlambda_types::enr::EnrForkId; use ethlambda_types::{ @@ -2969,4 +3005,31 @@ mod tests { peer_gone.fail_peer(&lone_peer); assert!(range_session_exhausted(&peer_gone)); } + + fn gloas_column(block_root: H256, index: u64) -> DataColumnSidecar { + DataColumnSidecar::Gloas(gloas::DataColumnSidecar { + index, + slot: 5, + beacon_block_root: block_root, + ..Default::default() + }) + } + + #[test] + fn a_by_root_answer_keeps_only_the_requested_block_and_columns() { + let asked = H256::repeat_byte(1); + let other = H256::repeat_byte(2); + let answer = vec![ + gloas_column(asked, 3), + // A column of the right block that was not asked for. + gloas_column(asked, 4), + // A column of a block that was not asked for. + gloas_column(other, 3), + ]; + + let (kept, dropped) = retain_requested_columns(answer, asked, &[3]); + + assert_eq!(kept, vec![gloas_column(asked, 3)]); + assert_eq!(dropped, 2); + } } From 2ec76c27cead08d2d184e5c9f5dba9686e31f9a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:22:06 -0300 Subject: [PATCH 049/132] feat(beacon): gloas attestation and aggregate gossip A gloas follower dropped every aggregate, subnet attestation and attester slashing as an unsupported fork. They now decode and validate under gloas's modified rules: data.index is the payload-present flag (0 or 1), and a vote for the full payload is ignored until the block's envelope is verified and its payload validated (verify_attestation_payload_status). Gloas shares electra's wire bytes but its Attestation is a progressive container, so the aggregator's signed root differs. The aggregate therefore gets its own SignedAggregateAndProof::Gloas variant rather than reusing electra's, and attester slashings get a matching AttesterSlashing::Gloas. Subnet attestations keep the SingleAttestation shape and are told apart by slot. apply_verified_aggregate took PreGloas rules unconditionally; it now reads the vote's fork from its slot so payload_present is recorded for gloas votes. Payload verdicts live in a new beacon-root-keyed scratch map, since a gloas block carries no execution hash in el_block_hashes; an absent root reads as NOT_VALIDATED, as the specification's default does. --- crates/blockchain/src/lib.rs | 2 + .../src/beacon/fork_choice.rs | 13 +-- .../src/beacon/gossip/aggregate.rs | 50 ++++++++++-- .../src/beacon/gossip/attestation.rs | 25 +++++- .../state_transition/src/beacon/gossip/mod.rs | 79 ++++++++++++++++++- .../tests/beacon_spec/gossip.rs | 50 ++++++++---- .../common/types/src/beacon/containers/mod.rs | 43 +++++++--- crates/net/p2p/src/beacon/decode.rs | 42 +++++----- crates/net/p2p/src/gossipsub/handler.rs | 48 ++++++++--- crates/storage/src/store.rs | 50 +++++++++++- docs/beacon_wire.md | 19 ++--- docs/metrics.md | 17 ++-- 12 files changed, 339 insertions(+), 99 deletions(-) diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 4af44423b..f0bf388a9 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -3185,6 +3185,8 @@ impl BlockChainServer { // takes one root per import and neither `mark_validated` nor // `invalidate_subtree` ever comes for them. self.store.prune_beacon_optimistic_roots(finalized_slot); + self.store + .prune_beacon_block_payload_statuses(finalized_slot); if self.blocks_awaiting_columns.is_empty() { return; diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 8680a54d2..75dbe04d5 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -4979,10 +4979,10 @@ pub fn on_block_attestation( /// records it against `attesting_indices`, resolved by the caller's gossip /// validation rather than recomputed here. /// -/// The rules are [`ForkRules::PreGloas`]: an aggregate reaches this function -/// only as a pre-gloas (phase0- or electra-shaped) one, since gloas's own -/// aggregate is refused when the gossip payload is decoded and never gets as -/// far as gossip validation. +/// The rules are those of the vote's own fork, read off `data.slot`: gloas +/// keeps electra's aggregate container, so the container cannot say, and under +/// gloas `data.index` is the payload flag that [`update_latest_messages`] +/// records. /// /// `is_from_block` is fixed at `false`, matching [`on_attestation`]'s call for /// this topic: an aggregate here is by definition not carried in a block, so @@ -4999,8 +4999,9 @@ pub fn apply_verified_aggregate( config: &Config, index: &HashMap, ) -> Result<()> { - validate_on_attestation_indexed(store, data, ForkRules::PreGloas, false, config, index)?; - update_latest_messages(store, attesting_indices, data, ForkRules::PreGloas); + let rules = ForkRules::of(config.fork_at_epoch(compute_epoch_at_slot(data.slot))); + validate_on_attestation_indexed(store, data, rules, false, config, index)?; + update_latest_messages(store, attesting_indices, data, rules); Ok(()) } diff --git a/crates/blockchain/state_transition/src/beacon/gossip/aggregate.rs b/crates/blockchain/state_transition/src/beacon/gossip/aggregate.rs index 260c7e794..188a01344 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/aggregate.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/aggregate.rs @@ -1,6 +1,9 @@ //! `beacon_aggregate_and_proof` gossip validation: electra's //! `validate_beacon_aggregate_and_proof_gossip` (`specs/electra/p2p-interface.md`), -//! which fulu (what mainnet runs) inherits unchanged. +//! which fulu inherits unchanged, and gloas's modification of it +//! (`specs/gloas/p2p-interface.md`). The aggregate's variant picks the +//! `data.index` rule and whether the payload-status check runs; it runs last, +//! as in the specification. //! //! # Deviations from the specification //! @@ -54,6 +57,7 @@ use lru::LruCache; use super::{ IgnoreReason, Outcome, RejectReason, ancestor_at, is_current_or_previous_epoch, is_future_slot, + verify_attestation_payload_status, }; use crate::beacon::bls; use crate::beacon::constants::{ @@ -240,6 +244,7 @@ fn aggregate_and_proof_root(aggregate: &SignedAggregateAndProof) -> Root { match aggregate { SignedAggregateAndProof::Phase0(signed) => signed.message.hash_tree_root(), SignedAggregateAndProof::Electra(signed) => signed.message.hash_tree_root(), + SignedAggregateAndProof::Gloas(signed) => signed.message.hash_tree_root(), } } @@ -256,12 +261,24 @@ pub fn cheap_checks( let config = store.config(); let data = aggregate.data(); - // [New in Electra:EIP7549] [REJECT] `data.index` is zero: the committee - // now lives in `committee_bits` instead. Phase0 has no `committee_bits` - // and carries the real committee in `data.index`, so this only applies - // to the electra shape. - if matches!(aggregate, SignedAggregateAndProof::Electra(_)) && data.index != 0 { - return Err(Outcome::Reject(RejectReason::NonZeroDataIndex)); + match aggregate { + // Phase0 has no `committee_bits` and carries the real committee in + // `data.index`, so there is nothing to require of it. + SignedAggregateAndProof::Phase0(_) => {} + // [New in Electra:EIP7549] [REJECT] `data.index` is zero: the committee + // now lives in `committee_bits` instead. + SignedAggregateAndProof::Electra(_) => { + if data.index != 0 { + return Err(Outcome::Reject(RejectReason::NonZeroDataIndex)); + } + } + // [New in Gloas:EIP7732] [REJECT] `data.index` is 0 or 1: it is the + // payload-present flag now. + SignedAggregateAndProof::Gloas(_) => { + if data.index > 1 { + return Err(Outcome::Reject(RejectReason::DataIndexOutOfRange)); + } + } } // [New in Electra:EIP7549] [REJECT] Exactly one committee is named. // Always `Some` for phase0, whose `data.index` alone names the committee. @@ -426,6 +443,19 @@ pub fn stateful_checks( } indexed.attesting_indices.to_vec() } + SignedAggregateAndProof::Gloas(signed) => { + let gloas_attestation = &signed.message.aggregate; + let indexed = crate::beacon::helpers::gloas::get_indexed_attestation( + &state, + gloas_attestation, + &committees, + ) + .map_err(|_| Outcome::Ignore(IgnoreReason::Internal))?; + if !crate::beacon::helpers::gloas::is_valid_indexed_attestation(&state, &indexed) { + return Err(Outcome::Reject(RejectReason::AggregateSignature)); + } + indexed.attesting_indices.to_vec() + } }; // Ancestry, via the vote state's own history; see this function's @@ -449,6 +479,12 @@ pub fn stateful_checks( return Err(Outcome::Ignore(IgnoreReason::FinalizedNotAncestor)); } + // [New in Gloas:EIP7732] The attested payload status is consistent with the + // block's execution payload. + if matches!(aggregate, SignedAggregateAndProof::Gloas(_)) { + verify_attestation_payload_status(store, &data)?; + } + Ok(attesting_indices) } diff --git a/crates/blockchain/state_transition/src/beacon/gossip/attestation.rs b/crates/blockchain/state_transition/src/beacon/gossip/attestation.rs index 4db9cd8bc..bb1975791 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/attestation.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/attestation.rs @@ -1,6 +1,8 @@ //! `beacon_attestation_{subnet_id}` gossip validation: electra's modified //! `validate_beacon_attestation_gossip` (`specs/electra/p2p-interface.md`), -//! which fulu (what mainnet runs) inherits unchanged. +//! which fulu inherits unchanged, and gloas's modification of it +//! (`specs/gloas/p2p-interface.md`), picked by the attestation's slot since +//! gloas keeps [`SingleAttestation`]. //! //! EIP-7549 replaced the wire type for this topic with [`SingleAttestation`]: //! one attester's vote, with its committee named explicitly @@ -43,6 +45,7 @@ use lru::LruCache; use super::{ IgnoreReason, Outcome, RejectReason, ancestor_at, is_current_or_previous_epoch, is_future_slot, + is_gloas_slot, verify_attestation_payload_status, }; use crate::beacon::bls; use crate::beacon::config::Config; @@ -134,9 +137,15 @@ pub fn cheap_checks( if seen.contains(target_epoch, attestation.attester_index) { return Err(Outcome::Ignore(IgnoreReason::AlreadySeen)); } - // [New in Electra:EIP7549] [REJECT] `data.index` is zero: the committee - // now travels in `committee_index` instead. - if data.index != 0 { + if is_gloas_slot(&config, data.slot) { + // [New in Gloas:EIP7732] [REJECT] `data.index` is 0 or 1: it is the + // payload-present flag now. + if data.index > 1 { + return Err(Outcome::Reject(RejectReason::DataIndexOutOfRange)); + } + } else if data.index != 0 { + // [New in Electra:EIP7549] [REJECT] `data.index` is zero: the committee + // now travels in `committee_index` instead. return Err(Outcome::Reject(RejectReason::NonZeroDataIndex)); } // [IGNORE] Not from a future slot. @@ -231,6 +240,14 @@ pub fn stateful_checks(store: &Store, attestation: &SingleAttestation, subnet_id return Outcome::Ignore(IgnoreReason::FinalizedNotAncestor); } + // [New in Gloas:EIP7732] The attested payload status is consistent with the + // block's execution payload. + if is_gloas_slot(&store.config(), data.slot) + && let Err(outcome) = verify_attestation_payload_status(store, data) + { + return outcome; + } + Outcome::Accept } diff --git a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs index d7f32bdd6..671fba423 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs @@ -25,10 +25,12 @@ use lru::LruCache; use crate::beacon::config::Config; use crate::beacon::constants::MAXIMUM_GOSSIP_CLOCK_DISPARITY; -use crate::beacon::containers::BeaconState; +use crate::beacon::containers::{AttestationData, BeaconState}; +use crate::beacon::fork::ForkName; use crate::beacon::fork_choice::{self, Store}; use crate::beacon::helpers::accessors::get_block_root_at_slot; -use crate::beacon::helpers::misc::compute_start_slot_at_epoch; +use crate::beacon::helpers::misc::{compute_epoch_at_slot, compute_start_slot_at_epoch}; +use crate::beacon::lean_boundary::lean_fork_unreachable; use crate::beacon::precheck::PrecheckError; use crate::beacon::primitives::{Epoch, Root, Slot, ValidatorIndex}; @@ -129,6 +131,12 @@ pub enum IgnoreReason { /// parent's envelope has not been seen and verified (the specification /// lets it be queued until it is). ParentPayloadUnverified, + /// A gloas vote names the full payload (`data.index == 1`) of a block whose + /// envelope has not been seen and verified (the specification lets it be + /// queued until it is). + PayloadEnvelopeUnseen, + /// A gloas vote names a payload the execution client has not validated. + PayloadOptimistic, } impl IgnoreReason { @@ -149,6 +157,8 @@ impl IgnoreReason { Self::FinalizedNotAncestor => "finalized_not_ancestor", Self::AncestryUnknown => "ancestry_unknown", Self::ParentPayloadUnverified => "parent_payload_unverified", + Self::PayloadEnvelopeUnseen => "payload_envelope_unseen", + Self::PayloadOptimistic => "payload_optimistic", } } } @@ -208,6 +218,12 @@ pub enum RejectReason { /// A gloas block builds on its parent's empty branch, but its bid's /// `parent_block_hash` is not the parent state's `latest_block_hash`. BidNotOnParentHead, + /// A gloas vote's `data.index` is neither zero nor one. + DataIndexOutOfRange, + /// A gloas vote cast in its block's own slot claims the payload is present. + SameSlotPayloadFlag, + /// A gloas vote names a payload the execution client found invalid. + PayloadInvalid, } impl RejectReason { @@ -244,6 +260,9 @@ impl RejectReason { Self::OperationLimit => "operation_limit", Self::BidParentMismatch => "bid_parent_mismatch", Self::BidNotOnParentHead => "bid_not_on_parent_head", + Self::DataIndexOutOfRange => "data_index_out_of_range", + Self::SameSlotPayloadFlag => "same_slot_payload_flag", + Self::PayloadInvalid => "payload_invalid", } } } @@ -384,6 +403,62 @@ pub(crate) fn is_current_or_previous_epoch(config: &Config, epoch: Epoch, now_ms is_within_epoch(config, epoch, now_ms) || is_within_epoch(config, epoch + 1, now_ms) } +/// Whether `slot` falls in gloas. +/// +/// Gloas keeps electra's aggregate and subnet-attestation containers, so the +/// message's shape cannot say which rules apply; its slot's fork does. +pub(crate) fn is_gloas_slot(config: &Config, slot: Slot) -> bool { + match config.fork_at_epoch(compute_epoch_at_slot(slot)) { + ForkName::Gloas => true, + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Fulu => false, + ForkName::Lean => lean_fork_unreachable("gossip::is_gloas_slot"), + } +} + +/// Gloas's `verify_attestation_payload_status` (`specs/gloas/p2p-interface.md`): +/// the payload flag a vote carries in `data.index` must agree with what this +/// node knows of the voted block's payload. +/// +/// The specification lets a vote on an unseen envelope be queued and asks the +/// node to request it by root; neither is done here, so it is an `IGNORE`. +pub(crate) fn verify_attestation_payload_status( + store: &Store, + data: &AttestationData, +) -> Result<(), Outcome> { + let block_root = data.beacon_block_root; + let Some((block_slot, _)) = store.block_slot_and_state_root(&block_root) else { + // The caller has already checked the block is known. + return Err(Outcome::Ignore(IgnoreReason::UnknownBlock)); + }; + // [REJECT] For same-slot attestations, the payload cannot yet be present. + if block_slot == data.slot && data.index != 0 { + return Err(Outcome::Reject(RejectReason::SameSlotPayloadFlag)); + } + if data.index != 1 { + return Ok(()); + } + // [IGNORE] The envelope has been seen and verified. + if !fork_choice::is_payload_verified(store, block_root) { + return Err(Outcome::Ignore(IgnoreReason::PayloadEnvelopeUnseen)); + } + let status = store.beacon_block_payload_status(block_root); + // [IGNORE] The attested payload is optimistic. + if status.is_not_validated() { + return Err(Outcome::Ignore(IgnoreReason::PayloadOptimistic)); + } + // [REJECT] The attested payload is processed and invalid. + if status.is_invalidated() { + return Err(Outcome::Reject(RejectReason::PayloadInvalid)); + } + Ok(()) +} + /// Which block `state`'s own history names as the ancestor at `slot`, given /// that `state` is `at_block_root`'s post-state. /// diff --git a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs index 471690e29..dd4fc8b03 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs @@ -20,7 +20,7 @@ use ethlambda_state_transition::beacon::containers::{ electra, gloas, phase0, }; use ethlambda_state_transition::beacon::fork_choice::{ - self, DataAvailability, PayloadValidity, Store, + self, DataAvailability, PayloadStatusEnum, PayloadValidity, Store, }; use ethlambda_state_transition::beacon::gossip::{ self as rules, Outcome, SeenAggregates, SeenAttestations, SeenBlockColumns, SeenBlocks, @@ -50,13 +50,15 @@ const HANDLERS: &[&str] = &[ /// `case.in_scope()` alone would pass every fork up to gloas, since the state /// transition handles them all; the gossip rules do not. The block rules /// (`beacon::gossip::block`) and the column rules (`beacon::gossip::column`) -/// have both fulu's and gloas's, and the aggregate and attestation rules -/// (`beacon::gossip::{aggregate, attestation}`) are electra's, which fulu -/// keeps. +/// have both fulu's and gloas's, and so do the aggregate and attestation +/// rules (`beacon::gossip::{aggregate, attestation}`): electra's, which fulu +/// keeps, and gloas's payload-flag modification of them. fn validated_forks(handler: &str) -> &'static [ForkName] { match handler { "gossip_data_column_sidecar" | "gossip_beacon_block" => &[ForkName::Fulu, ForkName::Gloas], - "gossip_beacon_aggregate_and_proof" | "gossip_beacon_attestation" => &[ForkName::Fulu], + "gossip_beacon_aggregate_and_proof" | "gossip_beacon_attestation" => { + &[ForkName::Fulu, ForkName::Gloas] + } other => panic!("{other} is not in HANDLERS, so it has no validated forks"), } } @@ -204,9 +206,7 @@ fn decode_block(case: &Case, name: &str) -> Result { /// needed one yet): every fork through deneb shares phase0's shape, and /// electra and fulu share electra's, exactly the split /// [`SignedAggregateAndProof`]'s own doc describes for -/// [`SignedBeaconBlock::Fulu`]. Gloas's own aggregate has no modeled variant, -/// and this runner does not run its cases (see [`trials`]), so it is refused -/// by name. +/// [`SignedBeaconBlock::Fulu`]. Gloas has its own, progressive one. fn decode_signed_aggregate(case: &Case, name: &str) -> Result { let bytes = case.ssz_bytes(name); match case.fork { @@ -222,9 +222,11 @@ fn decode_signed_aggregate(case: &Case, name: &str) -> Result { - Err(format!("no aggregate shape for fork {:?}", case.fork)) - } + ForkName::Gloas => Ok(SignedAggregateAndProof::Gloas( + gloas::SignedAggregateAndProof::from_ssz_bytes(&bytes) + .map_err(|err| format!("decoding {name}: {err:?}"))?, + )), + ForkName::Lean => Err(format!("no aggregate shape for fork {:?}", case.fork)), } } @@ -297,22 +299,31 @@ fn build_store( for entry in rest { let block = decode_block(case, &entry.block)?; let root = block.message_hash_tree_root(); + let block_slot = block.slot(); let block_start_s = (config.genesis_time_ms() + block.slot() * config.slot_duration_ms) / 1000; if block_start_s > clock_s { clock_s = block_start_s; fork_choice::on_tick(&mut store, clock_s, config); } - // Seen without a post-state. An `INVALIDATED` payload lands here too: - // this store never keeps a post-state for one, since `on_block` fails - // it and invalidates the branch. - if entry.failed || entry.pending || entry.payload_status.as_deref() == Some("INVALIDATED") { + // Gloas's `payload_status` is the verdict on the block's envelope, not + // on the block, which is imported regardless; it feeds the attestation + // rules' `block_payload_statuses` instead. + let gloas = case.fork == ForkName::Gloas; + // Seen without a post-state. A pre-gloas `INVALIDATED` payload lands + // here too: this store never keeps a post-state for one, since + // `on_block` fails it and invalidates the branch. + if entry.failed + || entry.pending + || (!gloas && entry.payload_status.as_deref() == Some("INVALIDATED")) + { store .insert_pending_block(root, block) .map_err(|err| format!("storing {}: {err}", entry.block))?; continue; } let validity = match entry.payload_status.as_deref() { + _ if gloas => PayloadValidity::NotRequired, None => PayloadValidity::NotRequired, Some("VALID") => PayloadValidity::Validated, Some("NOT_VALIDATED") => PayloadValidity::Optimistic, @@ -328,6 +339,15 @@ fn build_store( ) .map_err(|err| format!("importing {}: {err:?}", entry.block))?; deliver_payload(&mut store, case, entry, config)?; + if gloas && let Some(status) = entry.payload_status.as_deref() { + let status = match status { + "VALID" => PayloadStatusEnum::Valid, + "NOT_VALIDATED" => PayloadStatusEnum::Syncing, + "INVALIDATED" => PayloadStatusEnum::Invalid, + other => return Err(format!("unknown payload_status {other}")), + }; + store.insert_beacon_block_payload_status(root, block_slot, status); + } } if let Some(finalized) = &meta.finalized_checkpoint { diff --git a/crates/common/types/src/beacon/containers/mod.rs b/crates/common/types/src/beacon/containers/mod.rs index 89f466e73..e2c8b6275 100644 --- a/crates/common/types/src/beacon/containers/mod.rs +++ b/crates/common/types/src/beacon/containers/mod.rs @@ -1061,13 +1061,26 @@ impl BeaconState { } } +/// The one committee `bits` names, or `None` for none or several. +fn named_committee(bits: &electra::CommitteeBits) -> Option { + let mut named = (0..bits.len()).filter(|&index| bits.get(index).unwrap_or(false)); + let first = named.next()?; + // A second named committee disqualifies the aggregate outright. + match named.next() { + None => Some(first as CommitteeIndex), + Some(_) => None, + } +} + /// An aggregate attestation with the proof its aggregator was selected, in /// whichever fork's shape it currently has. /// -/// Two variants, not one per fork, for the reason +/// Three variants, not one per fork, for the reason /// [`SignedBeaconBlock::Fulu`] wraps electra's block: every fork through deneb /// shares [`phase0::SignedAggregateAndProof`] outright, and fulu shares -/// electra's the same way. +/// electra's the same way. Gloas has its own: same bytes on the wire, but its +/// `Attestation` is a progressive container (EIP-7688), so the +/// `hash_tree_root` the aggregator's signature covers differs. /// /// Here rather than beside the gossip decode in `ethlambda-p2p`, where it was /// first declared, because the gossip path no longer ends at that decode: an @@ -1083,6 +1096,7 @@ impl BeaconState { pub enum SignedAggregateAndProof { Phase0(phase0::SignedAggregateAndProof), Electra(electra::SignedAggregateAndProof), + Gloas(gloas::SignedAggregateAndProof), } impl SignedAggregateAndProof { @@ -1091,6 +1105,7 @@ impl SignedAggregateAndProof { match self { Self::Phase0(signed) => signed.message.aggregator_index, Self::Electra(signed) => signed.message.aggregator_index, + Self::Gloas(signed) => signed.message.aggregator_index, } } @@ -1099,6 +1114,7 @@ impl SignedAggregateAndProof { match self { Self::Phase0(signed) => signed.message.aggregate.data.slot, Self::Electra(signed) => signed.message.aggregate.data.slot, + Self::Gloas(signed) => signed.message.aggregate.data.slot, } } @@ -1107,6 +1123,7 @@ impl SignedAggregateAndProof { match self { Self::Phase0(signed) => signed.message.aggregate.data, Self::Electra(signed) => signed.message.aggregate.data, + Self::Gloas(signed) => signed.message.aggregate.data, } } @@ -1122,6 +1139,7 @@ impl SignedAggregateAndProof { match self { Self::Phase0(signed) => signed.message.selection_proof, Self::Electra(signed) => signed.message.selection_proof, + Self::Gloas(signed) => signed.message.selection_proof, } } @@ -1130,6 +1148,7 @@ impl SignedAggregateAndProof { match self { Self::Phase0(signed) => signed.signature, Self::Electra(signed) => signed.signature, + Self::Gloas(signed) => signed.signature, } } @@ -1152,16 +1171,9 @@ impl SignedAggregateAndProof { pub fn committee_index(&self) -> Option { match self { Self::Phase0(signed) => Some(signed.message.aggregate.data.index), - Self::Electra(signed) => { - let bits = &signed.message.aggregate.committee_bits; - let mut named = (0..bits.len()).filter(|&index| bits.get(index).unwrap_or(false)); - let first = named.next()?; - // A second named committee disqualifies the aggregate outright. - match named.next() { - None => Some(first as CommitteeIndex), - Some(_) => None, - } - } + // Gloas reuses electra's `CommitteeBits` outright. + Self::Electra(signed) => named_committee(&signed.message.aggregate.committee_bits), + Self::Gloas(signed) => named_committee(&signed.message.aggregate.committee_bits), } } @@ -1175,6 +1187,7 @@ impl SignedAggregateAndProof { match self { Self::Phase0(signed) => signed.message.aggregate.aggregation_bits.count_ones(), Self::Electra(signed) => signed.message.aggregate.aggregation_bits.count_ones(), + Self::Gloas(signed) => signed.message.aggregate.aggregation_bits.count_ones(), } } @@ -1198,6 +1211,12 @@ impl SignedAggregateAndProof { .map(|i| bits.get(i).unwrap_or(false)) .collect() } + Self::Gloas(signed) => { + let bits = &signed.message.aggregate.aggregation_bits; + (0..bits.len()) + .map(|i| bits.get(i).unwrap_or(false)) + .collect() + } } } } diff --git a/crates/net/p2p/src/beacon/decode.rs b/crates/net/p2p/src/beacon/decode.rs index a5ca1d6c7..ead92e2a7 100644 --- a/crates/net/p2p/src/beacon/decode.rs +++ b/crates/net/p2p/src/beacon/decode.rs @@ -26,7 +26,7 @@ use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::{ - DataColumnSidecar, SignedBeaconBlock, altair, capella, electra, phase0, shared, + DataColumnSidecar, SignedBeaconBlock, altair, capella, electra, gloas, phase0, shared, }; use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::preset; @@ -46,11 +46,12 @@ use super::topics; pub use ethlambda_types::beacon::containers::SignedAggregateAndProof; /// Slashing evidence, in whichever shape the slot's fork gives it. Electra -/// widened `IndexedAttestation`'s committee bound. +/// widened `IndexedAttestation`'s committee bound; gloas made it progressive. #[derive(Debug, Clone, PartialEq)] pub enum AttesterSlashing { Phase0(phase0::AttesterSlashing), Electra(electra::AttesterSlashing), + Gloas(gloas::AttesterSlashing), } /// A decoded gossip payload, one variant per subscribed topic. @@ -230,17 +231,16 @@ pub fn decode_aggregate_and_proof( electra::SignedAggregateAndProof::from_ssz_bytes(bytes) .map(SignedAggregateAndProof::Electra) } + // Electra's bytes, but a progressive `Attestation` (EIP-7688), so the + // root the aggregator signs differs; `data.index` is the payload flag. + ForkName::Gloas => gloas::SignedAggregateAndProof::from_ssz_bytes(bytes) + .map(SignedAggregateAndProof::Gloas), ForkName::Phase0 | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb => phase0::SignedAggregateAndProof::from_ssz_bytes(bytes) .map(SignedAggregateAndProof::Phase0), - // Gloas's own `AggregateAndProof` has no modeled variant here yet - // (its attestation is progressive-list-shaped, EIP-7549 continued), - // so this is refused rather than mis-decoded as electra's. - // `UnsupportedFork`, not `Ssz`: the sender did nothing wrong. - ForkName::Gloas => return Err(DecodeError::UnsupportedFork), ForkName::Lean => { unreachable!("fork_at_slot never returns Lean: it is absent from ForkName::ALL") } @@ -284,7 +284,9 @@ impl Attestation { /// subscribed digest was computed at. pub fn decode_attestation(fork: ForkName, bytes: &[u8]) -> Result { match fork { - ForkName::Electra | ForkName::Fulu => { + // Gloas's `SingleAttestation` has electra's bytes; its `data.index` + // carries the payload flag, which the gossip rules read. + ForkName::Electra | ForkName::Fulu | ForkName::Gloas => { electra::SingleAttestation::from_ssz_bytes(bytes).map(Attestation::Electra) } ForkName::Phase0 @@ -292,11 +294,6 @@ pub fn decode_attestation(fork: ForkName, bytes: &[u8]) -> Result phase0::Attestation::from_ssz_bytes(bytes).map(Attestation::Phase0), - // Gloas's `SingleAttestation` has the same bytes as electra's, but its - // `data.index` carries the payload-availability signal instead of - // being zero, so electra's gossip rules would reject an honest vote. - // `UnsupportedFork`, not `Ssz`: the sender did nothing wrong. - ForkName::Gloas => return Err(DecodeError::UnsupportedFork), ForkName::Lean => { unreachable!("a beacon topic's fork is never Lean: it is absent from ForkName::ALL") } @@ -326,6 +323,9 @@ pub fn decode_gossip( ForkName::Electra | ForkName::Fulu => { electra::AttesterSlashing::from_ssz_bytes(bytes).map(AttesterSlashing::Electra) } + ForkName::Gloas => { + gloas::AttesterSlashing::from_ssz_bytes(bytes).map(AttesterSlashing::Gloas) + } ForkName::Phase0 | ForkName::Altair | ForkName::Bellatrix @@ -333,9 +333,6 @@ pub fn decode_gossip( | ForkName::Deneb => { phase0::AttesterSlashing::from_ssz_bytes(bytes).map(AttesterSlashing::Phase0) } - // Gloas's own `AttesterSlashing` has no modeled variant here - // yet; see `decode_aggregate_and_proof`'s matching arm. - ForkName::Gloas => return Err(DecodeError::UnsupportedFork), ForkName::Lean => { unreachable!("fork_at_slot never returns Lean: it is absent from ForkName::ALL") } @@ -605,14 +602,13 @@ mod tests { } #[test] - fn a_subnet_attestation_at_gloas_is_unsupported_rather_than_malformed() { - // Gloas reads `data.index` as the payload-availability signal, so - // decoding these bytes with electra's rules would reject honest - // votes. The bytes are valid, so the refusal must not be `Ssz`. - let single = single_attestation(slot_of(10)).to_ssz(); + fn a_subnet_attestation_at_gloas_decodes_as_electras_shape() { + // Gloas keeps electra's bytes and reads `data.index` as the payload + // flag; the gossip rules, not the decoder, interpret it. + let single = single_attestation(slot_of(10)); assert_eq!( - decode_attestation(ForkName::Gloas, &single), - Err(DecodeError::UnsupportedFork) + decode_attestation(ForkName::Gloas, &single.to_ssz()), + Ok(Attestation::Electra(single)) ); } diff --git a/crates/net/p2p/src/gossipsub/handler.rs b/crates/net/p2p/src/gossipsub/handler.rs index f2cc0b163..e6c7615f3 100644 --- a/crates/net/p2p/src/gossipsub/handler.rs +++ b/crates/net/p2p/src/gossipsub/handler.rs @@ -693,6 +693,9 @@ pub async fn publish_beacon_aggregate( ethlambda_types::beacon::containers::SignedAggregateAndProof::Electra(signed) => { signed.to_ssz() } + ethlambda_types::beacon::containers::SignedAggregateAndProof::Gloas(signed) => { + signed.to_ssz() + } }; server.swarm_handle.publish(topic, compress_message(&ssz)); debug!(%slot, aggregator, "Published aggregate to gossipsub"); @@ -1100,11 +1103,11 @@ mod tests { )); } - /// Gloas's aggregate has no modeled container here, and an honest gloas - /// peer sends exactly this once the clock reaches the fork, so the decode - /// error must not become a `Reject`, which would penalize it. + /// Gloas repurposes `data.index` as the payload flag, so an honest `1` + /// passes the index rule electra would reject it on; it is turned away + /// later, here by the empty `committee_bits` of the test message. #[tokio::test] - async fn a_gloas_aggregate_is_ignored_as_an_unsupported_fork() { + async fn a_gloas_aggregate_with_a_payload_flag_passes_the_index_rule() { let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 0); let server = unconnected_beacon_server(config, 0).await; let wire = server @@ -1115,7 +1118,23 @@ mod tests { assert!(matches!( triage_aggregate(&server, wire, &payload, Instant::now()), - Dispatch::Report(Outcome::Ignore(IgnoreReason::UnsupportedFork)) + Dispatch::Report(Outcome::Reject(RejectReason::CommitteeBits)) + )); + } + + #[tokio::test] + async fn a_gloas_aggregate_with_a_data_index_above_one_is_rejected() { + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 0); + let server = unconnected_beacon_server(config, 0).await; + let wire = server + .wire + .beacon() + .expect("a beacon server has a beacon wire"); + let payload = electra_aggregate(4, 2).to_ssz(); + + assert!(matches!( + triage_aggregate(&server, wire, &payload, Instant::now()), + Dispatch::Report(Outcome::Reject(RejectReason::DataIndexOutOfRange)) )); } @@ -1172,21 +1191,28 @@ mod tests { } /// A gloas `SingleAttestation` has electra's bytes, but its `data.index` - /// is the payload flag, so an honest `1` must not reach electra's rules - /// and be rejected as a nonzero index. + /// is the payload flag: an honest `1` must not be rejected as a nonzero + /// index, while a value past the flag's range is. #[tokio::test] - async fn a_gloas_attestation_is_ignored_as_an_unsupported_fork() { + async fn a_gloas_attestation_is_judged_on_the_payload_flag_range() { let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 0); let server = unconnected_beacon_server(config, 0).await; let wire = server .wire .beacon() .expect("a beacon server has a beacon wire"); - let payload = electra_single_attestation(4, 1).to_ssz(); + let flagged = electra_single_attestation(4, 1).to_ssz(); + assert!(!matches!( + triage_attestation(&server, wire.fork, &flagged, 0), + Dispatch::Report(Outcome::Reject( + RejectReason::NonZeroDataIndex | RejectReason::DataIndexOutOfRange + )) + )); + let out_of_range = electra_single_attestation(4, 2).to_ssz(); assert!(matches!( - triage_attestation(&server, wire.fork, &payload, 0), - Dispatch::Report(Outcome::Ignore(IgnoreReason::UnsupportedFork)) + triage_attestation(&server, wire.fork, &out_of_range, 0), + Dispatch::Report(Outcome::Reject(RejectReason::DataIndexOutOfRange)) )); } diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index dfa012fae..465d38c8a 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -23,7 +23,9 @@ use ethlambda_types::{ gloas, }, fork::ForkName, - fork_choice::{BlockPayloadLink, LatestMessage, PayloadStatusV1, PowBlock}, + fork_choice::{ + BlockPayloadLink, LatestMessage, PayloadStatusEnum, PayloadStatusV1, PowBlock, + }, preset::{PTC_SIZE, Preset, SLOTS_PER_EPOCH}, primitives::ExecutionBlockHash, }, @@ -758,6 +760,13 @@ pub(crate) struct BeaconScratch { /// forkchoiceUpdated reads it once per head move for the whole life of the /// process. pub(crate) el_block_hashes: HashMap, + /// Gloas: the execution client's verdict on each beacon block's payload, + /// against the block's slot for pruning. Keyed by beacon root because a + /// gloas block's payload hash lives in its bid and envelope rather than in + /// `el_block_hashes`. Read by the attestation gossip rules as the + /// specification's `block_payload_statuses`; an absent root reads as + /// `NOT_VALIDATED`. + pub(crate) block_payload_statuses: HashMap, } /// Encode a LiveChain key (slot, root) to bytes. @@ -3619,6 +3628,45 @@ impl Store { .cloned() } + /// The execution client's verdict on a gloas block's payload, as the + /// specification's `block_payload_statuses.get(root, + /// PAYLOAD_STATUS_NOT_VALIDATED)`: a root with no recorded verdict reads as + /// `Syncing`, which is `NOT_VALIDATED`. + pub fn beacon_block_payload_status(&self, root: H256) -> PayloadStatusEnum { + self.beacon + .lock() + .unwrap() + .block_payload_statuses + .get(&root) + .map_or(PayloadStatusEnum::Syncing, |(_slot, status)| *status) + } + + /// Records the execution client's verdict on `root`'s payload, against the + /// block's slot. A later verdict replaces an earlier one, since an + /// optimistic payload is resolved one way or the other. + pub fn insert_beacon_block_payload_status( + &mut self, + root: H256, + slot: u64, + status: PayloadStatusEnum, + ) { + self.beacon + .lock() + .unwrap() + .block_payload_statuses + .insert(root, (slot, status)); + } + + /// Drops payload verdicts strictly below `finalized_slot`, which no + /// attestation rule can ask about any more. + pub fn prune_beacon_block_payload_statuses(&mut self, finalized_slot: u64) { + self.beacon + .lock() + .unwrap() + .block_payload_statuses + .retain(|_root, (slot, _status)| *slot >= finalized_slot); + } + /// Records an execution client's answer for a payload. The fixture format /// allows the same payload's status to be updated several times over a /// case, so this overwrites rather than preserving a first answer. diff --git a/docs/beacon_wire.md b/docs/beacon_wire.md index a770211d2..f18175794 100644 --- a/docs/beacon_wire.md +++ b/docs/beacon_wire.md @@ -164,15 +164,16 @@ reaches it at all, on any outcome: verifying and relaying it is the whole of what this node owes the topic (see above), so there is nothing further for the chain actor to do with one. -The rules above are fulu's, except that blocks and data columns also have -gloas rules. An aggregate, attester slashing or attestation from a fork this -build has no rules for, gloas today, is IGNOREd with the reason `unsupported_fork` and is never scored -against the peer: the decoders answer `DecodeError::UnsupportedFork`, not `Ssz`, -for a gloas aggregate, attester slashing or subnet attestation, or for a block -at a gloas slot that fails to decode, since an honest -gloas peer sends exactly those once this node's clock reaches the fork. A gloas -subnet attestation has the bytes of an electra one, but its `data.index` is the -payload flag, so decoding it under electra's rules would reject honest votes. +The rules above are fulu's, except that blocks, data columns, aggregates and +subnet attestations also have gloas rules. A gloas aggregate has electra's bytes +but its own progressive `Attestation`, so it decodes to its own +`SignedAggregateAndProof::Gloas` (the signed root differs); a gloas subnet +attestation is electra's `SingleAttestation` unchanged. In both, `data.index` is +the payload flag (0 or 1), and a vote for the full payload (1) is IGNOREd until +the block's envelope has been seen and its payload validated +(`verify_attestation_payload_status`). A block at a gloas slot that fails to +decode is IGNOREd as `unsupported_fork` rather than scored against the peer, +since the container this build models may trail the network's. The remaining five global topics are decoded, logged at `debug`, and IGNOREd, since nothing consumes them; an undecodable payload on any topic is REJECTed. diff --git a/docs/metrics.md b/docs/metrics.md index b8447663b..379d14f04 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -374,21 +374,20 @@ cache. The two topics add reasons the others do not, all `reason` label values on `lean_beacon_gossip_validation_total`: `outside_epoch_window`, `covered_bits` (aggregate only), `unknown_block`, `state_unavailable`, -`finalized_not_ancestor`, `ancestry_unknown` on the ignore side; +`finalized_not_ancestor`, `ancestry_unknown`, `payload_envelope_unseen` and +`payload_optimistic` (gloas only) on the ignore side; `epoch_mismatch`, `no_participants` (aggregate only), `non_zero_data_index`, `committee_bits` (aggregate only), `committee_index`, `bits_length` (aggregate only), `not_aggregator` (aggregate only), `not_in_committee`, `unknown_validator`, `selection_proof` (aggregate only), `aggregator_signature` (aggregate only), `aggregate_signature` (aggregate -only), `target_not_ancestor`, `wrong_subnet` (attestation only) on the reject -side. `already_seen`, `overloaded` and `unsupported_fork` are shared with the +only), `target_not_ancestor`, `wrong_subnet` (attestation only), and gloas's +`data_index_out_of_range`, `same_slot_payload_flag` and `payload_invalid` on the +reject side. `already_seen`, `overloaded` and `unsupported_fork` are shared with the other topics: `unsupported_fork` is an ignore reason for a message of a fork this -build has no gossip rules for (gloas today), on the block, data column, -aggregate, attestation and attester-slashing topics, so an honest peer past the -fork epoch is not scored as a bad decoder. The same label value appears as -`result` on `lean_beacon_gossip_messages_total` for the topics without gloas -rules. Gloas blocks and data columns are validated, so they carry their own -verdict reasons instead. +build has no gossip rules for, so an honest peer past the fork epoch is not +scored as a bad decoder. Gloas blocks, data columns, aggregates and attestations +are validated, so they carry their own verdict reasons instead. Two permit pools bound the blocking-thread half of validation: `gossip_validation_permits` for blocks and columns, From fe3a1fec0014ef81362a5e0fabd4dda51f04afbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 02:46:58 -0300 Subject: [PATCH 050/132] fix(beacon): bound gloas aggregation bits and judge pre-gloas payload votes Gloas aggregation bits are an unbounded progressive bitlist, and the seen-set verdict expands them into a Vec before any other check, so a large message forced a large allocation. A gloas aggregate longer than the electra type bound is now rejected first, reading the length without expanding it. An honest gloas-slot vote for the last pre-gloas block (FULL under the boundary rule) was ignored as optimistic because block_payload_statuses never holds a pre-gloas root. Such a root now reads its own block validity. Also drops the now-unreachable UnsupportedFork arms on the aggregate and attestation topics, fixes stale docs, tightens an assertion, and unit-tests the payload-status branches and the status map. --- .../src/beacon/fork_choice.rs | 53 ++++++++++- .../src/beacon/gossip/aggregate.rs | 66 +++++++++++++ .../state_transition/src/beacon/gossip/mod.rs | 93 ++++++++++++++++++- .../common/types/src/beacon/containers/mod.rs | 12 +++ crates/net/p2p/src/gossipsub/handler.rs | 21 +---- crates/storage/src/store.rs | 33 +++++++ 6 files changed, 253 insertions(+), 25 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 75dbe04d5..03b473d9a 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -2748,6 +2748,27 @@ pub fn is_payload_verified(store: &Store, root: Root) -> bool { store.has_verified_payload(&root) || is_known_pre_gloas_block(store, root) } +/// The execution verdict gloas's attestation gossip rules read for `root`'s +/// payload, as the specification's `block_payload_statuses`. +/// +/// A pre-gloas block's payload ran inside the block, so its verdict is the +/// block's own: `NOT_VALIDATED` while it sits in the optimistic set, `VALID` +/// otherwise (an invalidated block is dropped from the store, so is never +/// asked about). This keeps an honest gloas-slot vote for the last pre-gloas +/// block, which this node's boundary rule treats as FULL, from being ignored as +/// optimistic. A gloas root reads [`Store::beacon_block_payload_status`]. +pub fn block_payload_status(store: &Store, root: Root) -> PayloadStatusEnum { + if is_known_pre_gloas_block(store, root) { + if store.is_beacon_optimistic(root) { + PayloadStatusEnum::Syncing + } else { + PayloadStatusEnum::Valid + } + } else { + store.beacon_block_payload_status(root) + } +} + /// `payload_timeliness` (gloas `fork-choice.md`): whether `root`'s payload is /// considered `timely` (or not, when `timely` is `false`), taking into /// account both local availability and the payload timeliness committee's @@ -4979,10 +5000,11 @@ pub fn on_block_attestation( /// records it against `attesting_indices`, resolved by the caller's gossip /// validation rather than recomputed here. /// -/// The rules are those of the vote's own fork, read off `data.slot`: gloas -/// keeps electra's aggregate container, so the container cannot say, and under +/// The rules are those of the vote's own fork, read off `data.slot`: under /// gloas `data.index` is the payload flag that [`update_latest_messages`] -/// records. +/// records, and the electra-shaped callers that reach here (a gloas aggregate +/// has its own container but yields the same `AttestationData`) do not say +/// which fork they came from. /// /// `is_from_block` is fixed at `false`, matching [`on_attestation`]'s call for /// this topic: an aggregate here is by definition not carried in a block, so @@ -7186,6 +7208,31 @@ mod tests { ); } + #[test] + fn a_pre_gloas_root_reads_its_own_validity_as_a_payload_status() { + let mut store = empty_store(); + let root = Root::repeat_byte(0xa1); + store + .insert_signed_block(root, fulu_block(1, Root::ZERO)) + .unwrap(); + assert_eq!(block_payload_status(&store, root), PayloadStatusEnum::Valid); + + store.insert_beacon_optimistic_root(root, 1); + assert_eq!( + block_payload_status(&store, root), + PayloadStatusEnum::Syncing + ); + } + + #[test] + fn a_root_with_no_recorded_verdict_is_not_validated() { + let store = empty_store(); + assert!( + block_payload_status(&store, Root::repeat_byte(0xee)).is_not_validated(), + "the specification's default for a root absent from the map" + ); + } + #[test] fn is_payload_verified_is_false_for_an_unknown_root() { let store = empty_store(); diff --git a/crates/blockchain/state_transition/src/beacon/gossip/aggregate.rs b/crates/blockchain/state_transition/src/beacon/gossip/aggregate.rs index 188a01344..7578ce0d4 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/aggregate.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/aggregate.rs @@ -71,11 +71,17 @@ use crate::beacon::helpers::math::bytes_to_uint64; use crate::beacon::helpers::misc::{ compute_epoch_at_slot, compute_signing_root, compute_start_slot_at_epoch, }; +use crate::beacon::preset; use crate::beacon::primitives::{ BlsSignature, CommitteeIndex, Epoch, HashTreeRoot as _, Root, ValidatorIndex, }; use ethlambda_storage::CacheKey; +/// The most aggregation bits a gloas aggregate may carry: electra's type +/// bound (every committee of a slot at full size). Gloas's progressive bitlist +/// has no bound of its own, so the gossip rules enforce this one. +const MAX_AGGREGATION_BITS: usize = preset::MAX_VALIDATORS_PER_SLOT; + /// One committee's worth of aggregation bits, packed a `u64` at a time. /// /// The specification's `Seen.aggregate_data_roots` stores each accepted @@ -261,6 +267,18 @@ pub fn cheap_checks( let config = store.config(); let data = aggregate.data(); + // [REJECT] The bitfield fits the type bound electra gave it. Gloas's + // `AggregationBits` is a progressive bitlist with no bound of its own, and + // everything below (the seen-set verdict first) expands the bits, so an + // oversized one is turned away before anything reads them. The length is + // read without expanding the list. Earlier forks' bitlists carry this bound + // in their type. + if matches!(aggregate, SignedAggregateAndProof::Gloas(_)) + && aggregate.aggregation_bits_len() > MAX_AGGREGATION_BITS + { + return Err(Outcome::Reject(RejectReason::BitsLength)); + } + match aggregate { // Phase0 has no `committee_bits` and carries the real committee in // `data.index`, so there is nothing to require of it. @@ -554,6 +572,54 @@ mod tests { } } + #[test] + fn a_gloas_aggregate_with_an_oversized_bitfield_is_rejected_first() { + use crate::beacon::containers::gloas; + + let store = store(0); + let now_ms = slot_start_ms(&store, 1); + let mut committee_bits = electra::CommitteeBits::default(); + committee_bits + .set(0, true) + .expect("within MAX_COMMITTEES_PER_SLOT"); + let aggregate_with_len = |len: usize| { + SignedAggregateAndProof::Gloas(gloas::SignedAggregateAndProof { + message: gloas::AggregateAndProof { + aggregator_index: 0, + aggregate: gloas::Attestation { + aggregation_bits: gloas::AggregationBits::with_length(len), + data: shared_data(0), + signature: Default::default(), + committee_bits: committee_bits.clone(), + }, + selection_proof: Default::default(), + }, + signature: Default::default(), + }) + }; + + assert_eq!( + cheap_checks( + &seen_aggregates(), + &store, + &aggregate_with_len(MAX_AGGREGATION_BITS + 1), + now_ms + ), + Err(Outcome::Reject(RejectReason::BitsLength)) + ); + // At the bound it is no longer this check's to reject; it goes on to + // the participants rule, which an empty bitfield fails. + assert_eq!( + cheap_checks( + &seen_aggregates(), + &store, + &aggregate_with_len(MAX_AGGREGATION_BITS), + now_ms + ), + Err(Outcome::Reject(RejectReason::NoParticipants)) + ); + } + // -- PackedBits / superset semantics -------------------------------- #[test] diff --git a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs index 671fba423..5ee5289cc 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs @@ -27,7 +27,7 @@ use crate::beacon::config::Config; use crate::beacon::constants::MAXIMUM_GOSSIP_CLOCK_DISPARITY; use crate::beacon::containers::{AttestationData, BeaconState}; use crate::beacon::fork::ForkName; -use crate::beacon::fork_choice::{self, Store}; +use crate::beacon::fork_choice::{self, PayloadStatusEnum, Store}; use crate::beacon::helpers::accessors::get_block_root_at_slot; use crate::beacon::helpers::misc::{compute_epoch_at_slot, compute_start_slot_at_epoch}; use crate::beacon::lean_boundary::lean_fork_unreachable; @@ -405,8 +405,9 @@ pub(crate) fn is_current_or_previous_epoch(config: &Config, epoch: Epoch, now_ms /// Whether `slot` falls in gloas. /// -/// Gloas keeps electra's aggregate and subnet-attestation containers, so the -/// message's shape cannot say which rules apply; its slot's fork does. +/// A gloas subnet attestation is the same `SingleAttestation` as electra's, so +/// the message's shape cannot say which rules apply; its slot's fork does. +/// (A gloas aggregate has its own container, which says so itself.) pub(crate) fn is_gloas_slot(config: &Config, slot: Slot) -> bool { match config.fork_at_epoch(compute_epoch_at_slot(slot)) { ForkName::Gloas => true, @@ -436,6 +437,23 @@ pub(crate) fn verify_attestation_payload_status( // The caller has already checked the block is known. return Err(Outcome::Ignore(IgnoreReason::UnknownBlock)); }; + payload_vote_verdict( + block_slot, + data, + || fork_choice::is_payload_verified(store, block_root), + || fork_choice::block_payload_status(store, block_root), + ) +} + +/// The branches of [`verify_attestation_payload_status`], over the facts it +/// reads. The envelope and status lookups are lazy: only a vote for the full +/// payload needs either. +fn payload_vote_verdict( + block_slot: Slot, + data: &AttestationData, + is_payload_verified: impl FnOnce() -> bool, + payload_status: impl FnOnce() -> PayloadStatusEnum, +) -> Result<(), Outcome> { // [REJECT] For same-slot attestations, the payload cannot yet be present. if block_slot == data.slot && data.index != 0 { return Err(Outcome::Reject(RejectReason::SameSlotPayloadFlag)); @@ -444,10 +462,10 @@ pub(crate) fn verify_attestation_payload_status( return Ok(()); } // [IGNORE] The envelope has been seen and verified. - if !fork_choice::is_payload_verified(store, block_root) { + if !is_payload_verified() { return Err(Outcome::Ignore(IgnoreReason::PayloadEnvelopeUnseen)); } - let status = store.beacon_block_payload_status(block_root); + let status = payload_status(); // [IGNORE] The attested payload is optimistic. if status.is_not_validated() { return Err(Outcome::Ignore(IgnoreReason::PayloadOptimistic)); @@ -540,6 +558,71 @@ mod tests { NonZeroUsize::new(n).expect("non-zero") } + fn vote(slot: Slot, index: u64) -> AttestationData { + AttestationData { + slot, + index, + ..Default::default() + } + } + + fn never() -> bool { + unreachable!("a vote that does not claim the full payload reads no envelope") + } + + fn no_status() -> PayloadStatusEnum { + unreachable!("a vote that does not claim the full payload reads no status") + } + + #[test] + fn a_same_slot_vote_for_the_payload_is_rejected() { + assert_eq!( + payload_vote_verdict(5, &vote(5, 1), never, no_status), + Err(Outcome::Reject(RejectReason::SameSlotPayloadFlag)) + ); + } + + #[test] + fn a_vote_without_the_payload_flag_reads_no_payload_state() { + assert_eq!( + payload_vote_verdict(5, &vote(5, 0), never, no_status), + Ok(()) + ); + assert_eq!( + payload_vote_verdict(4, &vote(5, 0), never, no_status), + Ok(()) + ); + } + + #[test] + fn a_full_payload_vote_without_a_verified_envelope_is_ignored() { + assert_eq!( + payload_vote_verdict(4, &vote(5, 1), || false, no_status), + Err(Outcome::Ignore(IgnoreReason::PayloadEnvelopeUnseen)) + ); + } + + #[test] + fn a_full_payload_vote_judges_the_execution_verdict() { + let verdict = |status| payload_vote_verdict(4, &vote(5, 1), || true, move || status); + assert_eq!(verdict(PayloadStatusEnum::Valid), Ok(())); + for status in [PayloadStatusEnum::Syncing, PayloadStatusEnum::Accepted] { + assert_eq!( + verdict(status), + Err(Outcome::Ignore(IgnoreReason::PayloadOptimistic)) + ); + } + for status in [ + PayloadStatusEnum::Invalid, + PayloadStatusEnum::InvalidBlockHash, + ] { + assert_eq!( + verdict(status), + Err(Outcome::Reject(RejectReason::PayloadInvalid)) + ); + } + } + #[test] fn a_slot_is_future_only_past_the_clock_disparity() { let config = Config { diff --git a/crates/common/types/src/beacon/containers/mod.rs b/crates/common/types/src/beacon/containers/mod.rs index e2c8b6275..1e8b09e1c 100644 --- a/crates/common/types/src/beacon/containers/mod.rs +++ b/crates/common/types/src/beacon/containers/mod.rs @@ -1177,6 +1177,18 @@ impl SignedAggregateAndProof { } } + /// The length of the aggregation bitfield, read without expanding it. + /// + /// For callers that must bound the bitfield before anything iterates it: + /// gloas's is unbounded by its type. + pub fn aggregation_bits_len(&self) -> usize { + match self { + Self::Phase0(signed) => signed.message.aggregate.aggregation_bits.len(), + Self::Electra(signed) => signed.message.aggregate.aggregation_bits.len(), + Self::Gloas(signed) => signed.message.aggregate.aggregation_bits.len(), + } + } + /// How many attesters the aggregate covers. /// /// Counts set bits rather than reporting the bitfield's length: from diff --git a/crates/net/p2p/src/gossipsub/handler.rs b/crates/net/p2p/src/gossipsub/handler.rs index e6c7615f3..1328df4a1 100644 --- a/crates/net/p2p/src/gossipsub/handler.rs +++ b/crates/net/p2p/src/gossipsub/handler.rs @@ -372,13 +372,6 @@ fn triage_aggregate( const KIND: &str = beacon_topics::BEACON_AGGREGATE_AND_PROOF; let aggregate = match beacon_decode::decode_aggregate_and_proof(&wire.config, payload) { Ok(aggregate) => aggregate, - // `UnsupportedFork` is not the sender's fault (an honest gloas peer - // sends exactly this once this node's own clock reaches gloas), so it - // must not score like every other decode failure does. - Err(beacon_decode::DecodeError::UnsupportedFork) => { - metrics::inc_beacon_gossip(KIND, "unsupported_fork"); - return Dispatch::Report(Outcome::Ignore(IgnoreReason::UnsupportedFork)); - } Err(err) => { metrics::inc_beacon_gossip(KIND, "decode_failed"); debug!(kind = KIND, %err, bytes = payload.len(), "Beacon gossip decode failed"); @@ -449,12 +442,6 @@ fn triage_attestation( const KIND: &str = beacon_topics::BEACON_ATTESTATION_KIND; let attestation = match beacon_decode::decode_attestation(fork, payload) { Ok(attestation) => attestation, - // As on the aggregate topic: an honest peer on a fork this node has no - // attestation rules for must not be scored as a bad decoder. - Err(beacon_decode::DecodeError::UnsupportedFork) => { - metrics::inc_beacon_gossip(KIND, "unsupported_fork"); - return Dispatch::Report(Outcome::Ignore(IgnoreReason::UnsupportedFork)); - } Err(err) => { metrics::inc_beacon_gossip(KIND, "decode_failed"); debug!(kind = KIND, %err, bytes = payload.len(), "Beacon gossip decode failed"); @@ -1203,11 +1190,11 @@ mod tests { .expect("a beacon server has a beacon wire"); let flagged = electra_single_attestation(4, 1).to_ssz(); - assert!(!matches!( + // Past the index rule, and turned away by the clock instead: slot 4 is + // long outside the epoch window of a mainnet genesis. + assert!(matches!( triage_attestation(&server, wire.fork, &flagged, 0), - Dispatch::Report(Outcome::Reject( - RejectReason::NonZeroDataIndex | RejectReason::DataIndexOutOfRange - )) + Dispatch::Report(Outcome::Ignore(IgnoreReason::OutsideEpochWindow)) )); let out_of_range = electra_single_attestation(4, 2).to_ssz(); assert!(matches!( diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 465d38c8a..683e09e73 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -3632,6 +3632,13 @@ impl Store { /// specification's `block_payload_statuses.get(root, /// PAYLOAD_STATUS_NOT_VALIDATED)`: a root with no recorded verdict reads as /// `Syncing`, which is `NOT_VALIDATED`. + /// + /// Gloas roots only: a pre-gloas root's verdict is its block's own, which + /// `fork_choice::block_payload_status` derives. Envelope import is what + /// records a gloas root's verdict; nothing else writes it. The map is + /// scratch, so after a restart the verdicts of gloas blocks imported before + /// it are absent and votes for their full payload are ignored until those + /// blocks age out of the attestation window. A known, bounded gap. pub fn beacon_block_payload_status(&self, root: H256) -> PayloadStatusEnum { self.beacon .lock() @@ -7651,6 +7658,32 @@ mod tests { assert!(store.is_beacon_optimistic(above)); } + #[test] + fn block_payload_statuses_default_to_not_validated_and_prune_below_finality() { + let mut store = Store::test_store(); + let below = H256::repeat_byte(1); + let at = H256::repeat_byte(2); + assert!( + store.beacon_block_payload_status(below).is_not_validated(), + "an absent root reads as the specification's default" + ); + + store.insert_beacon_block_payload_status(below, 4, PayloadStatusEnum::Valid); + store.insert_beacon_block_payload_status(at, 5, PayloadStatusEnum::Invalid); + assert_eq!( + store.beacon_block_payload_status(below), + PayloadStatusEnum::Valid + ); + + store.prune_beacon_block_payload_statuses(5); + + assert!(store.beacon_block_payload_status(below).is_not_validated()); + assert_eq!( + store.beacon_block_payload_status(at), + PayloadStatusEnum::Invalid + ); + } + #[test] fn el_block_hashes_prune_strictly_below_the_finalized_slot() { let mut store = Store::test_store(); From 9edbcdf665a9b2b8fec70b3ab637378fb070b800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 02:44:35 -0300 Subject: [PATCH 051/132] feat(engine): newPayloadV5 for gloas envelopes Gloas moves the execution payload out of the block into a revealed envelope, and the Amsterdam execution fork asks about it with engine_newPayloadV5 and an ExecutionPayloadV4 (V3's fields plus blockAccessList and slotNumber). Add the JSON view of that payload, the client call (with the six second timeout amsterdam.md gives the method), the capability entry, and a request builder that takes the payload, parent root and requests from the envelope and the versioned hashes from the block's bid, as verify_execution_payload_envelope does. Nothing calls the new pieces yet; wiring them into envelope import is a separate change. --- crates/blockchain/src/beacon_engine.rs | 172 ++++++++++++++++++++++++- crates/net/engine/src/client.rs | 65 +++++++++- crates/net/engine/src/lib.rs | 1 + crates/net/engine/src/types.rs | 142 ++++++++++++++++++-- crates/net/engine/tests/wire_smoke.rs | 58 +++++++++ 5 files changed, 420 insertions(+), 18 deletions(-) diff --git a/crates/blockchain/src/beacon_engine.rs b/crates/blockchain/src/beacon_engine.rs index 510edbb9a..03bcd9e61 100644 --- a/crates/blockchain/src/beacon_engine.rs +++ b/crates/blockchain/src/beacon_engine.rs @@ -23,6 +23,7 @@ use ethlambda_state_transition::beacon::fork_choice::PayloadValidity; use ethlambda_state_transition::beacon::primitives::{Bytes32, Root}; use ethlambda_state_transition::beacon::stf::deneb::kzg_commitment_to_versioned_hash; use ethlambda_state_transition::beacon::stf::electra::get_execution_requests_list; +use ethlambda_state_transition::beacon::stf::gloas::get_execution_requests_list as get_gloas_execution_requests_list; /// Everything `engine_newPayloadV4` takes, derived from one block. pub struct NewPayloadRequest<'a> { @@ -32,6 +33,63 @@ pub struct NewPayloadRequest<'a> { pub execution_requests: Vec>, } +/// Everything `engine_newPayloadV5` takes, derived from a gloas block and the +/// envelope that reveals its payload. +pub struct GloasNewPayloadRequest<'a> { + pub execution_payload: &'a containers::gloas::ExecutionPayload, + pub versioned_hashes: Vec, + pub parent_beacon_block_root: Root, + pub execution_requests: Vec>, +} + +/// The question to ask about `block`'s revealed payload, or `None` if `block` +/// is not a gloas block. +/// +/// `NewPayloadRequest` as `verify_execution_payload_envelope` builds it +/// (gloas `beacon-chain.md`): the payload and the requests come from the +/// envelope, the parent root is the envelope's own `parent_beacon_block_root`, +/// and the versioned hashes come from the BLOCK, off its bid's +/// `blob_kzg_commitments`, since the envelope does not carry commitments. +/// Requests go through gloas's own list builder, which also encodes the +/// builder deposit and exit request types. +/// +/// The fork is chosen by an explicit match: a fork before gloas carries its +/// payload in the block and is asked about with [`new_payload_request`] and +/// `engine_newPayloadV4`, and no fork after gloas exists yet to need a V6. +pub fn gloas_new_payload_request<'a>( + block: &SignedBeaconBlock, + envelope: &'a containers::gloas::SignedExecutionPayloadEnvelope, +) -> Option> { + let inner = match block { + SignedBeaconBlock::Gloas(inner) => inner, + SignedBeaconBlock::Phase0(_) + | SignedBeaconBlock::Altair(_) + | SignedBeaconBlock::Bellatrix(_) + | SignedBeaconBlock::Capella(_) + | SignedBeaconBlock::Deneb(_) + | SignedBeaconBlock::Electra(_) + | SignedBeaconBlock::Fulu(_) + | SignedBeaconBlock::Lean(_) => return None, + }; + + let versioned_hashes = inner + .message + .body + .signed_execution_payload_bid + .message + .blob_kzg_commitments + .iter() + .map(kzg_commitment_to_versioned_hash) + .collect(); + + Some(GloasNewPayloadRequest { + execution_payload: &envelope.message.payload, + versioned_hashes, + parent_beacon_block_root: envelope.message.parent_beacon_block_root, + execution_requests: get_gloas_execution_requests_list(&envelope.message.execution_requests), + }) +} + /// The question to ask about `block`, or `None` if there is nothing to ask. /// /// `None` for phase0 and altair, which predate the merge and carry no payload, @@ -47,8 +105,7 @@ pub struct NewPayloadRequest<'a> { /// (see `containers::gloas::SignedExecutionPayloadEnvelope`), so a gloas /// block carries nothing this function could ask about at all. The engine /// question gloas does ask, whether a payload is valid, belongs to that -/// envelope rather than to the block, and this follower does not receive -/// envelopes yet, so nothing here asks it. +/// envelope rather than to the block: see [`gloas_new_payload_request`]. pub fn new_payload_request(block: &SignedBeaconBlock) -> Option> { let inner = match block { SignedBeaconBlock::Electra(inner) | SignedBeaconBlock::Fulu(inner) => inner, @@ -129,6 +186,29 @@ pub async fn ask( Ok(Some(verdict(&status))) } +/// Asks the execution client about the payload `envelope` reveals for `block`, +/// with `engine_newPayloadV5`. +/// +/// `Ok(None)` when `block` is not a gloas block; otherwise as [`ask`]. +pub async fn ask_envelope( + client: &EngineClient, + block: &SignedBeaconBlock, + envelope: &containers::gloas::SignedExecutionPayloadEnvelope, +) -> Result, EngineError> { + let Some(request) = gloas_new_payload_request(block, envelope) else { + return Ok(None); + }; + let status = client + .new_payload_v5( + request.execution_payload, + &request.versioned_hashes, + request.parent_beacon_block_root, + &request.execution_requests, + ) + .await?; + Ok(Some(verdict(&status))) +} + #[cfg(test)] mod tests { use ethlambda_state_transition::beacon::preset; @@ -254,6 +334,94 @@ mod tests { ); } + /// A gloas envelope whose payload is otherwise zero. + fn gloas_envelope( + parent_beacon_block_root: Root, + ) -> containers::gloas::SignedExecutionPayloadEnvelope { + let payload = containers::gloas::ExecutionPayload { + parent_hash: ExecutionBlockHash::ZERO, + fee_recipient: ExecutionAddress::ZERO, + state_root: Bytes32::ZERO, + receipts_root: Bytes32::ZERO, + logs_bloom: containers::bellatrix::LogsBloom::try_from(vec![ + 0u8; + preset::BYTES_PER_LOGS_BLOOM + ]) + .expect("built at exactly BYTES_PER_LOGS_BLOOM"), + prev_randao: Bytes32::ZERO, + block_number: 0, + gas_limit: 0, + gas_used: 0, + timestamp: 0, + extra_data: Default::default(), + base_fee_per_gas: Uint256::ZERO, + block_hash: ExecutionBlockHash::ZERO, + transactions: Default::default(), + withdrawals: Default::default(), + blob_gas_used: 0, + excess_blob_gas: 0, + block_access_list: Default::default(), + slot_number: 5, + }; + containers::gloas::SignedExecutionPayloadEnvelope { + message: containers::gloas::ExecutionPayloadEnvelope { + payload, + execution_requests: Default::default(), + builder_index: 0, + beacon_block_root: Root::repeat_byte(1), + parent_beacon_block_root, + }, + signature: BlsSignature::default(), + } + } + + #[test] + fn a_gloas_request_reads_each_input_from_the_place_the_spec_names() { + let mut block = containers::gloas::SignedBeaconBlock::default(); + // The block's own parent root must NOT be what the request carries. + block.message.parent_root = Root::repeat_byte(4); + block + .message + .body + .signed_execution_payload_bid + .message + .blob_kzg_commitments + .push(KzgCommitment::default()); + let block = SignedBeaconBlock::Gloas(block); + + let mut envelope = gloas_envelope(Root::repeat_byte(9)); + envelope + .message + .execution_requests + .builder_exits + .push(Default::default()); + + let request = + gloas_new_payload_request(&block, &envelope).expect("a gloas block has a request"); + + assert_eq!(request.parent_beacon_block_root, Root::repeat_byte(9)); + assert_eq!(request.execution_payload.slot_number, 5); + assert_eq!(request.versioned_hashes.len(), 1); + assert_eq!( + request.versioned_hashes[0].0[0], + ethlambda_state_transition::beacon::constants::VERSIONED_HASH_VERSION_KZG + ); + // The builder exit type is only in gloas's list builder. + assert_eq!(request.execution_requests.len(), 1); + assert_eq!( + request.execution_requests[0][0], + ethlambda_state_transition::beacon::constants::BUILDER_EXIT_REQUEST_TYPE + ); + } + + #[test] + fn a_pre_gloas_block_has_no_envelope_question() { + let block = SignedBeaconBlock::Fulu(electra_block(Root::ZERO)); + let envelope = gloas_envelope(Root::ZERO); + + assert!(gloas_new_payload_request(&block, &envelope).is_none()); + } + #[test] fn an_execution_status_becomes_the_matching_consensus_verdict() { let syncing = PayloadStatusV1 { diff --git a/crates/net/engine/src/client.rs b/crates/net/engine/src/client.rs index 00418181a..ba6f8525b 100644 --- a/crates/net/engine/src/client.rs +++ b/crates/net/engine/src/client.rs @@ -2,7 +2,7 @@ use std::time::Duration; -use ethlambda_types::beacon::containers::deneb; +use ethlambda_types::beacon::containers::{deneb, gloas}; use ethlambda_types::beacon::primitives::{Bytes32, Root}; use serde_json::json; use tracing::{debug, warn}; @@ -10,8 +10,8 @@ use tracing::{debug, warn}; use crate::auth::JwtSecret; use crate::error::EngineError; use crate::types::{ - ClientVersionV1, ExecutionPayloadV3, ForkchoiceStateV1, ForkchoiceUpdatedResponse, - PayloadStatusV1, data, + ClientVersionV1, ExecutionPayloadV3, ExecutionPayloadV4, ForkchoiceStateV1, + ForkchoiceUpdatedResponse, PayloadStatusV1, data, }; /// Per-attempt timeout. @@ -21,6 +21,11 @@ use crate::types::{ /// `newPayload` on mainnet answers in 50-500 ms. pub const ENGINE_TIMEOUT: Duration = Duration::from_secs(8); +/// Per-attempt timeout for `engine_newPayloadV5`, the value `amsterdam.md` +/// gives that method. Tighter than [`ENGINE_TIMEOUT`], which stays the default +/// for the older methods. +pub const ENGINE_NEW_PAYLOAD_V5_TIMEOUT: Duration = Duration::from_secs(6); + /// How many times one call is attempted before it is given up on. /// /// Three, not the ten `ethlambda-p2p` uses for a peer fetch. Those are different @@ -86,12 +91,22 @@ impl EngineClient { &self, method: &str, params: serde_json::Value, + ) -> Result { + self.call_with_timeout(method, params, ENGINE_TIMEOUT).await + } + + /// [`Self::call`] with a per-attempt `timeout` of the method's own. + async fn call_with_timeout( + &self, + method: &str, + params: serde_json::Value, + timeout: Duration, ) -> Result { let mut backoff = ENGINE_INITIAL_BACKOFF; let mut last = EngineError::Transport("no attempt was made".to_string()); for attempt in 1..=ENGINE_MAX_ATTEMPTS { - match self.call_once(method, ¶ms).await { + match self.call_once(method, ¶ms, timeout).await { Ok(value) => return Ok(value), // An answer, not a failure to get one. Do not retry. Err(err @ EngineError::Rpc { .. }) => return Err(err), @@ -113,6 +128,7 @@ impl EngineClient { &self, method: &str, params: &serde_json::Value, + timeout: Duration, ) -> Result { let body = json!({ "jsonrpc": "2.0", @@ -125,12 +141,13 @@ impl EngineClient { .http .post(&self.endpoint) .bearer_auth(self.secret.token()) + .timeout(timeout) .json(&body) .send() .await .map_err(|err| { if err.is_timeout() { - EngineError::Timeout(ENGINE_TIMEOUT) + EngineError::Timeout(timeout) } else { EngineError::Transport(err.to_string()) } @@ -202,6 +219,36 @@ impl EngineClient { self.call("engine_newPayloadV4", params).await } + /// `engine_newPayloadV5`: Amsterdam's `newPayload`, which gloas asks about + /// a revealed execution payload envelope. + /// + /// The parameters are the same four as V4's, with the payload as + /// `ExecutionPayloadV4` (adding `blockAccessList` and `slotNumber`). The + /// response is V4's. An execution client answers `-38005` when the + /// payload's timestamp is not Amsterdam's, which surfaces as + /// [`EngineError::Rpc`] and is not retried. + pub async fn new_payload_v5( + &self, + payload: &gloas::ExecutionPayload, + versioned_hashes: &[Bytes32], + parent_beacon_block_root: Root, + execution_requests: &[Vec], + ) -> Result { + let hashes: Vec = versioned_hashes.iter().map(|hash| data(&hash.0)).collect(); + let requests: Vec = execution_requests + .iter() + .map(|request| data(request)) + .collect(); + let params = json!([ + ExecutionPayloadV4(payload), + hashes, + data(&parent_beacon_block_root.0), + requests, + ]); + self.call_with_timeout("engine_newPayloadV5", params, ENGINE_NEW_PAYLOAD_V5_TIMEOUT) + .await + } + /// `engine_forkchoiceUpdatedV3`, always with a `null` `payloadAttributes`. /// /// A follower never proposes, so it never asks an execution client to start @@ -279,6 +326,14 @@ impl EngineClient { ); } } + // Kept out of the list above: V5 is only called from gloas on, and a + // node on an earlier fork never needs it. + if !theirs.iter().any(|method| method == "engine_newPayloadV5") { + warn!( + method = "engine_newPayloadV5", + "The execution client does not advertise a method this node needs from gloas on" + ); + } match self.client_version(ours).await { Ok(versions) => { for version in versions { diff --git a/crates/net/engine/src/lib.rs b/crates/net/engine/src/lib.rs index 922ec474a..69400139e 100644 --- a/crates/net/engine/src/lib.rs +++ b/crates/net/engine/src/lib.rs @@ -40,6 +40,7 @@ pub use types::{ForkchoiceStateV1, PayloadStatusV1, PayloadStatusValue}; /// requires `engine_exchangeCapabilities` itself not to appear. pub const ETHLAMBDA_ENGINE_CAPABILITIES: &[&str] = &[ "engine_newPayloadV4", + "engine_newPayloadV5", "engine_forkchoiceUpdatedV3", "engine_getPayloadV5", "engine_getClientVersionV1", diff --git a/crates/net/engine/src/types.rs b/crates/net/engine/src/types.rs index dee89790f..9bcfa26f6 100644 --- a/crates/net/engine/src/types.rs +++ b/crates/net/engine/src/types.rs @@ -9,7 +9,8 @@ //! number crossing this boundary is `QUANTITY`, `0x`-prefixed with leading zeros //! stripped: `0x0` for zero, never `0x00` and never `0x`. -use ethlambda_types::beacon::containers::deneb; +use ethlambda_types::beacon::containers::capella::Withdrawal; +use ethlambda_types::beacon::containers::{deneb, gloas}; use ethlambda_types::beacon::primitives::{ExecutionBlockHash, Uint256}; use ethlambda_types::beacon::serde_helpers::HexPrefixed; use serde::{Deserialize, Serialize, Serializer}; @@ -120,26 +121,74 @@ impl Serialize for ExecutionPayloadV3<'_> { .collect(); out.serialize_field("transactions", &transactions)?; - let withdrawals: Vec = payload - .withdrawals + let withdrawals: Vec = + payload.withdrawals.iter().map(withdrawal_json).collect(); + out.serialize_field("withdrawals", &withdrawals)?; + + out.serialize_field("blobGasUsed", &quantity(payload.blob_gas_used))?; + out.serialize_field("excessBlobGas", &quantity(payload.excess_blob_gas))?; + out.end() + } +} + +/// `ExecutionPayloadV4` (Amsterdam), borrowed from a decoded envelope. +/// +/// `ExecutionPayloadV3`'s fields followed by `blockAccessList` (the RLP bytes +/// the payload already carries, as `DATA`) and `slotNumber`. Gloas's payload +/// has progressive `transactions` and `withdrawals`, a different type from +/// deneb's, so it needs its own serializer; the wire shape of those two is +/// unchanged. +pub struct ExecutionPayloadV4<'a>(pub &'a gloas::ExecutionPayload); + +impl Serialize for ExecutionPayloadV4<'_> { + fn serialize(&self, serializer: S) -> Result { + use serde::ser::SerializeStruct as _; + + let payload = self.0; + let mut out = serializer.serialize_struct("ExecutionPayloadV4", 19)?; + out.serialize_field("parentHash", &data(&payload.parent_hash.0))?; + out.serialize_field("feeRecipient", &data(&payload.fee_recipient.0))?; + out.serialize_field("stateRoot", &data(&payload.state_root.0))?; + out.serialize_field("receiptsRoot", &data(&payload.receipts_root.0))?; + out.serialize_field("logsBloom", &data(payload.logs_bloom.as_ref()))?; + out.serialize_field("prevRandao", &data(&payload.prev_randao.0))?; + out.serialize_field("blockNumber", &quantity(payload.block_number))?; + out.serialize_field("gasLimit", &quantity(payload.gas_limit))?; + out.serialize_field("gasUsed", &quantity(payload.gas_used))?; + out.serialize_field("timestamp", &quantity(payload.timestamp))?; + out.serialize_field("extraData", &data(payload.extra_data.as_ref()))?; + out.serialize_field("baseFeePerGas", &uint256(&payload.base_fee_per_gas))?; + out.serialize_field("blockHash", &data(&payload.block_hash.0))?; + + let transactions: Vec = payload + .transactions .iter() - .map(|withdrawal| { - serde_json::json!({ - "index": quantity(withdrawal.index), - "validatorIndex": quantity(withdrawal.validator_index), - "address": data(&withdrawal.address.0), - "amount": quantity(withdrawal.amount), - }) - }) + .map(|transaction| data(transaction.as_ref())) .collect(); + out.serialize_field("transactions", &transactions)?; + + let withdrawals: Vec = + payload.withdrawals.iter().map(withdrawal_json).collect(); out.serialize_field("withdrawals", &withdrawals)?; out.serialize_field("blobGasUsed", &quantity(payload.blob_gas_used))?; out.serialize_field("excessBlobGas", &quantity(payload.excess_blob_gas))?; + out.serialize_field("blockAccessList", &data(payload.block_access_list.as_ref()))?; + out.serialize_field("slotNumber", &quantity(payload.slot_number))?; out.end() } } +/// `WithdrawalV1`. +fn withdrawal_json(withdrawal: &Withdrawal) -> serde_json::Value { + serde_json::json!({ + "index": quantity(withdrawal.index), + "validatorIndex": quantity(withdrawal.validator_index), + "address": data(&withdrawal.address.0), + "amount": quantity(withdrawal.amount), + }) +} + /// Encodes a `DATA`: `0x`-prefixed, every byte rendered, no stripping. pub fn data(bytes: &[u8]) -> String { HexPrefixed(bytes).to_string() @@ -294,6 +343,77 @@ mod tests { ); } + #[test] + fn a_gloas_payload_serializes_to_the_amsterdam_shape() { + let mut payload = gloas::ExecutionPayload { + parent_hash: ExecutionBlockHash::repeat_byte(1), + fee_recipient: ExecutionAddress::repeat_byte(2), + state_root: Bytes32::repeat_byte(3), + receipts_root: Bytes32::repeat_byte(4), + logs_bloom: bellatrix::LogsBloom::try_from(vec![0u8; preset::BYTES_PER_LOGS_BLOOM]) + .expect("built at exactly BYTES_PER_LOGS_BLOOM"), + prev_randao: Bytes32::repeat_byte(5), + block_number: 16, + gas_limit: 30_000_000, + gas_used: 0, + timestamp: 255, + extra_data: Default::default(), + base_fee_per_gas: Uint256::from_u128(7), + block_hash: ExecutionBlockHash::repeat_byte(6), + transactions: Default::default(), + withdrawals: Default::default(), + blob_gas_used: 131072, + excess_blob_gas: 0, + block_access_list: Default::default(), + slot_number: 4096, + }; + let transaction: gloas::Transaction = vec![0x02u8, 0xab].try_into().expect("fits"); + payload.transactions.push(transaction); + payload.withdrawals.push(Withdrawal { + index: 9, + validator_index: 10, + address: ExecutionAddress::repeat_byte(8), + amount: 32, + }); + for byte in [0xc1u8, 0x80] { + payload.block_access_list.push(byte); + } + + let json = serde_json::to_value(ExecutionPayloadV4(&payload)).expect("serializes"); + + let expected = serde_json::json!({ + "parentHash": format!("0x{}", "01".repeat(32)), + "feeRecipient": format!("0x{}", "02".repeat(20)), + "stateRoot": format!("0x{}", "03".repeat(32)), + "receiptsRoot": format!("0x{}", "04".repeat(32)), + "logsBloom": format!("0x{}", "00".repeat(256)), + "prevRandao": format!("0x{}", "05".repeat(32)), + "blockNumber": "0x10", + "gasLimit": "0x1c9c380", + "gasUsed": "0x0", + "timestamp": "0xff", + "extraData": "0x", + "baseFeePerGas": "0x7", + "blockHash": format!("0x{}", "06".repeat(32)), + "transactions": ["0x02ab"], + "withdrawals": [{ + "index": "0x9", + "validatorIndex": "0xa", + "address": format!("0x{}", "08".repeat(20)), + "amount": "0x20", + }], + "blobGasUsed": "0x20000", + "excessBlobGas": "0x0", + "blockAccessList": "0xc180", + "slotNumber": "0x1000", + }); + assert_eq!(json, expected); + + // The spec's order, with the two Amsterdam fields last. + let text = serde_json::to_string(&ExecutionPayloadV4(&payload)).expect("serializes"); + assert!(text.ends_with(r#""blockAccessList":"0xc180","slotNumber":"0x1000"}"#)); + } + #[test] fn quantities_strip_leading_zeros_and_zero_is_a_single_digit() { assert_eq!(quantity(0), "0x0"); diff --git a/crates/net/engine/tests/wire_smoke.rs b/crates/net/engine/tests/wire_smoke.rs index 78ed116bf..bf23227cc 100644 --- a/crates/net/engine/tests/wire_smoke.rs +++ b/crates/net/engine/tests/wire_smoke.rs @@ -134,3 +134,61 @@ async fn an_rpc_error_envelope_surfaces_typed_and_is_not_retried() { other => panic!("expected an Rpc error, got {other:?}"), } } + +#[tokio::test] +async fn new_payload_v5_sends_four_params_under_its_own_method() { + use ethlambda_types::beacon::containers::{bellatrix, gloas}; + use ethlambda_types::beacon::preset; + use ethlambda_types::beacon::primitives::{Bytes32, ExecutionAddress, Root, Uint256}; + + let (endpoint, seen) = serve_once( + r#"{"jsonrpc":"2.0","id":1,"result":{"status":"VALID","latestValidHash":"0x0505050505050505050505050505050505050505050505050505050505050505","validationError":null}}"#, + ) + .await; + + let payload = gloas::ExecutionPayload { + parent_hash: ExecutionBlockHash::ZERO, + fee_recipient: ExecutionAddress::ZERO, + state_root: Bytes32::ZERO, + receipts_root: Bytes32::ZERO, + logs_bloom: bellatrix::LogsBloom::try_from(vec![0u8; preset::BYTES_PER_LOGS_BLOOM]) + .expect("built at exactly BYTES_PER_LOGS_BLOOM"), + prev_randao: Bytes32::ZERO, + block_number: 1, + gas_limit: 1, + gas_used: 0, + timestamp: 1, + extra_data: Default::default(), + base_fee_per_gas: Uint256::ZERO, + block_hash: ExecutionBlockHash::repeat_byte(5), + transactions: Default::default(), + withdrawals: Default::default(), + blob_gas_used: 0, + excess_blob_gas: 0, + block_access_list: Default::default(), + slot_number: 77, + }; + let status = client(endpoint) + .new_payload_v5( + &payload, + &[Bytes32::repeat_byte(1)], + Root::repeat_byte(2), + &[vec![0x00, 0xaa]], + ) + .await + .expect("the mock answers"); + + assert_eq!(status.status, PayloadStatusValue::Valid); + + let request = seen.lock().await.clone().expect("the mock saw a request"); + let (_, body) = request.split_once("\r\n\r\n").expect("a request body"); + let body: serde_json::Value = serde_json::from_str(body).expect("a JSON body"); + assert_eq!(body["method"], "engine_newPayloadV5"); + let params = body["params"].as_array().expect("an array of params"); + assert_eq!(params.len(), 4); + assert_eq!(params[0]["slotNumber"], "0x4d"); + assert_eq!(params[0]["blockAccessList"], "0x"); + assert_eq!(params[1][0], format!("0x{}", "01".repeat(32))); + assert_eq!(params[2], format!("0x{}", "02".repeat(32))); + assert_eq!(params[3][0], "0x00aa"); +} From f0ecbf10010f1b927e34bc5bb47b126a6d674b45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 02:51:13 -0300 Subject: [PATCH 052/132] fix(engine): refuse a mismatched envelope and quiet the V5 warning ask_envelope returned Ok(None) for a non-gloas block, borrowing the "nothing to ask, import it" meaning from ask. Here it could only mean a block and envelope that do not belong together, which a caller copying ask would have accepted unchecked. It now returns an EnvelopeMismatch error, and the request builder checks the block is gloas, the envelope's beacon_block_root is the block's root, and the bid's block_hash is the payload's before building anything. The missing-V5 handshake warning now fires only when the network schedules gloas, so mainnet and Hoodi with a pre-Amsterdam execution client do not warn on every start. Also refresh docs that still said nothing calls the gloas request list or that V5 is absent, and use a well-formed empty block access list in the wire test. --- bin/ethlambda/src/main.rs | 4 +- crates/blockchain/src/beacon_engine.rs | 98 ++++++++++++++----- .../state_transition/src/beacon/stf/gloas.rs | 16 +-- crates/net/engine/src/client.rs | 16 ++- crates/net/engine/src/error.rs | 5 + crates/net/engine/src/lib.rs | 2 +- crates/net/engine/tests/wire_smoke.rs | 6 +- docs/beacon_engine.md | 5 +- 8 files changed, 109 insertions(+), 43 deletions(-) diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index e25dfacfa..af3a40f70 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -630,8 +630,10 @@ async fn run_node(options: Options) -> eyre::Result<()> { // A handshake failure is not a reason to refuse to run: the // execution client may simply be starting up, and every // call that matters has its own retry ladder. + let gloas_scheduled = store.config().gloas_fork_epoch + != ethlambda_types::beacon::constants::FAR_FUTURE_EPOCH; let _ = client - .handshake(&ours) + .handshake(&ours, gloas_scheduled) .await .inspect_err(|err| warn!(%err, "Engine API handshake failed")); diff --git a/crates/blockchain/src/beacon_engine.rs b/crates/blockchain/src/beacon_engine.rs index 03bcd9e61..377c71b56 100644 --- a/crates/blockchain/src/beacon_engine.rs +++ b/crates/blockchain/src/beacon_engine.rs @@ -7,7 +7,7 @@ //! pull in `reqwest`. This crate already depends on both, so the assembly that //! needs a helper from each lives here. //! -//! # Everything a request needs comes from the block alone +//! # Everything a pre-gloas request needs comes from the block alone //! //! `NewPayloadRequest` in the specification reads `parent_beacon_block_root` //! off `state.latest_block_header.parent_root`, which after @@ -15,15 +15,19 @@ //! fields are body fields. So the whole question is answerable before the state //! transition runs, which is what lets the engine round trip happen outside the //! state transition and the state transition stay synchronous. +//! +//! Gloas is the exception: its payload is revealed in an envelope, so its +//! request ([`gloas_new_payload_request`]) needs the block and the envelope. use ethlambda_engine::types::PayloadStatusValue; use ethlambda_engine::{EngineClient, EngineError, PayloadStatusV1}; use ethlambda_state_transition::beacon::containers::{self, SignedBeaconBlock}; use ethlambda_state_transition::beacon::fork_choice::PayloadValidity; +use ethlambda_state_transition::beacon::primitives::HashTreeRoot as _; use ethlambda_state_transition::beacon::primitives::{Bytes32, Root}; use ethlambda_state_transition::beacon::stf::deneb::kzg_commitment_to_versioned_hash; use ethlambda_state_transition::beacon::stf::electra::get_execution_requests_list; -use ethlambda_state_transition::beacon::stf::gloas::get_execution_requests_list as get_gloas_execution_requests_list; +use ethlambda_state_transition::beacon::stf::gloas; /// Everything `engine_newPayloadV4` takes, derived from one block. pub struct NewPayloadRequest<'a> { @@ -42,8 +46,14 @@ pub struct GloasNewPayloadRequest<'a> { pub execution_requests: Vec>, } -/// The question to ask about `block`'s revealed payload, or `None` if `block` -/// is not a gloas block. +/// The question to ask about `block`'s revealed payload. +/// +/// `Err(EnvelopeMismatch)` when `block` is not a gloas block, when the +/// envelope's `beacon_block_root` is not `block`'s root, or when the bid's +/// `block_hash` is not the payload's. Unlike [`new_payload_request`]'s `None`, +/// which means "nothing to ask, import it", none of these has an importable +/// reading, so a caller cannot mistake one for a pass. The checks are cheap +/// and stop an envelope being validated against a block it does not belong to. /// /// `NewPayloadRequest` as `verify_execution_payload_envelope` builds it /// (gloas `beacon-chain.md`): the payload and the requests come from the @@ -59,7 +69,8 @@ pub struct GloasNewPayloadRequest<'a> { pub fn gloas_new_payload_request<'a>( block: &SignedBeaconBlock, envelope: &'a containers::gloas::SignedExecutionPayloadEnvelope, -) -> Option> { +) -> Result, EngineError> { + let mismatch = |reason: &str| Err(EngineError::EnvelopeMismatch(reason.to_string())); let inner = match block { SignedBeaconBlock::Gloas(inner) => inner, SignedBeaconBlock::Phase0(_) @@ -69,24 +80,30 @@ pub fn gloas_new_payload_request<'a>( | SignedBeaconBlock::Deneb(_) | SignedBeaconBlock::Electra(_) | SignedBeaconBlock::Fulu(_) - | SignedBeaconBlock::Lean(_) => return None, + | SignedBeaconBlock::Lean(_) => return mismatch("the block is not a gloas block"), }; - let versioned_hashes = inner - .message - .body - .signed_execution_payload_bid - .message + if envelope.message.beacon_block_root != inner.message.hash_tree_root() { + return mismatch("the envelope's beacon_block_root is not the block's root"); + } + let bid = &inner.message.body.signed_execution_payload_bid.message; + if bid.block_hash != envelope.message.payload.block_hash { + return mismatch("the bid's block_hash is not the payload's block_hash"); + } + + let versioned_hashes = bid .blob_kzg_commitments .iter() .map(kzg_commitment_to_versioned_hash) .collect(); - Some(GloasNewPayloadRequest { + Ok(GloasNewPayloadRequest { execution_payload: &envelope.message.payload, versioned_hashes, parent_beacon_block_root: envelope.message.parent_beacon_block_root, - execution_requests: get_gloas_execution_requests_list(&envelope.message.execution_requests), + execution_requests: gloas::get_execution_requests_list( + &envelope.message.execution_requests, + ), }) } @@ -189,15 +206,16 @@ pub async fn ask( /// Asks the execution client about the payload `envelope` reveals for `block`, /// with `engine_newPayloadV5`. /// -/// `Ok(None)` when `block` is not a gloas block; otherwise as [`ask`]. +/// Never `Ok(None)`: unlike [`ask`] there is no "nothing to ask" reading, and a +/// block/envelope pair that does not match is an +/// [`EngineError::EnvelopeMismatch`], which the caller must treat as a refusal +/// to import. pub async fn ask_envelope( client: &EngineClient, block: &SignedBeaconBlock, envelope: &containers::gloas::SignedExecutionPayloadEnvelope, -) -> Result, EngineError> { - let Some(request) = gloas_new_payload_request(block, envelope) else { - return Ok(None); - }; +) -> Result { + let request = gloas_new_payload_request(block, envelope)?; let status = client .new_payload_v5( request.execution_payload, @@ -206,7 +224,7 @@ pub async fn ask_envelope( &request.execution_requests, ) .await?; - Ok(Some(verdict(&status))) + Ok(verdict(&status)) } #[cfg(test)] @@ -336,6 +354,7 @@ mod tests { /// A gloas envelope whose payload is otherwise zero. fn gloas_envelope( + beacon_block_root: Root, parent_beacon_block_root: Root, ) -> containers::gloas::SignedExecutionPayloadEnvelope { let payload = containers::gloas::ExecutionPayload { @@ -368,7 +387,7 @@ mod tests { payload, execution_requests: Default::default(), builder_index: 0, - beacon_block_root: Root::repeat_byte(1), + beacon_block_root, parent_beacon_block_root, }, signature: BlsSignature::default(), @@ -387,17 +406,18 @@ mod tests { .message .blob_kzg_commitments .push(KzgCommitment::default()); + let block_root = block.message.hash_tree_root(); let block = SignedBeaconBlock::Gloas(block); - let mut envelope = gloas_envelope(Root::repeat_byte(9)); + let mut envelope = gloas_envelope(block_root, Root::repeat_byte(9)); envelope .message .execution_requests .builder_exits .push(Default::default()); - let request = - gloas_new_payload_request(&block, &envelope).expect("a gloas block has a request"); + let request = gloas_new_payload_request(&block, &envelope) + .unwrap_or_else(|err| panic!("a matching pair has a request: {err}")); assert_eq!(request.parent_beacon_block_root, Root::repeat_byte(9)); assert_eq!(request.execution_payload.slot_number, 5); @@ -417,9 +437,37 @@ mod tests { #[test] fn a_pre_gloas_block_has_no_envelope_question() { let block = SignedBeaconBlock::Fulu(electra_block(Root::ZERO)); - let envelope = gloas_envelope(Root::ZERO); + let envelope = gloas_envelope(Root::ZERO, Root::ZERO); + + assert!(matches!( + gloas_new_payload_request(&block, &envelope), + Err(EngineError::EnvelopeMismatch(_)) + )); + } - assert!(gloas_new_payload_request(&block, &envelope).is_none()); + #[test] + fn an_envelope_for_another_block_is_refused() { + let block = SignedBeaconBlock::Gloas(containers::gloas::SignedBeaconBlock::default()); + let envelope = gloas_envelope(Root::repeat_byte(0xee), Root::ZERO); + + assert!(matches!( + gloas_new_payload_request(&block, &envelope), + Err(EngineError::EnvelopeMismatch(_)) + )); + } + + #[test] + fn a_payload_other_than_the_bids_is_refused() { + let inner = containers::gloas::SignedBeaconBlock::default(); + let block_root = inner.message.hash_tree_root(); + let block = SignedBeaconBlock::Gloas(inner); + let mut envelope = gloas_envelope(block_root, Root::ZERO); + envelope.message.payload.block_hash = ExecutionBlockHash::repeat_byte(0xab); + + assert!(matches!( + gloas_new_payload_request(&block, &envelope), + Err(EngineError::EnvelopeMismatch(_)) + )); } #[test] diff --git a/crates/blockchain/state_transition/src/beacon/stf/gloas.rs b/crates/blockchain/state_transition/src/beacon/stf/gloas.rs index 303dd403e..fe982ed14 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/gloas.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/gloas.rs @@ -512,14 +512,14 @@ pub fn process_withdrawals(state: &mut BeaconState) -> Result<()> { /// built from the revealed envelope's own `execution_requests`. Not from /// `state.latest_execution_payload_bid`: the bid commits to that list only /// by its hash ([`gloas::ExecutionPayloadBid::execution_requests_root`]), it -/// never carries the list itself. Nothing calls this: envelope verification -/// does not build the request list, since [`ExecutionEngine`] collapses the -/// whole `verify_and_notify_new_payload` interface to one boolean and never -/// inspects it (see [`crate::beacon::stf::deneb::process_execution_payload`]'s -/// own documentation). Electra's own -/// [`crate::beacon::stf::electra::get_execution_requests_list`] is called, by -/// the live follower's Engine API client (`ethlambda-blockchain`'s -/// `beacon_engine`), which sends the list to a real execution client. +/// never carries the list itself. [`verify_execution_payload_envelope`] does +/// not call this, since [`ExecutionEngine`] collapses the whole +/// `verify_and_notify_new_payload` interface to one boolean and never inspects +/// it (see [`crate::beacon::stf::deneb::process_execution_payload`]'s own +/// documentation). The live follower's Engine API client does: `ethlambda-blockchain`'s +/// `beacon_engine` builds the `engine_newPayloadV5` request list with this, +/// and electra's [`crate::beacon::stf::electra::get_execution_requests_list`] +/// for the earlier forks. pub fn get_execution_requests_list(requests: &gloas::ExecutionRequests) -> Vec> { let mut list = Vec::new(); diff --git a/crates/net/engine/src/client.rs b/crates/net/engine/src/client.rs index ba6f8525b..33e589aa3 100644 --- a/crates/net/engine/src/client.rs +++ b/crates/net/engine/src/client.rs @@ -311,10 +311,17 @@ impl EngineClient { /// Runs the startup handshake, logging what the execution client is and /// warning about any method this client needs that it does not advertise. /// + /// `gloas_scheduled` says whether the network's fork schedule includes + /// gloas, which is what makes `engine_newPayloadV5` a needed method. + /// /// Warns rather than refuses: an execution client that under-reports its /// capabilities still works, and refusing to start over a handshake would /// turn a cosmetic mismatch into an outage. - pub async fn handshake(&self, ours: &ClientVersionV1) -> Result<(), EngineError> { + pub async fn handshake( + &self, + ours: &ClientVersionV1, + gloas_scheduled: bool, + ) -> Result<(), EngineError> { let theirs = self .exchange_capabilities(crate::ETHLAMBDA_ENGINE_CAPABILITIES) .await?; @@ -326,9 +333,10 @@ impl EngineClient { ); } } - // Kept out of the list above: V5 is only called from gloas on, and a - // node on an earlier fork never needs it. - if !theirs.iter().any(|method| method == "engine_newPayloadV5") { + // Kept out of the list above: V5 is only called from gloas on, so a + // network that does not schedule gloas never needs it, and an + // execution client that predates Amsterdam is right not to offer it. + if gloas_scheduled && !theirs.iter().any(|method| method == "engine_newPayloadV5") { warn!( method = "engine_newPayloadV5", "The execution client does not advertise a method this node needs from gloas on" diff --git a/crates/net/engine/src/error.rs b/crates/net/engine/src/error.rs index 4d2c717cd..0e575e777 100644 --- a/crates/net/engine/src/error.rs +++ b/crates/net/engine/src/error.rs @@ -19,4 +19,9 @@ pub enum EngineError { Rpc { code: i64, message: String }, #[error("decoding the response: {0}")] Decode(String), + /// A caller asked about an envelope that does not belong to the block it + /// paired it with. Raised before any call is made, so it is no verdict on + /// the payload itself: the caller must not import either. + #[error("envelope does not match its block: {0}")] + EnvelopeMismatch(String), } diff --git a/crates/net/engine/src/lib.rs b/crates/net/engine/src/lib.rs index 69400139e..8dde3f530 100644 --- a/crates/net/engine/src/lib.rs +++ b/crates/net/engine/src/lib.rs @@ -11,7 +11,7 @@ //! //! Osaka introduces no new `newPayload`: its own document adds only //! `engine_getPayloadV5` and `engine_getBlobsV2`/`V3`, and `engine_newPayloadV5` -//! belongs to Amsterdam. So the Osaka-current call for a payload is Prague's +//! belongs to Amsterdam, where gloas's envelope import uses it. So the Osaka-current call for a payload is Prague's //! `engine_newPayloadV4`, and the Osaka-current fork choice notification is //! Cancun's `engine_forkchoiceUpdatedV3`. //! diff --git a/crates/net/engine/tests/wire_smoke.rs b/crates/net/engine/tests/wire_smoke.rs index bf23227cc..ff629c87a 100644 --- a/crates/net/engine/tests/wire_smoke.rs +++ b/crates/net/engine/tests/wire_smoke.rs @@ -165,7 +165,9 @@ async fn new_payload_v5_sends_four_params_under_its_own_method() { withdrawals: Default::default(), blob_gas_used: 0, excess_blob_gas: 0, - block_access_list: Default::default(), + // 0xc0 is the RLP of an empty list, the smallest BAL an execution + // client accepts as well-formed. + block_access_list: vec![0xc0u8].try_into().expect("fits"), slot_number: 77, }; let status = client(endpoint) @@ -187,7 +189,7 @@ async fn new_payload_v5_sends_four_params_under_its_own_method() { let params = body["params"].as_array().expect("an array of params"); assert_eq!(params.len(), 4); assert_eq!(params[0]["slotNumber"], "0x4d"); - assert_eq!(params[0]["blockAccessList"], "0x"); + assert_eq!(params[0]["blockAccessList"], "0xc0"); assert_eq!(params[1][0], format!("0x{}", "01".repeat(32))); assert_eq!(params[2], format!("0x{}", "02".repeat(32))); assert_eq!(params[3][0], "0x00aa"); diff --git a/docs/beacon_engine.md b/docs/beacon_engine.md index d7c557baa..27665762e 100644 --- a/docs/beacon_engine.md +++ b/docs/beacon_engine.md @@ -14,18 +14,19 @@ The wire is ## Which methods, and why so few -Four: +Five: | Method | Introduced | Why | |---|---|---| | `engine_newPayloadV4` | prague | Validate one block's payload | +| `engine_newPayloadV5` | amsterdam | Validate the payload a gloas envelope reveals | | `engine_forkchoiceUpdatedV3` | cancun | Say where the head, safe and finalized blocks are | | `engine_exchangeCapabilities` | common | Startup handshake | | `engine_getClientVersionV1` | identification | Log what the execution client is | Osaka introduces **no new `newPayload`**. Its own document adds only `engine_getPayloadV5` and `engine_getBlobsV2`/`V3`; `engine_newPayloadV5` belongs -to Amsterdam. So the Osaka-current call for a payload is prague's V4, and the +to Amsterdam, where gloas's envelope import uses it. So the Osaka-current call for a payload is prague's V4, and the Osaka-current fork choice notification is cancun's V3. Three method families a full client would have are deliberately absent: From 669283947f303be1f91ef51fa4d05b257544b4b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 02:56:45 -0300 Subject: [PATCH 053/132] feat(beacon): gloas execution_payload and payload_attestation_message gossip rules The two new gloas topics need validators before p2p can subscribe to them. Both follow the cheap/stateful split the other topics use, so the swarm loop only runs the seen-cache, clock and finalized-slot checks inline; store reads, the payload timeliness committee and signatures run on the blocking pool. A block seen without a post-state is queued (envelope) or ignored (payload attestation) where the spec rejects it, the deviation every other topic already makes until a bad-block cache exists; the two matching reject_block_failed_validation vectors are skipped for that reason. The envelope's finalized-slot IGNORE runs before the block lookups, which only changes the metric label between two IGNORE outcomes. verify_execution_requests_limits moves to gossip/mod.rs so the block and envelope rules share it. --- .../src/beacon/gossip/block.rs | 19 +- .../src/beacon/gossip/envelope.rs | 191 ++++++++++++++++++ .../state_transition/src/beacon/gossip/mod.rs | 91 ++++++++- .../src/beacon/gossip/payload_attestation.rs | 169 ++++++++++++++++ .../tests/beacon_spec/gossip.rs | 59 +++++- 5 files changed, 503 insertions(+), 26 deletions(-) create mode 100644 crates/blockchain/state_transition/src/beacon/gossip/envelope.rs create mode 100644 crates/blockchain/state_transition/src/beacon/gossip/payload_attestation.rs diff --git a/crates/blockchain/state_transition/src/beacon/gossip/block.rs b/crates/blockchain/state_transition/src/beacon/gossip/block.rs index 0fb31c2ab..13932e241 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/block.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/block.rs @@ -6,8 +6,8 @@ use ethlambda_storage::CacheKey; use tracing::warn; use super::{ - IgnoreReason, Outcome, QueueReason, RejectReason, SeenBlocks, finalized_ancestry, - finalized_start_slot, is_future_slot, + IgnoreReason, Outcome, QueueReason, RejectReason, SeenBlocks, execution_requests_within_limits, + finalized_ancestry, finalized_start_slot, is_future_slot, }; use crate::beacon::containers::{BeaconState, SignedBeaconBlock, gloas}; use crate::beacon::fork_choice::{self, Store}; @@ -134,21 +134,6 @@ fn body_operations_within_limits(body: &gloas::BeaconBlockBody) -> bool { && body.payload_attestations.len() as u64 <= preset::MAX_PAYLOAD_ATTESTATIONS } -/// `verify_execution_requests_limits` (`specs/gloas/p2p-interface.md`). -/// -/// The four lists the specification names: withdrawals, consolidations, -/// builder deposits and builder exits. Deposit requests are deliberately not -/// checked: gloas's `DepositRequests` is progressive and neither the -/// specification nor the state transition bounds it, so a limit here would -/// reject a block that import accepts. -fn execution_requests_within_limits(requests: &gloas::ExecutionRequests) -> bool { - requests.withdrawals.len() <= preset::MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD - && requests.consolidations.len() <= preset::MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD - && requests.builder_deposits.len() as u64 - <= preset::MAX_BUILDER_DEPOSIT_REQUESTS_PER_PAYLOAD - && requests.builder_exits.len() as u64 <= preset::MAX_BUILDER_EXIT_REQUESTS_PER_PAYLOAD -} - /// The rules that need the parent's post-state. Runs on a blocking thread. /// /// Reads states only through [`Store::cached_state`]: a miss would otherwise diff --git a/crates/blockchain/state_transition/src/beacon/gossip/envelope.rs b/crates/blockchain/state_transition/src/beacon/gossip/envelope.rs new file mode 100644 index 000000000..2e9da9255 --- /dev/null +++ b/crates/blockchain/state_transition/src/beacon/gossip/envelope.rs @@ -0,0 +1,191 @@ +//! Gossip validation for the gloas `execution_payload` topic: a builder's +//! `SignedExecutionPayloadEnvelope`, revealing the payload its bid committed to. +//! +//! The rules are the specification's `validate_execution_payload_envelope_gossip` +//! (`specs/gloas/p2p-interface.md`), split the way [`super::column`]'s are: +//! [`cheap_checks`] reads only the message, the seen cache and the store's +//! finalized checkpoint, so the p2p actor runs it inline; [`stateful_checks`] +//! reads the envelope's block and its post-state and verifies the signature, so +//! it runs on a blocking thread. +//! +//! Deliberate departures from the specification: +//! +//! - A block seen without a post-state is [`Outcome::Queue`] where the +//! specification rejects it, the same deviation every other topic makes until +//! a bad-block cache exists. A block never seen is `Queue` too (the +//! specification's "MAY be queued"). +//! - The finalized-slot `IGNORE` runs in [`cheap_checks`], ahead of the block +//! lookups the specification places before it. Both outcomes are `IGNORE`, so +//! only the metric label can differ, and an envelope too old to matter is +//! not parked waiting for its block. + +use std::num::NonZeroUsize; + +use ethlambda_storage::CacheKey; +use lru::LruCache; + +use super::column::judgeable_block_gloas; +use super::{ + IgnoreReason, Outcome, RejectReason, execution_requests_within_limits, finalized_start_slot, +}; +use crate::beacon::containers::{SignedBeaconBlock, gloas}; +use crate::beacon::fork_choice::Store; +use crate::beacon::lean_boundary::lean_block_unreachable; +use crate::beacon::preset; +use crate::beacon::primitives::{HashTreeRoot as _, Root}; +use crate::beacon::stf::gloas::verify_execution_payload_envelope_signature; + +/// The first valid envelope per `(block root, builder index)`: the +/// specification's `seen.execution_payload_envelopes`. +/// +/// Bounded by capacity rather than pruned on finality, like [`super::SeenBlocks`]. +pub struct SeenEnvelopes(LruCache<(Root, u64), ()>); + +impl SeenEnvelopes { + pub fn new(capacity: NonZeroUsize) -> Self { + Self(LruCache::new(capacity)) + } + + pub fn contains(&self, block_root: Root, builder_index: u64) -> bool { + self.0.contains(&(block_root, builder_index)) + } + + /// Record the first valid envelope for its key. Returns `false`, changing + /// nothing, when one is already recorded. + pub fn record(&mut self, block_root: Root, builder_index: u64) -> bool { + if self.0.contains(&(block_root, builder_index)) { + return false; + } + self.0.put((block_root, builder_index), ()); + true + } +} + +/// The rules that read only the message, the seen cache and the store's +/// finalized checkpoint. The caller records the envelope in `seen` once the +/// whole rule answers [`Outcome::Accept`]. +pub fn cheap_checks( + seen: &SeenEnvelopes, + store: &Store, + envelope: &gloas::SignedExecutionPayloadEnvelope, +) -> Result<(), Outcome> { + let message = &envelope.message; + // [IGNORE] The node has not seen another valid envelope for this block root + // from this builder. + if seen.contains(message.beacon_block_root, message.builder_index) { + return Err(Outcome::Ignore(IgnoreReason::AlreadySeen)); + } + // [IGNORE] The envelope is from a slot greater than or equal to the latest + // finalized slot. See the module documentation for its position. + if message.payload.slot_number < finalized_start_slot(store) { + return Err(Outcome::Ignore(IgnoreReason::Finalized)); + } + Ok(()) +} + +/// The rules that need the envelope's block and its post-state, then the +/// signature. Runs on a blocking thread. +pub fn stateful_checks(store: &Store, envelope: &gloas::SignedExecutionPayloadEnvelope) -> Outcome { + let message = &envelope.message; + let payload = &message.payload; + // [IGNORE] The envelope's block has been seen (MAY queue), and [REJECT] it + // passes validation: see the module documentation for the latter. + let block = match judgeable_block_gloas(store, &message.beacon_block_root) { + Ok(block) => block, + Err(outcome) => return outcome, + }; + let block = match &block { + SignedBeaconBlock::Gloas(block) => &block.message, + // A block of any other fork carries no bid, so no envelope is valid + // for it. + SignedBeaconBlock::Phase0(_) + | SignedBeaconBlock::Altair(_) + | SignedBeaconBlock::Bellatrix(_) + | SignedBeaconBlock::Capella(_) + | SignedBeaconBlock::Deneb(_) + | SignedBeaconBlock::Electra(_) + | SignedBeaconBlock::Fulu(_) => return Outcome::Reject(RejectReason::BlockNotGloas), + SignedBeaconBlock::Lean(_) => lean_block_unreachable("envelope::stateful_checks"), + }; + let bid = &block.body.signed_execution_payload_bid.message; + + // [REJECT] The block's slot matches the payload's slot number. + if block.slot != payload.slot_number { + return Outcome::Reject(RejectReason::SlotMismatch); + } + // [REJECT] The envelope is from the builder committed to by the bid. + if message.builder_index != bid.builder_index { + return Outcome::Reject(RejectReason::BuilderIndexMismatch); + } + // [REJECT] The payload's block hash matches the bid's block hash. + if payload.block_hash != bid.block_hash { + return Outcome::Reject(RejectReason::BlockHashMismatch); + } + // [REJECT] The execution requests root matches the bid's. + if message.execution_requests.hash_tree_root() != bid.execution_requests_root { + return Outcome::Reject(RejectReason::ExecutionRequestsRootMismatch); + } + // [REJECT] The execution request counts are within their limits. + if !execution_requests_within_limits(&message.execution_requests) { + return Outcome::Reject(RejectReason::OperationLimit); + } + // [REJECT] The number of withdrawals is within the limit. + if payload.withdrawals.len() > preset::MAX_WITHDRAWALS_PER_PAYLOAD { + return Outcome::Reject(RejectReason::TooManyWithdrawals); + } + // [REJECT] The envelope signature is valid. Read off the block's cached + // post-state, never rebuilt: a miss would stall the verdict gossipsub + // waits on. + let Some(state) = store.cached_state(CacheKey::BlockState(message.beacon_block_root)) else { + return Outcome::Ignore(IgnoreReason::StateUnavailable); + }; + // An error (a builder the state has no entry for) is a signature that + // cannot be valid. + if !matches!( + verify_execution_payload_envelope_signature(&state, envelope), + Ok(true) + ) { + return Outcome::Reject(RejectReason::BadSignature); + } + Outcome::Accept +} + +/// [`cheap_checks`] then [`stateful_checks`], for callers with no reason to +/// split them, such as the spec vectors: the specification's +/// `validate_execution_payload_envelope_gossip`. The caller records the +/// envelope in `seen` on [`Outcome::Accept`]. +pub fn validate( + seen: &SeenEnvelopes, + store: &Store, + envelope: &gloas::SignedExecutionPayloadEnvelope, +) -> Outcome { + if let Err(outcome) = cheap_checks(seen, store, envelope) { + return outcome; + } + stateful_checks(store, envelope) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_envelope_key_records_once_per_root_and_builder() { + let mut seen = SeenEnvelopes::new(NonZeroUsize::new(4).expect("non-zero")); + let root = Root::repeat_byte(1); + assert!(!seen.contains(root, 7)); + assert!(seen.record(root, 7)); + assert!(seen.contains(root, 7)); + assert!(!seen.record(root, 7)); + assert!(seen.record(root, 8)); + assert!(seen.record(Root::repeat_byte(2), 7)); + } + + #[test] + fn the_seen_envelopes_forget_their_oldest_entry_past_capacity() { + let mut seen = SeenEnvelopes::new(NonZeroUsize::new(1).expect("non-zero")); + seen.record(Root::repeat_byte(1), 0); + seen.record(Root::repeat_byte(2), 0); + assert!(!seen.contains(Root::repeat_byte(1), 0)); + } +} diff --git a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs index 5ee5289cc..63e430eec 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs @@ -10,6 +10,8 @@ pub mod aggregate; pub mod attestation; pub mod block; pub mod column; +pub mod envelope; +pub mod payload_attestation; #[cfg(test)] pub(crate) mod test_support; @@ -18,6 +20,8 @@ pub(crate) mod test_support; // topic's submodule defines it. pub use aggregate::SeenAggregates; pub use attestation::SeenAttestations; +pub use envelope::SeenEnvelopes; +pub use payload_attestation::SeenPayloadAttestations; use std::num::NonZeroUsize; @@ -25,13 +29,14 @@ use lru::LruCache; use crate::beacon::config::Config; use crate::beacon::constants::MAXIMUM_GOSSIP_CLOCK_DISPARITY; -use crate::beacon::containers::{AttestationData, BeaconState}; +use crate::beacon::containers::{AttestationData, BeaconState, gloas}; use crate::beacon::fork::ForkName; use crate::beacon::fork_choice::{self, PayloadStatusEnum, Store}; use crate::beacon::helpers::accessors::get_block_root_at_slot; use crate::beacon::helpers::misc::{compute_epoch_at_slot, compute_start_slot_at_epoch}; use crate::beacon::lean_boundary::lean_fork_unreachable; use crate::beacon::precheck::PrecheckError; +use crate::beacon::preset; use crate::beacon::primitives::{Epoch, Root, Slot, ValidatorIndex}; /// A gossip message's verdict. @@ -137,6 +142,10 @@ pub enum IgnoreReason { PayloadEnvelopeUnseen, /// A gloas vote names a payload the execution client has not validated. PayloadOptimistic, + /// A payload attestation's slot is not the current slot. + NotCurrentSlot, + /// A payload attestation names a block that is not at the attested slot. + BlockNotAtSlot, } impl IgnoreReason { @@ -159,6 +168,8 @@ impl IgnoreReason { Self::ParentPayloadUnverified => "parent_payload_unverified", Self::PayloadEnvelopeUnseen => "payload_envelope_unseen", Self::PayloadOptimistic => "payload_optimistic", + Self::NotCurrentSlot => "not_current_slot", + Self::BlockNotAtSlot => "block_not_at_slot", } } } @@ -224,6 +235,19 @@ pub enum RejectReason { SameSlotPayloadFlag, /// A gloas vote names a payload the execution client found invalid. PayloadInvalid, + /// A gloas envelope's builder is not the one its block's bid committed to. + BuilderIndexMismatch, + /// A gloas envelope's payload block hash is not its bid's. + BlockHashMismatch, + /// The root of a gloas envelope's execution requests is not its bid's. + ExecutionRequestsRootMismatch, + /// A gloas envelope's payload carries more withdrawals than its limit. + TooManyWithdrawals, + /// A payload attestation's slot lies before the gloas fork. + PreGloasSlot, + /// A payload attestation's validator is not in its slot's payload + /// timeliness committee. + NotInPtc, } impl RejectReason { @@ -263,6 +287,12 @@ impl RejectReason { Self::DataIndexOutOfRange => "data_index_out_of_range", Self::SameSlotPayloadFlag => "same_slot_payload_flag", Self::PayloadInvalid => "payload_invalid", + Self::BuilderIndexMismatch => "builder_index_mismatch", + Self::BlockHashMismatch => "block_hash_mismatch", + Self::ExecutionRequestsRootMismatch => "execution_requests_root_mismatch", + Self::TooManyWithdrawals => "too_many_withdrawals", + Self::PreGloasSlot => "pre_gloas_slot", + Self::NotInPtc => "not_in_ptc", } } } @@ -358,6 +388,23 @@ impl SeenBlockColumns { } } +/// `verify_execution_requests_limits` (`specs/gloas/p2p-interface.md`). +/// +/// The four lists the specification names: withdrawals, consolidations, +/// builder deposits and builder exits. Deposit requests are deliberately not +/// checked: gloas's `DepositRequests` is progressive and neither the +/// specification nor the state transition bounds it, so a limit here would +/// reject a block that import accepts. +/// +/// Shared by [`block`] (a block's parent execution requests) and [`envelope`]. +pub(crate) fn execution_requests_within_limits(requests: &gloas::ExecutionRequests) -> bool { + requests.withdrawals.len() <= preset::MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD + && requests.consolidations.len() <= preset::MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD + && requests.builder_deposits.len() as u64 + <= preset::MAX_BUILDER_DEPOSIT_REQUESTS_PER_PAYLOAD + && requests.builder_exits.len() as u64 <= preset::MAX_BUILDER_EXIT_REQUESTS_PER_PAYLOAD +} + /// The specification's `compute_time_at_slot_ms`: the clock reading at the /// start of `slot`. pub(crate) fn slot_start_ms(config: &Config, slot: Slot) -> u64 { @@ -372,6 +419,18 @@ pub(crate) fn is_future_slot(config: &Config, slot: Slot, now_ms: u64) -> bool { slot_start_ms(config, slot) > now_ms.saturating_add(MAXIMUM_GOSSIP_CLOCK_DISPARITY) } +/// The specification's `is_current_slot` (`altair/p2p-interface.md`): the +/// clock, with the gossip clock disparity allowance on both ends, falls within +/// `slot`'s own span. That is `is_within_slot_range` with a range of zero, so +/// the far edge is the *next* slot's start. +pub(crate) fn is_current_slot(config: &Config, slot: Slot, now_ms: u64) -> bool { + if now_ms.saturating_add(MAXIMUM_GOSSIP_CLOCK_DISPARITY) < slot_start_ms(config, slot) { + return false; + } + slot_start_ms(config, slot.saturating_add(1)).saturating_add(MAXIMUM_GOSSIP_CLOCK_DISPARITY) + >= now_ms +} + /// The specification's `is_within_epoch`: the clock, with the gossip clock /// disparity allowance on both ends, falls somewhere in `epoch`'s own span of /// slots. @@ -642,6 +701,36 @@ mod tests { )); } + #[test] + fn the_current_slot_holds_within_the_clock_disparity_at_either_edge() { + let config = Config { + genesis_time: 0, + ..Config::mainnet() + }; + let start_ms = slot_start_ms(&config, 3); + let next_start_ms = slot_start_ms(&config, 4); + assert!(is_current_slot( + &config, + 3, + start_ms - MAXIMUM_GOSSIP_CLOCK_DISPARITY + )); + assert!(!is_current_slot( + &config, + 3, + start_ms - MAXIMUM_GOSSIP_CLOCK_DISPARITY - 1 + )); + assert!(is_current_slot( + &config, + 3, + next_start_ms + MAXIMUM_GOSSIP_CLOCK_DISPARITY + )); + assert!(!is_current_slot( + &config, + 3, + next_start_ms + MAXIMUM_GOSSIP_CLOCK_DISPARITY + 1 + )); + } + #[test] fn a_block_key_records_once() { let mut seen = SeenBlocks::new(capacity(4)); diff --git a/crates/blockchain/state_transition/src/beacon/gossip/payload_attestation.rs b/crates/blockchain/state_transition/src/beacon/gossip/payload_attestation.rs new file mode 100644 index 000000000..736fb492a --- /dev/null +++ b/crates/blockchain/state_transition/src/beacon/gossip/payload_attestation.rs @@ -0,0 +1,169 @@ +//! Gossip validation for the gloas `payload_attestation_message` topic: a +//! payload timeliness committee member's vote on whether a block's payload +//! arrived in time. +//! +//! The rules are the specification's `validate_payload_attestation_message_gossip` +//! (`specs/gloas/p2p-interface.md`), split like [`super::envelope`]'s: +//! [`cheap_checks`] reads the message, the clock, the seen cache and the +//! config; [`stateful_checks`] reads the voted block and the head state's +//! committee and verifies the signature. +//! +//! Deliberate departures from the specification: +//! +//! - A voted block seen without a post-state is `IGNORE` where the +//! specification rejects it, until a bad-block cache exists. An unseen block +//! is `IGNORE` as the specification says; nothing is queued, since the vote +//! is only valid within its own slot. +//! - The head state is read from the state cache, never rebuilt. A miss is +//! `IGNORE`, like the other topics' uncached states. + +use std::num::NonZeroUsize; + +use ethlambda_storage::CacheKey; +use lru::LruCache; + +use super::{IgnoreReason, Outcome, RejectReason, is_current_slot}; +use crate::beacon::bls; +use crate::beacon::constants::DOMAIN_PTC_ATTESTER; +use crate::beacon::containers::gloas; +use crate::beacon::fork_choice::Store; +use crate::beacon::helpers::accessors::get_domain; +use crate::beacon::helpers::gloas::get_ptc; +use crate::beacon::helpers::misc::{compute_epoch_at_slot, compute_signing_root}; +use crate::beacon::primitives::{HashTreeRoot as _, Slot, ValidatorIndex}; + +/// The first valid payload attestation per `(slot, validator index)`: the +/// specification's `seen.payload_attestation_validators`. +/// +/// Bounded the same way as [`super::SeenBlocks`]. +pub struct SeenPayloadAttestations(LruCache<(Slot, ValidatorIndex), ()>); + +impl SeenPayloadAttestations { + pub fn new(capacity: NonZeroUsize) -> Self { + Self(LruCache::new(capacity)) + } + + pub fn contains(&self, slot: Slot, validator_index: ValidatorIndex) -> bool { + self.0.contains(&(slot, validator_index)) + } + + /// Record the first valid attestation for its key. Returns `false`, + /// changing nothing, when one is already recorded. + pub fn record(&mut self, slot: Slot, validator_index: ValidatorIndex) -> bool { + if self.0.contains(&(slot, validator_index)) { + return false; + } + self.0.put((slot, validator_index), ()); + true + } +} + +/// The rules that read only the message, the clock, the seen cache and the +/// config. The caller records the message in `seen` once the whole rule +/// answers [`Outcome::Accept`]. +pub fn cheap_checks( + seen: &SeenPayloadAttestations, + store: &Store, + message: &gloas::PayloadAttestationMessage, + now_ms: u64, +) -> Result<(), Outcome> { + let config = store.config(); + let slot = message.data.slot; + // [REJECT] The payload attestation's slot is at or after the gloas fork. + if compute_epoch_at_slot(slot) < config.gloas_fork_epoch { + return Err(Outcome::Reject(RejectReason::PreGloasSlot)); + } + // [IGNORE] This is the first valid payload attestation from this validator. + if seen.contains(slot, message.validator_index) { + return Err(Outcome::Ignore(IgnoreReason::AlreadySeen)); + } + // [IGNORE] The payload attestation's slot is the current slot. + if !is_current_slot(&config, slot, now_ms) { + return Err(Outcome::Ignore(IgnoreReason::NotCurrentSlot)); + } + Ok(()) +} + +/// The rules that need the voted block and the head state. Runs on a blocking +/// thread. +pub fn stateful_checks(store: &Store, message: &gloas::PayloadAttestationMessage) -> Outcome { + let data = &message.data; + let validator_index = message.validator_index; + // [IGNORE] The voted block has been seen. + let Some((block_slot, _)) = store.block_slot_and_state_root(&data.beacon_block_root) else { + return Outcome::Ignore(IgnoreReason::UnknownBlock); + }; + // [REJECT] The voted block passes validation: see the module documentation. + match store.has_state(&data.beacon_block_root) { + Ok(true) => {} + Ok(false) => return Outcome::Ignore(IgnoreReason::StateUnavailable), + Err(_) => return Outcome::Ignore(IgnoreReason::Internal), + } + // [IGNORE] The voted block is at the assigned slot. + if block_slot != data.slot { + return Outcome::Ignore(IgnoreReason::BlockNotAtSlot); + } + + // The committee is read off the head state, as the specification says. + let Some((_, head_root)) = store.beacon_head() else { + return Outcome::Ignore(IgnoreReason::Internal); + }; + let Some(state) = store.cached_state(CacheKey::BlockState(head_root)) else { + return Outcome::Ignore(IgnoreReason::StateUnavailable); + }; + // [REJECT] The validator index is valid. + let Ok(validator) = state.validator(validator_index) else { + return Outcome::Reject(RejectReason::UnknownValidator); + }; + // [REJECT] The validator is a member of the payload timeliness committee. + // A slot the head's committee window cannot answer is not the sender's fault. + let Ok(ptc) = get_ptc(&state, data.slot, &store.config()) else { + return Outcome::Ignore(IgnoreReason::StateUnavailable); + }; + if !ptc.contains(&validator_index) { + return Outcome::Reject(RejectReason::NotInPtc); + } + // [REJECT] The signature is valid. + let domain = get_domain( + &state, + DOMAIN_PTC_ATTESTER, + Some(compute_epoch_at_slot(data.slot)), + ); + let signing_root = compute_signing_root(data.hash_tree_root(), domain); + if !bls::verify(&validator.pubkey, signing_root, &message.signature) { + return Outcome::Reject(RejectReason::BadSignature); + } + Outcome::Accept +} + +/// [`cheap_checks`] then [`stateful_checks`], for callers with no reason to +/// split them, such as the spec vectors: the specification's +/// `validate_payload_attestation_message_gossip`. The caller records the +/// message in `seen` on [`Outcome::Accept`]. +pub fn validate( + seen: &SeenPayloadAttestations, + store: &Store, + message: &gloas::PayloadAttestationMessage, + now_ms: u64, +) -> Outcome { + if let Err(outcome) = cheap_checks(seen, store, message, now_ms) { + return outcome; + } + stateful_checks(store, message) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_payload_attestation_key_records_once_per_slot_and_validator() { + let mut seen = SeenPayloadAttestations::new(NonZeroUsize::new(4).expect("non-zero")); + assert!(!seen.contains(5, 9)); + assert!(seen.record(5, 9)); + assert!(seen.contains(5, 9)); + assert!(!seen.record(5, 9)); + assert!(seen.record(5, 10)); + assert!(seen.record(6, 9)); + } +} diff --git a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs index dd4fc8b03..e98ec5ed2 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs @@ -24,7 +24,7 @@ use ethlambda_state_transition::beacon::fork_choice::{ }; use ethlambda_state_transition::beacon::gossip::{ self as rules, Outcome, SeenAggregates, SeenAttestations, SeenBlockColumns, SeenBlocks, - SeenColumns, + SeenColumns, SeenEnvelopes, SeenPayloadAttestations, }; use ethlambda_state_transition::beacon::helpers::accessors::CommitteeCache; use ethlambda_state_transition::beacon::primitives::Root; @@ -41,6 +41,8 @@ const HANDLERS: &[&str] = &[ "gossip_data_column_sidecar", "gossip_beacon_aggregate_and_proof", "gossip_beacon_attestation", + "gossip_execution_payload_envelope", + "gossip_payload_attestation_message", ]; /// The forks each of [`HANDLERS`] validates. A case from any other fork is @@ -59,6 +61,9 @@ fn validated_forks(handler: &str) -> &'static [ForkName] { "gossip_beacon_aggregate_and_proof" | "gossip_beacon_attestation" => { &[ForkName::Fulu, ForkName::Gloas] } + "gossip_execution_payload_envelope" | "gossip_payload_attestation_message" => { + &[ForkName::Gloas] + } other => panic!("{other} is not in HANDLERS, so it has no validated forks"), } } @@ -72,11 +77,10 @@ fn validated_forks(handler: &str) -> &'static [ForkName] { /// list, the same way [`super::UNMODELED_FORKS`] forces a decision on a new /// fork directory. /// -/// `gossip_execution_payload_bid`, `gossip_execution_payload_envelope`, -/// `gossip_payload_attestation_message`, and `gossip_proposer_preferences` -/// are gloas's own topics (EIP-7732 ePBS): the builder's bid, the revealed -/// payload envelope, the payload timeliness committee's vote, and a -/// builder's advertised preferences, respectively. None of them existed +/// `gossip_execution_payload_bid` and `gossip_proposer_preferences` are +/// gloas's own topics (EIP-7732 ePBS): the builder's bid and a builder's +/// advertised preferences, respectively. (Its envelope and payload +/// attestation topics are in [`HANDLERS`].) None of them existed /// until gloas's fixture directory started parsing (`ForkName::Gloas`), so /// they land here rather than silently in `unknown` the first time this /// runner sees them. Alphabetized with the rest rather than kept together. @@ -85,9 +89,7 @@ const IGNORED_HANDLERS: &[&str] = &[ "gossip_blob_sidecar", "gossip_bls_to_execution_change", "gossip_execution_payload_bid", - "gossip_execution_payload_envelope", "gossip_partial_data_column_sidecar", - "gossip_payload_attestation_message", "gossip_proposer_preferences", "gossip_proposer_slashing", "gossip_sync_committee_contribution_and_proof", @@ -121,6 +123,14 @@ const SKIPPED: &[(&str, &str)] = &[ "gossip_beacon_attestation__reject_block_failed_validation", "a vote block seen without a post-state is ignored, not rejected, until a bad-block cache exists", ), + ( + "gossip_execution_payload_envelope__reject_block_failed_validation", + "a block seen without a post-state is queued, not rejected, until a bad-block cache exists", + ), + ( + "gossip_payload_attestation_message__reject_block_failed_validation", + "a vote block seen without a post-state is ignored, not rejected, until a bad-block cache exists", + ), ]; /// Capacity for a case's seen caches. A case sends a handful of messages. @@ -403,6 +413,8 @@ fn run_case(case: &Case) -> Result<(), String> { let mut seen_block_columns = SeenBlockColumns::new(capacity); let mut seen_aggregates = SeenAggregates::new(capacity, capacity); let mut seen_attestations = SeenAttestations::new(capacity); + let mut seen_envelopes = SeenEnvelopes::new(capacity); + let mut seen_payload_attestations = SeenPayloadAttestations::new(capacity); for (index, message) in meta.messages.iter().enumerate() { let now_ms = config.genesis_time_ms() @@ -491,6 +503,37 @@ fn run_case(case: &Case) -> Result<(), String> { } outcome } + "execution_payload" => { + let envelope = gloas::SignedExecutionPayloadEnvelope::from_ssz_bytes( + &case.ssz_bytes(&message.message), + ) + .map_err(|err| format!("decoding {}: {err:?}", message.message))?; + let outcome = rules::envelope::validate(&seen_envelopes, &store, &envelope); + if outcome == Outcome::Accept { + seen_envelopes.record( + envelope.message.beacon_block_root, + envelope.message.builder_index, + ); + } + outcome + } + "payload_attestation_message" => { + let attestation = gloas::PayloadAttestationMessage::from_ssz_bytes( + &case.ssz_bytes(&message.message), + ) + .map_err(|err| format!("decoding {}: {err:?}", message.message))?; + let outcome = rules::payload_attestation::validate( + &seen_payload_attestations, + &store, + &attestation, + now_ms, + ); + if outcome == Outcome::Accept { + seen_payload_attestations + .record(attestation.data.slot, attestation.validator_index); + } + outcome + } other => return Err(format!("topic {other} has no runner")), }; check(message, outcome).map_err(|err| format!("message {index}: {err}"))?; From 0304c1c67c198c4d1895859157d76d866b6c2eb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:06:45 -0300 Subject: [PATCH 054/132] fix(beacon): gate queued envelopes on their shape Withdrawals and the execution request lists are progressive, so only the gossip size cap bounded an envelope that could be queued for an unknown block. The withdrawal and request count limits do not depend on the block, so they now run in cheap_checks, before anything can be queued. The payload attestation path also decoded the whole head block just to get its root; head() returns the root alone. A committee-window miss now has its own ignore reason instead of sharing StateUnavailable, and the deviation paths (no post-state, uncached head, window miss, non-gloas block) are pinned by unit tests since the fixtures treat Queue as ignore. --- .../src/beacon/gossip/envelope.rs | 166 ++++++++++++++++-- .../state_transition/src/beacon/gossip/mod.rs | 4 + .../src/beacon/gossip/payload_attestation.rs | 142 ++++++++++++++- .../src/beacon/gossip/test_support.rs | 5 + 4 files changed, 304 insertions(+), 13 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/gossip/envelope.rs b/crates/blockchain/state_transition/src/beacon/gossip/envelope.rs index 2e9da9255..6557301cc 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/envelope.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/envelope.rs @@ -14,6 +14,9 @@ //! specification rejects it, the same deviation every other topic makes until //! a bad-block cache exists. A block never seen is `Queue` too (the //! specification's "MAY be queued"). +//! - The block-independent limits (withdrawal and request counts) run in +//! [`cheap_checks`], so a message that no block could make valid is never +//! queued. //! - The finalized-slot `IGNORE` runs in [`cheap_checks`], ahead of the block //! lookups the specification places before it. Both outcomes are `IGNORE`, so //! only the metric label can differ, and an envelope too old to matter is @@ -28,6 +31,7 @@ use super::column::judgeable_block_gloas; use super::{ IgnoreReason, Outcome, RejectReason, execution_requests_within_limits, finalized_start_slot, }; +use crate::beacon::containers::gloas::BuilderIndex; use crate::beacon::containers::{SignedBeaconBlock, gloas}; use crate::beacon::fork_choice::Store; use crate::beacon::lean_boundary::lean_block_unreachable; @@ -39,20 +43,20 @@ use crate::beacon::stf::gloas::verify_execution_payload_envelope_signature; /// specification's `seen.execution_payload_envelopes`. /// /// Bounded by capacity rather than pruned on finality, like [`super::SeenBlocks`]. -pub struct SeenEnvelopes(LruCache<(Root, u64), ()>); +pub struct SeenEnvelopes(LruCache<(Root, BuilderIndex), ()>); impl SeenEnvelopes { pub fn new(capacity: NonZeroUsize) -> Self { Self(LruCache::new(capacity)) } - pub fn contains(&self, block_root: Root, builder_index: u64) -> bool { + pub fn contains(&self, block_root: Root, builder_index: BuilderIndex) -> bool { self.0.contains(&(block_root, builder_index)) } /// Record the first valid envelope for its key. Returns `false`, changing /// nothing, when one is already recorded. - pub fn record(&mut self, block_root: Root, builder_index: u64) -> bool { + pub fn record(&mut self, block_root: Root, builder_index: BuilderIndex) -> bool { if self.0.contains(&(block_root, builder_index)) { return false; } @@ -80,6 +84,18 @@ pub fn cheap_checks( if message.payload.slot_number < finalized_start_slot(store) { return Err(Outcome::Ignore(IgnoreReason::Finalized)); } + // [REJECT] The number of withdrawals is within the limit, and [REJECT] the + // execution request counts are within theirs. Neither depends on the + // block, so both run before anything can be queued: the lists are + // progressive, and without this only the gossip size cap would bound what + // a parked envelope holds. The specification checks them after the block + // lookups; an envelope invalid under every block is rejected either way. + if message.payload.withdrawals.len() > preset::MAX_WITHDRAWALS_PER_PAYLOAD { + return Err(Outcome::Reject(RejectReason::TooManyWithdrawals)); + } + if !execution_requests_within_limits(&message.execution_requests) { + return Err(Outcome::Reject(RejectReason::OperationLimit)); + } Ok(()) } @@ -125,14 +141,7 @@ pub fn stateful_checks(store: &Store, envelope: &gloas::SignedExecutionPayloadEn if message.execution_requests.hash_tree_root() != bid.execution_requests_root { return Outcome::Reject(RejectReason::ExecutionRequestsRootMismatch); } - // [REJECT] The execution request counts are within their limits. - if !execution_requests_within_limits(&message.execution_requests) { - return Outcome::Reject(RejectReason::OperationLimit); - } - // [REJECT] The number of withdrawals is within the limit. - if payload.withdrawals.len() > preset::MAX_WITHDRAWALS_PER_PAYLOAD { - return Outcome::Reject(RejectReason::TooManyWithdrawals); - } + // The request-count and withdrawal-count limits ran in `cheap_checks`. // [REJECT] The envelope signature is valid. Read off the block's cached // post-state, never rebuilt: a miss would stall the verdict gossipsub // waits on. @@ -168,6 +177,141 @@ pub fn validate( #[cfg(test)] mod tests { use super::*; + use crate::beacon::containers::electra; + use crate::beacon::gossip::QueueReason; + use crate::beacon::gossip::test_support::{fulu_parent, seen_envelopes, store}; + use crate::beacon::primitives::Slot; + + /// A fulu block at `slot`: a block of the wrong fork for either topic. + fn fulu_block(slot: Slot) -> SignedBeaconBlock { + SignedBeaconBlock::Fulu(electra::SignedBeaconBlock { + message: electra::BeaconBlock { + slot, + proposer_index: 0, + parent_root: Root::ZERO, + state_root: Root::ZERO, + body: electra::BeaconBlockBody::empty(), + }, + signature: Default::default(), + }) + } + + const BLOCK_ROOT: Root = Root::repeat_byte(7); + + fn withdrawal() -> crate::beacon::containers::capella::Withdrawal { + crate::beacon::containers::capella::Withdrawal { + index: 0, + validator_index: 0, + address: Default::default(), + amount: 0, + } + } + + /// An envelope naming `BLOCK_ROOT`, with `withdrawals` withdrawals in its + /// payload. Nothing else in it is meant to be valid. + fn envelope(slot: Slot, withdrawals: usize) -> gloas::SignedExecutionPayloadEnvelope { + let payload = gloas::ExecutionPayload { + parent_hash: Default::default(), + fee_recipient: Default::default(), + state_root: Default::default(), + receipts_root: Default::default(), + logs_bloom: vec![0u8; preset::BYTES_PER_LOGS_BLOOM] + .try_into() + .expect("an exact-length bloom"), + prev_randao: Default::default(), + block_number: 0, + gas_limit: 0, + gas_used: 0, + timestamp: 0, + extra_data: Default::default(), + base_fee_per_gas: Default::default(), + block_hash: Default::default(), + transactions: Default::default(), + withdrawals: vec![withdrawal(); withdrawals] + .try_into() + .expect("progressive lists are unbounded"), + blob_gas_used: 0, + excess_blob_gas: 0, + block_access_list: Default::default(), + slot_number: slot, + }; + gloas::SignedExecutionPayloadEnvelope { + message: gloas::ExecutionPayloadEnvelope { + payload, + execution_requests: Default::default(), + builder_index: 0, + beacon_block_root: BLOCK_ROOT, + parent_beacon_block_root: Root::ZERO, + }, + signature: Default::default(), + } + } + + #[test] + fn too_many_withdrawals_are_rejected_before_any_block_lookup() { + let store = store(0); + let envelope = envelope(5, preset::MAX_WITHDRAWALS_PER_PAYLOAD + 1); + assert_eq!( + cheap_checks(&seen_envelopes(), &store, &envelope), + Err(Outcome::Reject(RejectReason::TooManyWithdrawals)) + ); + // The block is unknown, which would queue a well-formed envelope. + assert_eq!( + validate(&seen_envelopes(), &store, &envelope), + Outcome::Reject(RejectReason::TooManyWithdrawals) + ); + } + + #[test] + fn too_many_requests_are_rejected_before_any_block_lookup() { + let store = store(0); + let mut envelope = envelope(5, 0); + envelope.message.execution_requests.builder_exits = + vec![Default::default(); preset::MAX_BUILDER_EXIT_REQUESTS_PER_PAYLOAD as usize + 1] + .try_into() + .expect("progressive lists are unbounded"); + assert_eq!( + validate(&seen_envelopes(), &store, &envelope), + Outcome::Reject(RejectReason::OperationLimit) + ); + } + + #[test] + fn an_envelope_within_the_limits_for_an_unseen_block_is_queued() { + let store = store(0); + let envelope = envelope(5, preset::MAX_WITHDRAWALS_PER_PAYLOAD); + assert_eq!( + validate(&seen_envelopes(), &store, &envelope), + Outcome::Queue(QueueReason::BlockUnknown) + ); + } + + #[test] + fn an_envelope_whose_block_has_no_post_state_is_queued_not_rejected() { + let mut store = store(0); + store + .insert_pending_block(BLOCK_ROOT, fulu_block(5)) + .expect("insert pending block"); + assert_eq!( + validate(&seen_envelopes(), &store, &envelope(5, 0)), + Outcome::Queue(QueueReason::BlockNotReady) + ); + } + + #[test] + fn an_envelope_naming_a_non_gloas_block_is_rejected() { + let mut store = store(0); + store + .insert_pending_block(BLOCK_ROOT, fulu_block(5)) + .expect("insert the block"); + store + .insert_state(BLOCK_ROOT, fulu_parent(3)) + .expect("insert the state"); + assert_eq!( + validate(&seen_envelopes(), &store, &envelope(5, 0)), + Outcome::Reject(RejectReason::BlockNotGloas) + ); + } #[test] fn an_envelope_key_records_once_per_root_and_builder() { diff --git a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs index 63e430eec..1a25b5756 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs @@ -146,6 +146,9 @@ pub enum IgnoreReason { NotCurrentSlot, /// A payload attestation names a block that is not at the attested slot. BlockNotAtSlot, + /// The head state's payload timeliness committee window cannot answer for + /// the attested slot. + PtcUnavailable, } impl IgnoreReason { @@ -170,6 +173,7 @@ impl IgnoreReason { Self::PayloadOptimistic => "payload_optimistic", Self::NotCurrentSlot => "not_current_slot", Self::BlockNotAtSlot => "block_not_at_slot", + Self::PtcUnavailable => "ptc_unavailable", } } } diff --git a/crates/blockchain/state_transition/src/beacon/gossip/payload_attestation.rs b/crates/blockchain/state_transition/src/beacon/gossip/payload_attestation.rs index 736fb492a..1ba89616a 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/payload_attestation.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/payload_attestation.rs @@ -105,7 +105,9 @@ pub fn stateful_checks(store: &Store, message: &gloas::PayloadAttestationMessage } // The committee is read off the head state, as the specification says. - let Some((_, head_root)) = store.beacon_head() else { + // `head` is the root alone: `beacon_head` would decode the whole head block + // for a slot nothing here reads. + let Ok(head_root) = store.head() else { return Outcome::Ignore(IgnoreReason::Internal); }; let Some(state) = store.cached_state(CacheKey::BlockState(head_root)) else { @@ -118,7 +120,7 @@ pub fn stateful_checks(store: &Store, message: &gloas::PayloadAttestationMessage // [REJECT] The validator is a member of the payload timeliness committee. // A slot the head's committee window cannot answer is not the sender's fault. let Ok(ptc) = get_ptc(&state, data.slot, &store.config()) else { - return Outcome::Ignore(IgnoreReason::StateUnavailable); + return Outcome::Ignore(IgnoreReason::PtcUnavailable); }; if !ptc.contains(&validator_index) { return Outcome::Reject(RejectReason::NotInPtc); @@ -154,7 +156,143 @@ pub fn validate( #[cfg(test)] mod tests { + use std::sync::Arc; + + use ethlambda_storage::backend::InMemoryBackend; + use ethlambda_types::checkpoint::Checkpoint; + use super::*; + use crate::beacon::config::Config; + use crate::beacon::containers::{SignedBeaconBlock, electra}; + use crate::beacon::fork::ForkName; + use crate::beacon::gossip::test_support::{GENESIS_TIME, store}; + use crate::beacon::helpers::test_state::with_validators_at; + use crate::beacon::preset; + use crate::beacon::primitives::Root; + + /// A fulu block at `slot`: a block of the wrong fork for either topic. + fn fulu_block(slot: Slot) -> SignedBeaconBlock { + SignedBeaconBlock::Fulu(electra::SignedBeaconBlock { + message: electra::BeaconBlock { + slot, + proposer_index: 0, + parent_root: Root::ZERO, + state_root: Root::ZERO, + body: electra::BeaconBlockBody::empty(), + }, + signature: Default::default(), + }) + } + + const BLOCK_ROOT: Root = Root::repeat_byte(7); + + fn message(slot: Slot) -> gloas::PayloadAttestationMessage { + gloas::PayloadAttestationMessage { + validator_index: 0, + data: gloas::PayloadAttestationData { + beacon_block_root: BLOCK_ROOT, + slot, + payload_present: true, + blob_data_available: true, + }, + signature: Default::default(), + } + } + + /// A store that is gloas from genesis, so the committee lookup is reached. + fn gloas_store() -> Store { + Store::init_beacon( + Arc::new(InMemoryBackend::new()), + GENESIS_TIME, + Config::mainnet() + .with_fork_epoch(ForkName::Fulu, 0) + .with_fork_epoch(ForkName::Gloas, 0), + Root::ZERO, + Checkpoint { + root: Root::ZERO, + slot: 0, + }, + 0, + ) + } + + #[test] + fn a_vote_for_an_unseen_block_is_ignored() { + let store = store(0); + assert_eq!( + stateful_checks(&store, &message(5)), + Outcome::Ignore(IgnoreReason::UnknownBlock) + ); + } + + #[test] + fn a_vote_for_a_block_without_a_post_state_is_ignored_not_rejected() { + let mut store = store(0); + store + .insert_pending_block(BLOCK_ROOT, fulu_block(5)) + .expect("insert pending block"); + assert_eq!( + stateful_checks(&store, &message(5)), + Outcome::Ignore(IgnoreReason::StateUnavailable) + ); + } + + #[test] + fn a_vote_is_ignored_when_the_head_state_is_not_cached() { + let mut store = store(0); + store + .insert_pending_block(BLOCK_ROOT, fulu_block(5)) + .expect("insert the block"); + // Only the voted block has a state; the head (the anchor root) has none. + store + .insert_state(BLOCK_ROOT, with_validators_at(ForkName::Gloas, 8)) + .expect("insert the state"); + assert_eq!( + stateful_checks(&store, &message(5)), + Outcome::Ignore(IgnoreReason::StateUnavailable) + ); + } + + #[test] + fn a_vote_outside_the_head_states_committee_window_is_ignored_with_its_own_reason() { + let mut store = gloas_store(); + let slot = 10 * preset::SLOTS_PER_EPOCH; + store + .insert_pending_block(BLOCK_ROOT, fulu_block(slot)) + .expect("insert the block"); + store + .insert_state(BLOCK_ROOT, with_validators_at(ForkName::Gloas, 8)) + .expect("insert the voted block's state"); + let head = store.head().expect("head root"); + store + .insert_state(head, with_validators_at(ForkName::Gloas, 8)) + .expect("insert the head state"); + assert_eq!( + stateful_checks(&store, &message(slot)), + Outcome::Ignore(IgnoreReason::PtcUnavailable) + ); + } + + #[test] + fn a_validator_the_head_state_lacks_is_rejected() { + let mut store = gloas_store(); + store + .insert_pending_block(BLOCK_ROOT, fulu_block(5)) + .expect("insert the block"); + store + .insert_state(BLOCK_ROOT, with_validators_at(ForkName::Gloas, 8)) + .expect("insert the voted block's state"); + let head = store.head().expect("head root"); + store + .insert_state(head, with_validators_at(ForkName::Gloas, 8)) + .expect("insert the head state"); + let mut message = message(5); + message.validator_index = 8; + assert_eq!( + stateful_checks(&store, &message), + Outcome::Reject(RejectReason::UnknownValidator) + ); + } #[test] fn a_payload_attestation_key_records_once_per_slot_and_validator() { diff --git a/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs b/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs index dd35862cb..78527bc26 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs @@ -66,6 +66,11 @@ pub(crate) fn seen_aggregates() -> SeenAggregates { SeenAggregates::new(capacity, capacity) } +/// The [`seen_blocks`] counterpart for gloas execution payload envelopes. +pub(crate) fn seen_envelopes() -> super::SeenEnvelopes { + super::SeenEnvelopes::new(NonZeroUsize::new(8).expect("non-zero")) +} + /// The [`seen_blocks`] counterpart for subnet attestations. pub(crate) fn seen_attestations() -> SeenAttestations { SeenAttestations::new(NonZeroUsize::new(8).expect("non-zero")) From ce29681deaf52efd7de9ec0fd8d90b2e68ad22cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:01:07 -0300 Subject: [PATCH 055/132] feat(p2p): serve execution payload envelopes over req/resp Gloas peers recover missed payloads through execution_payload_envelopes_by_range/1 and by_root/1. Register both protocols (bidirectional at the libp2p level) and answer them from the store: the codec frames each envelope under the digest of its block's epoch and rejects any non-gloas or unscheduled digest, by-range serves only envelopes the next canonical block builds on (the head's when held, since fork choice does not persist which payload branch the head picked), and both are capped at MAX_REQUEST_PAYLOADS. No requests are issued yet; a response nothing asked for is logged and dropped. --- crates/net/p2p/src/beacon/encoding.rs | 148 +++++++++ crates/net/p2p/src/beacon/messages.rs | 16 +- crates/net/p2p/src/beacon/protocols.rs | 38 +++ crates/net/p2p/src/lib.rs | 14 + crates/net/p2p/src/req_resp/behaviour.rs | 22 +- crates/net/p2p/src/req_resp/codec.rs | 183 ++++++++++- crates/net/p2p/src/req_resp/handlers.rs | 391 ++++++++++++++++++++++- crates/net/p2p/src/req_resp/messages.rs | 21 +- crates/net/p2p/src/swarm_adapter.rs | 6 + crates/storage/src/store.rs | 104 ++++++ 10 files changed, 933 insertions(+), 10 deletions(-) diff --git a/crates/net/p2p/src/beacon/encoding.rs b/crates/net/p2p/src/beacon/encoding.rs index 575e36d7d..cb1124b09 100644 --- a/crates/net/p2p/src/beacon/encoding.rs +++ b/crates/net/p2p/src/beacon/encoding.rs @@ -36,6 +36,8 @@ use std::io; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::DataColumnSidecar; use ethlambda_types::beacon::containers::SignedBeaconBlock; +use ethlambda_types::beacon::containers::gloas::SignedExecutionPayloadEnvelope; +use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::fork_digest::compute_fork_digest; use ethlambda_types::beacon::preset; use ethlambda_types::beacon::primitives::Root; @@ -336,3 +338,149 @@ where }) .await } + +/// Write an envelope response: one result code, one `ForkDigest` and one +/// payload per envelope. +/// +/// The counterpart of [`write_blocks_response`]. The spec keys the context +/// epoch off the block the envelope fulfills; an envelope carries no slot of +/// its own, but `verify_execution_payload_envelope` pins +/// `payload.slot_number` to that block's slot, so a stored envelope's +/// `slot_number` is the block's slot. +pub async fn write_execution_payload_envelopes_response( + io: &mut T, + label: &'static str, + config: &Config, + genesis_validators_root: Root, + envelopes: &[SignedExecutionPayloadEnvelope], +) -> io::Result<()> +where + T: AsyncWrite + Unpin + Send, +{ + for envelope in envelopes { + let encoded = envelope.to_ssz(); + let slot = envelope.message.payload.slot_number; + if encoded.len() > MAX_PAYLOAD_SIZE - 1024 { + warn!( + slot, + size = encoded.len(), + "Skipping oversized execution payload envelope in response" + ); + continue; + } + let digest = compute_fork_digest( + config, + genesis_validators_root, + slot / preset::SLOTS_PER_EPOCH, + ); + write_success_chunk(io, label, &digest, encoded).await?; + } + Ok(()) +} + +/// Read an envelope response: one `SignedExecutionPayloadEnvelope` per chunk, +/// until the peer closes. +/// +/// The counterpart of [`decode_data_column_sidecars_response`]: the fork comes +/// from the context bytes, and a digest no scheduled fork uses ends the stream. +/// Envelopes exist only from gloas, so any other fork's digest is refused too. +/// The context is then checked against the digest the payload's own +/// `slot_number` implies. +pub async fn decode_execution_payload_envelopes_response( + io: &mut T, + protocol_label: &str, + config: &Config, + genesis_validators_root: Root, +) -> io::Result> +where + T: AsyncRead + Unpin + Send, +{ + let limits = ChunkLimits { + has_context: true, + max_chunks: protocols::MAX_REQUEST_PAYLOADS as usize, + }; + let schedule = ForkSchedule::new(config, genesis_validators_root); + read_chunked_response(io, protocol_label, limits, |context, payload| { + let fork = <[u8; 4]>::try_from(context) + .ok() + .and_then(|digest| schedule.fork_for_digest(digest)); + if fork != Some(ForkName::Gloas) { + return Err(invalid(format!( + "execution payload envelope chunk context {} is not the gloas fork digest", + hex::encode(context), + ))); + } + let envelope = SignedExecutionPayloadEnvelope::from_ssz_bytes(payload) + .map_err(|err| invalid(format!("execution payload envelope chunk: {err:?}")))?; + let slot = envelope.message.payload.slot_number; + let expected = schedule.digest_at(slot / preset::SLOTS_PER_EPOCH); + if context != expected { + warn!( + slot, + peer_context = %hex::encode(context), + our_context = %hex::encode(expected), + "Execution payload envelope chunk names another fork digest" + ); + return Err(invalid(format!( + "execution payload envelope chunk context {} does not match {} for slot {}", + hex::encode(context), + hex::encode(expected), + slot, + ))); + } + Ok(envelope) + }) + .await +} + +/// Test fixtures shared by the codec and handler tests. +#[cfg(test)] +pub(crate) mod test_support { + use ethlambda_types::beacon::containers::gloas::{ + ExecutionPayload, ExecutionPayloadEnvelope, SignedExecutionPayloadEnvelope, + }; + use ethlambda_types::beacon::preset; + use ethlambda_types::beacon::primitives::{ExecutionBlockHash, Root}; + + /// An otherwise-empty envelope for the block `beacon_block_root` at `slot`, + /// revealing the payload `block_hash`. + pub(crate) fn envelope( + beacon_block_root: Root, + slot: u64, + block_hash: ExecutionBlockHash, + ) -> SignedExecutionPayloadEnvelope { + let payload = ExecutionPayload { + parent_hash: Default::default(), + fee_recipient: Default::default(), + state_root: Default::default(), + receipts_root: Default::default(), + logs_bloom: vec![0u8; preset::BYTES_PER_LOGS_BLOOM] + .try_into() + .expect("built at exactly BYTES_PER_LOGS_BLOOM"), + prev_randao: Default::default(), + block_number: 0, + gas_limit: 0, + gas_used: 0, + timestamp: 0, + extra_data: Default::default(), + base_fee_per_gas: Default::default(), + block_hash, + transactions: Default::default(), + withdrawals: Default::default(), + blob_gas_used: 0, + excess_blob_gas: 0, + block_access_list: Default::default(), + slot_number: slot, + }; + SignedExecutionPayloadEnvelope { + message: ExecutionPayloadEnvelope { + payload, + execution_requests: Default::default(), + builder_index: 0, + beacon_block_root, + parent_beacon_block_root: Root::ZERO, + }, + signature: Default::default(), + } + } +} diff --git a/crates/net/p2p/src/beacon/messages.rs b/crates/net/p2p/src/beacon/messages.rs index e32d26256..2a89713ae 100644 --- a/crates/net/p2p/src/beacon/messages.rs +++ b/crates/net/p2p/src/beacon/messages.rs @@ -14,7 +14,7 @@ use libssz_derive::{SszDecode, SszEncode}; use libssz_types::{SszBitvector, SszList}; use super::constants::{ATTESTATION_SUBNET_COUNT, SYNC_COMMITTEE_SUBNET_COUNT}; -use super::protocols::MAX_REQUEST_BLOCKS_DENEB; +use super::protocols::{MAX_REQUEST_BLOCKS_DENEB, MAX_REQUEST_PAYLOADS}; /// `attnets`: which attestation subnets a node serves. pub type AttnetsBits = SszBitvector<{ ATTESTATION_SUBNET_COUNT as usize }>; @@ -206,6 +206,20 @@ pub struct DataColumnsByRangeRequest { pub columns: ColumnIndices, } +/// `ExecutionPayloadEnvelopeRoots`: `ExecutionPayloadEnvelopesByRoot` v1's +/// request body, the beacon block roots of the envelopes asked for, bare as an +/// SSZ field. +pub type ExecutionPayloadEnvelopeRoots = SszList; + +/// `ExecutionPayloadEnvelopesByRange` v1's request body. +/// +/// Fixed-size like the block range body, minus the deprecated `step`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, SszEncode, SszDecode)] +pub struct ExecutionPayloadEnvelopesByRangeRequest { + pub start_slot: Slot, + pub count: u64, +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/net/p2p/src/beacon/protocols.rs b/crates/net/p2p/src/beacon/protocols.rs index 3c376ba58..7498420de 100644 --- a/crates/net/p2p/src/beacon/protocols.rs +++ b/crates/net/p2p/src/beacon/protocols.rs @@ -9,6 +9,10 @@ //! into, and an unregistered protocol is still refused at stream negotiation //! rather than answered with a lie. //! +//! The two gloas execution payload envelope protocols are registered for the +//! same reason the column ones are: the server side answers out of +//! `Table::ExecutionPayloadEnvelopes`. Nothing requests them yet. +//! //! Only version 2 of the two block protocols is registered. Version 1 is //! deprecated by the spec, which lets a client answer it with an empty list, //! and its chunks carry no ``, so serving it would mean a second @@ -31,6 +35,14 @@ pub const DATA_COLUMN_SIDECARS_BY_RANGE_V1: &str = "/eth2/beacon_chain/req/data_column_sidecars_by_range/1/ssz_snappy"; pub const DATA_COLUMN_SIDECARS_BY_ROOT_V1: &str = "/eth2/beacon_chain/req/data_column_sidecars_by_root/1/ssz_snappy"; +pub const EXECUTION_PAYLOAD_ENVELOPES_BY_RANGE_V1: &str = + "/eth2/beacon_chain/req/execution_payload_envelopes_by_range/1/ssz_snappy"; +pub const EXECUTION_PAYLOAD_ENVELOPES_BY_ROOT_V1: &str = + "/eth2/beacon_chain/req/execution_payload_envelopes_by_root/1/ssz_snappy"; + +/// `MAX_REQUEST_PAYLOADS`: the ceiling on either gloas envelope request, and +/// on the list a response carries. +pub const MAX_REQUEST_PAYLOADS: u64 = 128; /// `MAX_REQUEST_BLOCKS`: the ceiling phase0 put on either block request. /// @@ -99,6 +111,14 @@ pub fn registrations() -> Vec<(StreamProtocol, ProtocolSupport)> { StreamProtocol::new(DATA_COLUMN_SIDECARS_BY_ROOT_V1), ProtocolSupport::Full, ), + ( + StreamProtocol::new(EXECUTION_PAYLOAD_ENVELOPES_BY_RANGE_V1), + ProtocolSupport::Full, + ), + ( + StreamProtocol::new(EXECUTION_PAYLOAD_ENVELOPES_BY_ROOT_V1), + ProtocolSupport::Full, + ), ] } @@ -116,6 +136,12 @@ pub fn label(protocol: &str) -> Option<&'static str> { BLOCKS_BY_ROOT_V2 => Some("beacon_blocks_by_root_v2"), DATA_COLUMN_SIDECARS_BY_RANGE_V1 => Some("beacon_data_column_sidecars_by_range"), DATA_COLUMN_SIDECARS_BY_ROOT_V1 => Some("beacon_data_column_sidecars_by_root"), + EXECUTION_PAYLOAD_ENVELOPES_BY_RANGE_V1 => { + Some("beacon_execution_payload_envelopes_by_range") + } + EXECUTION_PAYLOAD_ENVELOPES_BY_ROOT_V1 => { + Some("beacon_execution_payload_envelopes_by_root") + } _ => None, } } @@ -158,6 +184,18 @@ mod tests { ); } + #[test] + fn the_envelope_protocol_ids_are_the_spec_strings() { + assert_eq!( + EXECUTION_PAYLOAD_ENVELOPES_BY_RANGE_V1, + "/eth2/beacon_chain/req/execution_payload_envelopes_by_range/1/ssz_snappy" + ); + assert_eq!( + EXECUTION_PAYLOAD_ENVELOPES_BY_ROOT_V1, + "/eth2/beacon_chain/req/execution_payload_envelopes_by_root/1/ssz_snappy" + ); + } + #[test] fn a_sidecar_request_is_bounded_by_the_block_ceiling_times_the_columns() { assert_eq!( diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index 92fee32a1..e86425a65 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -392,6 +392,8 @@ pub enum ReqRespProtocol { BeaconBlocksByRoot, DataColumnSidecarsByRange, DataColumnSidecarsByRoot, + ExecutionPayloadEnvelopesByRange, + ExecutionPayloadEnvelopesByRoot, } /// An outbound request id, namespaced by the protocol it was sent on. @@ -1910,6 +1912,12 @@ async fn handle_behaviour_event( ReqRespEvent::DataColumnSidecarsByRoot(e) => { (ReqRespProtocol::DataColumnSidecarsByRoot, e) } + ReqRespEvent::ExecutionPayloadEnvelopesByRange(e) => { + (ReqRespProtocol::ExecutionPayloadEnvelopesByRange, e) + } + ReqRespEvent::ExecutionPayloadEnvelopesByRoot(e) => { + (ReqRespProtocol::ExecutionPayloadEnvelopesByRoot, e) + } }, }; @@ -2982,6 +2990,12 @@ mod tests { ReqRespEvent::DataColumnSidecarsByRoot(e) => { (ReqRespProtocol::DataColumnSidecarsByRoot, e) } + ReqRespEvent::ExecutionPayloadEnvelopesByRange(e) => { + (ReqRespProtocol::ExecutionPayloadEnvelopesByRange, e) + } + ReqRespEvent::ExecutionPayloadEnvelopesByRoot(e) => { + (ReqRespProtocol::ExecutionPayloadEnvelopesByRoot, e) + } }) } diff --git a/crates/net/p2p/src/req_resp/behaviour.rs b/crates/net/p2p/src/req_resp/behaviour.rs index cae1002b8..8242337e7 100644 --- a/crates/net/p2p/src/req_resp/behaviour.rs +++ b/crates/net/p2p/src/req_resp/behaviour.rs @@ -61,6 +61,8 @@ pub(crate) struct ReqResp { pub(crate) beacon_blocks_by_root: request_response::Behaviour, pub(crate) data_column_sidecars_by_range: request_response::Behaviour, pub(crate) data_column_sidecars_by_root: request_response::Behaviour, + pub(crate) execution_payload_envelopes_by_range: request_response::Behaviour, + pub(crate) execution_payload_envelopes_by_root: request_response::Behaviour, } impl ReqResp { @@ -197,7 +199,7 @@ impl ReqResp { fetch_protocol_config(), ), data_column_sidecars_by_root: request_response::Behaviour::with_codec( - codec, + codec.clone(), one_protocol( is_beacon, beacon::protocols::DATA_COLUMN_SIDECARS_BY_ROOT_V1, @@ -205,6 +207,24 @@ impl ReqResp { ), fetch_protocol_config(), ), + execution_payload_envelopes_by_range: request_response::Behaviour::with_codec( + codec.clone(), + one_protocol( + is_beacon, + beacon::protocols::EXECUTION_PAYLOAD_ENVELOPES_BY_RANGE_V1, + request_response::ProtocolSupport::Full, + ), + fetch_protocol_config(), + ), + execution_payload_envelopes_by_root: request_response::Behaviour::with_codec( + codec, + one_protocol( + is_beacon, + beacon::protocols::EXECUTION_PAYLOAD_ENVELOPES_BY_ROOT_V1, + request_response::ProtocolSupport::Full, + ), + fetch_protocol_config(), + ), } } } diff --git a/crates/net/p2p/src/req_resp/codec.rs b/crates/net/p2p/src/req_resp/codec.rs index b77359d2e..9bd77f103 100644 --- a/crates/net/p2p/src/req_resp/codec.rs +++ b/crates/net/p2p/src/req_resp/codec.rs @@ -11,8 +11,8 @@ use super::{ }; use crate::beacon::messages::{ - BeaconBlocksByRangeRequest, DataColumnsByRangeRequest, DataColumnsByRootIdentifiers, Goodbye, - Ping, + BeaconBlocksByRangeRequest, DataColumnsByRangeRequest, DataColumnsByRootIdentifiers, + ExecutionPayloadEnvelopeRoots, ExecutionPayloadEnvelopesByRangeRequest, Goodbye, Ping, }; use crate::beacon::{BeaconContext, encoding as beacon_encoding, protocols}; use crate::lean::messages::{BlocksByRootRequest, RequestedBlockRoots}; @@ -20,8 +20,8 @@ use crate::lean::{encoding as lean_encoding, protocols as lean_protocols}; use crate::metrics; /// Protocols whose response payload has one shape forever write no context -/// bytes, which is every protocol here except the two beacon block ones and -/// the two beacon data column sidecar ones. +/// bytes, which is every protocol here except the two beacon block ones, the +/// two beacon data column sidecar ones and the two gloas envelope ones. const NO_CONTEXT: &[u8] = &[]; /// Short label extracted from a libp2p protocol id, used as the `protocol` @@ -197,6 +197,17 @@ impl libp2p::request_response::Codec for Codec { DataColumnsByRangeRequest::from_ssz_bytes(&payload) .map_err(|err| invalid(format!("{err:?}")))?, )), + protocols::EXECUTION_PAYLOAD_ENVELOPES_BY_RANGE_V1 => { + Ok(Request::ExecutionPayloadEnvelopesByRange( + ExecutionPayloadEnvelopesByRangeRequest::from_ssz_bytes(&payload) + .map_err(|err| invalid(format!("{err:?}")))?, + )) + } + protocols::EXECUTION_PAYLOAD_ENVELOPES_BY_ROOT_V1 => { + let roots = ExecutionPayloadEnvelopeRoots::from_ssz_bytes(&payload) + .map_err(|err| invalid(format!("{err:?}")))?; + Ok(Request::ExecutionPayloadEnvelopesByRoot(roots.into_inner())) + } _ => Err(invalid(format!("unknown protocol: {}", protocol.as_ref()))), } } @@ -266,6 +277,20 @@ impl libp2p::request_response::Codec for Codec { sidecars, ))) } + protocols::EXECUTION_PAYLOAD_ENVELOPES_BY_RANGE_V1 + | protocols::EXECUTION_PAYLOAD_ENVELOPES_BY_ROOT_V1 => { + let context = self.beacon_context(protocol.as_ref())?; + let envelopes = beacon_encoding::decode_execution_payload_envelopes_response( + io, + label, + &context.config, + context.genesis_validators_root, + ) + .await?; + Ok(Response::success( + ResponsePayload::ExecutionPayloadEnvelopes(envelopes), + )) + } _ => Err(invalid(format!("unknown protocol: {}", protocol.as_ref()))), } } @@ -333,6 +358,12 @@ impl libp2p::request_response::Codec for Codec { .to_ssz() } Request::DataColumnsByRange(request) => request.to_ssz(), + Request::ExecutionPayloadEnvelopesByRange(request) => request.to_ssz(), + Request::ExecutionPayloadEnvelopesByRoot(roots) => { + ExecutionPayloadEnvelopeRoots::try_from(roots.clone()) + .map_err(|err| invalid(format!("{err:?}")))? + .to_ssz() + } }; let compressed_size = write_payload(io, &encoded).await?; @@ -398,6 +429,17 @@ impl libp2p::request_response::Codec for Codec { ) .await } + ResponsePayload::ExecutionPayloadEnvelopes(envelopes) => { + let context = self.beacon_context(protocol.as_ref())?; + beacon_encoding::write_execution_payload_envelopes_response( + io, + label, + &context.config, + context.genesis_validators_root, + envelopes, + ) + .await + } }, Response::Error { code, message } => { // Send error code @@ -904,4 +946,137 @@ mod tests { "a fork digest mismatch must abort the stream rather than decode successfully" ); } + + fn envelope_at(slot: u64, byte: u8) -> gloas::SignedExecutionPayloadEnvelope { + crate::beacon::encoding::test_support::envelope( + Root::repeat_byte(byte), + slot, + Root::repeat_byte(byte), + ) + } + + #[tokio::test] + async fn an_envelope_by_range_request_round_trips() { + let request = Request::ExecutionPayloadEnvelopesByRange( + crate::beacon::messages::ExecutionPayloadEnvelopesByRangeRequest { + start_slot: 9, + count: 128, + }, + ); + let decoded = + request_round_trip(protocols::EXECUTION_PAYLOAD_ENVELOPES_BY_RANGE_V1, request).await; + match decoded { + Request::ExecutionPayloadEnvelopesByRange(wire) => { + assert_eq!((wire.start_slot, wire.count), (9, 128)); + } + other => panic!("decoded as {other:?}"), + } + } + + #[tokio::test] + async fn an_envelope_by_root_request_round_trips_as_a_bare_list() { + let roots = vec![Root::repeat_byte(1), Root::repeat_byte(2)]; + let decoded = request_round_trip( + protocols::EXECUTION_PAYLOAD_ENVELOPES_BY_ROOT_V1, + Request::ExecutionPayloadEnvelopesByRoot(roots.clone()), + ) + .await; + match decoded { + Request::ExecutionPayloadEnvelopesByRoot(decoded) => assert_eq!(decoded, roots), + other => panic!("decoded as {other:?}"), + } + } + + #[tokio::test] + async fn an_envelope_by_root_request_over_the_limit_is_refused_on_write() { + let roots = vec![Root::ZERO; protocols::MAX_REQUEST_PAYLOADS as usize + 1]; + let stream_protocol = + StreamProtocol::new(protocols::EXECUTION_PAYLOAD_ENVELOPES_BY_ROOT_V1); + let mut buffer = Cursor::new(Vec::new()); + let result = codec() + .write_request( + &stream_protocol, + &mut buffer, + Request::ExecutionPayloadEnvelopesByRoot(roots), + ) + .await; + assert!(result.is_err()); + } + + /// Each chunk's context is the digest of its own envelope's epoch; for + /// gloas envelopes that is the gloas digest. + #[tokio::test] + async fn an_envelope_response_round_trips_under_the_gloas_digest() { + let envelopes = vec![ + envelope_at(gloas_slot(), 1), + envelope_at(gloas_slot() + 1, 2), + ]; + let stream_protocol = + StreamProtocol::new(protocols::EXECUTION_PAYLOAD_ENVELOPES_BY_RANGE_V1); + + let mut buffer = Cursor::new(Vec::new()); + codec_for(gloas_config()) + .write_response( + &stream_protocol, + &mut buffer, + Response::success(ResponsePayload::ExecutionPayloadEnvelopes( + envelopes.clone(), + )), + ) + .await + .expect("writes"); + let bytes = buffer.into_inner(); + // The result code, then the four context bytes. + let config = gloas_config(); + let expected = compute_fork_digest(&config, mainnet_gvr(), config.gloas_fork_epoch); + assert_eq!(bytes[0], 0); + assert_eq!(bytes[1..5], expected); + + let decoded = codec_for(gloas_config()) + .read_response(&stream_protocol, &mut Cursor::new(bytes)) + .await + .expect("reads"); + match decoded { + Response::Success { + payload: ResponsePayload::ExecutionPayloadEnvelopes(decoded), + } => assert_eq!(decoded, envelopes), + other => panic!("decoded as {other:?}"), + } + } + + #[tokio::test] + async fn an_envelope_chunk_under_a_fulu_digest_aborts_the_stream() { + let config = gloas_config(); + let fulu_digest = compute_fork_digest(&config, mainnet_gvr(), config.fulu_fork_epoch); + let payload = envelope_at(gloas_slot(), 1).to_ssz(); + let stream_protocol = + StreamProtocol::new(protocols::EXECUTION_PAYLOAD_ENVELOPES_BY_ROOT_V1); + + let mut buffer = Cursor::new(Vec::new()); + write_success_chunk(&mut buffer, "test", &fulu_digest, payload) + .await + .expect("writes"); + let result = codec_for(config) + .read_response(&stream_protocol, &mut Cursor::new(buffer.into_inner())) + .await; + + assert!(result.is_err()); + } + + #[tokio::test] + async fn an_envelope_chunk_under_an_unscheduled_digest_aborts_the_stream() { + let payload = envelope_at(gloas_slot(), 1).to_ssz(); + let stream_protocol = + StreamProtocol::new(protocols::EXECUTION_PAYLOAD_ENVELOPES_BY_ROOT_V1); + + let mut buffer = Cursor::new(Vec::new()); + write_success_chunk(&mut buffer, "test", &[0xde, 0xad, 0xbe, 0xef], payload) + .await + .expect("writes"); + let result = codec_for(gloas_config()) + .read_response(&stream_protocol, &mut Cursor::new(buffer.into_inner())) + .await; + + assert!(result.is_err()); + } } diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index 5c938e76a..fec1ba7c3 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -24,7 +24,7 @@ use ethlambda_state_transition::beacon::das; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::constants; use ethlambda_types::beacon::containers::fulu::{ColumnIndices, DataColumnsByRootIdentifier}; -use ethlambda_types::beacon::containers::{DataColumnSidecar, SignedBeaconBlock}; +use ethlambda_types::beacon::containers::{DataColumnSidecar, SignedBeaconBlock, gloas}; use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::checkpoint::Checkpoint; use ethlambda_types::primitives::HashTreeRoot as _; @@ -39,10 +39,11 @@ use crate::beacon::column_checks; use crate::beacon::decode::fork_at_slot; use crate::beacon::handler::{self as beacon_handler, StatusVersion}; use crate::beacon::messages::{ - BeaconMetaData, BeaconStatus, DataColumnsByRangeRequest, Goodbye, Ping, + BeaconMetaData, BeaconStatus, DataColumnsByRangeRequest, + ExecutionPayloadEnvelopesByRangeRequest, Goodbye, Ping, }; use crate::beacon::protocols::{ - MAX_REQUEST_BLOCKS as MAX_BEACON_REQUEST_BLOCKS, MAX_REQUEST_BLOCKS_DENEB, + MAX_REQUEST_BLOCKS as MAX_BEACON_REQUEST_BLOCKS, MAX_REQUEST_BLOCKS_DENEB, MAX_REQUEST_PAYLOADS, }; use crate::beacon::transition; use crate::discovery::enr::node_id_from_peer_id; @@ -167,6 +168,26 @@ pub async fn handle_req_resp_message( ) .await; } + Request::ExecutionPayloadEnvelopesByRange(request) => { + trace!( + kind = "execution_payload_envelopes_by_range_request", + peer_count, "P2P message received" + ); + handle_execution_payload_envelopes_by_range_request( + server, peer, request, channel, + ) + .await; + } + Request::ExecutionPayloadEnvelopesByRoot(roots) => { + trace!( + kind = "execution_payload_envelopes_by_root_request", + peer_count, "P2P message received" + ); + handle_execution_payload_envelopes_by_root_request( + server, peer, roots, channel, + ) + .await; + } } } request_response::Message::Response { @@ -267,6 +288,21 @@ pub async fn handle_req_resp_message( } } } + ResponsePayload::ExecutionPayloadEnvelopes(envelopes) => { + trace!( + kind = "execution_payload_envelopes_response", + peer_count, "P2P message received" + ); + // Nothing requests envelopes yet, so no id can + // legitimately be waiting for this answer. + server.outbound_requests.remove(&request_id); + debug!( + %peer, + ?request_id, + count = envelopes.len(), + "Received an execution payload envelopes response nothing asked for" + ); + } ResponsePayload::Blocks(blocks) => { trace!(kind = "blocks_response", peer_count, "P2P message received"); // Dispatched on what was asked for first and on the @@ -2092,6 +2128,154 @@ async fn handle_data_column_sidecars_by_range_request( ); } +/// Answer `execution_payload_envelopes_by_range/1` off the canonical chain. +/// +/// The spec makes it equivalent to `BeaconBlocksByRange` v2 with another +/// response type, so the window rules are the block ones: an empty window is +/// `INVALID_REQUEST`, a `count` above `MAX_REQUEST_PAYLOADS` is truncated to it +/// ("Clients MAY limit the number of ... in the response"), and a `start_slot` +/// below the anchor is `RESOURCE_UNAVAILABLE`. A window before gloas simply +/// holds no envelopes, so it is an empty answer. Which envelopes count as +/// canonical is [`Store::canonical_execution_payload_envelopes`]'s business. +async fn handle_execution_payload_envelopes_by_range_request( + server: &mut P2PServer, + peer: PeerId, + request: ExecutionPayloadEnvelopesByRangeRequest, + channel: ResponseChannel, +) { + let Some(channel) = beacon_block_store_or_refuse(server, peer, channel) else { + return; + }; + + if request.count == 0 { + refuse( + server, + channel, + ResponseCode::INVALID_REQUEST, + "invalid ExecutionPayloadEnvelopesByRange request", + ); + return; + } + let envelopes = match envelopes_by_range(&server.store, request.start_slot, request.count) { + Ok(envelopes) => envelopes, + Err(reason) => { + debug!( + %peer, + start_slot = request.start_slot, + "ExecutionPayloadEnvelopesByRange request starts before this node's chain" + ); + refuse(server, channel, ResponseCode::RESOURCE_UNAVAILABLE, reason); + return; + } + }; + + trace!( + %peer, + start_slot = request.start_slot, + count = request.count, + found = envelopes.len(), + "Responding to ExecutionPayloadEnvelopesByRange request" + ); + + respond( + server, + channel, + ResponsePayload::ExecutionPayloadEnvelopes(envelopes), + ); +} + +/// The envelopes `ExecutionPayloadEnvelopesByRange` answers a non-empty window +/// with: the canonical ones in `[start_slot, start_slot + count)`, truncated +/// to `MAX_REQUEST_PAYLOADS` slots. +/// +/// `Err` carries the `RESOURCE_UNAVAILABLE` reason for a `start_slot` below +/// the store's anchor, which this chain does not reach back to. +fn envelopes_by_range( + store: &Store, + start_slot: u64, + count: u64, +) -> Result, &'static str> { + if start_slot < store.anchor_slot() { + return Err("requested range starts before this node's earliest available slot"); + } + let count = count.min(MAX_REQUEST_PAYLOADS); + // An overflowing window is attacker-supplied and answered empty, as is a + // zero `count`, which has no last offset to add. + let Some(end_slot) = count + .checked_sub(1) + .and_then(|last_offset| start_slot.checked_add(last_offset)) + else { + return Ok(Vec::new()); + }; + Ok(store + .canonical_execution_payload_envelopes(start_slot, end_slot) + .inspect_err(|err| { + warn!( + start_slot, + end_slot, + ?err, + "Failed to get execution payload envelopes by slot range" + ) + }) + .unwrap_or_default()) +} + +/// Answer `execution_payload_envelopes_by_root/1` with whichever of the named +/// envelopes this node holds. +/// +/// The counterpart of [`handle_beacon_blocks_by_root_request`]: a root with no +/// stored envelope is left out rather than answered with an error, and the +/// answer follows the order asked in. Roots past `MAX_REQUEST_PAYLOADS` are +/// ignored (the wire list type already bounds a decoded request to it). +async fn handle_execution_payload_envelopes_by_root_request( + server: &mut P2PServer, + peer: PeerId, + roots: Vec, + channel: ResponseChannel, +) { + let Some(channel) = beacon_block_store_or_refuse(server, peer, channel) else { + return; + }; + + let requested = roots.len(); + let envelopes = envelopes_by_root(&server.store, &roots); + + trace!( + %peer, + requested, + found = envelopes.len(), + "Responding to ExecutionPayloadEnvelopesByRoot request" + ); + + respond( + server, + channel, + ResponsePayload::ExecutionPayloadEnvelopes(envelopes), + ); +} + +/// The held envelopes among `roots`, in the order asked, looking at no more +/// than `MAX_REQUEST_PAYLOADS` of them. +fn envelopes_by_root(store: &Store, roots: &[H256]) -> Vec { + roots + .iter() + .take(MAX_REQUEST_PAYLOADS as usize) + .filter_map(|root| { + store + .get_execution_payload_envelope(root) + .inspect_err(|err| { + error!( + root = %ethlambda_types::ShortRoot(&root.0), + %err, + "Stored execution payload envelope failed to read" + ) + }) + .ok() + .flatten() + }) + .collect() +} + /// Ask `peer` for the beacon blocks in `[start_slot, start_slot + count)`. /// /// Returns the request id, so a caller can pair the answer with what it asked @@ -3032,4 +3216,205 @@ mod tests { assert_eq!(kept, vec![gloas_column(asked, 3)]); assert_eq!(dropped, 2); } + + /// A gloas block whose bid builds on the payload `parent_block_hash`. + fn gloas_block(slot: u64, parent_root: H256, parent_block_hash: H256) -> SignedBeaconBlock { + let mut block = gloas::SignedBeaconBlock { + message: gloas::BeaconBlock { + slot, + proposer_index: 0, + parent_root, + state_root: H256::ZERO, + body: Default::default(), + }, + signature: Default::default(), + }; + block + .message + .body + .signed_execution_payload_bid + .message + .parent_block_hash = parent_block_hash; + SignedBeaconBlock::Gloas(block) + } + + fn beacon_store() -> Store { + let mut store = Store::init_beacon( + Arc::new(InMemoryBackend::new()), + 0, + Config::mainnet(), + H256::ZERO, + Checkpoint::default(), + 0, + ); + store + .insert_signed_block(H256::ZERO, gloas_block(0, H256::ZERO, H256::ZERO)) + .expect("insert anchor"); + store + } + + /// Inserts `block` and moves the head to it; returns its root. + fn extend_chain(store: &mut Store, block: SignedBeaconBlock) -> H256 { + let root = block.message_hash_tree_root(); + store.insert_signed_block(root, block).expect("insert"); + store + .update_checkpoints(ForkCheckpoints::head_only(root)) + .expect("advance head"); + root + } + + fn reveal( + store: &mut Store, + slot: u64, + root: H256, + hash: H256, + ) -> gloas::SignedExecutionPayloadEnvelope { + let envelope = crate::beacon::encoding::test_support::envelope(root, slot, hash); + store.insert_verified_payload(slot, &envelope); + envelope + } + + fn hash(byte: u8) -> H256 { + H256::repeat_byte(byte) + } + + /// Canonical slots 1 to 3, every payload revealed and built on, plus a + /// sibling of slot 2 with its own envelope. Returns the three canonical + /// envelopes. + fn chain_with_every_payload_built_on() -> (Store, Vec) { + let mut store = beacon_store(); + let root_1 = extend_chain(&mut store, gloas_block(1, H256::ZERO, hash(0))); + let root_2 = extend_chain(&mut store, gloas_block(2, root_1, hash(1))); + let root_3 = extend_chain(&mut store, gloas_block(3, root_2, hash(2))); + // A fork at slot 2 that the head does not descend from. + let sibling = gloas_block(2, root_1, hash(9)); + let sibling_root = sibling.message_hash_tree_root(); + store + .insert_signed_block(sibling_root, sibling) + .expect("insert"); + reveal(&mut store, 2, sibling_root, hash(0x99)); + let envelopes = vec![ + reveal(&mut store, 1, root_1, hash(1)), + reveal(&mut store, 2, root_2, hash(2)), + reveal(&mut store, 3, root_3, hash(3)), + ]; + (store, envelopes) + } + + #[test] + fn by_range_serves_only_canonical_envelopes_in_slot_order() { + let (store, envelopes) = chain_with_every_payload_built_on(); + + let served = envelopes_by_range(&store, 1, 3).expect("in window"); + + assert_eq!(served, envelopes); + assert_eq!( + envelopes_by_range(&store, 2, 1).expect("in window"), + envelopes[1..2] + ); + } + + #[test] + fn by_range_omits_a_payload_the_next_block_does_not_build_on() { + let mut store = beacon_store(); + let root_1 = extend_chain(&mut store, gloas_block(1, H256::ZERO, hash(0))); + // Slot 2 builds on slot 1's empty branch, not on the payload revealed. + let root_2 = extend_chain(&mut store, gloas_block(2, root_1, hash(0))); + reveal(&mut store, 1, root_1, hash(1)); + let head_envelope = reveal(&mut store, 2, root_2, hash(2)); + + let served = envelopes_by_range(&store, 1, 2).expect("in window"); + + assert_eq!(served, vec![head_envelope]); + } + + #[test] + fn by_range_judges_the_last_block_of_a_window_by_its_successor_past_the_window() { + let mut store = beacon_store(); + let root_1 = extend_chain(&mut store, gloas_block(1, H256::ZERO, hash(0))); + // Slot 3 follows an empty slot 2 and builds on slot 1's payload. + extend_chain(&mut store, gloas_block(3, root_1, hash(1))); + let envelope = reveal(&mut store, 1, root_1, hash(1)); + + assert_eq!( + envelopes_by_range(&store, 1, 1).expect("in window"), + vec![envelope] + ); + } + + #[test] + fn by_range_caps_the_window_at_max_request_payloads() { + let (store, envelopes) = chain_with_every_payload_built_on(); + + // A window far wider than the cap still answers from the first + // MAX_REQUEST_PAYLOADS slots only, which here includes every envelope; + // the cap is observable on where the window ends. + assert_eq!( + envelopes_by_range(&store, 1, u64::MAX).expect("in window"), + envelopes + ); + let beyond_cap = 1 + MAX_REQUEST_PAYLOADS; + assert_eq!( + envelopes_by_range(&store, beyond_cap, u64::MAX).expect("in window"), + Vec::new() + ); + } + + #[test] + fn by_range_answers_empty_for_a_window_before_gloas_and_for_overflow() { + let (store, _) = chain_with_every_payload_built_on(); + + assert!( + envelopes_by_range(&store, 0, 1) + .expect("in window") + .is_empty() + ); + assert!( + envelopes_by_range(&store, u64::MAX, 5) + .expect("in window") + .is_empty() + ); + } + + #[test] + fn by_range_refuses_a_start_below_the_anchor() { + let mut store = Store::init_beacon( + Arc::new(InMemoryBackend::new()), + 0, + Config::mainnet(), + H256::ZERO, + Checkpoint::default(), + 10, + ); + store + .insert_signed_block(H256::ZERO, gloas_block(10, H256::ZERO, H256::ZERO)) + .expect("insert anchor"); + + assert!(envelopes_by_range(&store, 9, 1).is_err()); + assert!(envelopes_by_range(&store, 10, 1).is_ok()); + } + + #[test] + fn by_root_serves_known_envelopes_in_request_order_and_skips_unknown_roots() { + let (store, envelopes) = chain_with_every_payload_built_on(); + let roots: Vec = envelopes + .iter() + .map(|e| e.message.beacon_block_root) + .collect(); + let asked = vec![roots[2], H256::repeat_byte(0xee), roots[0]]; + + let served = envelopes_by_root(&store, &asked); + + assert_eq!(served, vec![envelopes[2].clone(), envelopes[0].clone()]); + } + + #[test] + fn by_root_looks_at_no_more_than_the_limit() { + let (store, envelopes) = chain_with_every_payload_built_on(); + let known = envelopes[0].message.beacon_block_root; + let mut asked = vec![H256::repeat_byte(0xee); MAX_REQUEST_PAYLOADS as usize]; + asked.push(known); + + assert!(envelopes_by_root(&store, &asked).is_empty()); + } } diff --git a/crates/net/p2p/src/req_resp/messages.rs b/crates/net/p2p/src/req_resp/messages.rs index f0a8df344..a94f6dac6 100644 --- a/crates/net/p2p/src/req_resp/messages.rs +++ b/crates/net/p2p/src/req_resp/messages.rs @@ -2,10 +2,13 @@ use ethlambda_types::ShortRoot; use ethlambda_types::beacon::containers::DataColumnSidecar; use ethlambda_types::beacon::containers::SignedBeaconBlock; use ethlambda_types::beacon::containers::fulu::DataColumnsByRootIdentifier; +use ethlambda_types::beacon::containers::gloas::SignedExecutionPayloadEnvelope; +use ethlambda_types::beacon::primitives::Root; use libssz_types::SszList; use crate::beacon::messages::{ - BeaconMetaData, BeaconStatus, DataColumnsByRangeRequest, Goodbye, Ping, + BeaconMetaData, BeaconStatus, DataColumnsByRangeRequest, + ExecutionPayloadEnvelopesByRangeRequest, Goodbye, Ping, }; use crate::lean::messages::{BlocksByRootRequest, Status}; @@ -90,6 +93,13 @@ pub enum Request { /// codec already enforces on encode. DataColumnsByRoot(Vec), DataColumnsByRange(DataColumnsByRangeRequest), + /// Gloas envelopes in a slot window. Beacon-only. + ExecutionPayloadEnvelopesByRange(ExecutionPayloadEnvelopesByRangeRequest), + /// The beacon block roots of the envelopes asked for. A plain `Vec` for the + /// same reason as [`Self::DataColumnsByRoot`]: the wire type + /// ([`crate::beacon::messages::ExecutionPayloadEnvelopeRoots`]) carries the + /// bound on decode and the codec re-applies it on encode. + ExecutionPayloadEnvelopesByRoot(Vec), } #[derive(Debug, Clone)] @@ -187,6 +197,13 @@ impl std::fmt::Display for Response { // `Debug` alone runs to tens of kilobytes of cell bytes. write!(f, "Success(DataColumnSidecars count={})", sidecars.len()) } + Self::Success { + payload: ResponsePayload::ExecutionPayloadEnvelopes(envelopes), + } => write!( + f, + "Success(ExecutionPayloadEnvelopes count={})", + envelopes.len() + ), Self::Error { code, message } => { let message = String::from_utf8_lossy(message); write!(f, "Error({code:?}: {message})") @@ -273,6 +290,8 @@ pub enum ResponsePayload { /// protocols: which one produced the answer is known from the outbound /// request id, and the chunk framing is the same either way. DataColumnSidecars(Vec), + /// The envelopes answering either gloas envelope protocol. + ExecutionPayloadEnvelopes(Vec), } /// Error message type for non-success responses. diff --git a/crates/net/p2p/src/swarm_adapter.rs b/crates/net/p2p/src/swarm_adapter.rs index f060c3fb0..1303c52d7 100644 --- a/crates/net/p2p/src/swarm_adapter.rs +++ b/crates/net/p2p/src/swarm_adapter.rs @@ -366,6 +366,12 @@ fn execute_command(swarm: &mut libp2p::Swarm, cmd: SwarmCommand) { ReqRespProtocol::DataColumnSidecarsByRoot => behaviour .data_column_sidecars_by_root .send_request(&peer, request), + ReqRespProtocol::ExecutionPayloadEnvelopesByRange => behaviour + .execution_payload_envelopes_by_range + .send_request(&peer, request), + ReqRespProtocol::ExecutionPayloadEnvelopesByRoot => behaviour + .execution_payload_envelopes_by_root + .send_request(&peer, request), }; if let Some(tx) = request_id_tx { let _ = tx.send(ReqRespRequestId { protocol, id }); diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 683e09e73..09c5ce79f 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -834,6 +834,31 @@ fn decode_beacon_block_value(bytes: &[u8]) -> SignedBeaconBlock { SignedBeaconBlock::from_ssz(fork, ssz).expect("valid signed block") } +/// Whether `block` builds on the payload whose hash is `payload_block_hash`: +/// gloas's bid `parent_block_hash` rule for a payload being part of the chain. +fn builds_on_payload(block: &SignedBeaconBlock, payload_block_hash: &ExecutionBlockHash) -> bool { + match block { + SignedBeaconBlock::Gloas(block) => { + block + .message + .body + .signed_execution_payload_bid + .message + .parent_block_hash + == *payload_block_hash + } + // A pre-gloas block cannot follow a gloas one. + SignedBeaconBlock::Phase0(_) + | SignedBeaconBlock::Altair(_) + | SignedBeaconBlock::Bellatrix(_) + | SignedBeaconBlock::Capella(_) + | SignedBeaconBlock::Deneb(_) + | SignedBeaconBlock::Electra(_) + | SignedBeaconBlock::Fulu(_) => false, + SignedBeaconBlock::Lean(_) => Store::lean_only("builds_on_payload"), + } +} + /// `root`'s slot on a beacon chain, read without a `Store`, for the writer /// thread's anchor decision. /// @@ -3373,6 +3398,85 @@ impl Store { })) } + /// Gloas: the stored envelopes of payloads on the canonical chain, for the + /// slots `start_slot..=end_slot`, in ascending slot order. + /// + /// Serves `ExecutionPayloadEnvelopesByRange`. A payload is part of the + /// chain when the next canonical block builds on it: that block's bid + /// `parent_block_hash` equals the payload's `block_hash`. A block whose + /// successor builds on its empty branch has its envelope left out, even + /// when this node holds one. The head has no successor and fork choice does + /// not persist which payload branch it picked, so its envelope is served + /// when held: only a verified envelope is ever stored. + /// + /// Canonical blocks come from `BlockRoots` and the whole answer is read + /// from one view, like [`get_signed_blocks_by_slot_range`](Self::get_signed_blocks_by_slot_range). + /// The successor of the last block in the window may lie past `end_slot`, + /// so the index is read on beyond it, up to the head, until one is found. + pub fn canonical_execution_payload_envelopes( + &self, + start_slot: u64, + end_slot: u64, + ) -> Result, Error> { + if self.chain != Chain::Beacon { + Self::lean_only("canonical_execution_payload_envelopes"); + } + let view = self.backend.begin_read().expect("read view"); + let canonical_root = |slot: u64| -> Option { + view.get(Table::BlockRoots, &encode_block_root_key(slot)) + .expect("get block root") + .map(|bytes| H256::from_ssz_bytes(&bytes).expect("valid block root")) + }; + + let mut window: Vec<(u64, H256)> = (start_slot..=end_slot) + .filter_map(|slot| canonical_root(slot).map(|root| (slot, root))) + .collect(); + if !window.is_empty() + && let Some(head_slot) = beacon_block_slot(self.backend.as_ref(), &self.head()?) + { + let successor = (end_slot.saturating_add(1)..=head_slot) + .find_map(|slot| canonical_root(slot).map(|root| (slot, root))); + window.extend(successor); + } + + let mut envelopes = Vec::new(); + for (index, &(slot, root)) in window.iter().enumerate() { + if slot > end_slot { + break; + } + let Some(bytes) = view + .get( + Table::ExecutionPayloadEnvelopes, + &encode_slot_root_key(slot, &root), + ) + .expect("get") + else { + continue; + }; + let envelope = gloas::SignedExecutionPayloadEnvelope::from_ssz_bytes(&bytes) + .expect("a stored envelope decodes"); + let builds_on_it = match window.get(index + 1) { + None => true, + Some((_, next_root)) => { + let next = view + .get(Table::BlockHeaders, &next_root.to_ssz()) + .expect("get") + .map(|bytes| decode_beacon_block_value(&bytes)); + match next { + Some(next) => { + builds_on_payload(&next, &envelope.message.payload.block_hash) + } + None => false, + } + } + }; + if builds_on_it { + envelopes.push(envelope); + } + } + Ok(envelopes) + } + /// Rebuilds the in-memory `verified_payloads` and `block_timeliness` from /// their tables, for every block fork choice can still reach. /// From 1580cd529fe32150b2daafe483c85768f92b845e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:06:44 -0300 Subject: [PATCH 056/132] refactor(p2p): move the envelope req/resp handlers into their own module handlers.rs is already past three thousand lines and the client side of the envelope protocols lands next, so give the server handlers, their pure functions and tests a module of their own. Pure move. --- crates/net/p2p/src/req_resp/envelopes.rs | 377 +++++++++++++++++++++++ crates/net/p2p/src/req_resp/handlers.rs | 372 +--------------------- crates/net/p2p/src/req_resp/mod.rs | 1 + 3 files changed, 391 insertions(+), 359 deletions(-) create mode 100644 crates/net/p2p/src/req_resp/envelopes.rs diff --git a/crates/net/p2p/src/req_resp/envelopes.rs b/crates/net/p2p/src/req_resp/envelopes.rs new file mode 100644 index 000000000..b6670914c --- /dev/null +++ b/crates/net/p2p/src/req_resp/envelopes.rs @@ -0,0 +1,377 @@ +//! The gloas execution payload envelope protocols +//! (`execution_payload_envelopes_by_range/1` and `_by_root/1`): the server side, +//! answering peers out of `Table::ExecutionPayloadEnvelopes`. + +use ethlambda_storage::Store; +use ethlambda_types::beacon::containers::gloas; +use ethlambda_types::primitives::H256; +use libp2p::PeerId; +use libp2p::request_response::ResponseChannel; +use tracing::{debug, error, trace, warn}; + +use super::handlers::{beacon_block_store_or_refuse, refuse, respond}; +use super::messages::ResponseCode; +use super::{Response, ResponsePayload}; +use crate::P2PServer; +use crate::beacon::messages::ExecutionPayloadEnvelopesByRangeRequest; +use crate::beacon::protocols::MAX_REQUEST_PAYLOADS; + +/// Answer `execution_payload_envelopes_by_range/1` off the canonical chain. +/// +/// The spec makes it equivalent to `BeaconBlocksByRange` v2 with another +/// response type, so the window rules are the block ones: an empty window is +/// `INVALID_REQUEST`, a `count` above `MAX_REQUEST_PAYLOADS` is truncated to it +/// ("Clients MAY limit the number of ... in the response"), and a `start_slot` +/// below the anchor is `RESOURCE_UNAVAILABLE`. A window before gloas simply +/// holds no envelopes, so it is an empty answer. Which envelopes count as +/// canonical is [`Store::canonical_execution_payload_envelopes`]'s business. +pub(super) async fn handle_execution_payload_envelopes_by_range_request( + server: &mut P2PServer, + peer: PeerId, + request: ExecutionPayloadEnvelopesByRangeRequest, + channel: ResponseChannel, +) { + let Some(channel) = beacon_block_store_or_refuse(server, peer, channel) else { + return; + }; + + if request.count == 0 { + refuse( + server, + channel, + ResponseCode::INVALID_REQUEST, + "invalid ExecutionPayloadEnvelopesByRange request", + ); + return; + } + let envelopes = match envelopes_by_range(&server.store, request.start_slot, request.count) { + Ok(envelopes) => envelopes, + Err(reason) => { + debug!( + %peer, + start_slot = request.start_slot, + "ExecutionPayloadEnvelopesByRange request starts before this node's chain" + ); + refuse(server, channel, ResponseCode::RESOURCE_UNAVAILABLE, reason); + return; + } + }; + + trace!( + %peer, + start_slot = request.start_slot, + count = request.count, + found = envelopes.len(), + "Responding to ExecutionPayloadEnvelopesByRange request" + ); + + respond( + server, + channel, + ResponsePayload::ExecutionPayloadEnvelopes(envelopes), + ); +} + +/// The envelopes `ExecutionPayloadEnvelopesByRange` answers a non-empty window +/// with: the canonical ones in `[start_slot, start_slot + count)`, truncated +/// to `MAX_REQUEST_PAYLOADS` slots. +/// +/// `Err` carries the `RESOURCE_UNAVAILABLE` reason for a `start_slot` below +/// the store's anchor, which this chain does not reach back to. +fn envelopes_by_range( + store: &Store, + start_slot: u64, + count: u64, +) -> Result, &'static str> { + if start_slot < store.anchor_slot() { + return Err("requested range starts before this node's earliest available slot"); + } + let count = count.min(MAX_REQUEST_PAYLOADS); + // An overflowing window is attacker-supplied and answered empty, as is a + // zero `count`, which has no last offset to add. + let Some(end_slot) = count + .checked_sub(1) + .and_then(|last_offset| start_slot.checked_add(last_offset)) + else { + return Ok(Vec::new()); + }; + Ok(store + .canonical_execution_payload_envelopes(start_slot, end_slot) + .inspect_err(|err| { + warn!( + start_slot, + end_slot, + ?err, + "Failed to get execution payload envelopes by slot range" + ) + }) + .unwrap_or_default()) +} + +/// Answer `execution_payload_envelopes_by_root/1` with whichever of the named +/// envelopes this node holds. +/// +/// The counterpart of [`handle_beacon_blocks_by_root_request`]: a root with no +/// stored envelope is left out rather than answered with an error, and the +/// answer follows the order asked in. Roots past `MAX_REQUEST_PAYLOADS` are +/// ignored (the wire list type already bounds a decoded request to it). +pub(super) async fn handle_execution_payload_envelopes_by_root_request( + server: &mut P2PServer, + peer: PeerId, + roots: Vec, + channel: ResponseChannel, +) { + let Some(channel) = beacon_block_store_or_refuse(server, peer, channel) else { + return; + }; + + let requested = roots.len(); + let envelopes = envelopes_by_root(&server.store, &roots); + + trace!( + %peer, + requested, + found = envelopes.len(), + "Responding to ExecutionPayloadEnvelopesByRoot request" + ); + + respond( + server, + channel, + ResponsePayload::ExecutionPayloadEnvelopes(envelopes), + ); +} + +/// The held envelopes among `roots`, in the order asked, looking at no more +/// than `MAX_REQUEST_PAYLOADS` of them. +fn envelopes_by_root(store: &Store, roots: &[H256]) -> Vec { + roots + .iter() + .take(MAX_REQUEST_PAYLOADS as usize) + .filter_map(|root| { + store + .get_execution_payload_envelope(root) + .inspect_err(|err| { + error!( + root = %ethlambda_types::ShortRoot(&root.0), + %err, + "Stored execution payload envelope failed to read" + ) + }) + .ok() + .flatten() + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::beacon::encoding::test_support; + use ethlambda_storage::{ForkCheckpoints, backend::InMemoryBackend}; + use ethlambda_types::beacon::config::Config; + use ethlambda_types::beacon::containers::SignedBeaconBlock; + use ethlambda_types::checkpoint::Checkpoint; + use std::sync::Arc; + + /// A gloas block whose bid builds on the payload `parent_block_hash`. + fn gloas_block(slot: u64, parent_root: H256, parent_block_hash: H256) -> SignedBeaconBlock { + let mut block = gloas::SignedBeaconBlock { + message: gloas::BeaconBlock { + slot, + proposer_index: 0, + parent_root, + state_root: H256::ZERO, + body: Default::default(), + }, + signature: Default::default(), + }; + block + .message + .body + .signed_execution_payload_bid + .message + .parent_block_hash = parent_block_hash; + SignedBeaconBlock::Gloas(block) + } + + fn beacon_store() -> Store { + let mut store = Store::init_beacon( + Arc::new(InMemoryBackend::new()), + 0, + Config::mainnet(), + H256::ZERO, + Checkpoint::default(), + 0, + ); + store + .insert_signed_block(H256::ZERO, gloas_block(0, H256::ZERO, H256::ZERO)) + .expect("insert anchor"); + store + } + + /// Inserts `block` and moves the head to it; returns its root. + fn extend_chain(store: &mut Store, block: SignedBeaconBlock) -> H256 { + let root = block.message_hash_tree_root(); + store.insert_signed_block(root, block).expect("insert"); + store + .update_checkpoints(ForkCheckpoints::head_only(root)) + .expect("advance head"); + root + } + + fn reveal( + store: &mut Store, + slot: u64, + root: H256, + hash: H256, + ) -> gloas::SignedExecutionPayloadEnvelope { + let envelope = test_support::envelope(root, slot, hash); + store.insert_verified_payload(slot, &envelope); + envelope + } + + fn hash(byte: u8) -> H256 { + H256::repeat_byte(byte) + } + + /// Canonical slots 1 to 3, every payload revealed and built on, plus a + /// sibling of slot 2 with its own envelope. Returns the three canonical + /// envelopes. + fn chain_with_every_payload_built_on() -> (Store, Vec) { + let mut store = beacon_store(); + let root_1 = extend_chain(&mut store, gloas_block(1, H256::ZERO, hash(0))); + let root_2 = extend_chain(&mut store, gloas_block(2, root_1, hash(1))); + let root_3 = extend_chain(&mut store, gloas_block(3, root_2, hash(2))); + // A fork at slot 2 that the head does not descend from. + let sibling = gloas_block(2, root_1, hash(9)); + let sibling_root = sibling.message_hash_tree_root(); + store + .insert_signed_block(sibling_root, sibling) + .expect("insert"); + reveal(&mut store, 2, sibling_root, hash(0x99)); + let envelopes = vec![ + reveal(&mut store, 1, root_1, hash(1)), + reveal(&mut store, 2, root_2, hash(2)), + reveal(&mut store, 3, root_3, hash(3)), + ]; + (store, envelopes) + } + + #[test] + fn by_range_serves_only_canonical_envelopes_in_slot_order() { + let (store, envelopes) = chain_with_every_payload_built_on(); + + let served = envelopes_by_range(&store, 1, 3).expect("in window"); + + assert_eq!(served, envelopes); + assert_eq!( + envelopes_by_range(&store, 2, 1).expect("in window"), + envelopes[1..2] + ); + } + + #[test] + fn by_range_omits_a_payload_the_next_block_does_not_build_on() { + let mut store = beacon_store(); + let root_1 = extend_chain(&mut store, gloas_block(1, H256::ZERO, hash(0))); + // Slot 2 builds on slot 1's empty branch, not on the payload revealed. + let root_2 = extend_chain(&mut store, gloas_block(2, root_1, hash(0))); + reveal(&mut store, 1, root_1, hash(1)); + let head_envelope = reveal(&mut store, 2, root_2, hash(2)); + + let served = envelopes_by_range(&store, 1, 2).expect("in window"); + + assert_eq!(served, vec![head_envelope]); + } + + #[test] + fn by_range_judges_the_last_block_of_a_window_by_its_successor_past_the_window() { + let mut store = beacon_store(); + let root_1 = extend_chain(&mut store, gloas_block(1, H256::ZERO, hash(0))); + // Slot 3 follows an empty slot 2 and builds on slot 1's payload. + extend_chain(&mut store, gloas_block(3, root_1, hash(1))); + let envelope = reveal(&mut store, 1, root_1, hash(1)); + + assert_eq!( + envelopes_by_range(&store, 1, 1).expect("in window"), + vec![envelope] + ); + } + + #[test] + fn by_range_caps_the_window_at_max_request_payloads() { + let (store, envelopes) = chain_with_every_payload_built_on(); + + // A window far wider than the cap still answers from the first + // MAX_REQUEST_PAYLOADS slots only, which here includes every envelope; + // the cap is observable on where the window ends. + assert_eq!( + envelopes_by_range(&store, 1, u64::MAX).expect("in window"), + envelopes + ); + let beyond_cap = 1 + MAX_REQUEST_PAYLOADS; + assert_eq!( + envelopes_by_range(&store, beyond_cap, u64::MAX).expect("in window"), + Vec::new() + ); + } + + #[test] + fn by_range_answers_empty_for_a_window_before_gloas_and_for_overflow() { + let (store, _) = chain_with_every_payload_built_on(); + + assert!( + envelopes_by_range(&store, 0, 1) + .expect("in window") + .is_empty() + ); + assert!( + envelopes_by_range(&store, u64::MAX, 5) + .expect("in window") + .is_empty() + ); + } + + #[test] + fn by_range_refuses_a_start_below_the_anchor() { + let mut store = Store::init_beacon( + Arc::new(InMemoryBackend::new()), + 0, + Config::mainnet(), + H256::ZERO, + Checkpoint::default(), + 10, + ); + store + .insert_signed_block(H256::ZERO, gloas_block(10, H256::ZERO, H256::ZERO)) + .expect("insert anchor"); + + assert!(envelopes_by_range(&store, 9, 1).is_err()); + assert!(envelopes_by_range(&store, 10, 1).is_ok()); + } + + #[test] + fn by_root_serves_known_envelopes_in_request_order_and_skips_unknown_roots() { + let (store, envelopes) = chain_with_every_payload_built_on(); + let roots: Vec = envelopes + .iter() + .map(|e| e.message.beacon_block_root) + .collect(); + let asked = vec![roots[2], H256::repeat_byte(0xee), roots[0]]; + + let served = envelopes_by_root(&store, &asked); + + assert_eq!(served, vec![envelopes[2].clone(), envelopes[0].clone()]); + } + + #[test] + fn by_root_looks_at_no_more_than_the_limit() { + let (store, envelopes) = chain_with_every_payload_built_on(); + let known = envelopes[0].message.beacon_block_root; + let mut asked = vec![H256::repeat_byte(0xee); MAX_REQUEST_PAYLOADS as usize]; + asked.push(known); + + assert!(envelopes_by_root(&store, &asked).is_empty()); + } +} diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index fec1ba7c3..d21514dfa 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -24,14 +24,14 @@ use ethlambda_state_transition::beacon::das; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::constants; use ethlambda_types::beacon::containers::fulu::{ColumnIndices, DataColumnsByRootIdentifier}; -use ethlambda_types::beacon::containers::{DataColumnSidecar, SignedBeaconBlock, gloas}; +use ethlambda_types::beacon::containers::{DataColumnSidecar, SignedBeaconBlock}; use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::checkpoint::Checkpoint; use ethlambda_types::primitives::HashTreeRoot as _; use ethlambda_types::{block::SignedBlock, primitives::H256}; use super::{ - Request, Response, ResponsePayload, + Request, Response, ResponsePayload, envelopes, messages::{ResponseCode, error_message}, }; use crate::beacon::BeaconWire; @@ -39,11 +39,10 @@ use crate::beacon::column_checks; use crate::beacon::decode::fork_at_slot; use crate::beacon::handler::{self as beacon_handler, StatusVersion}; use crate::beacon::messages::{ - BeaconMetaData, BeaconStatus, DataColumnsByRangeRequest, - ExecutionPayloadEnvelopesByRangeRequest, Goodbye, Ping, + BeaconMetaData, BeaconStatus, DataColumnsByRangeRequest, Goodbye, Ping, }; use crate::beacon::protocols::{ - MAX_REQUEST_BLOCKS as MAX_BEACON_REQUEST_BLOCKS, MAX_REQUEST_BLOCKS_DENEB, MAX_REQUEST_PAYLOADS, + MAX_REQUEST_BLOCKS as MAX_BEACON_REQUEST_BLOCKS, MAX_REQUEST_BLOCKS_DENEB, }; use crate::beacon::transition; use crate::discovery::enr::node_id_from_peer_id; @@ -173,7 +172,7 @@ pub async fn handle_req_resp_message( kind = "execution_payload_envelopes_by_range_request", peer_count, "P2P message received" ); - handle_execution_payload_envelopes_by_range_request( + envelopes::handle_execution_payload_envelopes_by_range_request( server, peer, request, channel, ) .await; @@ -183,7 +182,7 @@ pub async fn handle_req_resp_message( kind = "execution_payload_envelopes_by_root_request", peer_count, "P2P message received" ); - handle_execution_payload_envelopes_by_root_request( + envelopes::handle_execution_payload_envelopes_by_root_request( server, peer, roots, channel, ) .await; @@ -498,14 +497,18 @@ fn lean_blocks(blocks: Vec) -> Vec { /// /// Every request handler on either chain ends here, which is most of what the /// two have in common above encoding. -fn respond(server: &mut P2PServer, channel: ResponseChannel, payload: ResponsePayload) { +pub(super) fn respond( + server: &mut P2PServer, + channel: ResponseChannel, + payload: ResponsePayload, +) { server .swarm_handle .send_response(channel, Response::success(payload)); } /// Answer a request with an error code and a reason. -fn refuse( +pub(super) fn refuse( server: &mut P2PServer, channel: ResponseChannel, code: ResponseCode, @@ -1832,7 +1835,7 @@ fn handle_metadata_response(server: &mut P2PServer, peer: PeerId, metadata: Beac /// to block requests" (data column sidecar requests name the same code for /// the same reason), where an empty stream claims we looked and had nothing, /// and `INVALID_REQUEST` would blame the asker for a request that was fine. -fn beacon_block_store_or_refuse( +pub(super) fn beacon_block_store_or_refuse( server: &mut P2PServer, peer: PeerId, channel: ResponseChannel, @@ -2128,154 +2131,6 @@ async fn handle_data_column_sidecars_by_range_request( ); } -/// Answer `execution_payload_envelopes_by_range/1` off the canonical chain. -/// -/// The spec makes it equivalent to `BeaconBlocksByRange` v2 with another -/// response type, so the window rules are the block ones: an empty window is -/// `INVALID_REQUEST`, a `count` above `MAX_REQUEST_PAYLOADS` is truncated to it -/// ("Clients MAY limit the number of ... in the response"), and a `start_slot` -/// below the anchor is `RESOURCE_UNAVAILABLE`. A window before gloas simply -/// holds no envelopes, so it is an empty answer. Which envelopes count as -/// canonical is [`Store::canonical_execution_payload_envelopes`]'s business. -async fn handle_execution_payload_envelopes_by_range_request( - server: &mut P2PServer, - peer: PeerId, - request: ExecutionPayloadEnvelopesByRangeRequest, - channel: ResponseChannel, -) { - let Some(channel) = beacon_block_store_or_refuse(server, peer, channel) else { - return; - }; - - if request.count == 0 { - refuse( - server, - channel, - ResponseCode::INVALID_REQUEST, - "invalid ExecutionPayloadEnvelopesByRange request", - ); - return; - } - let envelopes = match envelopes_by_range(&server.store, request.start_slot, request.count) { - Ok(envelopes) => envelopes, - Err(reason) => { - debug!( - %peer, - start_slot = request.start_slot, - "ExecutionPayloadEnvelopesByRange request starts before this node's chain" - ); - refuse(server, channel, ResponseCode::RESOURCE_UNAVAILABLE, reason); - return; - } - }; - - trace!( - %peer, - start_slot = request.start_slot, - count = request.count, - found = envelopes.len(), - "Responding to ExecutionPayloadEnvelopesByRange request" - ); - - respond( - server, - channel, - ResponsePayload::ExecutionPayloadEnvelopes(envelopes), - ); -} - -/// The envelopes `ExecutionPayloadEnvelopesByRange` answers a non-empty window -/// with: the canonical ones in `[start_slot, start_slot + count)`, truncated -/// to `MAX_REQUEST_PAYLOADS` slots. -/// -/// `Err` carries the `RESOURCE_UNAVAILABLE` reason for a `start_slot` below -/// the store's anchor, which this chain does not reach back to. -fn envelopes_by_range( - store: &Store, - start_slot: u64, - count: u64, -) -> Result, &'static str> { - if start_slot < store.anchor_slot() { - return Err("requested range starts before this node's earliest available slot"); - } - let count = count.min(MAX_REQUEST_PAYLOADS); - // An overflowing window is attacker-supplied and answered empty, as is a - // zero `count`, which has no last offset to add. - let Some(end_slot) = count - .checked_sub(1) - .and_then(|last_offset| start_slot.checked_add(last_offset)) - else { - return Ok(Vec::new()); - }; - Ok(store - .canonical_execution_payload_envelopes(start_slot, end_slot) - .inspect_err(|err| { - warn!( - start_slot, - end_slot, - ?err, - "Failed to get execution payload envelopes by slot range" - ) - }) - .unwrap_or_default()) -} - -/// Answer `execution_payload_envelopes_by_root/1` with whichever of the named -/// envelopes this node holds. -/// -/// The counterpart of [`handle_beacon_blocks_by_root_request`]: a root with no -/// stored envelope is left out rather than answered with an error, and the -/// answer follows the order asked in. Roots past `MAX_REQUEST_PAYLOADS` are -/// ignored (the wire list type already bounds a decoded request to it). -async fn handle_execution_payload_envelopes_by_root_request( - server: &mut P2PServer, - peer: PeerId, - roots: Vec, - channel: ResponseChannel, -) { - let Some(channel) = beacon_block_store_or_refuse(server, peer, channel) else { - return; - }; - - let requested = roots.len(); - let envelopes = envelopes_by_root(&server.store, &roots); - - trace!( - %peer, - requested, - found = envelopes.len(), - "Responding to ExecutionPayloadEnvelopesByRoot request" - ); - - respond( - server, - channel, - ResponsePayload::ExecutionPayloadEnvelopes(envelopes), - ); -} - -/// The held envelopes among `roots`, in the order asked, looking at no more -/// than `MAX_REQUEST_PAYLOADS` of them. -fn envelopes_by_root(store: &Store, roots: &[H256]) -> Vec { - roots - .iter() - .take(MAX_REQUEST_PAYLOADS as usize) - .filter_map(|root| { - store - .get_execution_payload_envelope(root) - .inspect_err(|err| { - error!( - root = %ethlambda_types::ShortRoot(&root.0), - %err, - "Stored execution payload envelope failed to read" - ) - }) - .ok() - .flatten() - }) - .collect() -} - /// Ask `peer` for the beacon blocks in `[start_slot, start_slot + count)`. /// /// Returns the request id, so a caller can pair the answer with what it asked @@ -3216,205 +3071,4 @@ mod tests { assert_eq!(kept, vec![gloas_column(asked, 3)]); assert_eq!(dropped, 2); } - - /// A gloas block whose bid builds on the payload `parent_block_hash`. - fn gloas_block(slot: u64, parent_root: H256, parent_block_hash: H256) -> SignedBeaconBlock { - let mut block = gloas::SignedBeaconBlock { - message: gloas::BeaconBlock { - slot, - proposer_index: 0, - parent_root, - state_root: H256::ZERO, - body: Default::default(), - }, - signature: Default::default(), - }; - block - .message - .body - .signed_execution_payload_bid - .message - .parent_block_hash = parent_block_hash; - SignedBeaconBlock::Gloas(block) - } - - fn beacon_store() -> Store { - let mut store = Store::init_beacon( - Arc::new(InMemoryBackend::new()), - 0, - Config::mainnet(), - H256::ZERO, - Checkpoint::default(), - 0, - ); - store - .insert_signed_block(H256::ZERO, gloas_block(0, H256::ZERO, H256::ZERO)) - .expect("insert anchor"); - store - } - - /// Inserts `block` and moves the head to it; returns its root. - fn extend_chain(store: &mut Store, block: SignedBeaconBlock) -> H256 { - let root = block.message_hash_tree_root(); - store.insert_signed_block(root, block).expect("insert"); - store - .update_checkpoints(ForkCheckpoints::head_only(root)) - .expect("advance head"); - root - } - - fn reveal( - store: &mut Store, - slot: u64, - root: H256, - hash: H256, - ) -> gloas::SignedExecutionPayloadEnvelope { - let envelope = crate::beacon::encoding::test_support::envelope(root, slot, hash); - store.insert_verified_payload(slot, &envelope); - envelope - } - - fn hash(byte: u8) -> H256 { - H256::repeat_byte(byte) - } - - /// Canonical slots 1 to 3, every payload revealed and built on, plus a - /// sibling of slot 2 with its own envelope. Returns the three canonical - /// envelopes. - fn chain_with_every_payload_built_on() -> (Store, Vec) { - let mut store = beacon_store(); - let root_1 = extend_chain(&mut store, gloas_block(1, H256::ZERO, hash(0))); - let root_2 = extend_chain(&mut store, gloas_block(2, root_1, hash(1))); - let root_3 = extend_chain(&mut store, gloas_block(3, root_2, hash(2))); - // A fork at slot 2 that the head does not descend from. - let sibling = gloas_block(2, root_1, hash(9)); - let sibling_root = sibling.message_hash_tree_root(); - store - .insert_signed_block(sibling_root, sibling) - .expect("insert"); - reveal(&mut store, 2, sibling_root, hash(0x99)); - let envelopes = vec![ - reveal(&mut store, 1, root_1, hash(1)), - reveal(&mut store, 2, root_2, hash(2)), - reveal(&mut store, 3, root_3, hash(3)), - ]; - (store, envelopes) - } - - #[test] - fn by_range_serves_only_canonical_envelopes_in_slot_order() { - let (store, envelopes) = chain_with_every_payload_built_on(); - - let served = envelopes_by_range(&store, 1, 3).expect("in window"); - - assert_eq!(served, envelopes); - assert_eq!( - envelopes_by_range(&store, 2, 1).expect("in window"), - envelopes[1..2] - ); - } - - #[test] - fn by_range_omits_a_payload_the_next_block_does_not_build_on() { - let mut store = beacon_store(); - let root_1 = extend_chain(&mut store, gloas_block(1, H256::ZERO, hash(0))); - // Slot 2 builds on slot 1's empty branch, not on the payload revealed. - let root_2 = extend_chain(&mut store, gloas_block(2, root_1, hash(0))); - reveal(&mut store, 1, root_1, hash(1)); - let head_envelope = reveal(&mut store, 2, root_2, hash(2)); - - let served = envelopes_by_range(&store, 1, 2).expect("in window"); - - assert_eq!(served, vec![head_envelope]); - } - - #[test] - fn by_range_judges_the_last_block_of_a_window_by_its_successor_past_the_window() { - let mut store = beacon_store(); - let root_1 = extend_chain(&mut store, gloas_block(1, H256::ZERO, hash(0))); - // Slot 3 follows an empty slot 2 and builds on slot 1's payload. - extend_chain(&mut store, gloas_block(3, root_1, hash(1))); - let envelope = reveal(&mut store, 1, root_1, hash(1)); - - assert_eq!( - envelopes_by_range(&store, 1, 1).expect("in window"), - vec![envelope] - ); - } - - #[test] - fn by_range_caps_the_window_at_max_request_payloads() { - let (store, envelopes) = chain_with_every_payload_built_on(); - - // A window far wider than the cap still answers from the first - // MAX_REQUEST_PAYLOADS slots only, which here includes every envelope; - // the cap is observable on where the window ends. - assert_eq!( - envelopes_by_range(&store, 1, u64::MAX).expect("in window"), - envelopes - ); - let beyond_cap = 1 + MAX_REQUEST_PAYLOADS; - assert_eq!( - envelopes_by_range(&store, beyond_cap, u64::MAX).expect("in window"), - Vec::new() - ); - } - - #[test] - fn by_range_answers_empty_for_a_window_before_gloas_and_for_overflow() { - let (store, _) = chain_with_every_payload_built_on(); - - assert!( - envelopes_by_range(&store, 0, 1) - .expect("in window") - .is_empty() - ); - assert!( - envelopes_by_range(&store, u64::MAX, 5) - .expect("in window") - .is_empty() - ); - } - - #[test] - fn by_range_refuses_a_start_below_the_anchor() { - let mut store = Store::init_beacon( - Arc::new(InMemoryBackend::new()), - 0, - Config::mainnet(), - H256::ZERO, - Checkpoint::default(), - 10, - ); - store - .insert_signed_block(H256::ZERO, gloas_block(10, H256::ZERO, H256::ZERO)) - .expect("insert anchor"); - - assert!(envelopes_by_range(&store, 9, 1).is_err()); - assert!(envelopes_by_range(&store, 10, 1).is_ok()); - } - - #[test] - fn by_root_serves_known_envelopes_in_request_order_and_skips_unknown_roots() { - let (store, envelopes) = chain_with_every_payload_built_on(); - let roots: Vec = envelopes - .iter() - .map(|e| e.message.beacon_block_root) - .collect(); - let asked = vec![roots[2], H256::repeat_byte(0xee), roots[0]]; - - let served = envelopes_by_root(&store, &asked); - - assert_eq!(served, vec![envelopes[2].clone(), envelopes[0].clone()]); - } - - #[test] - fn by_root_looks_at_no_more_than_the_limit() { - let (store, envelopes) = chain_with_every_payload_built_on(); - let known = envelopes[0].message.beacon_block_root; - let mut asked = vec![H256::repeat_byte(0xee); MAX_REQUEST_PAYLOADS as usize]; - asked.push(known); - - assert!(envelopes_by_root(&store, &asked).is_empty()); - } } diff --git a/crates/net/p2p/src/req_resp/mod.rs b/crates/net/p2p/src/req_resp/mod.rs index 005fbbc0f..d8e256645 100644 --- a/crates/net/p2p/src/req_resp/mod.rs +++ b/crates/net/p2p/src/req_resp/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod behaviour; pub(crate) mod codec; pub(crate) mod encoding; +mod envelopes; pub mod handlers; pub(crate) mod messages; From 4af0326cab003d7348864adec23c034b945e9da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:09:33 -0300 Subject: [PATCH 057/132] fix(p2p): pin MAX_REQUEST_PAYLOADS and record the head-envelope deviation check_constants now refuses a config whose MAX_REQUEST_PAYLOADS differs from the ceiling the envelope protocols run on, so /eth/v1/config/spec cannot report a value the node does not use. The head envelope is served whenever held since the head node payload status is not stored; document that in spec_deviations.md and the store method, and add both protocols to beacon_wire.md. Also: accept any gloas-or-later digest on an envelope chunk instead of gloas alone, read the head inside the same view as the block index, skip the successor search when the last block has no envelope, correct the by-range handler doc about the count ceiling, and make two tests exercise what their names say (a pre-gloas window that has blocks, a chain past the cap). --- bin/ethlambda/src/network/mod.rs | 10 +++ crates/net/p2p/src/beacon/encoding.rs | 14 +++- crates/net/p2p/src/req_resp/envelopes.rs | 94 ++++++++++++++++++------ crates/storage/src/store.rs | 36 ++++++--- docs/beacon_wire.md | 23 ++++++ docs/spec_deviations.md | 21 ++++++ 6 files changed, 163 insertions(+), 35 deletions(-) diff --git a/bin/ethlambda/src/network/mod.rs b/bin/ethlambda/src/network/mod.rs index 570fdb40b..594564e95 100644 --- a/bin/ethlambda/src/network/mod.rs +++ b/bin/ethlambda/src/network/mod.rs @@ -197,6 +197,7 @@ pub(crate) fn check_constants(config: &Config) -> Result<(), ConstantsMismatch> maximum_gossip_clock_disparity == constants::MAXIMUM_GOSSIP_CLOCK_DISPARITY, max_request_blocks == protocols::MAX_REQUEST_BLOCKS, max_request_blocks_deneb == protocols::MAX_REQUEST_BLOCKS_DENEB, + max_request_payloads == protocols::MAX_REQUEST_PAYLOADS, max_request_data_column_sidecars == protocols::max_request_data_column_sidecars(), max_payload_size == ethlambda_p2p::MAX_PAYLOAD_SIZE as u64, message_domain_invalid_snappy == ethlambda_p2p::MESSAGE_DOMAIN_INVALID_SNAPPY, @@ -460,6 +461,15 @@ mod tests { ); } + #[test] + fn a_config_that_changes_the_envelope_request_ceiling_is_refused() { + let mut config = Config::mainnet(); + config.max_request_payloads = 64; + + let err = check_constants(&config).unwrap_err().to_string(); + assert!(err.contains("MAX_REQUEST_PAYLOADS is 64"), "got {err}"); + } + #[test] fn an_absent_preset_base_is_a_distinct_error_from_a_mismatch() { // `Config::preset_base` defaults an absent PRESET_BASE to "". diff --git a/crates/net/p2p/src/beacon/encoding.rs b/crates/net/p2p/src/beacon/encoding.rs index cb1124b09..ea7df9bee 100644 --- a/crates/net/p2p/src/beacon/encoding.rs +++ b/crates/net/p2p/src/beacon/encoding.rs @@ -378,12 +378,20 @@ where Ok(()) } +/// Whether `fork` has execution payload envelopes: gloas and every beacon fork +/// after it, which the schedule's digests can name. Lean is not on the beacon +/// timeline, though it sorts after every beacon fork. +fn carries_envelopes(fork: ForkName) -> bool { + fork >= ForkName::Gloas && fork != ForkName::Lean +} + /// Read an envelope response: one `SignedExecutionPayloadEnvelope` per chunk, /// until the peer closes. /// /// The counterpart of [`decode_data_column_sidecars_response`]: the fork comes /// from the context bytes, and a digest no scheduled fork uses ends the stream. -/// Envelopes exist only from gloas, so any other fork's digest is refused too. +/// Envelopes exist only from gloas on, so the digest of an earlier fork is +/// refused too ([`carries_envelopes`]). /// The context is then checked against the digest the payload's own /// `slot_number` implies. pub async fn decode_execution_payload_envelopes_response( @@ -404,9 +412,9 @@ where let fork = <[u8; 4]>::try_from(context) .ok() .and_then(|digest| schedule.fork_for_digest(digest)); - if fork != Some(ForkName::Gloas) { + if !fork.is_some_and(carries_envelopes) { return Err(invalid(format!( - "execution payload envelope chunk context {} is not the gloas fork digest", + "execution payload envelope chunk context {} is not a gloas-or-later fork digest", hex::encode(context), ))); } diff --git a/crates/net/p2p/src/req_resp/envelopes.rs b/crates/net/p2p/src/req_resp/envelopes.rs index b6670914c..8b1a22c44 100644 --- a/crates/net/p2p/src/req_resp/envelopes.rs +++ b/crates/net/p2p/src/req_resp/envelopes.rs @@ -19,11 +19,13 @@ use crate::beacon::protocols::MAX_REQUEST_PAYLOADS; /// Answer `execution_payload_envelopes_by_range/1` off the canonical chain. /// /// The spec makes it equivalent to `BeaconBlocksByRange` v2 with another -/// response type, so the window rules are the block ones: an empty window is -/// `INVALID_REQUEST`, a `count` above `MAX_REQUEST_PAYLOADS` is truncated to it -/// ("Clients MAY limit the number of ... in the response"), and a `start_slot` -/// below the anchor is `RESOURCE_UNAVAILABLE`. A window before gloas simply -/// holds no envelopes, so it is an empty answer. Which envelopes count as +/// response type. An empty window is `INVALID_REQUEST` and a `start_slot` +/// below the anchor is `RESOURCE_UNAVAILABLE`, as for blocks. A `count` above +/// `MAX_REQUEST_PAYLOADS` is truncated to it rather than refused, which the +/// specification's "Clients MAY limit the number of payload envelopes in the +/// response" allows; the block handler refuses a `count` past its phase0 +/// ceiling instead, because a request that wide is a protocol violation there. +/// A window before gloas simply holds no envelopes, so it is an empty answer. Which envelopes count as /// canonical is [`Store::canonical_execution_payload_envelopes`]'s business. pub(super) async fn handle_execution_payload_envelopes_by_range_request( server: &mut P2PServer, @@ -299,33 +301,83 @@ mod tests { ); } + /// A phase0 block, standing in for any block before gloas. + fn pre_gloas_block(slot: u64, parent_root: H256) -> SignedBeaconBlock { + use ethlambda_types::beacon::containers::phase0; + + SignedBeaconBlock::Phase0(phase0::SignedBeaconBlock { + message: phase0::BeaconBlock { + slot, + proposer_index: 0, + parent_root, + state_root: H256::ZERO, + body: phase0::BeaconBlockBody { + randao_reveal: Default::default(), + eth1_data: Default::default(), + graffiti: H256::ZERO, + proposer_slashings: Default::default(), + attester_slashings: Default::default(), + attestations: Default::default(), + deposits: Default::default(), + voluntary_exits: Default::default(), + }, + }, + signature: Default::default(), + }) + } + #[test] fn by_range_caps_the_window_at_max_request_payloads() { - let (store, envelopes) = chain_with_every_payload_built_on(); + let mut store = beacon_store(); + let total = MAX_REQUEST_PAYLOADS + 2; + let payload_hash = |slot: u64| H256::repeat_byte(slot as u8); + let mut parent = H256::ZERO; + for slot in 1..=total { + // Each block builds on the previous slot's payload, so every + // envelope is on the chain and only the cap can cut the answer. + let block = gloas_block(slot, parent, payload_hash(slot - 1)); + parent = extend_chain(&mut store, block); + reveal(&mut store, slot, parent, payload_hash(slot)); + } - // A window far wider than the cap still answers from the first - // MAX_REQUEST_PAYLOADS slots only, which here includes every envelope; - // the cap is observable on where the window ends. - assert_eq!( - envelopes_by_range(&store, 1, u64::MAX).expect("in window"), - envelopes - ); - let beyond_cap = 1 + MAX_REQUEST_PAYLOADS; - assert_eq!( - envelopes_by_range(&store, beyond_cap, u64::MAX).expect("in window"), - Vec::new() - ); + let served = envelopes_by_range(&store, 1, total).expect("in window"); + + assert_eq!(served.len() as u64, MAX_REQUEST_PAYLOADS); + let last_slot = served + .last() + .expect("non-empty") + .message + .payload + .slot_number; + assert_eq!(last_slot, MAX_REQUEST_PAYLOADS); + // A count past the cap is cut to it, not refused. + let wide = envelopes_by_range(&store, 1, u64::MAX).expect("in window"); + assert_eq!(wide, served); } #[test] - fn by_range_answers_empty_for_a_window_before_gloas_and_for_overflow() { - let (store, _) = chain_with_every_payload_built_on(); + fn by_range_answers_empty_for_a_pre_gloas_window_that_has_blocks() { + let mut store = beacon_store(); + let root_1 = extend_chain(&mut store, pre_gloas_block(1, H256::ZERO)); + let root_2 = extend_chain(&mut store, pre_gloas_block(2, root_1)); + let root_3 = extend_chain(&mut store, gloas_block(3, root_2, hash(0))); + let envelope = reveal(&mut store, 3, root_3, hash(3)); assert!( - envelopes_by_range(&store, 0, 1) + envelopes_by_range(&store, 1, 2) .expect("in window") .is_empty() ); + assert_eq!( + envelopes_by_range(&store, 1, 3).expect("in window"), + vec![envelope] + ); + } + + #[test] + fn by_range_answers_empty_for_an_overflowing_window() { + let (store, _) = chain_with_every_payload_built_on(); + assert!( envelopes_by_range(&store, u64::MAX, 5) .expect("in window") diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 09c5ce79f..30967afc5 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -3407,12 +3407,15 @@ impl Store { /// successor builds on its empty branch has its envelope left out, even /// when this node holds one. The head has no successor and fork choice does /// not persist which payload branch it picked, so its envelope is served - /// when held: only a verified envelope is ever stored. + /// when held: only a verified envelope is ever stored. See + /// `docs/spec_deviations.md`, "The head's payload envelope is served + /// whenever it is held". /// /// Canonical blocks come from `BlockRoots` and the whole answer is read /// from one view, like [`get_signed_blocks_by_slot_range`](Self::get_signed_blocks_by_slot_range). /// The successor of the last block in the window may lie past `end_slot`, - /// so the index is read on beyond it, up to the head, until one is found. + /// so the index is read on beyond it, up to the head, until one is found; + /// that search is skipped when the last block holds no envelope to judge. pub fn canonical_execution_payload_envelopes( &self, start_slot: u64, @@ -3428,11 +3431,28 @@ impl Store { .map(|bytes| H256::from_ssz_bytes(&bytes).expect("valid block root")) }; + let envelope_row = |slot: u64, root: &H256| { + view.get( + Table::ExecutionPayloadEnvelopes, + &encode_slot_root_key(slot, root), + ) + .expect("get") + }; + let mut window: Vec<(u64, H256)> = (start_slot..=end_slot) .filter_map(|slot| canonical_root(slot).map(|root| (slot, root))) .collect(); - if !window.is_empty() - && let Some(head_slot) = beacon_block_slot(self.backend.as_ref(), &self.head()?) + // Head and its slot come out of the same view as the rest, so a head + // move partway through cannot pair one branch's index with another's. + let head_slot = view + .get(Table::Metadata, KEY_HEAD) + .expect("get head") + .map(|bytes| H256::from_ssz_bytes(&bytes).expect("valid head root")) + .and_then(|head| view.get(Table::BlockHeaders, &head.to_ssz()).expect("get")) + .map(|bytes| decode_beacon_block_value(&bytes).slot()); + if let Some(&(last_slot, last_root)) = window.last() + && envelope_row(last_slot, &last_root).is_some() + && let Some(head_slot) = head_slot { let successor = (end_slot.saturating_add(1)..=head_slot) .find_map(|slot| canonical_root(slot).map(|root| (slot, root))); @@ -3444,13 +3464,7 @@ impl Store { if slot > end_slot { break; } - let Some(bytes) = view - .get( - Table::ExecutionPayloadEnvelopes, - &encode_slot_root_key(slot, &root), - ) - .expect("get") - else { + let Some(bytes) = envelope_row(slot, &root) else { continue; }; let envelope = gloas::SignedExecutionPayloadEnvelope::from_ssz_bytes(&bytes) diff --git a/docs/beacon_wire.md b/docs/beacon_wire.md index f18175794..8a4e23e94 100644 --- a/docs/beacon_wire.md +++ b/docs/beacon_wire.md @@ -274,6 +274,8 @@ choice. | `beacon_blocks_by_root/2` | both | | `data_column_sidecars_by_root/1` | both | | `data_column_sidecars_by_range/1` | both | +| `execution_payload_envelopes_by_range/1` | both; served only, nothing asks yet | +| `execution_payload_envelopes_by_root/1` | both; served only, nothing asks yet | The two data column sidecar protocols are registered because this node custodies the columns its node id selects and can answer for them out of @@ -381,6 +383,27 @@ fails. A mismatch ends the stream, logged at `warn` with both digests: the one w reach it in good faith is a blob schedule of ours that has fallen behind the network's. +### Execution payload envelopes + +Both gloas envelope protocols are registered `Full` so the server side can +answer; no request is issued yet. A chunk carries one +`SignedExecutionPayloadEnvelope` under the fork digest of its block's epoch. +The envelope has no slot of its own, so the epoch comes from +`payload.slot_number`, which `verify_execution_payload_envelope` pins to the +block's slot. A chunk whose digest names no gloas-or-later scheduled fork, or +not the digest its slot implies, ends the stream. + +| Protocol | Served from | +| --- | --- | +| `execution_payload_envelopes_by_root/1` | a point lookup per root in `Table::ExecutionPayloadEnvelopes`; unknown roots are left out, answers follow the order asked | +| `execution_payload_envelopes_by_range/1` | the canonical blocks of the window (`BlockRoots`), each envelope kept only when the next canonical block builds on its payload; see [spec_deviations.md](./spec_deviations.md) for the head | + +Both are bounded by `MAX_REQUEST_PAYLOADS`: a by-root list over it fails to +decode, and a by-range `count` over it is truncated, which the specification's +"Clients MAY limit the number of payload envelopes in the response" allows. +A by-range request starting below `Store::anchor_slot` gets +`RESOURCE_UNAVAILABLE`, and a window before gloas is simply empty. + ### Data column sidecars Both protocols are registered `Full`, since with the columns on disk this node diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 6a85d7372..776364f86 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -216,6 +216,27 @@ the root of the state's latest block header. shapes give the same root. - **Equivalence:** the same check, on both kinds of state. +## The head's payload envelope is served whenever it is held + +`ExecutionPayloadEnvelopesByRange` (gloas `p2p-interface.md`) says clients +MUST NOT include the envelope of a block whose `ForkChoiceNode` in the +served chain has `payload_status` equal to `PAYLOAD_STATUS_EMPTY`. + +- **The specification:** for the head block, the envelope is part of the chain + only when the head node's payload status is FULL. +- **ethlambda:** for every block but the head, the envelope is served when the + next canonical block's bid `parent_block_hash` equals the payload's + `block_hash`, which is the specification's rule. For the head there is no + next block, and the head node's payload status is not yet exposed by the + store (`KEY_HEAD` holds a root only), so the envelope is served whenever it + is held. Only verified envelopes are ever stored. +- **Effect:** when fork choice picked the head's EMPTY branch despite a + verified envelope being held, a peer receives an envelope fork choice + currently does not build on. The next block resolves it, since that block's + bid then decides the envelope's membership like any other. +- **Fixed when:** the head node's payload status is stored. + `Store::canonical_execution_payload_envelopes` is the one place to change. + ## The attestation deadline takes the epoch that picks the fork's rule - **The specification:** `get_attestation_due_ms()` takes no argument, and From 894f793a923510587534e7dc2caefa80850f8791 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:16:32 -0300 Subject: [PATCH 058/132] feat(engine): forkchoiceUpdatedV4 for gloas Amsterdam's fork choice notification is the method a gloas follower must call once the execution client is past Osaka. Adds the client method (null payloadAttributes, optional custody bitmap), the CustodyColumns type with SSZ Bitvector bit order (matches ethrex), the capability entry, and a handshake warning when it is missing and gloas is scheduled. --- crates/net/engine/src/client.rs | 79 ++++++++++++++++++++++++--- crates/net/engine/src/lib.rs | 3 +- crates/net/engine/src/types.rs | 61 ++++++++++++++++++++- crates/net/engine/tests/wire_smoke.rs | 29 +++++++++- 4 files changed, 162 insertions(+), 10 deletions(-) diff --git a/crates/net/engine/src/client.rs b/crates/net/engine/src/client.rs index 33e589aa3..25a4db2e9 100644 --- a/crates/net/engine/src/client.rs +++ b/crates/net/engine/src/client.rs @@ -10,7 +10,7 @@ use tracing::{debug, warn}; use crate::auth::JwtSecret; use crate::error::EngineError; use crate::types::{ - ClientVersionV1, ExecutionPayloadV3, ExecutionPayloadV4, ForkchoiceStateV1, + ClientVersionV1, CustodyColumns, ExecutionPayloadV3, ExecutionPayloadV4, ForkchoiceStateV1, ForkchoiceUpdatedResponse, PayloadStatusV1, data, }; @@ -264,6 +264,24 @@ impl EngineClient { Ok(response.payload_status) } + /// `engine_forkchoiceUpdatedV4`: Amsterdam's fork choice notification, with + /// a `null` `payloadAttributes` and the consensus client's custody set. + /// + /// Follower only: there is no variant taking `PayloadAttributesV4`, since a + /// follower never asks for a build. `custody_columns` of `None` sends + /// `null`, meaning the consensus client provides no custody. The response + /// is V3's. + pub async fn forkchoice_updated_v4( + &self, + state: &ForkchoiceStateV1, + custody_columns: Option, + ) -> Result { + let params = forkchoice_updated_v4_params(state, custody_columns); + let response: ForkchoiceUpdatedResponse = + self.call("engine_forkchoiceUpdatedV4", params).await?; + Ok(response.payload_status) + } + /// `engine_forkchoiceUpdatedV3` with `payloadAttributes`: the same fork /// choice notification, plus a request to start building a payload on the /// head for the slot the attributes describe. @@ -333,14 +351,18 @@ impl EngineClient { ); } } - // Kept out of the list above: V5 is only called from gloas on, so a + // Kept out of the list above: these are only called from gloas on, so a // network that does not schedule gloas never needs it, and an // execution client that predates Amsterdam is right not to offer it. - if gloas_scheduled && !theirs.iter().any(|method| method == "engine_newPayloadV5") { - warn!( - method = "engine_newPayloadV5", - "The execution client does not advertise a method this node needs from gloas on" - ); + if gloas_scheduled { + for required in ["engine_newPayloadV5", "engine_forkchoiceUpdatedV4"] { + if !theirs.iter().any(|method| method == required) { + warn!( + method = required, + "The execution client does not advertise a method this node needs from gloas on" + ); + } + } } match self.client_version(ours).await { Ok(versions) => { @@ -360,3 +382,46 @@ impl EngineClient { Ok(()) } } + +/// The three positional parameters of `engine_forkchoiceUpdatedV4`: the state, +/// a `null` `payloadAttributes`, and the custody bitmap or `null`. +fn forkchoice_updated_v4_params( + state: &ForkchoiceStateV1, + custody_columns: Option, +) -> serde_json::Value { + json!([state, serde_json::Value::Null, custody_columns]) +} + +#[cfg(test)] +mod tests { + use ethlambda_types::beacon::primitives::ExecutionBlockHash; + + use super::*; + + fn state() -> ForkchoiceStateV1 { + ForkchoiceStateV1 { + head_block_hash: ExecutionBlockHash::repeat_byte(1), + safe_block_hash: ExecutionBlockHash::repeat_byte(2), + finalized_block_hash: ExecutionBlockHash::repeat_byte(3), + } + } + + #[test] + fn forkchoice_updated_v4_params_are_state_null_and_bitmap() { + let columns = CustodyColumns::from_indices([0, 9]); + let params = forkchoice_updated_v4_params(&state(), columns); + let params = params.as_array().expect("an array of params"); + assert_eq!(params.len(), 3); + assert!(params[0].get("headBlockHash").is_some()); + assert!(params[1].is_null()); + assert_eq!(params[2], "0x01020000000000000000000000000000"); + } + + #[test] + fn forkchoice_updated_v4_without_custody_sends_a_third_null() { + let params = forkchoice_updated_v4_params(&state(), None); + let params = params.as_array().expect("an array of params"); + assert_eq!(params.len(), 3); + assert!(params[2].is_null()); + } +} diff --git a/crates/net/engine/src/lib.rs b/crates/net/engine/src/lib.rs index 8dde3f530..f3cf1d737 100644 --- a/crates/net/engine/src/lib.rs +++ b/crates/net/engine/src/lib.rs @@ -31,7 +31,7 @@ pub mod types; pub use auth::JwtSecret; pub use client::EngineClient; pub use error::EngineError; -pub use types::{ForkchoiceStateV1, PayloadStatusV1, PayloadStatusValue}; +pub use types::{CustodyColumns, ForkchoiceStateV1, PayloadStatusV1, PayloadStatusValue}; /// The methods this client will call, sent in the `engine_exchangeCapabilities` /// handshake. @@ -42,6 +42,7 @@ pub const ETHLAMBDA_ENGINE_CAPABILITIES: &[&str] = &[ "engine_newPayloadV4", "engine_newPayloadV5", "engine_forkchoiceUpdatedV3", + "engine_forkchoiceUpdatedV4", "engine_getPayloadV5", "engine_getClientVersionV1", ]; diff --git a/crates/net/engine/src/types.rs b/crates/net/engine/src/types.rs index 9bcfa26f6..6e142812e 100644 --- a/crates/net/engine/src/types.rs +++ b/crates/net/engine/src/types.rs @@ -11,6 +11,7 @@ use ethlambda_types::beacon::containers::capella::Withdrawal; use ethlambda_types::beacon::containers::{deneb, gloas}; +use ethlambda_types::beacon::preset; use ethlambda_types::beacon::primitives::{ExecutionBlockHash, Uint256}; use ethlambda_types::beacon::serde_helpers::HexPrefixed; use serde::{Deserialize, Serialize, Serializer}; @@ -194,6 +195,44 @@ pub fn data(bytes: &[u8]) -> String { HexPrefixed(bytes).to_string() } +/// The `custodyColumns` parameter of `engine_forkchoiceUpdatedV4`: a bitarray +/// of `CELLS_PER_EXT_BLOB` bits marking the columns the consensus client +/// custodies. +/// +/// Little-endian bit order, as an SSZ `Bitvector` has it: column `i` is bit +/// `i % 8` of byte `i / 8`. ethrex parses it the same way (`u128::from_le_bytes`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct CustodyColumns([u8; Self::BYTES]); + +impl CustodyColumns { + /// Wire length of the bitmap, which the specification fixes at 16 bytes. + pub const BYTES: usize = 16; + + /// Builds the bitmap from column indices, or `None` when any index is not + /// below `NUMBER_OF_COLUMNS`. Duplicates are harmless. + pub fn from_indices(indices: impl IntoIterator) -> Option { + let mut bits = [0u8; Self::BYTES]; + for index in indices { + if index >= preset::NUMBER_OF_COLUMNS as u64 { + return None; + } + bits[(index / 8) as usize] |= 1 << (index % 8); + } + Some(Self(bits)) + } + + /// The `DATA` encoding: `0x` and 32 hex digits. + pub fn to_data(self) -> String { + data(&self.0) + } +} + +impl Serialize for CustodyColumns { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.to_data()) + } +} + /// Encodes a `QUANTITY`: `0x`-prefixed, leading zeros stripped, `0x0` for zero. pub fn quantity(value: u64) -> String { format!("0x{value:x}") @@ -219,11 +258,31 @@ pub fn uint256(value: &Uint256) -> String { #[cfg(test)] mod tests { use ethlambda_types::beacon::containers::bellatrix; - use ethlambda_types::beacon::preset; use ethlambda_types::beacon::primitives::{Bytes32, ExecutionAddress, U256}; use super::*; + #[test] + fn custody_columns_use_little_endian_bit_order() { + // Column 0 is bit 0 of byte 0; column 9 is bit 1 of byte 1; column 127 + // is bit 7 of byte 15. + let columns = CustodyColumns::from_indices([0, 9, 127]).unwrap(); + assert_eq!(columns.to_data(), "0x01020000000000000000000000000080"); + } + + #[test] + fn custody_columns_reject_an_out_of_range_index() { + let bound = preset::NUMBER_OF_COLUMNS as u64; + assert!(CustodyColumns::from_indices([bound - 1]).is_some()); + assert!(CustodyColumns::from_indices([0, bound]).is_none()); + } + + #[test] + fn empty_custody_columns_encode_as_sixteen_zero_bytes() { + let columns = CustodyColumns::from_indices([]).unwrap(); + assert_eq!(columns.to_data(), format!("0x{}", "00".repeat(16))); + } + /// An otherwise-zero deneb execution payload whose `block_hash` is /// `block_hash`. /// diff --git a/crates/net/engine/tests/wire_smoke.rs b/crates/net/engine/tests/wire_smoke.rs index ff629c87a..72d943efd 100644 --- a/crates/net/engine/tests/wire_smoke.rs +++ b/crates/net/engine/tests/wire_smoke.rs @@ -9,7 +9,7 @@ use std::sync::Arc; -use ethlambda_engine::types::{ForkchoiceStateV1, PayloadStatusValue}; +use ethlambda_engine::types::{CustodyColumns, ForkchoiceStateV1, PayloadStatusValue}; use ethlambda_engine::{EngineClient, EngineError, JwtSecret}; use ethlambda_types::beacon::primitives::ExecutionBlockHash; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; @@ -194,3 +194,30 @@ async fn new_payload_v5_sends_four_params_under_its_own_method() { assert_eq!(params[2], format!("0x{}", "02".repeat(32))); assert_eq!(params[3][0], "0x00aa"); } + +#[tokio::test] +async fn forkchoice_updated_v4_sends_three_params_and_parses_syncing() { + let (endpoint, seen) = serve_once( + r#"{"jsonrpc":"2.0","id":1,"result":{"payloadStatus":{"status":"SYNCING","latestValidHash":null,"validationError":null},"payloadId":null}}"#, + ) + .await; + + let state = ForkchoiceStateV1 { + head_block_hash: ExecutionBlockHash::repeat_byte(1), + safe_block_hash: ExecutionBlockHash::repeat_byte(2), + finalized_block_hash: ExecutionBlockHash::repeat_byte(3), + }; + let columns = CustodyColumns::from_indices([0, 9]); + let status = client(endpoint) + .forkchoice_updated_v4(&state, columns) + .await + .expect("the mock answers"); + + assert_eq!(status.status, PayloadStatusValue::Syncing); + + let request = seen.lock().await.clone().expect("the mock saw a request"); + assert!(request.contains("engine_forkchoiceUpdatedV4")); + assert!(request.contains("headBlockHash")); + // The attributes are null and the bitmap follows them. + assert!(request.contains(",null,\"0x01020000000000000000000000000000\"]")); +} From 1311ce444cbbc37052ee336d6006682c76b76964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:02:25 -0300 Subject: [PATCH 059/132] feat(beacon): import gloas blocks and their execution payload envelopes A gloas block carries only the builder's bid; the payload arrives in a separate envelope and fork choice weighs a FULL and an EMPTY node per block. The chain actor refused every gloas block; it now imports them and drives the envelope path: - data availability moves from the block to the envelope, so data_availability_for no longer gates a gloas block on columns; - an envelope waits for its block's post-state, then for every sampled column when the bid has commitments, then goes through on_execution_payload_envelope and records the payload VALID (the follower trusts payloads when no execution client is configured); - a block whose parent is FULL with an unverified payload is held and re-imported when that envelope verifies; - all three queues are capped and swept at finality, and each has a gauge. Parked gloas columns now park regardless of is_followed, and the T1b review leftovers are fixed: a root's slot is recorded on its own so a forged fulu sidecar naming it as parent cannot decide it, stale bookkeeping is swept on every eviction, and roots the actor already holds as blocks are exempt from the per-slot cap. --- Cargo.lock | 1 + crates/blockchain/Cargo.toml | 1 + crates/blockchain/src/beacon_columns.rs | 191 ++-- crates/blockchain/src/beacon_envelope.rs | 981 +++++++++++++++++++++ crates/blockchain/src/lib.rs | 105 ++- crates/blockchain/src/metrics.rs | 49 + crates/net/api/src/lib.rs | 14 + crates/net/p2p/src/beacon/column_checks.rs | 9 + crates/net/p2p/src/beacon/verdict.rs | 9 + 9 files changed, 1265 insertions(+), 95 deletions(-) create mode 100644 crates/blockchain/src/beacon_envelope.rs diff --git a/Cargo.lock b/Cargo.lock index d0b5404ca..33b2009e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1990,6 +1990,7 @@ dependencies = [ "libssz 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", "libssz-types 0.3.0 (git+https://github.com/lambdaclass/libssz?rev=5cb14378ffdadcb51b387c003567e2630d0c7073)", "serde", + "snap", "spawned-concurrency", "thiserror 2.0.18", "tokio", diff --git a/crates/blockchain/Cargo.toml b/crates/blockchain/Cargo.toml index 8e55fc0e0..908852460 100644 --- a/crates/blockchain/Cargo.toml +++ b/crates/blockchain/Cargo.toml @@ -39,6 +39,7 @@ hex = { workspace = true } libssz.workspace = true libssz-types.workspace = true datatest-stable = "0.3.3" +snap = "1.1.1" [[test]] name = "forkchoice_spectests" diff --git a/crates/blockchain/src/beacon_columns.rs b/crates/blockchain/src/beacon_columns.rs index f6ee4bfd6..99fc471f2 100644 --- a/crates/blockchain/src/beacon_columns.rs +++ b/crates/blockchain/src/beacon_columns.rs @@ -11,8 +11,8 @@ //! A fulu sidecar carries a signed header, which the p2p checks verify against //! the head state before it is ever sent here. A gloas sidecar carries no //! signature at all, so a peer can invent `(slot, root, index)` triples as fast -//! as it can send them. Gloas sidecars are therefore parked only while the node -//! follows gloas, only when their shape passes +//! as it can send them. Gloas sidecars are therefore parked only when their shape +//! passes //! [`is_parkable_gloas`](ethlambda_state_transition::beacon::gossip::column::is_parkable_gloas), //! and at most [`MAX_PARKED_GLOAS_ROOTS_PER_SLOT`] distinct roots per slot. @@ -58,6 +58,12 @@ pub(crate) struct ColumnParking { pub(crate) count: usize, /// The gloas roots with sidecars parked, by the slot they were parked at. pub(crate) gloas_roots: HashMap>, + /// The slot each root in [`Self::gloas_roots`] was parked at, kept as its + /// own map rather than read back off any parked sidecar: the parked set + /// for a root is keyed by the awaited root alone, so a fulu sidecar whose + /// parent is a gloas root shares it, and its slot says nothing about the + /// gloas root's. + pub(crate) gloas_root_slots: HashMap, } /// The root whose post-state `sidecar` cannot be checked without. @@ -84,24 +90,7 @@ impl BlockChainServer { /// finality evicts it. Asked with the same predicate the checks use /// ([`Self::is_ready_to_judge`]), so a sidecar sent back is one they will /// find everything for. - /// - /// A sidecar of a fork the node does not follow is dropped: no block of - /// that fork will import, so nothing would ever drain it. pub(crate) fn park_data_columns(&mut self, sidecars: Vec) { - let (followed, unfollowed): (Vec<_>, Vec<_>) = sidecars - .into_iter() - .partition(|sidecar| sidecar.fork().is_followed()); - if !unfollowed.is_empty() { - trace!( - count = unfollowed.len(), - "Dropping data column sidecars of a fork this node does not follow" - ); - } - self.park_followed_columns(followed); - } - - /// [`Self::park_data_columns`] for sidecars of a followed fork. - pub(crate) fn park_followed_columns(&mut self, sidecars: Vec) { let config = self.store.config(); let mut ready = Vec::new(); for sidecar in sidecars { @@ -252,6 +241,9 @@ impl BlockChainServer { .entry(parked.slot) .or_default() .insert(awaited); + self.column_parking + .gloas_root_slots + .insert(awaited, parked.slot); } let inserted = self .sidecars_awaiting_parent @@ -266,23 +258,38 @@ impl BlockChainServer { /// Whether a gloas sidecar for `root` at `slot` may be parked: the root is /// not already parked at another slot, and this slot has room for it. + /// + /// A root this actor already holds as a pending or held block is exempt + /// from the per-slot cap: an honest block is known, so four forged roots + /// at its slot cannot shut its columns out. It still has to match the slot + /// it was first parked at. fn gloas_root_fits(&self, root: H256, slot: u64) -> bool { - let parked_slot = self - .sidecars_awaiting_parent + if self + .column_parking + .gloas_root_slots .get(&root) - .and_then(|parked| parked.iter().next()) - .map(|parked| parked.slot); - if parked_slot.is_some_and(|parked_slot| parked_slot != slot) { + .is_some_and(|&parked_slot| parked_slot != slot) + { return false; } self.column_parking .gloas_roots .get(&slot) .is_none_or(|roots| { - roots.contains(&root) || roots.len() < MAX_PARKED_GLOAS_ROOTS_PER_SLOT + roots.contains(&root) + || roots.len() < MAX_PARKED_GLOAS_ROOTS_PER_SLOT + || self.holds_block(&root) }) } + /// Whether this actor holds the block `root` names as pending (missing a + /// parent) or held (for columns, or for its parent's payload). + fn holds_block(&self, root: &H256) -> bool { + self.pending_block_parents.contains_key(root) + || self.blocks_awaiting_columns.contains_key(root) + || self.envelopes.is_block_held(root) + } + /// Republish how many sidecars are parked, from the running count. pub(crate) fn publish_sidecars_awaiting_parent(&self) { metrics::set_sidecars_awaiting_parent(self.column_parking.count as u64); @@ -308,7 +315,7 @@ impl BlockChainServer { .column_parking .count .saturating_sub(parked_columns.len()); - if let Some(slot) = parked_columns.iter().next().map(|parked| parked.slot) + if let Some(slot) = self.column_parking.gloas_root_slots.remove(&block_root) && let Some(roots) = self.column_parking.gloas_roots.get_mut(&slot) { roots.remove(&block_root); @@ -357,15 +364,26 @@ impl BlockChainServer { /// uptime. A sidecar at or below the finalized slot can never be needed /// again, since the chain checks would drop it outright now. pub(crate) fn evict_sidecars_awaiting_parent_at_or_below_finality(&mut self) { - if self.sidecars_awaiting_parent.is_empty() { - return; - } let finalized_slot = self .store .latest_finalized() .expect("finalized checkpoint exists") .slot; + // Swept on every call, not only when a row is dropped: a root whose + // sidecars were all replayed or refused leaves its bookkeeping behind, + // and nothing else would clear it. + self.column_parking + .gloas_roots + .retain(|slot, _| *slot > finalized_slot); + self.column_parking + .gloas_root_slots + .retain(|_, slot| *slot > finalized_slot); + + if self.sidecars_awaiting_parent.is_empty() { + return; + } + let mut dropped: Vec = Vec::new(); self.sidecars_awaiting_parent.retain(|_, parked_columns| { parked_columns.retain(|parked| { @@ -380,9 +398,6 @@ impl BlockChainServer { if !dropped.is_empty() { self.column_parking.count = self.column_parking.count.saturating_sub(dropped.len()); - self.column_parking - .gloas_roots - .retain(|slot, _| *slot > finalized_slot); info!( finalized_slot, count = dropped.len(), @@ -527,7 +542,7 @@ mod tests { let block_root = H256::repeat_byte(5); let sidecar = gloas_sidecar_at(10, block_root, 3); - server.park_followed_columns(vec![sidecar.clone()]); + server.park_data_columns(vec![sidecar.clone()]); assert_eq!( server.sidecars_awaiting_parent.get(&block_root), @@ -573,7 +588,7 @@ mod tests { .expect("insert"); let sidecar = gloas_sidecar_at(10, block_root, 3); - server.park_followed_columns(vec![sidecar.clone()]); + server.park_data_columns(vec![sidecar.clone()]); assert!(server.sidecars_awaiting_parent.is_empty()); assert_eq!(*p2p.checks.lock().unwrap(), vec![vec![sidecar]]); @@ -591,31 +606,107 @@ mod tests { .insert_state(block_root, bare_state()) .expect("insert"); - server.park_followed_columns(vec![gloas_sidecar_at(10, block_root, 3)]); + server.park_data_columns(vec![gloas_sidecar_at(10, block_root, 3)]); assert!(server.sidecars_awaiting_parent.contains_key(&block_root)); assert!(p2p.checks.lock().unwrap().is_empty()); } #[test] - fn a_gloas_sidecar_is_dropped_while_the_node_does_not_follow_gloas() { - // `ForkName::is_followed` is false for gloas today: no gloas block - // imports, so nothing would ever drain a parked row. + fn a_gloas_sidecar_parks_although_gloas_is_not_in_the_followed_set() { + // Gloas blocks import now, so a parked row has a block to drain it; + // `ForkName::is_followed` belongs to the anchor, RPC and gossip side + // and no longer decides whether the parking lot keeps a column. let (mut server, p2p) = gloas_server(); let block_root = H256::repeat_byte(5); server.park_data_columns(vec![gloas_sidecar_at(10, block_root, 3)]); - assert!(server.sidecars_awaiting_parent.is_empty()); - assert_eq!(server.column_parking.count, 0); + assert!(server.sidecars_awaiting_parent.contains_key(&block_root)); + assert_eq!(server.column_parking.count, 1); + assert!(p2p.checks.lock().unwrap().is_empty()); + } + + #[test] + fn a_fulu_sidecar_naming_a_gloas_root_as_parent_does_not_decide_that_roots_slot() { + // The parked set is keyed by the awaited root alone, so a forged fulu + // sidecar whose parent is the gloas root R shares R's set. R's slot + // must come from its own record, not from whichever element the set + // yields first. + let (mut server, _p2p) = gloas_server(); + let root = H256::repeat_byte(5); + server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar_at(7, root))]); + + server.park_data_columns(vec![ + gloas_sidecar_at(10, root, 0), + gloas_sidecar_at(10, root, 1), + ]); + + assert_eq!(server.column_parking.gloas_root_slots.get(&root), Some(&10)); + assert_eq!(server.sidecars_awaiting_parent[&root].len(), 3); + + // Draining the root frees its bookkeeping even though the first parked + // element was the fulu one. + server.drain_sidecars_awaiting_parent(root); + assert!(server.column_parking.gloas_roots.is_empty()); + assert!(server.column_parking.gloas_root_slots.is_empty()); + } + + #[test] + fn a_gloas_root_parked_at_one_slot_refuses_a_sidecar_naming_another() { + let (mut server, _p2p) = gloas_server(); + let root = H256::repeat_byte(5); + server.park_data_columns(vec![DataColumnSidecar::Fulu(sidecar_at(7, root))]); + server.park_data_columns(vec![gloas_sidecar_at(10, root, 0)]); + + server.park_data_columns(vec![gloas_sidecar_at(9, root, 1)]); + assert!( - server - .store - .take_pending_data_column(10, &block_root, 3) - .expect("DB read should succeed") - .is_none() + !server.sidecars_awaiting_parent[&root] + .iter() + .any(|parked| parked.slot == 9) ); - assert!(p2p.checks.lock().unwrap().is_empty()); + assert_count_matches(&server); + } + + #[test] + fn eviction_clears_gloas_bookkeeping_even_when_no_sidecar_is_left_to_drop() { + // A root whose sidecars were all replayed leaves its entries behind; + // the sweep must not return early on an empty parking map. + let mut server = beacon_server(beacon_store(GENESIS_TIME, 10)); + let root = H256::repeat_byte(5); + server + .column_parking + .gloas_roots + .insert(10, HashSet::from([root])); + server.column_parking.gloas_root_slots.insert(root, 10); + assert!(server.sidecars_awaiting_parent.is_empty()); + + server.evict_sidecars_awaiting_parent_at_or_below_finality(); + + assert!(server.column_parking.gloas_roots.is_empty()); + assert!(server.column_parking.gloas_root_slots.is_empty()); + } + + #[test] + fn a_root_the_actor_already_holds_as_a_block_is_exempt_from_the_slot_cap() { + let (mut server, _p2p) = gloas_server(); + for byte in 1..=MAX_PARKED_GLOAS_ROOTS_PER_SLOT as u8 { + server.park_data_columns(vec![gloas_sidecar_at(10, H256::repeat_byte(byte), 0)]); + } + let honest = H256::repeat_byte(0xaa); + let forged = H256::repeat_byte(0xbb); + // The honest block is pending on a parent this node does not have. + server.pending_block_parents.insert(honest, H256::ZERO); + + server.park_data_columns(vec![ + gloas_sidecar_at(10, forged, 0), + gloas_sidecar_at(10, honest, 0), + ]); + + assert!(!server.sidecars_awaiting_parent.contains_key(&forged)); + assert!(server.sidecars_awaiting_parent.contains_key(&honest)); + assert_count_matches(&server); } #[test] @@ -631,7 +722,7 @@ mod tests { sidecar.kzg_proofs = Default::default(); } - server.park_followed_columns(vec![uncustodied, empty]); + server.park_data_columns(vec![uncustodied, empty]); assert!(server.sidecars_awaiting_parent.is_empty()); assert_eq!(server.column_parking.count, 0); @@ -653,7 +744,7 @@ mod tests { .map(H256::repeat_byte) .collect(); for &root in &roots { - server.park_followed_columns(vec![gloas_sidecar_at(10, root, 0)]); + server.park_data_columns(vec![gloas_sidecar_at(10, root, 0)]); } let (admitted, refused) = roots.split_at(MAX_PARKED_GLOAS_ROOTS_PER_SLOT); @@ -675,7 +766,7 @@ mod tests { // A root already parked keeps taking its other columns, and another // slot has a cap of its own. - server.park_followed_columns(vec![ + server.park_data_columns(vec![ gloas_sidecar_at(10, admitted[0], 1), gloas_sidecar_at(9, refused[0], 0), ]); @@ -688,7 +779,7 @@ mod tests { fn every_sidecar_parked_for_a_gloas_root_shares_its_slot() { let (mut server, _p2p) = gloas_server(); let root = H256::repeat_byte(5); - server.park_followed_columns(vec![ + server.park_data_columns(vec![ gloas_sidecar_at(10, root, 0), gloas_sidecar_at(9, root, 1), ]); diff --git a/crates/blockchain/src/beacon_envelope.rs b/crates/blockchain/src/beacon_envelope.rs new file mode 100644 index 000000000..2387ac14d --- /dev/null +++ b/crates/blockchain/src/beacon_envelope.rs @@ -0,0 +1,981 @@ +//! The chain actor's import path for gloas execution payload envelopes. +//! +//! A gloas block carries only the builder's bid; the payload arrives apart from +//! it, in a `SignedExecutionPayloadEnvelope`, and fork choice weighs a FULL and +//! an EMPTY node per block. Three things can make the order of arrival wrong, +//! and each has a bounded queue here: +//! +//! ```text +//! envelope ─► block known (with post-state)? ─no─► awaiting_block[root] +//! │yes +//! ▼ +//! bid has commitments and a sampled column missing? ─yes─► awaiting_columns[root] +//! │no +//! ▼ +//! advance clock; on_execution_payload_envelope ─► record payload status +//! ─► redrive blocks_awaiting_parent_payload[root] ─► recompute head +//! +//! gloas block ─► parent FULL and its payload not verified ─yes─► blocks_awaiting_parent_payload[parent] +//! ``` +//! +//! Data availability moves here from the block: a gloas block's commitments are +//! in its bid, and the columns they commit to gate the envelope that reveals the +//! payload, not the block's own import. +//! +//! # Driving the queues +//! +//! Nothing here awaits inside the import path. A block import, a stored column +//! or a tick only pushes the root whose envelope may now be ready onto +//! [`EnvelopeQueues::work`]; the handler that ran it then calls +//! [`BlockChainServer::settle_envelopes`], which drains that list. That keeps +//! the cycle envelope, block import, envelope from being a recursion between +//! async functions. +//! +//! # What bounds this +//! +//! An envelope for a block this node has not imported cannot be judged, so +//! nobody has vouched for it: at most [`MAX_ENVELOPES_AWAITING_BLOCK_PER_SLOT`] +//! are kept for the slot they arrived in, and one per root. Blocks held for a +//! parent's payload are capped per parent by +//! [`MAX_BLOCKS_HELD_PER_PARENT_PAYLOAD`]. All three queues are swept at +//! finality by [`BlockChainServer::evict_envelope_queues_at_or_below_finality`]. + +use std::collections::{HashMap, VecDeque}; + +use ethlambda_state_transition::beacon::fork::ForkName; +use ethlambda_state_transition::beacon::fork_choice::{self, PayloadStatus, PayloadStatusEnum}; +use ethlambda_state_transition::beacon::stf::ExecutionEngine; +use ethlambda_types::ShortRoot; +use ethlambda_types::beacon::containers::{DataColumnSidecar, SignedBeaconBlock, gloas}; +use ethlambda_types::primitives::H256; +use tracing::{debug, error, info, trace, warn}; + +use crate::import_timing::ImportTimings; +use crate::{BlockChainServer, custody_columns_present, metrics}; + +/// How many envelopes for blocks this node has not imported may be held for one +/// slot. +/// +/// A slot has one canonical block, and a short fork or an equivocating +/// proposer adds a few more. The envelope names its block by an unverified +/// root, so past this cap the extra roots at a slot are ones nobody can vouch +/// for. +pub(crate) const MAX_ENVELOPES_AWAITING_BLOCK_PER_SLOT: usize = 4; + +/// How many blocks may wait on one parent's payload. +/// +/// The children of a block that all name it as a FULL parent: one per slot the +/// chain skipped past it, plus equivocations. +pub(crate) const MAX_BLOCKS_HELD_PER_PARENT_PAYLOAD: usize = 4; + +/// An envelope the actor is holding until it can be verified. +pub(crate) struct HeldEnvelope { + envelope: gloas::SignedExecutionPayloadEnvelope, + /// The slot the hold is evicted by: the slot the envelope arrived in while + /// its block is unknown (the envelope names no slot), the block's own once + /// it is. + slot: u64, + /// Unix milliseconds the envelope reached this node, which the store clock + /// is advanced to before verifying so the timeliness deadlines are judged + /// at arrival rather than at redrive. + event_ms: u64, +} + +impl HeldEnvelope { + pub(crate) fn new( + envelope: gloas::SignedExecutionPayloadEnvelope, + slot: u64, + event_ms: u64, + ) -> Self { + Self { + envelope, + slot, + event_ms, + } + } +} + +/// The three queues, and the list of roots whose envelope may have become +/// ready. +#[derive(Default)] +pub(crate) struct EnvelopeQueues { + /// Envelopes whose block has no post-state yet, by the block's root. + pub(crate) awaiting_block: HashMap, + /// Envelopes whose bid has commitments and whose sampled columns are not + /// all stored, by the block's root. + pub(crate) awaiting_columns: HashMap, + /// Blocks whose parent is FULL with a payload not yet verified, by the + /// parent's root, each with its own slot. The block's bytes are in + /// `Table::BlockHeaders`/`BlockBodies` as for any other held block. + pub(crate) blocks_awaiting_parent_payload: HashMap>, + /// Roots to look at in [`BlockChainServer::settle_envelopes`]. + pub(crate) work: VecDeque, + /// Roots whose envelope was verified since the last settle, whose waiting + /// children [`BlockChainServer::settle_envelopes`] then releases. + pub(crate) verified: VecDeque, +} + +impl EnvelopeQueues { + /// Whether `block_root` is a block held for its parent's payload. + pub(crate) fn is_block_held(&self, block_root: &H256) -> bool { + self.blocks_awaiting_parent_payload + .values() + .any(|children| children.contains_key(block_root)) + } + + fn is_empty(&self) -> bool { + self.awaiting_block.is_empty() + && self.awaiting_columns.is_empty() + && self.blocks_awaiting_parent_payload.is_empty() + } + + fn held_blocks(&self) -> usize { + self.blocks_awaiting_parent_payload + .values() + .map(HashMap::len) + .sum() + } +} + +/// The sampled columns an envelope still needs, or `None` if nothing is +/// outstanding. +/// +/// A bid without commitments has no columns to wait for. With commitments, +/// every column this node samples must be stored: the spec's +/// `is_data_available` is vacuous over an empty list, so a partial set must +/// never reach fork choice as evidence. +pub(crate) fn missing_columns_for_envelope( + commitment_count: usize, + custody_columns: &[u64], + present: &[u64], +) -> Option> { + if commitment_count == 0 { + return None; + } + let missing: Vec = custody_columns + .iter() + .copied() + .filter(|index| !present.contains(index)) + .collect(); + (!missing.is_empty()).then_some(missing) +} + +impl BlockChainServer { + /// Take an envelope that passed gossip validation: verify it now, or hold + /// it until what it waits on arrives. The caller then settles the queues. + pub(crate) fn receive_envelope( + &mut self, + envelope: gloas::SignedExecutionPayloadEnvelope, + event_ms: u64, + ) { + let config = self.store.config(); + let arrival_slot = fork_choice::get_current_slot(&self.store, &config); + self.apply_or_hold_envelope(HeldEnvelope::new(envelope, arrival_slot, event_ms)); + } + + /// Verify `held` against its block, or put it back in the queue for what it + /// is waiting on. `true` when the envelope was verified and recorded. + fn apply_or_hold_envelope(&mut self, held: HeldEnvelope) -> bool { + let root = held.envelope.message.beacon_block_root; + if self.store.has_verified_payload(&root) { + trace!( + block_root = %ShortRoot(&root.0), + "Dropping an envelope whose payload is already verified" + ); + return false; + } + + let has_state = self.store.has_state(&root).expect("DB read should succeed"); + let block = if has_state { + self.store.get_signed_block(&root).ok().flatten() + } else { + None + }; + let Some(block) = block else { + self.hold_envelope_awaiting_block(root, held); + return false; + }; + + let slot = block.slot(); + let finalized_slot = self + .store + .latest_finalized() + .expect("finalized checkpoint exists") + .slot; + // Strictly below: the finalized block's own children may still name it + // as their FULL parent. + if slot < finalized_slot { + trace!(%slot, "Dropping an envelope for a block below the finalized slot"); + return false; + } + + let commitment_count = match &block { + SignedBeaconBlock::Gloas(inner) => inner + .message + .body + .signed_execution_payload_bid + .message + .blob_kzg_commitments + .len(), + SignedBeaconBlock::Phase0(_) + | SignedBeaconBlock::Altair(_) + | SignedBeaconBlock::Bellatrix(_) + | SignedBeaconBlock::Capella(_) + | SignedBeaconBlock::Deneb(_) + | SignedBeaconBlock::Electra(_) + | SignedBeaconBlock::Fulu(_) + | SignedBeaconBlock::Lean(_) => { + warn!( + %slot, + block_root = %ShortRoot(&root.0), + "Dropping an envelope: its block is not a gloas block" + ); + return false; + } + }; + + let present = self + .store + .data_column_indices_for(slot, &root) + .expect("DB read should succeed"); + if let Some(missing) = + missing_columns_for_envelope(commitment_count, &self.custody_columns, &present) + { + self.hold_envelope_awaiting_columns(root, slot, held, missing); + return false; + } + + let sidecars = if commitment_count == 0 { + Vec::new() + } else { + match self.stored_gloas_columns(slot, &root) { + Some(sidecars) => sidecars, + None => { + // Present a moment ago and unreadable now: not worth + // guessing at, so wait for them to be stored again. + self.hold_envelope_awaiting_columns(root, slot, held, Vec::new()); + return false; + } + } + }; + + self.advance_beacon_clock_to(held.event_ms); + let config = self.store.config(); + // Without an execution client the follower trusts the payload, as it + // already does for every pre-gloas block it imports (the engine's + // verdict is `NotRequired` there), so the engine's part of + // `verify_execution_payload_envelope` is answered "valid". + let verdict = fork_choice::on_execution_payload_envelope( + &mut self.store, + &held.envelope, + &config, + &sidecars, + &ExecutionEngine::valid(), + ); + match verdict { + Ok(()) => { + info!( + %slot, + block_root = %ShortRoot(&root.0), + "Execution payload envelope verified" + ); + // With a client configured the verdict is the client's to give + // (a later change asks it); recording VALID without asking + // would claim a validation that never happened. + if self.engine.is_none() { + self.store.insert_beacon_block_payload_status( + root, + slot, + PayloadStatusEnum::Valid, + ); + } + self.envelopes.verified.push_back(root); + true + } + Err(err) => { + // p2p judged the envelope before it got here, so no peer is + // penalized from this side. + warn!( + %slot, + block_root = %ShortRoot(&root.0), + ?err, + "Dropping an execution payload envelope that failed verification" + ); + false + } + } + } + + /// Every sampled column stored for `root`, decoded as gloas sidecars, or + /// `None` if any is missing or not one. + fn stored_gloas_columns( + &self, + slot: u64, + root: &H256, + ) -> Option> { + self.custody_columns + .iter() + .map( + |&index| match self.store.get_data_column(slot, root, index) { + Ok(Some(DataColumnSidecar::Gloas(sidecar))) => Some(sidecar), + Ok(Some(DataColumnSidecar::Fulu(_))) | Ok(None) => None, + Err(err) => { + error!(%err, "Failed to read back a stored data column sidecar"); + None + } + }, + ) + .collect() + } + + fn hold_envelope_awaiting_block(&mut self, root: H256, held: HeldEnvelope) { + let queues = &mut self.envelopes; + if queues.awaiting_block.contains_key(&root) { + return; + } + let at_slot = queues + .awaiting_block + .values() + .filter(|other| other.slot == held.slot) + .count(); + if at_slot >= MAX_ENVELOPES_AWAITING_BLOCK_PER_SLOT { + trace!( + slot = held.slot, + block_root = %ShortRoot(&root.0), + "Dropping an envelope: its slot already holds the most unmatched envelopes allowed" + ); + return; + } + debug!( + block_root = %ShortRoot(&root.0), + "Holding an envelope until its block is imported" + ); + queues.awaiting_block.insert(root, held); + self.publish_envelope_queues(); + } + + fn hold_envelope_awaiting_columns( + &mut self, + root: H256, + slot: u64, + mut held: HeldEnvelope, + missing: Vec, + ) { + held.slot = slot; + // One envelope per root; a re-delivery replaces nothing. + if self.envelopes.awaiting_columns.contains_key(&root) { + return; + } + info!( + %slot, + block_root = %ShortRoot(&root.0), + missing = ?missing, + "Holding an envelope: its custody columns have not all arrived yet" + ); + self.envelopes.awaiting_columns.insert(root, held); + if !missing.is_empty() { + self.request_missing_columns(root, missing); + } + self.publish_envelope_queues(); + } + + /// Queue `root`'s envelope for a look in [`Self::settle_envelopes`], if one + /// is waiting for the block to import. + pub(crate) fn note_block_imported_for_envelope(&mut self, root: H256) { + if self.envelopes.awaiting_block.contains_key(&root) { + self.envelopes.work.push_back(root); + } + } + + /// Queue `root`'s envelope for a look, if one is waiting for columns. + pub(crate) fn note_column_stored_for_envelope(&mut self, root: H256) { + if self.envelopes.awaiting_columns.contains_key(&root) { + self.envelopes.work.push_back(root); + } + } + + /// Once a slot: look at every envelope waiting for columns, and ask peers + /// for whatever is still missing. + pub(crate) fn redrive_envelopes_awaiting_columns(&mut self) { + let held: Vec<(H256, u64)> = self + .envelopes + .awaiting_columns + .iter() + .map(|(&root, held)| (root, held.slot)) + .collect(); + for (root, slot) in held { + let present = self + .store + .data_column_indices_for(slot, &root) + .expect("DB read should succeed"); + let missing: Vec = self + .custody_columns + .iter() + .copied() + .filter(|index| !present.contains(index)) + .collect(); + if missing.is_empty() { + self.envelopes.work.push_back(root); + } else { + self.request_missing_columns(root, missing); + } + } + } + + /// The held envelope for `root` if what it waited on is now there. + fn take_ready_envelope(&mut self, root: H256) -> Option { + if self.envelopes.awaiting_block.contains_key(&root) + && self.store.has_state(&root).expect("DB read should succeed") + { + return self.envelopes.awaiting_block.remove(&root); + } + let slot = self.envelopes.awaiting_columns.get(&root)?.slot; + if custody_columns_present(&self.store, slot, &root, &self.custody_columns) { + return self.envelopes.awaiting_columns.remove(&root); + } + None + } + + /// Drain the roots pushed since the last call: verify each ready envelope, + /// import the blocks that were waiting on each verified payload, and + /// recompute the head once if anything changed. + pub(crate) async fn settle_envelopes(&mut self) { + let mut changed = false; + loop { + if let Some(root) = self.envelopes.work.pop_front() { + if let Some(held) = self.take_ready_envelope(root) { + self.apply_or_hold_envelope(held); + } + } else if let Some(root) = self.envelopes.verified.pop_front() { + changed = true; + self.release_blocks_awaiting_parent_payload(root).await; + } else { + break; + } + } + self.publish_envelope_queues(); + if changed { + self.recompute_beacon_head().await; + } + } + + /// Whether `block` is a gloas block whose parent is FULL while that + /// parent's payload is not yet verified: `on_block` would reject it with + /// `is_payload_verified(store, block.parent_root)`. + pub(crate) fn parent_payload_unverified(&self, block: &SignedBeaconBlock) -> bool { + if block.fork_name() != ForkName::Gloas { + return false; + } + matches!( + fork_choice::get_parent_payload_status(&self.store, block), + Ok(PayloadStatus::Full) + ) && !fork_choice::is_payload_verified(&self.store, block.parent_root()) + } + + /// Keep `block` until its parent's envelope is verified. + /// + /// Written the way a block held for a missing parent is: no `LiveChain` + /// entry, so it stays invisible to fork choice, and readable back by root. + pub(crate) fn hold_block_for_parent_payload( + &mut self, + block: SignedBeaconBlock, + timings: ImportTimings, + ) { + let slot = block.slot(); + let block_root = block.message_hash_tree_root(); + let parent_root = block.parent_root(); + + let held_for_parent = self + .envelopes + .blocks_awaiting_parent_payload + .get(&parent_root); + if held_for_parent.is_some_and(|held| held.contains_key(&block_root)) { + return; + } + if held_for_parent.is_some_and(|held| held.len() >= MAX_BLOCKS_HELD_PER_PARENT_PAYLOAD) { + warn!( + %slot, + block_root = %ShortRoot(&block_root.0), + parent_root = %ShortRoot(&parent_root.0), + "Dropping a block: its parent already has the most children held for its payload" + ); + return; + } + + info!( + %slot, + block_root = %ShortRoot(&block_root.0), + parent_root = %ShortRoot(&parent_root.0), + "Holding block: its parent's execution payload is not verified yet" + ); + self.store + .insert_pending_block(block_root, block) + .expect("DB insert should succeed"); + self.envelopes + .blocks_awaiting_parent_payload + .entry(parent_root) + .or_default() + .insert(block_root, slot); + self.held_timings.insert(block_root, timings); + self.publish_envelope_queues(); + } + + /// Re-import every block that waited on `parent_root`'s payload, now that + /// it is verified. + async fn release_blocks_awaiting_parent_payload(&mut self, parent_root: H256) { + let Some(children) = self + .envelopes + .blocks_awaiting_parent_payload + .remove(&parent_root) + else { + return; + }; + self.publish_envelope_queues(); + for (block_root, slot) in children { + let timings = self + .held_timings + .remove(&block_root) + .unwrap_or_else(ImportTimings::starting_now); + let Ok(Some(block)) = self.store.get_signed_block(&block_root) else { + error!( + %slot, + block_root = %ShortRoot(&block_root.0), + "A block held for its parent's payload vanished from the store" + ); + continue; + }; + debug!( + %slot, + block_root = %ShortRoot(&block_root.0), + "Parent payload verified; re-importing a held block" + ); + self.on_block(block, timings).await; + } + } + + /// Forget `block_root` as a block held for a parent's payload. Called from + /// the funnel every eviction goes through, so `held_timings` and the queue + /// cannot outlive the block they describe. + pub(crate) fn forget_block_held_for_parent_payload(&mut self, block_root: H256) { + if !self.envelopes.is_block_held(&block_root) { + return; + } + self.envelopes + .blocks_awaiting_parent_payload + .retain(|_, children| { + children.remove(&block_root); + !children.is_empty() + }); + self.publish_envelope_queues(); + } + + /// Drop every queued envelope and held block that finality has superseded. + /// + /// The counterpart of [`Self::evict_held_blocks_at_or_below_finality`], run + /// beside it: an envelope whose block never arrives, or a parent whose + /// payload never does, would otherwise hold its entry for the node's whole + /// uptime. Envelopes are kept at the finalized slot itself, since its + /// children may still name it as their FULL parent. + pub(crate) fn evict_envelope_queues_at_or_below_finality(&mut self) { + if self.envelopes.is_empty() { + return; + } + let finalized_slot = self + .store + .latest_finalized() + .expect("finalized checkpoint exists") + .slot; + + self.envelopes + .awaiting_block + .retain(|_, held| held.slot >= finalized_slot); + self.envelopes + .awaiting_columns + .retain(|_, held| held.slot >= finalized_slot); + + let stale: Vec = self + .envelopes + .blocks_awaiting_parent_payload + .values() + .flat_map(|children| children.iter()) + .filter(|&(_, &slot)| slot <= finalized_slot) + .map(|(&root, _)| root) + .collect(); + for root in stale { + self.discard_pending_subtree(root); + } + self.publish_envelope_queues(); + } + + pub(crate) fn publish_envelope_queues(&self) { + metrics::set_envelopes_awaiting_block(self.envelopes.awaiting_block.len() as u64); + metrics::set_envelopes_awaiting_columns(self.envelopes.awaiting_columns.len() as u64); + metrics::set_blocks_awaiting_parent_payload(self.envelopes.held_blocks() as u64); + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + use std::sync::Arc; + + use ethlambda_state_transition::beacon::config::Config; + use ethlambda_storage::backend::InMemoryBackend; + use ethlambda_types::beacon::containers::BeaconState; + use libssz::SszDecode as _; + + use super::*; + use crate::tests::{GENESIS_TIME, beacon_server, beacon_store}; + + /// The mainnet preset with every fork active from genesis, as the fork + /// choice fixtures that carry no `config.yaml` of their own are run. + fn fixture_config() -> Config { + ForkName::ALL + .into_iter() + .fold(Config::mainnet(), |config, fork| { + config.with_fork_epoch(fork, 0) + }) + } + + /// One fork choice fixture case, read the way the state transition crate's + /// spec runner reads it. + struct Case(PathBuf); + + impl Case { + fn new(handler: &str, case: &str) -> Self { + Self( + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../consensus-spec-tests/tests/mainnet/gloas/fork_choice") + .join(handler) + .join("pyspec_tests") + .join(case), + ) + } + + fn bytes(&self, name: &str) -> Vec { + let path = self.0.join(format!("{name}.ssz_snappy")); + let compressed = std::fs::read(&path) + .unwrap_or_else(|err| panic!("reading {}: {err}", path.display())); + snap::raw::Decoder::new() + .decompress_vec(&compressed) + .unwrap_or_else(|err| panic!("decompressing {}: {err}", path.display())) + } + + fn block(&self, root_hex: &str) -> SignedBeaconBlock { + let bytes = self.bytes(&format!("block_0x{root_hex}")); + SignedBeaconBlock::from_ssz(ForkName::Gloas, &bytes).expect("block decodes") + } + + fn envelope(&self, root_hex: &str) -> gloas::SignedExecutionPayloadEnvelope { + let bytes = self.bytes(&format!("execution_payload_envelope_0x{root_hex}")); + gloas::SignedExecutionPayloadEnvelope::from_ssz_bytes(&bytes).expect("envelope decodes") + } + + /// A server over the case's anchor, its clock at `time` seconds. + fn server(&self, time: u64) -> BlockChainServer { + let state = BeaconState::from_ssz(ForkName::Gloas, &self.bytes("anchor_state")) + .expect("anchor state decodes"); + let anchor = SignedBeaconBlock::Gloas(gloas::SignedBeaconBlock { + message: libssz::SszDecode::from_ssz_bytes(&self.bytes("anchor_block")) + .expect("anchor block decodes"), + signature: Default::default(), + }); + let config = fixture_config(); + let backend = Arc::new(InMemoryBackend::new()); + let store = fork_choice::get_forkchoice_store(backend, state, anchor, &config) + .expect("anchor store"); + let mut server = beacon_server(store); + fork_choice::on_tick(&mut server.store, time, &config); + server + } + } + + const TIEBREAK: &str = "get_head_full_payload_tiebreak"; + const PARENT: &str = "780218dcb4c1afc8ba72f325eb4f9ad43523356242345752d0e7a52391c15582"; + const CHILD: &str = "14f9d9ee25f47e0d8dd8ea7d01b206737e2aede3a7732d953546348511f58128"; + const PARENT_ENVELOPE: &str = + "bba7bbe6c4fb93af68a15e1048436396821dfaf7af6e2a5049dbef906df6801e"; + + fn root(hex_root: &str) -> H256 { + let bytes: [u8; 32] = hex::decode(hex_root).unwrap().try_into().unwrap(); + H256(bytes) + } + + fn head(server: &BlockChainServer) -> (H256, PayloadStatus) { + let node = fork_choice::get_head_node(&server.store, &fixture_config()).expect("head"); + (node.root, node.payload_status) + } + + #[tokio::test] + async fn a_gloas_block_imports_without_columns_and_is_not_held() { + let case = Case::new("get_head", TIEBREAK); + let mut server = case.server(12); + server.custody_columns = (0..8).collect(); + + server + .on_block(case.block(PARENT), ImportTimings::default()) + .await; + + assert!(server.store.has_state(&root(PARENT)).unwrap()); + assert!(server.blocks_awaiting_columns.is_empty()); + assert!(server.envelopes.blocks_awaiting_parent_payload.is_empty()); + } + + #[tokio::test] + async fn an_envelope_ahead_of_its_block_is_held_then_applied_when_the_block_imports() { + let case = Case::new("get_head", TIEBREAK); + let mut server = case.server(12); + + server.receive_envelope(case.envelope(PARENT_ENVELOPE), 0); + server.settle_envelopes().await; + assert!(server.envelopes.awaiting_block.contains_key(&root(PARENT))); + assert!(!server.store.has_verified_payload(&root(PARENT))); + + server + .on_block(case.block(PARENT), ImportTimings::default()) + .await; + // The import only queues the look; settling is the handler's step. + assert!(!server.store.has_verified_payload(&root(PARENT))); + server.settle_envelopes().await; + + assert!(server.envelopes.awaiting_block.is_empty()); + assert!(server.store.has_verified_payload(&root(PARENT))); + assert_eq!( + server.store.beacon_block_payload_status(root(PARENT)), + PayloadStatusEnum::Valid + ); + assert_eq!(head(&server), (root(PARENT), PayloadStatus::Full)); + } + + /// The `on_execution_payload_envelope` cases share one anchor and one + /// first block, so the valid case's envelope fits the invalid-full-child + /// case's chain: the one place a block built on a FULL parent exists. + const FULL_CHILD_CASE: &str = "on_execution_payload_envelope_invalid_full_child"; + const FULL_PARENT: &str = "76bf14e70fc96a5a3442a46dff3bfb897d5568cdb4dc7e94419f625df2fcd9e5"; + const FULL_CHILD: &str = "7a25db19479cdcc24bd59bb8f5feaeee2e9b24bd25f5eddd54dd809c567b5e5b"; + + fn valid_envelope_for_full_parent() -> gloas::SignedExecutionPayloadEnvelope { + Case::new( + "on_execution_payload_envelope", + "on_execution_payload_envelope_valid", + ) + .envelope("240ec2c17454f3e653fd3b43961e3c9a16770d336b0e882535c01f99b9223b41") + } + + #[tokio::test] + async fn a_block_on_a_full_parent_waits_for_the_parents_envelope_then_imports() { + let case = Case::new("on_execution_payload_envelope", FULL_CHILD_CASE); + let mut server = case.server(12); + server + .on_block(case.block(FULL_PARENT), ImportTimings::default()) + .await; + fork_choice::on_tick(&mut server.store, 24, &fixture_config()); + + server + .on_block(case.block(FULL_CHILD), ImportTimings::default()) + .await; + + // Held, not failed, and invisible to fork choice. + assert!(!server.store.has_state(&root(FULL_CHILD)).unwrap()); + assert!(server.envelopes.is_block_held(&root(FULL_CHILD))); + assert!(server.held_timings.contains_key(&root(FULL_CHILD))); + assert_eq!(head(&server).0, root(FULL_PARENT)); + + server.receive_envelope(valid_envelope_for_full_parent(), 0); + server.settle_envelopes().await; + + assert!(server.store.has_state(&root(FULL_CHILD)).unwrap()); + assert!(server.envelopes.blocks_awaiting_parent_payload.is_empty()); + assert!(!server.held_timings.contains_key(&root(FULL_CHILD))); + assert_eq!(head(&server).0, root(FULL_CHILD)); + } + + #[tokio::test] + async fn an_envelope_that_fails_verification_is_dropped_and_its_child_stays_held() { + let case = Case::new( + "on_execution_payload_envelope", + "on_execution_payload_envelope_invalid_full_child", + ); + let parent = "76bf14e70fc96a5a3442a46dff3bfb897d5568cdb4dc7e94419f625df2fcd9e5"; + let child = "7a25db19479cdcc24bd59bb8f5feaeee2e9b24bd25f5eddd54dd809c567b5e5b"; + let bad_envelope = "73c5948d041ccb171b9331c0eac4a3bb8cdcf01994916fee4c7493e63e890559"; + let mut server = case.server(12); + server + .on_block(case.block(parent), ImportTimings::default()) + .await; + fork_choice::on_tick(&mut server.store, 24, &fixture_config()); + server + .on_block(case.block(child), ImportTimings::default()) + .await; + assert!(server.envelopes.is_block_held(&root(child))); + + server.receive_envelope(case.envelope(bad_envelope), 0); + server.settle_envelopes().await; + + assert!(!server.store.has_verified_payload(&root(parent))); + assert!(server.envelopes.awaiting_block.is_empty()); + assert!(server.envelopes.awaiting_columns.is_empty()); + assert!(server.envelopes.is_block_held(&root(child))); + assert!(!server.store.has_state(&root(child)).unwrap()); + } + + #[tokio::test] + async fn an_envelope_for_an_already_verified_payload_is_ignored() { + let case = Case::new("get_head", TIEBREAK); + let mut server = case.server(12); + server + .on_block(case.block(PARENT), ImportTimings::default()) + .await; + server.receive_envelope(case.envelope(PARENT_ENVELOPE), 0); + assert!(server.store.has_verified_payload(&root(PARENT))); + + server.receive_envelope(case.envelope(PARENT_ENVELOPE), 0); + + assert!(server.envelopes.awaiting_block.is_empty()); + assert!(server.envelopes.awaiting_columns.is_empty()); + } + + #[test] + fn an_envelope_waits_for_every_sampled_column_when_the_bid_has_commitments() { + let custody = [2, 5, 9]; + // Nothing outstanding without commitments, whatever is stored. + assert_eq!(missing_columns_for_envelope(0, &custody, &[]), None); + // Some stored, some not: the rest are named, in custody order. + assert_eq!( + missing_columns_for_envelope(3, &custody, &[5]), + Some(vec![2, 9]) + ); + // A column outside the sample does not count. + assert_eq!( + missing_columns_for_envelope(1, &custody, &[1, 2, 5]), + Some(vec![9]) + ); + assert_eq!(missing_columns_for_envelope(1, &custody, &[9, 5, 2]), None); + } + + #[tokio::test] + async fn a_stored_column_queues_only_an_envelope_that_waits_for_columns() { + let case = Case::new("get_head", TIEBREAK); + let mut server = case.server(12); + let envelope = case.envelope(PARENT_ENVELOPE); + + server.note_column_stored_for_envelope(root(PARENT)); + assert!(server.envelopes.work.is_empty()); + + server + .envelopes + .awaiting_columns + .insert(root(PARENT), HeldEnvelope::new(envelope, 1, 0)); + server.note_column_stored_for_envelope(root(PARENT)); + assert_eq!(server.envelopes.work, VecDeque::from([root(PARENT)])); + + // Columns still missing: the look finds nothing ready, and the + // envelope stays where it is. + server.custody_columns = vec![3]; + server.settle_envelopes().await; + assert!( + server + .envelopes + .awaiting_columns + .contains_key(&root(PARENT)) + ); + } + + #[tokio::test] + async fn unmatched_envelopes_are_capped_per_slot_and_per_root() { + let case = Case::new("get_head", TIEBREAK); + let mut server = case.server(12); + let envelope = case.envelope(PARENT_ENVELOPE); + let with_root = |byte: u8| { + let mut envelope = envelope.clone(); + envelope.message.beacon_block_root = H256::repeat_byte(byte); + envelope + }; + + for byte in 1..=MAX_ENVELOPES_AWAITING_BLOCK_PER_SLOT as u8 + 2 { + server.receive_envelope(with_root(byte), 0); + } + // A second delivery of a held root changes nothing. + server.receive_envelope(with_root(1), 0); + + assert_eq!( + server.envelopes.awaiting_block.len(), + MAX_ENVELOPES_AWAITING_BLOCK_PER_SLOT + ); + assert!( + server + .envelopes + .awaiting_block + .contains_key(&H256::repeat_byte(1)) + ); + } + + #[tokio::test] + async fn blocks_held_for_one_parents_payload_are_capped() { + let case = Case::new("get_head", TIEBREAK); + let mut server = case.server(12); + let child = case.block(CHILD); + for extra in 0..MAX_BLOCKS_HELD_PER_PARENT_PAYLOAD as u64 + 2 { + let mut block = child.clone(); + if let SignedBeaconBlock::Gloas(inner) = &mut block { + inner.message.slot += extra; + } + server.hold_block_for_parent_payload(block, ImportTimings::default()); + } + + let held = &server.envelopes.blocks_awaiting_parent_payload; + assert_eq!(held.len(), 1); + assert_eq!( + held[&child.parent_root()].len(), + MAX_BLOCKS_HELD_PER_PARENT_PAYLOAD + ); + assert_eq!( + server.held_timings.len(), + MAX_BLOCKS_HELD_PER_PARENT_PAYLOAD + ); + } + + #[test] + fn finality_evicts_all_three_queues() { + // The finalized slot is fixed at init, so the queues are filled + // directly around it. + let mut server = beacon_server(beacon_store(GENESIS_TIME, 10)); + let envelope = Case::new("get_head", TIEBREAK).envelope(PARENT_ENVELOPE); + let held = |slot: u64| HeldEnvelope::new(envelope.clone(), slot, 0); + let queues = &mut server.envelopes; + queues.awaiting_block.insert(H256::repeat_byte(1), held(9)); + queues.awaiting_block.insert(H256::repeat_byte(2), held(11)); + queues + .awaiting_columns + .insert(H256::repeat_byte(3), held(9)); + queues + .awaiting_columns + .insert(H256::repeat_byte(4), held(10)); + let parent = H256::repeat_byte(5); + queues.blocks_awaiting_parent_payload.insert( + parent, + HashMap::from([(H256::repeat_byte(6), 10), (H256::repeat_byte(7), 11)]), + ); + server + .held_timings + .insert(H256::repeat_byte(6), ImportTimings::default()); + + server.evict_envelope_queues_at_or_below_finality(); + + let queues = &server.envelopes; + assert_eq!( + queues.awaiting_block.keys().collect::>(), + vec![&H256::repeat_byte(2)] + ); + // The finalized slot itself stays: its children may name it as FULL. + assert_eq!( + queues.awaiting_columns.keys().collect::>(), + vec![&H256::repeat_byte(4)] + ); + assert_eq!( + queues.blocks_awaiting_parent_payload[&parent], + HashMap::from([(H256::repeat_byte(7), 11)]) + ); + assert!(!server.held_timings.contains_key(&H256::repeat_byte(6))); + } +} diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index f0bf388a9..d7df5adf3 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -45,6 +45,7 @@ use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, trace, warn}; use crate::beacon_columns::{ColumnParking, ParkedColumn}; +use crate::beacon_envelope::EnvelopeQueues; use crate::block_builder::ProposerConfig; use crate::events::ChainEventSnapshot; use crate::import_timing::{BlockImportReport, CascadeTimings, HeadTimings, ImportTimings}; @@ -56,6 +57,7 @@ pub mod aggregation; mod beacon_aggregates; mod beacon_columns; pub mod beacon_engine; +mod beacon_envelope; pub mod block_builder; pub(crate) mod coverage; pub mod events; @@ -404,6 +406,7 @@ impl BlockChain { held_timings: HashMap::new(), sidecars_awaiting_parent: HashMap::new(), column_parking: ColumnParking::default(), + envelopes: EnvelopeQueues::default(), beacon_aggregates: Default::default(), custody_columns, engine, @@ -511,6 +514,10 @@ pub struct BlockChainServer { /// which gloas roots are parked at each slot. column_parking: ColumnParking, + /// Gloas execution payload envelopes held for their block or columns, and + /// blocks held for their parent's envelope. See `beacon_envelope`. + envelopes: EnvelopeQueues, + /// The columns this node samples, computed once at startup from its node /// id (see `das::custody_columns`). Empty on lean. custody_columns: Vec, @@ -845,17 +852,12 @@ fn data_availability_for( | SignedBeaconBlock::Bellatrix(_) | SignedBeaconBlock::Capella(_) => Some(fork_choice::DataAvailability::NotRequired), // ePBS (EIP-7732) moves the availability question onto the payload - // envelope's data columns, not this block's own (nonexistent) - // `blob_kzg_commitments`, so this gate has nothing gloas-shaped to - // check yet. Unreachable from network input today: - // `process_or_pend_block` refuses every gloas block before this - // function is ever called, because this node cannot yet deliver the - // payload envelopes and payload attestations a gloas chain needs. - // `on_block` itself accepts a gloas block and reads no availability - // evidence for it. `None` here regardless, defensively: were that - // gate ever bypassed, `None` keeps a block this node cannot follow - // out of `on_block` entirely. - SignedBeaconBlock::Gloas(_) => None, + // envelope: the commitments are in the block's bid, and the columns + // they commit to gate the envelope that reveals the payload, not the + // block's own import (see `beacon_envelope`). So a gloas block has + // nothing to wait for here, and `on_block` reads no availability + // evidence for it. + SignedBeaconBlock::Gloas(_) => Some(fork_choice::DataAvailability::NotRequired), // `process_block` dispatches a lean block to `store::on_block` before // this function is ever reached, so this arm is never observed for // one; named on its own rather than folded into the group above so @@ -1042,7 +1044,10 @@ impl BlockChainServer { // why nothing else evicts a held block nobody redelivers. self.evict_held_blocks_at_or_below_finality(); self.evict_sidecars_awaiting_parent_at_or_below_finality(); + self.evict_envelope_queues_at_or_below_finality(); self.redrive_held_blocks().await; + self.redrive_envelopes_awaiting_columns(); + self.settle_envelopes().await; // Per-interval duties for this tick. Lean-only, so this is where a // beacon follower's tick ends: it has no validator duties (see @@ -1910,6 +1915,7 @@ impl BlockChainServer { // held block nobody redelivers; see the method's own documentation. self.evict_held_blocks_at_or_below_finality(); self.evict_sidecars_awaiting_parent_at_or_below_finality(); + self.evict_envelope_queues_at_or_below_finality(); self.refresh_chain_metrics(); @@ -1962,6 +1968,7 @@ impl BlockChainServer { held_timings: HashMap::new(), sidecars_awaiting_parent: HashMap::new(), column_parking: ColumnParking::default(), + envelopes: EnvelopeQueues::default(), custody_columns: Vec::new(), engine, safe_slots_to_import_optimistically, @@ -2598,38 +2605,6 @@ impl BlockChainServer { let proposer = signed_block.proposer_index(); timings.guards_start = Some(Instant::now()); - // Refused before anything else, including the columns/finalized-slot - // checks below: this node cannot follow a gloas chain yet, since - // nothing delivers the payload envelopes a block's full branch needs - // (`fork_choice::on_execution_payload_envelope`) or the payload - // attestations that vote on them, so persisting a gloas block as - // pending is pure waste. Worse than waste were this not here - // first: a gloas block whose parent is missing would otherwise be - // `insert_pending_block`-ed and tracked in `pending_block_parents`, - // and every later block naming it as an ancestor would walk back to - // it, fetch it out of storage, and requeue it, paying that same - // round trip again on every single delivery, forever, since nothing - // ever marks it imported. `discard_pending_subtree` clears any - // children already queued under it before this landed; logged once, - // here, rather than once per retry, since after this there is no - // retry. - // - // Decided by chain first: a lean block is not a point on the beacon - // fork schedule, and `ForkName::is_followed` refuses to answer for it. - let refused = match self.store.chain() { - Chain::Lean => false, - Chain::Beacon => !signed_block.fork_name().is_followed(), - }; - if refused { - warn!( - %slot, - block_root = %ShortRoot(&block_root.0), - "Refusing a gloas block: this build cannot follow a gloas chain yet" - ); - self.discard_pending_subtree(block_root); - return None; - } - // Asked before the parent check, so that an absent `columns_wait` row // can be read two ways rather than one: the columns were never // missing, or they landed while this block was held for its parent. @@ -2769,7 +2744,9 @@ impl BlockChainServer { // a checkpoint sync, the 68 blocks behind the first one // re-held it 68 times over 54 s while its columns were // already on their way. - if self.blocks_awaiting_columns.contains_key(&missing_root) { + if self.blocks_awaiting_columns.contains_key(&missing_root) + || self.envelopes.is_block_held(&missing_root) + { return None; } // Parent state available — enqueue for processing, cascade @@ -2807,6 +2784,17 @@ impl BlockChainServer { return None; } + // A gloas block built on its parent's FULL node cannot be imported + // until that parent's envelope is verified (`on_block` asserts + // `is_payload_verified`). Held until the envelope arrives rather than + // failed, since it is a block the network accepted. `None`: the + // payload queue is now responsible for it. + if self.parent_payload_unverified(&signed_block) { + timings.guards_end = Some(Instant::now()); + self.hold_block_for_parent_payload(signed_block, timings); + return None; + } + // Parent exists, proceed with processing. Clone the block so we // can run post-import reaggregation against its merged proof — // `process_block` consumes the original for the storage layer. @@ -2856,6 +2844,9 @@ impl BlockChainServer { // sidecar parked under it was waiting for. self.drain_sidecars_awaiting_parent(block_root); + // An envelope that arrived ahead of this block can be judged now. + self.note_block_imported_for_envelope(block_root); + Some(ImportOutcome::Imported) } // A hold writes no post-state, so nothing pending on this root is @@ -3107,6 +3098,8 @@ impl BlockChainServer { metrics::set_blocks_held_for_columns(self.blocks_awaiting_columns.len() as u64); } + self.forget_block_held_for_parent_payload(block_root); + // Both eviction paths reach this function, so removing here is what // keeps `held_timings` from outliving the blocks it describes. self.held_timings.remove(&block_root); @@ -3313,6 +3306,8 @@ impl BlockChainServer { // A held block may now be complete. self.release_block_if_columns_complete(block_root).await; + // And so may a held gloas envelope. + self.note_column_stored_for_envelope(block_root); } /// Once a slot, revisit every held block: release the ones whose columns @@ -3659,7 +3654,7 @@ impl BlockChainServer { use ethlambda_network_api::p2p_to_block_chain::{ DataColumnSidecarsAwaitingParent, NewAggregatedAttestation, NewAttestation, NewBeaconAggregate, - NewBlock, NewDataColumnSidecars, + NewBlock, NewDataColumnSidecars, NewExecutionPayloadEnvelope, }; impl Handler for BlockChainServer { @@ -3776,6 +3771,24 @@ impl Handler for BlockChainServer { .unwrap_or(u64::MAX); self.advance_beacon_clock_to(arrival_ms.saturating_sub(in_flight_ms)); self.on_block(msg.block, timings).await; + // A gloas envelope that arrived ahead of this block is judgeable now. + self.settle_envelopes().await; + } +} + +impl Handler for BlockChainServer { + async fn handle(&mut self, msg: NewExecutionPayloadEnvelope, _ctx: &Context) { + // Beacon-only: lean has no execution payload envelopes. + let ChainDuties::Beacon = &self.duties else { + return; + }; + let arrival_ms = unix_now_ms(); + let picked_up = Instant::now(); + let received = msg.arrival.decode_start.unwrap_or(msg.arrival.handed_off); + let in_flight_ms = u64::try_from(picked_up.saturating_duration_since(received).as_millis()) + .unwrap_or(u64::MAX); + self.receive_envelope(*msg.envelope, arrival_ms.saturating_sub(in_flight_ms)); + self.settle_envelopes().await; } } @@ -3826,6 +3839,7 @@ impl Handler for BlockChainServer { impl Handler for BlockChainServer { async fn handle(&mut self, msg: NewDataColumnSidecars, _ctx: &Context) { self.on_checked_data_columns(msg.sidecars).await; + self.settle_envelopes().await; } } @@ -4278,6 +4292,7 @@ mod tests { held_timings: HashMap::new(), sidecars_awaiting_parent: HashMap::new(), column_parking: ColumnParking::default(), + envelopes: EnvelopeQueues::default(), beacon_aggregates: Default::default(), custody_columns: Vec::new(), engine: None, diff --git a/crates/blockchain/src/metrics.rs b/crates/blockchain/src/metrics.rs index 3a40d0d7f..8d5cf1483 100644 --- a/crates/blockchain/src/metrics.rs +++ b/crates/blockchain/src/metrics.rs @@ -949,6 +949,33 @@ static LEAN_BLOCKS_HELD_FOR_COLUMNS: std::sync::LazyLock = .unwrap() }); +static LEAN_ENVELOPES_AWAITING_BLOCK: std::sync::LazyLock = + std::sync::LazyLock::new(|| { + register_int_gauge!( + "lean_envelopes_awaiting_block", + "Execution payload envelopes held until their block is imported" + ) + .unwrap() + }); + +static LEAN_ENVELOPES_AWAITING_COLUMNS: std::sync::LazyLock = + std::sync::LazyLock::new(|| { + register_int_gauge!( + "lean_envelopes_awaiting_columns", + "Execution payload envelopes held until their custody columns are stored" + ) + .unwrap() + }); + +static LEAN_BLOCKS_AWAITING_PARENT_PAYLOAD: std::sync::LazyLock = + std::sync::LazyLock::new(|| { + register_int_gauge!( + "lean_blocks_awaiting_parent_payload", + "Blocks held until their FULL parent's execution payload envelope is verified" + ) + .unwrap() + }); + static LEAN_SIDECARS_AWAITING_PARENT: std::sync::LazyLock = std::sync::LazyLock::new(|| { register_int_gauge!( @@ -1073,6 +1100,9 @@ pub fn init() { // Data column sidecars. std::sync::LazyLock::force(&LEAN_DATA_COLUMNS_STORED_TOTAL); LEAN_BLOCKS_HELD_FOR_COLUMNS.set(0); + LEAN_ENVELOPES_AWAITING_BLOCK.set(0); + LEAN_ENVELOPES_AWAITING_COLUMNS.set(0); + LEAN_BLOCKS_AWAITING_PARENT_PAYLOAD.set(0); } // --- Public API --- @@ -1424,6 +1454,25 @@ pub fn set_blocks_held_for_columns(count: u64) { LEAN_BLOCKS_HELD_FOR_COLUMNS.set(count as i64); } +/// Execution payload envelopes held until their block is imported. +/// +/// Nobody has vouched for these (the block that would judge them is not here +/// yet), so a value that stays up means envelopes are arriving for blocks this +/// node never gets. +pub fn set_envelopes_awaiting_block(count: u64) { + LEAN_ENVELOPES_AWAITING_BLOCK.set(count as i64); +} + +/// Execution payload envelopes held until every sampled column is stored. +pub fn set_envelopes_awaiting_columns(count: u64) { + LEAN_ENVELOPES_AWAITING_COLUMNS.set(count as i64); +} + +/// Blocks held until their FULL parent's execution payload is verified. +pub fn set_blocks_awaiting_parent_payload(count: u64) { + LEAN_BLOCKS_AWAITING_PARENT_PAYLOAD.set(count as i64); +} + /// Sidecars currently parked against a parent root with no post-state. /// /// Reads as the queue depth of the recovery path the availability gate depends diff --git a/crates/net/api/src/lib.rs b/crates/net/api/src/lib.rs index f42786280..73a2758e9 100644 --- a/crates/net/api/src/lib.rs +++ b/crates/net/api/src/lib.rs @@ -4,6 +4,7 @@ use ethlambda_types::{ attestation::{SignedAggregatedAttestation, SignedAttestation}, beacon::containers::{ DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock, electra::SingleAttestation, + gloas::SignedExecutionPayloadEnvelope, }, beacon::primitives::ValidatorIndex, block::SignedBlock, @@ -252,6 +253,19 @@ pub trait P2PToBlockChain: Send + Sync { attesting_indices: Vec, arrival: AggregateArrival, ) -> Result<(), ActorError>; + /// A gloas execution payload envelope that passed gossip validation (or + /// was fetched), for the chain actor to verify against its block's + /// post-state and apply to fork choice. + /// + /// Boxed for the reason [`Self::new_beacon_aggregate`] is: the envelope + /// carries a whole execution payload. `arrival` is the same shape a block + /// brings, since the actor reads the clock the envelope arrived at to + /// judge its payload timeliness. + fn new_execution_payload_envelope( + &self, + envelope: Box, + arrival: BlockArrival, + ) -> Result<(), ActorError>; } /// When an aggregate reached this node, for the one metric that cannot be diff --git a/crates/net/p2p/src/beacon/column_checks.rs b/crates/net/p2p/src/beacon/column_checks.rs index 945b6c21f..070787b56 100644 --- a/crates/net/p2p/src/beacon/column_checks.rs +++ b/crates/net/p2p/src/beacon/column_checks.rs @@ -170,6 +170,15 @@ mod tests { ) -> Result<(), ActorError> { Ok(()) } + fn new_execution_payload_envelope( + &self, + _envelope: Box< + ethlambda_types::beacon::containers::gloas::SignedExecutionPayloadEnvelope, + >, + _arrival: BlockArrival, + ) -> Result<(), ActorError> { + Ok(()) + } } /// A sidecar the checks cannot judge yet goes back to the chain actor to diff --git a/crates/net/p2p/src/beacon/verdict.rs b/crates/net/p2p/src/beacon/verdict.rs index c5d178400..d922990a8 100644 --- a/crates/net/p2p/src/beacon/verdict.rs +++ b/crates/net/p2p/src/beacon/verdict.rs @@ -592,6 +592,15 @@ mod tests { self.0.store(true, Ordering::SeqCst); Ok(()) } + fn new_execution_payload_envelope( + &self, + _envelope: Box< + ethlambda_types::beacon::containers::gloas::SignedExecutionPayloadEnvelope, + >, + _arrival: BlockArrival, + ) -> Result<(), ActorError> { + Ok(()) + } } #[tokio::test] From 2bfb08ad4b96c23a1091a84b07c439365b391ffb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:12:20 -0300 Subject: [PATCH 060/132] test(beacon): gate the fixture-driven envelope tests The envelope tests read consensus-spec-tests, which the consensus CI job and a plain make test do not download. Gate them behind beacon-spec-tests like the BLS and KZG vectors, and run them from test-beacon-mainnet. --- Makefile | 1 + crates/blockchain/Cargo.toml | 6 ++++ crates/blockchain/src/beacon_envelope.rs | 36 ++++++++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/Makefile b/Makefile index f8fe3c95d..10796aff3 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,7 @@ test-beacon: test-beacon-mainnet test-beacon-minimal ## 🧪 Run the Beacon Chai test-beacon-mainnet: consensus-spec-tests cryptography-specs ## 🧪 Run the Beacon Chain spec tests, mainnet preset $(BEACON_TEST) --features beacon-spec-tests + cargo test -p ethlambda-blockchain --lib --profile release-fast --features beacon-spec-tests test-beacon-minimal: consensus-spec-tests cryptography-specs ## 🧪 Run the Beacon Chain spec tests, minimal preset $(BEACON_TEST) --features beacon-spec-tests,preset-minimal diff --git a/crates/blockchain/Cargo.toml b/crates/blockchain/Cargo.toml index 908852460..3f9ade160 100644 --- a/crates/blockchain/Cargo.toml +++ b/crates/blockchain/Cargo.toml @@ -41,6 +41,12 @@ libssz-types.workspace = true datatest-stable = "0.3.3" snap = "1.1.1" +[features] +# Run the fork choice fixture cases the actor tests read from +# `consensus-spec-tests`. Off by default so `cargo test` needs no fixture +# download; `make test-beacon-mainnet` turns it on. +beacon-spec-tests = [] + [[test]] name = "forkchoice_spectests" path = "tests/forkchoice_spectests.rs" diff --git a/crates/blockchain/src/beacon_envelope.rs b/crates/blockchain/src/beacon_envelope.rs index 2387ac14d..4b73b28b4 100644 --- a/crates/blockchain/src/beacon_envelope.rs +++ b/crates/blockchain/src/beacon_envelope.rs @@ -707,6 +707,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] async fn a_gloas_block_imports_without_columns_and_is_not_held() { let case = Case::new("get_head", TIEBREAK); let mut server = case.server(12); @@ -722,6 +726,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] async fn an_envelope_ahead_of_its_block_is_held_then_applied_when_the_block_imports() { let case = Case::new("get_head", TIEBREAK); let mut server = case.server(12); @@ -763,6 +771,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] async fn a_block_on_a_full_parent_waits_for_the_parents_envelope_then_imports() { let case = Case::new("on_execution_payload_envelope", FULL_CHILD_CASE); let mut server = case.server(12); @@ -791,6 +803,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] async fn an_envelope_that_fails_verification_is_dropped_and_its_child_stays_held() { let case = Case::new( "on_execution_payload_envelope", @@ -820,6 +836,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] async fn an_envelope_for_an_already_verified_payload_is_ignored() { let case = Case::new("get_head", TIEBREAK); let mut server = case.server(12); @@ -854,6 +874,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] async fn a_stored_column_queues_only_an_envelope_that_waits_for_columns() { let case = Case::new("get_head", TIEBREAK); let mut server = case.server(12); @@ -882,6 +906,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] async fn unmatched_envelopes_are_capped_per_slot_and_per_root() { let case = Case::new("get_head", TIEBREAK); let mut server = case.server(12); @@ -911,6 +939,10 @@ mod tests { } #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] async fn blocks_held_for_one_parents_payload_are_capped() { let case = Case::new("get_head", TIEBREAK); let mut server = case.server(12); @@ -936,6 +968,10 @@ mod tests { } #[test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] fn finality_evicts_all_three_queues() { // The finalized slot is fixed at init, so the queues are filled // directly around it. From 7738f2ba22fb925edd3ea915a56f7796cc8bc90a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:16:11 -0300 Subject: [PATCH 061/132] fix(beacon): keep an honest envelope recoverable A forged envelope for an unknown root used to take the root's only hold and lose the honest one. A root now keeps a few distinct envelopes, tried in turn when the block imports, and an envelope for a known block must match its bid before it may occupy a hold. Also: ask p2p for an envelope by root (FetchRequest.needs_envelope) when blocks wait on one; exempt the finalized block's own envelope from eviction; skip the settle pass and its gauge writes when there is no work (every lean tick); discard the pending subtree of a block dropped by the per-parent cap; drop an envelope whose stored columns cannot be read back instead of redriving it every slot; refresh stale docs. --- bin/ethlambda/src/beacon.rs | 6 +- bin/ethlambda/src/main.rs | 10 +- crates/blockchain/src/beacon_columns.rs | 5 +- crates/blockchain/src/beacon_envelope.rs | 357 ++++++++++++++++++++--- crates/blockchain/src/lib.rs | 6 +- crates/common/types/src/beacon/fork.rs | 9 +- crates/net/api/src/lib.rs | 4 + crates/net/p2p/src/lib.rs | 4 + docs/metrics.md | 3 + 9 files changed, 342 insertions(+), 62 deletions(-) diff --git a/bin/ethlambda/src/beacon.rs b/bin/ethlambda/src/beacon.rs index ea340a1b1..2965a834b 100644 --- a/bin/ethlambda/src/beacon.rs +++ b/bin/ethlambda/src/beacon.rs @@ -182,9 +182,9 @@ pub fn wire_params( // `SUBSCRIBE_LEAD_EPOCHS` and `UNSUBSCRIBE_LAG_EPOCHS`. Say what is coming. // // The gloas boundary keeps its own warning: crossing its digest works like - // any other, but the chain actor refuses every gloas block (nothing - // delivers payload envelopes or payload attestations to it yet), so the - // node stops tracking the chain there regardless. + // any other, but the network side delivers neither gloas blocks nor + // payload envelopes or payload attestations to the chain actor yet, so + // the node stops tracking the chain there regardless. let schedule = ForkSchedule::new(&chain, genesis.genesis_validators_root); match schedule.next_boundary_after(epoch) { Some(next) if !next.fork.is_followed() => warn!( diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index af3a40f70..86325e970 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -1697,11 +1697,11 @@ fn first_config_difference(persisted: &Config, supplied: &Config) -> Option bool { if self diff --git a/crates/blockchain/src/beacon_envelope.rs b/crates/blockchain/src/beacon_envelope.rs index 4b73b28b4..de1b081c8 100644 --- a/crates/blockchain/src/beacon_envelope.rs +++ b/crates/blockchain/src/beacon_envelope.rs @@ -42,12 +42,13 @@ use std::collections::{HashMap, VecDeque}; +use ethlambda_network_api::FetchRequest; use ethlambda_state_transition::beacon::fork::ForkName; use ethlambda_state_transition::beacon::fork_choice::{self, PayloadStatus, PayloadStatusEnum}; use ethlambda_state_transition::beacon::stf::ExecutionEngine; use ethlambda_types::ShortRoot; use ethlambda_types::beacon::containers::{DataColumnSidecar, SignedBeaconBlock, gloas}; -use ethlambda_types::primitives::H256; +use ethlambda_types::primitives::{H256, HashTreeRoot as _}; use tracing::{debug, error, info, trace, warn}; use crate::import_timing::ImportTimings; @@ -62,6 +63,14 @@ use crate::{BlockChainServer, custody_columns_present, metrics}; /// for. pub(crate) const MAX_ENVELOPES_AWAITING_BLOCK_PER_SLOT: usize = 4; +/// How many distinct envelopes may be held for one block root this node has not +/// imported. +/// +/// Nobody has checked these against a bid yet, so a forged envelope for a real +/// root must not be able to evict the honest one: the root keeps a few, and +/// when the block imports each is tried until one verifies. +pub(crate) const MAX_ENVELOPES_PER_UNMATCHED_ROOT: usize = 3; + /// How many blocks may wait on one parent's payload. /// /// The children of a block that all name it as a FULL parent: one per slot the @@ -71,6 +80,9 @@ pub(crate) const MAX_BLOCKS_HELD_PER_PARENT_PAYLOAD: usize = 4; /// An envelope the actor is holding until it can be verified. pub(crate) struct HeldEnvelope { envelope: gloas::SignedExecutionPayloadEnvelope, + /// The envelope's own root, which tells a re-delivery from a different + /// envelope for the same block. + envelope_root: H256, /// The slot the hold is evicted by: the slot the envelope arrived in while /// its block is unknown (the envelope names no slot), the block's own once /// it is. @@ -88,6 +100,7 @@ impl HeldEnvelope { event_ms: u64, ) -> Self { Self { + envelope_root: envelope.hash_tree_root(), envelope, slot, event_ms, @@ -99,8 +112,9 @@ impl HeldEnvelope { /// ready. #[derive(Default)] pub(crate) struct EnvelopeQueues { - /// Envelopes whose block has no post-state yet, by the block's root. - pub(crate) awaiting_block: HashMap, + /// Envelopes whose block has no post-state yet, by the block's root, up to + /// [`MAX_ENVELOPES_PER_UNMATCHED_ROOT`] distinct ones each. + pub(crate) awaiting_block: HashMap>, /// Envelopes whose bid has commitments and whose sampled columns are not /// all stored, by the block's root. pub(crate) awaiting_columns: HashMap, @@ -160,6 +174,21 @@ pub(crate) fn missing_columns_for_envelope( (!missing.is_empty()).then_some(missing) } +/// The checks of `verify_execution_payload_envelope` that need only the block's +/// bid, not its post-state: the envelope is the one the bid promised. +fn envelope_matches_bid( + envelope: &gloas::ExecutionPayloadEnvelope, + bid: &gloas::ExecutionPayloadBid, + block_slot: u64, +) -> bool { + envelope.builder_index == bid.builder_index + && envelope.payload.block_hash == bid.block_hash + && envelope.payload.prev_randao == bid.prev_randao + && envelope.payload.gas_limit == bid.gas_limit + && envelope.execution_requests.hash_tree_root() == bid.execution_requests_root + && envelope.payload.slot_number == block_slot +} + impl BlockChainServer { /// Take an envelope that passed gossip validation: verify it now, or hold /// it until what it waits on arrives. The caller then settles the queues. @@ -197,26 +226,22 @@ impl BlockChainServer { }; let slot = block.slot(); - let finalized_slot = self + let finalized = self .store .latest_finalized() - .expect("finalized checkpoint exists") - .slot; - // Strictly below: the finalized block's own children may still name it - // as their FULL parent. - if slot < finalized_slot { + .expect("finalized checkpoint exists"); + // Strictly below, and the finalized block exempted by name: its + // children may still name it as their FULL parent, and the checkpoint + // slot is an epoch boundary the finalized block can sit before. + if slot < finalized.slot && root != finalized.root { trace!(%slot, "Dropping an envelope for a block below the finalized slot"); return false; } - let commitment_count = match &block { - SignedBeaconBlock::Gloas(inner) => inner - .message - .body - .signed_execution_payload_bid - .message - .blob_kzg_commitments - .len(), + let bid = match &block { + SignedBeaconBlock::Gloas(inner) => { + &inner.message.body.signed_execution_payload_bid.message + } SignedBeaconBlock::Phase0(_) | SignedBeaconBlock::Altair(_) | SignedBeaconBlock::Bellatrix(_) @@ -233,6 +258,20 @@ impl BlockChainServer { return false; } }; + let commitment_count = bid.blob_kzg_commitments.len(); + + // The cheap half of verification, before anything is held: only an + // envelope that matches the bid its block committed to may occupy a + // hold or cost a fetch. The signature is p2p's gossip check. + if !envelope_matches_bid(&held.envelope.message, bid, slot) { + warn!( + %slot, + block_root = %ShortRoot(&root.0), + "Dropping an envelope that does not match its block's bid" + ); + self.refetch_envelope_if_children_wait(root); + return false; + } let present = self .store @@ -251,9 +290,18 @@ impl BlockChainServer { match self.stored_gloas_columns(slot, &root) { Some(sidecars) => sidecars, None => { - // Present a moment ago and unreadable now: not worth - // guessing at, so wait for them to be stored again. - self.hold_envelope_awaiting_columns(root, slot, held, Vec::new()); + // Every sampled column was listed as stored a moment ago, + // so failing to read one back is a storage or decode + // fault, not a wait. Holding would name nothing missing + // and be redriven every slot for as long as the fault + // lasts, so the envelope is dropped (and asked for again + // if children wait on it). + warn!( + %slot, + block_root = %ShortRoot(&root.0), + "Dropping an envelope: a stored sampled column could not be read back" + ); + self.refetch_envelope_if_children_wait(root); return false; } } @@ -301,6 +349,7 @@ impl BlockChainServer { ?err, "Dropping an execution payload envelope that failed verification" ); + self.refetch_envelope_if_children_wait(root); false } } @@ -330,19 +379,28 @@ impl BlockChainServer { fn hold_envelope_awaiting_block(&mut self, root: H256, held: HeldEnvelope) { let queues = &mut self.envelopes; - if queues.awaiting_block.contains_key(&root) { + if let Some(held_for_root) = queues.awaiting_block.get_mut(&root) { + if held_for_root.len() >= MAX_ENVELOPES_PER_UNMATCHED_ROOT + || held_for_root + .iter() + .any(|other| other.envelope_root == held.envelope_root) + { + return; + } + held_for_root.push(held); + self.publish_envelope_queues(); return; } - let at_slot = queues + let roots_at_slot = queues .awaiting_block .values() - .filter(|other| other.slot == held.slot) + .filter(|others| others.iter().any(|other| other.slot == held.slot)) .count(); - if at_slot >= MAX_ENVELOPES_AWAITING_BLOCK_PER_SLOT { + if roots_at_slot >= MAX_ENVELOPES_AWAITING_BLOCK_PER_SLOT { trace!( slot = held.slot, block_root = %ShortRoot(&root.0), - "Dropping an envelope: its slot already holds the most unmatched envelopes allowed" + "Dropping an envelope: its slot already holds the most unmatched roots allowed" ); return; } @@ -350,7 +408,7 @@ impl BlockChainServer { block_root = %ShortRoot(&root.0), "Holding an envelope until its block is imported" ); - queues.awaiting_block.insert(root, held); + queues.awaiting_block.insert(root, vec![held]); self.publish_envelope_queues(); } @@ -369,8 +427,8 @@ impl BlockChainServer { info!( %slot, block_root = %ShortRoot(&root.0), - missing = ?missing, - "Holding an envelope: its custody columns have not all arrived yet" + ?missing, + "Holding an envelope: its sampled columns have not all arrived yet" ); self.envelopes.awaiting_columns.insert(root, held); if !missing.is_empty() { @@ -422,29 +480,54 @@ impl BlockChainServer { } } - /// The held envelope for `root` if what it waited on is now there. - fn take_ready_envelope(&mut self, root: H256) -> Option { + /// The held envelopes for `root` if what they waited on is now there. + fn take_ready_envelopes(&mut self, root: H256) -> Vec { if self.envelopes.awaiting_block.contains_key(&root) && self.store.has_state(&root).expect("DB read should succeed") { - return self.envelopes.awaiting_block.remove(&root); + return self + .envelopes + .awaiting_block + .remove(&root) + .unwrap_or_default(); } - let slot = self.envelopes.awaiting_columns.get(&root)?.slot; + let Some(slot) = self + .envelopes + .awaiting_columns + .get(&root) + .map(|held| held.slot) + else { + return Vec::new(); + }; if custody_columns_present(&self.store, slot, &root, &self.custody_columns) { - return self.envelopes.awaiting_columns.remove(&root); + return self + .envelopes + .awaiting_columns + .remove(&root) + .into_iter() + .collect(); } - None + Vec::new() } /// Drain the roots pushed since the last call: verify each ready envelope, /// import the blocks that were waiting on each verified payload, and /// recompute the head once if anything changed. pub(crate) async fn settle_envelopes(&mut self) { + // Nothing to do on most ticks, and none at all on lean: leave the + // gauges alone rather than rewriting them every interval. + if self.envelopes.work.is_empty() && self.envelopes.verified.is_empty() { + return; + } let mut changed = false; loop { if let Some(root) = self.envelopes.work.pop_front() { - if let Some(held) = self.take_ready_envelope(root) { - self.apply_or_hold_envelope(held); + // The first that verifies wins; the rest are forgeries or + // duplicates of it. + for held in self.take_ready_envelopes(root) { + if self.apply_or_hold_envelope(held) { + break; + } } } else if let Some(root) = self.envelopes.verified.pop_front() { changed = true; @@ -499,6 +582,9 @@ impl BlockChainServer { parent_root = %ShortRoot(&parent_root.0), "Dropping a block: its parent already has the most children held for its payload" ); + // Its own pending children would otherwise wait on a root that + // never imports, and their descendants re-queue it. + self.discard_pending_subtree(block_root); return; } @@ -518,6 +604,35 @@ impl BlockChainServer { .insert(block_root, slot); self.held_timings.insert(block_root, timings); self.publish_envelope_queues(); + self.request_missing_envelope(parent_root); + } + + /// Ask p2p for `root`'s envelope by root. The request carries nothing but + /// the flag, since an envelope names its block and nothing else. + fn request_missing_envelope(&self, root: H256) { + if let Some(ref p2p) = self.p2p { + let _ = p2p + .fetch_block(FetchRequest { + block_root: root, + needs_block: false, + needs_envelope: true, + columns: Vec::new(), + }) + .inspect_err(|err| { + error!(block_root = %ShortRoot(&root.0), %err, "Failed to request a missing envelope") + }); + } + } + + /// After an envelope was dropped: ask again if blocks wait on exactly it. + fn refetch_envelope_if_children_wait(&self, root: H256) { + if self + .envelopes + .blocks_awaiting_parent_payload + .contains_key(&root) + { + self.request_missing_envelope(root); + } } /// Re-import every block that waited on `parent_root`'s payload, now that @@ -580,18 +695,23 @@ impl BlockChainServer { if self.envelopes.is_empty() { return; } - let finalized_slot = self + let finalized = self .store .latest_finalized() - .expect("finalized checkpoint exists") - .slot; - - self.envelopes - .awaiting_block - .retain(|_, held| held.slot >= finalized_slot); + .expect("finalized checkpoint exists"); + let finalized_slot = finalized.slot; + + // The finalized block's own envelope is exempt by name, as its + // execution hash is from `prune_beacon_el_block_hashes`: the + // checkpoint slot is an epoch boundary, and the block can sit before + // a skipped one. + self.envelopes.awaiting_block.retain(|root, held| { + held.retain(|other| other.slot >= finalized_slot || *root == finalized.root); + !held.is_empty() + }); self.envelopes .awaiting_columns - .retain(|_, held| held.slot >= finalized_slot); + .retain(|root, held| held.slot >= finalized_slot || *root == finalized.root); let stale: Vec = self .envelopes @@ -979,8 +1099,12 @@ mod tests { let envelope = Case::new("get_head", TIEBREAK).envelope(PARENT_ENVELOPE); let held = |slot: u64| HeldEnvelope::new(envelope.clone(), slot, 0); let queues = &mut server.envelopes; - queues.awaiting_block.insert(H256::repeat_byte(1), held(9)); - queues.awaiting_block.insert(H256::repeat_byte(2), held(11)); + queues + .awaiting_block + .insert(H256::repeat_byte(1), vec![held(9)]); + queues + .awaiting_block + .insert(H256::repeat_byte(2), vec![held(11)]); queues .awaiting_columns .insert(H256::repeat_byte(3), held(9)); @@ -1014,4 +1138,145 @@ mod tests { ); assert!(!server.held_timings.contains_key(&H256::repeat_byte(6))); } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn a_forged_envelope_for_a_root_does_not_evict_the_honest_one() { + let case = Case::new("get_head", TIEBREAK); + let mut server = case.server(12); + let honest = case.envelope(PARENT_ENVELOPE); + let forged = |gas_limit: u64| { + let mut envelope = honest.clone(); + envelope.message.payload.gas_limit = gas_limit; + envelope + }; + + // The forgeries arrive first and fill what the root may hold, bar one. + for gas_limit in 1..MAX_ENVELOPES_PER_UNMATCHED_ROOT as u64 { + server.receive_envelope(forged(gas_limit), 0); + } + server.receive_envelope(honest.clone(), 0); + // A re-delivery of the honest one and a surplus forgery add nothing. + server.receive_envelope(honest.clone(), 0); + server.receive_envelope(forged(99), 0); + assert_eq!( + server.envelopes.awaiting_block[&root(PARENT)].len(), + MAX_ENVELOPES_PER_UNMATCHED_ROOT + ); + + server + .on_block(case.block(PARENT), ImportTimings::default()) + .await; + server.settle_envelopes().await; + + assert!(server.store.has_verified_payload(&root(PARENT))); + assert!(server.envelopes.awaiting_block.is_empty()); + } + + #[test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + fn the_finalized_blocks_own_envelope_survives_eviction() { + // Finalized at slot 10, so the finalized root is the zero anchor of + // `beacon_store`. + let mut server = beacon_server(beacon_store(GENESIS_TIME, 10)); + let envelope = Case::new("get_head", TIEBREAK).envelope(PARENT_ENVELOPE); + let finalized_root = server.store.latest_finalized().unwrap().root; + server.envelopes.awaiting_block.insert( + finalized_root, + vec![HeldEnvelope::new(envelope.clone(), 3, 0)], + ); + server + .envelopes + .awaiting_columns + .insert(finalized_root, HeldEnvelope::new(envelope, 3, 0)); + + server.evict_envelope_queues_at_or_below_finality(); + + assert!( + server + .envelopes + .awaiting_block + .contains_key(&finalized_root) + ); + assert!( + server + .envelopes + .awaiting_columns + .contains_key(&finalized_root) + ); + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn a_block_held_for_its_parents_payload_asks_p2p_for_the_envelope() { + let case = Case::new("on_execution_payload_envelope", FULL_CHILD_CASE); + let state_server = case.server(12); + let (mut server, p2p) = crate::tests::beacon_server_recording(state_server.store.clone()); + server + .on_block(case.block(FULL_PARENT), ImportTimings::default()) + .await; + fork_choice::on_tick(&mut server.store, 24, &fixture_config()); + + server + .on_block(case.block(FULL_CHILD), ImportTimings::default()) + .await; + + let fetches = p2p.fetches.lock().unwrap(); + assert!(fetches.iter().any(|request| request.needs_envelope + && !request.needs_block + && request.block_root == root(FULL_PARENT))); + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn a_dropped_block_takes_its_pending_children_with_it() { + let case = Case::new("get_head", TIEBREAK); + let mut server = case.server(12); + let child = case.block(CHILD); + let held_roots: Vec = (0..MAX_BLOCKS_HELD_PER_PARENT_PAYLOAD as u64) + .map(|extra| { + let mut block = child.clone(); + if let SignedBeaconBlock::Gloas(inner) = &mut block { + inner.message.slot += extra; + } + let block_root = block.message_hash_tree_root(); + server.hold_block_for_parent_payload(block, ImportTimings::default()); + block_root + }) + .collect(); + assert_eq!(held_roots.len(), MAX_BLOCKS_HELD_PER_PARENT_PAYLOAD); + + // One more, with a child pending on it: the cap drops the block and + // the child goes too. + let mut surplus = child.clone(); + if let SignedBeaconBlock::Gloas(inner) = &mut surplus { + inner.message.slot += 50; + } + let surplus_root = surplus.message_hash_tree_root(); + let pending_child = H256::repeat_byte(0xcc); + server.pending_blocks.insert( + surplus_root, + std::collections::HashSet::from([pending_child]), + ); + server + .pending_block_parents + .insert(pending_child, surplus_root); + + server.hold_block_for_parent_payload(surplus, ImportTimings::default()); + + assert!(server.pending_blocks.is_empty()); + assert!(server.pending_block_parents.is_empty()); + } } diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index d7df5adf3..f22f2d17d 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -2929,6 +2929,7 @@ impl BlockChainServer { .fetch_block(FetchRequest { block_root, needs_block: true, + needs_envelope: false, // Nothing to name: a block this node has never seen has // told it nothing about what it committed to. columns: Vec::new(), @@ -2951,6 +2952,7 @@ impl BlockChainServer { let request = FetchRequest { block_root, needs_block: false, + needs_envelope: false, columns: missing, }; let _ = p2p.fetch_block(request).inspect_err( @@ -3485,7 +3487,7 @@ impl BlockChainServer { /// Whether the clock is at a fork this node does not follow while the /// node's own head is still from one it does. /// - /// Blocks past the fork never reach the live chain, so the freshest block + /// Blocks past the fork do not reach the live chain from the network, so the freshest block /// the store knows stays at the head and the sync tracker would read the /// network as stalled `NETWORK_STALL_THRESHOLD` slots after the fork and /// report `Synced` for good on a node that follows nothing. Nothing else @@ -4474,7 +4476,7 @@ mod tests { #[derive(Default)] pub(crate) struct RecordingP2P { pub(crate) checks: std::sync::Mutex>>, - fetches: std::sync::Mutex>, + pub(crate) fetches: std::sync::Mutex>, } impl ethlambda_network_api::BlockChainToP2P for RecordingP2P { diff --git a/crates/common/types/src/beacon/fork.rs b/crates/common/types/src/beacon/fork.rs index 79584df4c..6347e0d60 100644 --- a/crates/common/types/src/beacon/fork.rs +++ b/crates/common/types/src/beacon/fork.rs @@ -112,10 +112,11 @@ impl ForkName { /// Whether this node follows a chain that has reached this fork. /// - /// Phase0 through fulu are followed. Gloas is not: nothing delivers the - /// payload envelopes or payload attestations a gloas chain needs, so the - /// node refuses gloas blocks and anchors, ignores gloas gossip and reports - /// syncing once the clock reaches the fork. This is the one place to + /// Phase0 through fulu are followed. Gloas is not yet: the chain actor + /// imports gloas blocks and payload envelopes, but the network side does + /// not deliver envelopes or payload attestations, so the node still + /// refuses gloas anchors, ignores gloas gossip and reports syncing once + /// the clock reaches the fork. This is the one place to /// change when the node starts following gloas; every site whose rule is /// "does this node follow the fork" calls it. /// diff --git a/crates/net/api/src/lib.rs b/crates/net/api/src/lib.rs index 73a2758e9..269ee39da 100644 --- a/crates/net/api/src/lib.rs +++ b/crates/net/api/src/lib.rs @@ -63,6 +63,10 @@ pub struct FetchRequest { /// reading its own store to find out would pay a DB read per request to /// re-derive what the caller already knew. pub needs_block: bool, + /// Whether the block's execution payload envelope is missing (gloas): the + /// block is known, and it names a payload this node has no envelope for. + /// Fetched by root, like the block. + pub needs_envelope: bool, /// Columns of this block that this node custodies and does not have. /// /// Empty when nothing is missing, or when the block itself is, since a diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index e86425a65..d613ba15a 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -1548,8 +1548,12 @@ async fn fetch_missing(server: &mut P2PServer, request: FetchRequest) { let FetchRequest { block_root, needs_block, + needs_envelope, columns, } = request; + if needs_envelope { + debug!(%block_root, "Envelope fetch by root is not served yet; ignoring"); + } if needs_block { fetch_missing_block(server, block_root).await; } diff --git a/docs/metrics.md b/docs/metrics.md index 379d14f04..d66d15944 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -466,6 +466,9 @@ spec. | `lean_data_column_fetch_failures_total` | Counter | `DataColumnsByRoot` lookups this node gave up on, by reason | On lookup abandonment | reason=no_peers,max_retries | | | `lean_blocks_held_for_columns` | Gauge | Blocks held out of fork choice pending their custody columns | On every hold, release, and finality eviction of the held-block set | | | | `lean_sidecars_awaiting_parent` | Gauge | Sidecars parked until their block's parent has a post-state | On every park, replay, and finality eviction of the parked set | | | +| `lean_envelopes_awaiting_block` | Gauge | Gloas execution payload envelopes held until their block is imported | On every hold, release, and finality eviction of the held envelopes | | | +| `lean_envelopes_awaiting_columns` | Gauge | Gloas execution payload envelopes held until every sampled column is stored | On every hold, release, and finality eviction of the held envelopes | | | +| `lean_blocks_awaiting_parent_payload` | Gauge | Gloas blocks held until their FULL parent's payload envelope is verified | On every hold, release, and finality eviction of the held blocks | | | `lean_data_columns_rejected_total` counts the chain checks, which run in the p2p layer on every sidecar gossip did not accept: every fetched sidecar, a From 61f9fa4998c51600b87a009a2bf34affcc1a9872 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:21:12 -0300 Subject: [PATCH 062/132] fix(beacon): re-ask for missing envelopes and bound the unmatched queue The fetch request for a parent's envelope was sent once per event, so one failed fetch stalled the held child for good; the tick now re-asks once a slot for every parent with held children and no envelope in hand. Unmatched envelopes also age out after a small number of slots, so the queue stays finite while the chain does not finalize. The awaiting-block gauge counts envelopes, matching its help text. --- crates/blockchain/src/beacon_envelope.rs | 143 ++++++++++++++++++++++- crates/blockchain/src/lib.rs | 1 + 2 files changed, 141 insertions(+), 3 deletions(-) diff --git a/crates/blockchain/src/beacon_envelope.rs b/crates/blockchain/src/beacon_envelope.rs index de1b081c8..164b518a7 100644 --- a/crates/blockchain/src/beacon_envelope.rs +++ b/crates/blockchain/src/beacon_envelope.rs @@ -71,6 +71,16 @@ pub(crate) const MAX_ENVELOPES_AWAITING_BLOCK_PER_SLOT: usize = 4; /// when the block imports each is tried until one verifies. pub(crate) const MAX_ENVELOPES_PER_UNMATCHED_ROOT: usize = 3; +/// How many slots an envelope for a block this node has not imported is kept. +/// +/// A builder reveals its payload after the block it bid in, so the block +/// should reach this node within the slot or the next; an envelope still +/// unmatched after that is for a block that is not coming, or one the fetch +/// path (`FetchRequest::needs_envelope`) recovers later. Bounds the queue by +/// time where the per-slot and per-root caps bound it by count, so the total +/// stays finite while the chain does not finalize. +pub(crate) const ENVELOPE_AWAITING_BLOCK_TTL_SLOTS: u64 = 2; + /// How many blocks may wait on one parent's payload. /// /// The children of a block that all name it as a FULL parent: one per slot the @@ -452,6 +462,20 @@ impl BlockChainServer { } } + /// Once a slot: ask for the envelope of every parent that has blocks held + /// for its payload and no envelope held or verified, so one failed fetch + /// does not stall the child for good. + pub(crate) fn redrive_missing_envelopes(&self) { + for &parent_root in self.envelopes.blocks_awaiting_parent_payload.keys() { + if !self.envelopes.awaiting_block.contains_key(&parent_root) + && !self.envelopes.awaiting_columns.contains_key(&parent_root) + && !self.store.has_verified_payload(&parent_root) + { + self.request_missing_envelope(parent_root); + } + } + } + /// Once a slot: look at every envelope waiting for columns, and ask peers /// for whatever is still missing. pub(crate) fn redrive_envelopes_awaiting_columns(&mut self) { @@ -523,7 +547,10 @@ impl BlockChainServer { loop { if let Some(root) = self.envelopes.work.pop_front() { // The first that verifies wins; the rest are forgeries or - // duplicates of it. + // duplicates of it. One parked for columns is enough too: it + // matched the bid, whose block hash commits to the whole + // payload, so a second bid-consistent candidate differs only + // in a signature p2p already checked. for held in self.take_ready_envelopes(root) { if self.apply_or_hold_envelope(held) { break; @@ -705,8 +732,14 @@ impl BlockChainServer { // execution hash is from `prune_beacon_el_block_hashes`: the // checkpoint slot is an epoch boundary, and the block can sit before // a skipped one. + let config = self.store.config(); + let current_slot = fork_choice::get_current_slot(&self.store, &config); self.envelopes.awaiting_block.retain(|root, held| { - held.retain(|other| other.slot >= finalized_slot || *root == finalized.root); + held.retain(|other| { + let fresh = + other.slot.saturating_add(ENVELOPE_AWAITING_BLOCK_TTL_SLOTS) >= current_slot; + fresh && (other.slot >= finalized_slot || *root == finalized.root) + }); !held.is_empty() }); self.envelopes @@ -728,7 +761,8 @@ impl BlockChainServer { } pub(crate) fn publish_envelope_queues(&self) { - metrics::set_envelopes_awaiting_block(self.envelopes.awaiting_block.len() as u64); + let unmatched: usize = self.envelopes.awaiting_block.values().map(Vec::len).sum(); + metrics::set_envelopes_awaiting_block(unmatched as u64); metrics::set_envelopes_awaiting_columns(self.envelopes.awaiting_columns.len() as u64); metrics::set_blocks_awaiting_parent_payload(self.envelopes.held_blocks() as u64); } @@ -1279,4 +1313,107 @@ mod tests { assert!(server.pending_blocks.is_empty()); assert!(server.pending_block_parents.is_empty()); } + + #[test] + fn each_slot_re_asks_for_the_envelope_of_a_parent_with_held_children() { + let (mut server, p2p) = + crate::tests::beacon_server_recording(beacon_store(GENESIS_TIME, 0)); + let parent = H256::repeat_byte(5); + server + .envelopes + .blocks_awaiting_parent_payload + .insert(parent, HashMap::from([(H256::repeat_byte(6), 3)])); + + server.redrive_missing_envelopes(); + server.redrive_missing_envelopes(); + + let fetches = p2p.fetches.lock().unwrap(); + assert_eq!(fetches.len(), 2, "one ask per call, so one per slot"); + assert!(fetches.iter().all(|request| request.needs_envelope + && !request.needs_block + && request.block_root == parent)); + } + + #[test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + fn a_parent_whose_envelope_is_already_held_is_not_asked_for() { + let (mut server, p2p) = + crate::tests::beacon_server_recording(beacon_store(GENESIS_TIME, 0)); + let envelope = Case::new("get_head", TIEBREAK).envelope(PARENT_ENVELOPE); + let parent = H256::repeat_byte(5); + server + .envelopes + .blocks_awaiting_parent_payload + .insert(parent, HashMap::from([(H256::repeat_byte(6), 3)])); + server + .envelopes + .awaiting_block + .insert(parent, vec![HeldEnvelope::new(envelope, 0, 0)]); + + server.redrive_missing_envelopes(); + + assert!(p2p.fetches.lock().unwrap().is_empty()); + } + + #[test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + fn an_unmatched_envelope_ages_out_after_the_ttl() { + // The store clock reads slot 10. + let mut server = beacon_server(crate::tests::beacon_store_at_slot_10()); + let envelope = Case::new("get_head", TIEBREAK).envelope(PARENT_ENVELOPE); + let fresh = H256::repeat_byte(1); + let stale = H256::repeat_byte(2); + let fresh_slot = 10 - ENVELOPE_AWAITING_BLOCK_TTL_SLOTS; + server.envelopes.awaiting_block.insert( + fresh, + vec![HeldEnvelope::new(envelope.clone(), fresh_slot, 0)], + ); + server + .envelopes + .awaiting_block + .insert(stale, vec![HeldEnvelope::new(envelope, fresh_slot - 1, 0)]); + + server.evict_envelope_queues_at_or_below_finality(); + + assert!(server.envelopes.awaiting_block.contains_key(&fresh)); + assert!(!server.envelopes.awaiting_block.contains_key(&stale)); + } + + #[test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + fn the_unmatched_queue_is_bounded_across_slots_by_the_ttl() { + // Fill every slot the TTL keeps to the per-slot cap and per-root cap, + // then read off the most the queue can hold. + let mut server = beacon_server(crate::tests::beacon_store_at_slot_10()); + let envelope = Case::new("get_head", TIEBREAK).envelope(PARENT_ENVELOPE); + for slot in 0..=10u64 { + for byte in 0..MAX_ENVELOPES_AWAITING_BLOCK_PER_SLOT as u8 { + let mut root_bytes = [0u8; 32]; + root_bytes[0] = slot as u8; + root_bytes[1] = byte; + server.envelopes.awaiting_block.insert( + H256(root_bytes), + vec![HeldEnvelope::new(envelope.clone(), slot, 0)], + ); + } + } + + server.evict_envelope_queues_at_or_below_finality(); + + let kept: usize = server.envelopes.awaiting_block.values().map(Vec::len).sum(); + assert_eq!( + kept, + (ENVELOPE_AWAITING_BLOCK_TTL_SLOTS as usize + 1) + * MAX_ENVELOPES_AWAITING_BLOCK_PER_SLOT + ); + } } diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index f22f2d17d..f4ba92946 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -1047,6 +1047,7 @@ impl BlockChainServer { self.evict_envelope_queues_at_or_below_finality(); self.redrive_held_blocks().await; self.redrive_envelopes_awaiting_columns(); + self.redrive_missing_envelopes(); self.settle_envelopes().await; // Per-interval duties for this tick. Lean-only, so this is where a From 8deb2cadc76977ed23b60696b8e19921e138fb7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:33:42 -0300 Subject: [PATCH 063/132] feat(beacon): follow execution_payload and payload_attestation_message gossip A gloas follower needs the builder's envelope and the payload timeliness committee's votes to see payloads become FULL, and both only travel on two topics that gloas digests add. BeaconTopics::for_fork now subscribes them under gloas digests only, so the fork boundary window joins them ahead of the fork and earlier digests never carry them. Both topics run the existing verdict pipeline: decode and cheap checks inline, stateful checks on the blocking pool (envelopes beside blocks, votes on the attestation pool), seen-cache recheck and recording on Accept only. An envelope is forwarded to the chain actor on Accept and on Queue, since the actor holds one whose block is not imported yet. A payload attestation is forwarded on Accept only and applied to fork choice after advancing the store clock to its arrival; a vote for an unimported block is dropped, as it is valid only within its slot. --- crates/blockchain/src/beacon_envelope.rs | 43 ++++ crates/blockchain/src/lib.rs | 82 +++++++- crates/net/api/src/lib.rs | 13 +- crates/net/p2p/src/beacon/column_checks.rs | 7 + crates/net/p2p/src/beacon/decode.rs | 45 ++++ crates/net/p2p/src/beacon/topics.rs | 71 ++++++- crates/net/p2p/src/beacon/verdict.rs | 231 ++++++++++++++++++++- crates/net/p2p/src/gossipsub/handler.rs | 139 ++++++++++++- crates/net/p2p/src/lib.rs | 76 ++++++- crates/net/p2p/src/req_resp/handlers.rs | 7 + docs/beacon_wire.md | 11 +- docs/metrics.md | 6 +- 12 files changed, 712 insertions(+), 19 deletions(-) diff --git a/crates/blockchain/src/beacon_envelope.rs b/crates/blockchain/src/beacon_envelope.rs index 164b518a7..7693e024a 100644 --- a/crates/blockchain/src/beacon_envelope.rs +++ b/crates/blockchain/src/beacon_envelope.rs @@ -1416,4 +1416,47 @@ mod tests { * MAX_ENVELOPES_AWAITING_BLOCK_PER_SLOT ); } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn a_gossip_payload_attestation_message_is_applied_to_its_blocks_votes() { + let case = Case::new( + "on_payload_attestation_message", + "on_payload_attestation_message_valid", + ); + let block = "76bf14e70fc96a5a3442a46dff3bfb897d5568cdb4dc7e94419f625df2fcd9e5"; + let message_name = "payload_attestation_message_0xeaae966a9a1931c94efc5ca0280b35bac86166ebf8d47a936156bbeb1e7179d1"; + let mut server = case.server(12); + server + .on_block(case.block(block), ImportTimings::default()) + .await; + let message = gloas::PayloadAttestationMessage::from_ssz_bytes(&case.bytes(message_name)) + .expect("message decodes"); + let before = server.store.time_ms().unwrap(); + assert!( + server + .store + .payload_timeliness_vote(&root(block)) + .expect("votes exist") + .iter() + .all(Option::is_none) + ); + + server.apply_payload_attestation_message( + &message, + ðlambda_network_api::BlockArrival::now(), + ); + + assert!(server.store.time_ms().unwrap() >= before); + assert!( + server + .store + .payload_timeliness_vote(&root(block)) + .expect("votes exist") + .contains(&Some(true)) + ); + } } diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index f4ba92946..b4bf4ac9f 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -15,7 +15,7 @@ use ethlambda_types::{ beacon::{ config::Config, constants, - containers::{DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock}, + containers::{DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock, gloas}, preset, primitives::ValidatorIndex, signing::compute_epoch_at_slot, @@ -3540,6 +3540,46 @@ impl BlockChainServer { } } + /// Applies a gossip payload attestation message to fork choice, with the + /// clock first advanced to the instant it arrived: the specification's + /// `on_payload_attestation_message` rejects a vote outside its own slot, + /// and only the arrival says which slot that is. + /// + /// An error is logged at `debug` and the vote dropped. The common cause + /// is a vote for a block not imported here yet, which per the + /// specification is only valid within its slot, so holding it would + /// serve nothing. + fn apply_payload_attestation_message( + &mut self, + message: &gloas::PayloadAttestationMessage, + arrival: &BlockArrival, + ) { + let arrival_ms = unix_now_ms(); + let received = arrival.decode_start.unwrap_or(arrival.handed_off); + let in_flight_ms = u64::try_from( + Instant::now() + .saturating_duration_since(received) + .as_millis(), + ) + .unwrap_or(u64::MAX); + self.advance_beacon_clock_to(arrival_ms.saturating_sub(in_flight_ms)); + let config = self.store.config(); + let slot = message.data.slot; + let validator = message.validator_index; + let block_root = message.data.beacon_block_root; + let _ = + fork_choice::on_payload_attestation_message(&mut self.store, message, false, &config) + .inspect_err(|err| { + debug!( + slot, + validator, + block_root = %ShortRoot(&block_root.0), + %err, + "Dropping a payload attestation message" + ) + }); + } + /// Milliseconds until this actor's next tick, dispatched by chain: lean's /// interval grid via [`ms_until_next_interval`], beacon's once-per-slot /// cadence via [`ms_until_next_beacon_slot`]. @@ -3657,7 +3697,7 @@ impl BlockChainServer { use ethlambda_network_api::p2p_to_block_chain::{ DataColumnSidecarsAwaitingParent, NewAggregatedAttestation, NewAttestation, NewBeaconAggregate, - NewBlock, NewDataColumnSidecars, NewExecutionPayloadEnvelope, + NewBlock, NewDataColumnSidecars, NewExecutionPayloadEnvelope, NewPayloadAttestationMessage, }; impl Handler for BlockChainServer { @@ -3795,6 +3835,16 @@ impl Handler for BlockChainServer { } } +impl Handler for BlockChainServer { + async fn handle(&mut self, msg: NewPayloadAttestationMessage, _ctx: &Context) { + // Beacon-only: lean has no payload timeliness committee. + let ChainDuties::Beacon = &self.duties else { + return; + }; + self.apply_payload_attestation_message(&msg.message, &msg.arrival); + } +} + impl Handler for BlockChainServer { async fn handle(&mut self, msg: NewAttestation, ctx: &Context) { // Lean-only. A beacon node subscribes to no attestation subnet, so @@ -4541,6 +4591,34 @@ mod tests { server.store.ms_since_genesis() % server.store.config().slot_duration_ms } + /// A payload attestation message advances the clock to its arrival before + /// it is judged, and one naming a block that is not imported here is + /// dropped without touching the store. + #[tokio::test] + async fn a_payload_attestation_for_an_unknown_block_advances_the_clock_and_is_dropped() { + let mut server = beacon_server(beacon_store_at_slot_10()); + assert_eq!(ms_into_slot(&server), 0); + let mut message = gloas::PayloadAttestationMessage { + validator_index: 0, + data: Default::default(), + signature: Default::default(), + }; + message.data.slot = 10; + message.data.beacon_block_root = H256::repeat_byte(9); + + server.apply_payload_attestation_message(&message, &BlockArrival::now()); + + // The wall clock is long past the store's slot, so the clock is + // clamped to that slot's last second. + assert!(ms_into_slot(&server) > 0); + assert!( + server + .store + .payload_timeliness_vote(&message.data.beacon_block_root) + .is_none() + ); + } + /// A fulu-era block arriving after the attestation deadline used to be /// judged against the slot-start clock the once-per-slot tick leaves the /// store on, so `block_timeliness` saw zero milliseconds into the slot and diff --git a/crates/net/api/src/lib.rs b/crates/net/api/src/lib.rs index 269ee39da..a9332f578 100644 --- a/crates/net/api/src/lib.rs +++ b/crates/net/api/src/lib.rs @@ -3,8 +3,9 @@ use std::time::Instant; use ethlambda_types::{ attestation::{SignedAggregatedAttestation, SignedAttestation}, beacon::containers::{ - DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock, electra::SingleAttestation, - gloas::SignedExecutionPayloadEnvelope, + DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock, + electra::SingleAttestation, + gloas::{PayloadAttestationMessage, SignedExecutionPayloadEnvelope}, }, beacon::primitives::ValidatorIndex, block::SignedBlock, @@ -270,6 +271,14 @@ pub trait P2PToBlockChain: Send + Sync { envelope: Box, arrival: BlockArrival, ) -> Result<(), ActorError>; + /// A gloas payload attestation message that passed gossip validation, for + /// the chain actor to apply to fork choice. Not queued when its block is + /// not imported yet: a vote is only valid within its own slot. + fn new_payload_attestation_message( + &self, + message: PayloadAttestationMessage, + arrival: BlockArrival, + ) -> Result<(), ActorError>; } /// When an aggregate reached this node, for the one metric that cannot be diff --git a/crates/net/p2p/src/beacon/column_checks.rs b/crates/net/p2p/src/beacon/column_checks.rs index 070787b56..80d484ac4 100644 --- a/crates/net/p2p/src/beacon/column_checks.rs +++ b/crates/net/p2p/src/beacon/column_checks.rs @@ -179,6 +179,13 @@ mod tests { ) -> Result<(), ActorError> { Ok(()) } + fn new_payload_attestation_message( + &self, + _message: ethlambda_types::beacon::containers::gloas::PayloadAttestationMessage, + _arrival: BlockArrival, + ) -> Result<(), ActorError> { + Ok(()) + } } /// A sidecar the checks cannot judge yet goes back to the chain actor to diff --git a/crates/net/p2p/src/beacon/decode.rs b/crates/net/p2p/src/beacon/decode.rs index ead92e2a7..57736b4f5 100644 --- a/crates/net/p2p/src/beacon/decode.rs +++ b/crates/net/p2p/src/beacon/decode.rs @@ -23,6 +23,7 @@ //! | `bls_to_execution_change` | No, capella onward | //! | `sync_committee_contribution_and_proof` | No, altair onward | //! | `data_column_sidecar_{subnet_id}` | Yes, at gloas, by topic digest | +//! | `execution_payload`, `payload_attestation_message` | No, gloas onward | use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::{ @@ -220,6 +221,22 @@ pub fn decode_data_column_sidecar( DataColumnSidecar::from_ssz(fork, bytes).map_err(|_| DecodeError::Ssz) } +/// Decode an `execution_payload` payload. The topic exists from gloas on, so +/// the container needs no fork lookup. +pub fn decode_execution_payload_envelope( + bytes: &[u8], +) -> Result { + gloas::SignedExecutionPayloadEnvelope::from_ssz_bytes(bytes).map_err(|_| DecodeError::Ssz) +} + +/// Decode a `payload_attestation_message` payload. Gloas on, like +/// [`decode_execution_payload_envelope`]. +pub fn decode_payload_attestation_message( + bytes: &[u8], +) -> Result { + gloas::PayloadAttestationMessage::from_ssz_bytes(bytes).map_err(|_| DecodeError::Ssz) +} + /// Decode a `beacon_aggregate_and_proof` payload, at the fork its slot names. pub fn decode_aggregate_and_proof( config: &Config, @@ -620,6 +637,34 @@ mod tests { } } + #[test] + fn a_payload_attestation_message_round_trips() { + let message = gloas::PayloadAttestationMessage { + validator_index: 17, + data: gloas::PayloadAttestationData { + beacon_block_root: Root::repeat_byte(3), + slot: slot_of(10), + payload_present: true, + blob_data_available: false, + }, + signature: Default::default(), + }; + let bytes = message.to_ssz(); + assert_eq!(decode_payload_attestation_message(&bytes), Ok(message)); + for length in 0..bytes.len() { + assert!(decode_payload_attestation_message(&bytes[..length]).is_err()); + } + } + + #[test] + fn an_execution_payload_envelope_round_trips() { + let envelope = crate::test_support::envelope(5, 9); + let bytes = envelope.to_ssz(); + assert_eq!(decode_execution_payload_envelope(&bytes), Ok(envelope)); + assert!(decode_execution_payload_envelope(&bytes[..bytes.len() - 1]).is_err()); + assert!(decode_execution_payload_envelope(&[0xff; 3]).is_err()); + } + #[test] fn an_unsubscribed_topic_is_refused() { let config = Config::mainnet(); diff --git a/crates/net/p2p/src/beacon/topics.rs b/crates/net/p2p/src/beacon/topics.rs index b10a0a22d..ff4e6da60 100644 --- a/crates/net/p2p/src/beacon/topics.rs +++ b/crates/net/p2p/src/beacon/topics.rs @@ -47,7 +47,17 @@ pub const BLS_TO_EXECUTION_CHANGE: &str = "bls_to_execution_change"; /// Topic kind for aggregated sync committee contributions. pub const SYNC_COMMITTEE_CONTRIBUTION_AND_PROOF: &str = "sync_committee_contribution_and_proof"; -/// Every topic kind this node subscribes to, in the order they are subscribed. +/// Topic kind for gloas execution payload envelopes. +pub const EXECUTION_PAYLOAD: &str = "execution_payload"; +/// Topic kind for gloas payload timeliness committee votes. +pub const PAYLOAD_ATTESTATION_MESSAGE: &str = "payload_attestation_message"; + +/// The topic kinds gloas adds to [`SUBSCRIBED_TOPIC_KINDS`], subscribed from +/// the gloas digest on and never under an earlier one. +pub const GLOAS_TOPIC_KINDS: [&str; 2] = [EXECUTION_PAYLOAD, PAYLOAD_ATTESTATION_MESSAGE]; + +/// Every topic kind this node subscribes to at every fork, in the order they +/// are subscribed. [`GLOAS_TOPIC_KINDS`] follow from gloas. pub const SUBSCRIBED_TOPIC_KINDS: [&str; 7] = [ BEACON_BLOCK, BEACON_AGGREGATE_AND_PROOF, @@ -73,7 +83,11 @@ pub const BEACON_ATTESTATION_KIND: &str = "beacon_attestation"; /// `None` for anything else, lean kinds included, which is what tells the /// gossip handler a message needs no verdict. pub fn metric_kind(kind: &str) -> Option<&'static str> { - if let Some(&global) = SUBSCRIBED_TOPIC_KINDS.iter().find(|&&known| known == kind) { + if let Some(&global) = SUBSCRIBED_TOPIC_KINDS + .iter() + .chain(GLOAS_TOPIC_KINDS.iter()) + .find(|&&known| known == kind) + { return Some(global); } if data_column_subnet(kind).is_some() { @@ -228,10 +242,14 @@ impl BeaconTopics { | ForkName::Deneb | ForkName::Electra | ForkName::Fulu => Self::new(fork_digest, column_subnets, attestation_subnets), - // Gloas adds topics of its own (payload envelopes, payload - // attestations) that the follower does not yet consume, so for now - // it holds the same set. - ForkName::Gloas => Self::new(fork_digest, column_subnets, attestation_subnets), + // Gloas adds the payload envelope and the payload attestation + // topics, which the follower consumes. + ForkName::Gloas => Self::with_extra_kinds( + fork_digest, + &GLOAS_TOPIC_KINDS, + column_subnets, + attestation_subnets, + ), ForkName::Lean => { unreachable!("a beacon topic's fork is never Lean: it is absent from ForkName::ALL") } @@ -242,6 +260,17 @@ impl BeaconTopics { fork_digest: ForkDigest, column_subnets: &[u64], attestation_subnets: &[u64], + ) -> Self { + Self::with_extra_kinds(fork_digest, &[], column_subnets, attestation_subnets) + } + + /// [`Self::new`] plus the global topic `extra_kinds` a fork adds, placed + /// after [`SUBSCRIBED_TOPIC_KINDS`] and ahead of the subnet families. + fn with_extra_kinds( + fork_digest: ForkDigest, + extra_kinds: &[&str], + column_subnets: &[u64], + attestation_subnets: &[u64], ) -> Self { // Built first so the map's own key semantics do the deduplication; // `topics` below only ever sees what survived that. @@ -261,6 +290,7 @@ impl BeaconTopics { let mut topics: Vec = SUBSCRIBED_TOPIC_KINDS .iter() + .chain(extra_kinds) .map(|kind| IdentTopic::new(topic_name(fork_digest, kind))) .collect(); topics.extend(column_topics.values().cloned()); @@ -313,6 +343,35 @@ mod tests { assert_eq!(topics.topics.len(), 7); } + #[test] + fn gloas_adds_the_envelope_and_payload_attestation_topics() { + let fork_topics = |fork| { + BeaconTopics::for_fork(fork, MAINNET, &[], &[]) + .topics + .iter() + .map(|topic| topic.to_string()) + .collect::>() + }; + let gloas = fork_topics(ForkName::Gloas); + let fulu = fork_topics(ForkName::Fulu); + assert_eq!(gloas.len(), SUBSCRIBED_TOPIC_KINDS.len() + 2); + assert_eq!(fulu.len(), SUBSCRIBED_TOPIC_KINDS.len()); + for kind in GLOAS_TOPIC_KINDS { + let name = topic_name(MAINNET, kind); + assert!(gloas.contains(&name), "gloas lacks {name}"); + assert!(!fulu.contains(&name), "fulu holds {name}"); + assert_eq!(metric_kind(kind), Some(kind)); + } + assert_eq!( + topic_name(MAINNET, EXECUTION_PAYLOAD), + "/eth2/8c9f62fe/execution_payload/ssz_snappy" + ); + assert_eq!( + topic_name(MAINNET, PAYLOAD_ATTESTATION_MESSAGE), + "/eth2/8c9f62fe/payload_attestation_message/ssz_snappy" + ); + } + #[test] fn no_subnet_family_is_subscribed() { // The narrow subscription set is a design decision, not an accident of diff --git a/crates/net/p2p/src/beacon/verdict.rs b/crates/net/p2p/src/beacon/verdict.rs index d922990a8..af995e22b 100644 --- a/crates/net/p2p/src/beacon/verdict.rs +++ b/crates/net/p2p/src/beacon/verdict.rs @@ -19,6 +19,9 @@ use ethlambda_state_transition::beacon::gossip::{self, IgnoreReason, Outcome}; use ethlambda_state_transition::beacon::helpers::accessors::CommitteeCacheExt as _; use ethlambda_storage::{CacheKey, Store}; use ethlambda_types::beacon::containers::electra::SingleAttestation; +use ethlambda_types::beacon::containers::gloas::{ + PayloadAttestationMessage, SignedExecutionPayloadEnvelope, +}; use ethlambda_types::beacon::containers::{ DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock, }; @@ -72,6 +75,11 @@ pub(crate) enum Validated { attestation: Box, subnet_id: u64, }, + /// A gloas `execution_payload`. Boxed for the reason `Block` is: it + /// carries a whole execution payload. + Envelope(Box), + /// A gloas `payload_attestation_message`. + PayloadAttestation(PayloadAttestationMessage), } impl Validated { @@ -106,6 +114,10 @@ impl Validated { attestation, subnet_id, } => gossip::attestation::stateful_checks(store, attestation, *subnet_id), + Self::Envelope(envelope) => gossip::envelope::stateful_checks(store, envelope), + Self::PayloadAttestation(message) => { + gossip::payload_attestation::stateful_checks(store, message) + } } } @@ -131,6 +143,13 @@ impl Validated { }, Self::Aggregate { aggregate, .. } => server.seen_aggregates.record(aggregate), Self::Attestation { attestation, .. } => server.seen_attestations.record(attestation), + Self::Envelope(envelope) => server.seen_envelopes.record( + envelope.message.beacon_block_root, + envelope.message.builder_index, + ), + Self::PayloadAttestation(message) => server + .seen_payload_attestations + .record(message.data.slot, message.validator_index), } } @@ -151,6 +170,14 @@ impl Validated { /// validators, which verifies and relays its own backbone subnets but /// never calls `apply_attestation_to_fork_choice` for them either. /// + /// An envelope goes on when accepted, and also when queued: the chain + /// actor holds an envelope whose block it has not imported yet, which is + /// exactly what `Queue` means, and its own checks judge it again once the + /// block is there. Any other outcome keeps it away, since the actor trusts + /// the signature gossip verified. A payload attestation goes on only when + /// accepted: a vote for a block not imported here is only valid within its + /// own slot, so there is nothing to hold it for. + /// /// What an accepted subnet attestation does feed is the attestation pool, /// when its subnet is one a validator client's aggregator had this node /// join: that aggregator will ask for exactly these votes. See @@ -206,7 +233,30 @@ impl Validated { .new_beacon_aggregate(aggregate, attesting_indices, arrival) .inspect_err(|err| warn!(%err, "Failed to forward a gossip aggregate")); } - Self::Aggregate { .. } | Self::Attestation { .. } => {} + Self::Envelope(envelope) if matches!(outcome, Outcome::Accept | Outcome::Queue(_)) => { + let arrival = BlockArrival { + decode_start: Some(received_at), + handed_off: Instant::now(), + deferred_from: None, + }; + let _ = blockchain + .new_execution_payload_envelope(envelope, arrival) + .inspect_err(|err| warn!(%err, "Failed to forward a gossip envelope")); + } + Self::PayloadAttestation(message) if outcome == Outcome::Accept => { + let arrival = BlockArrival { + decode_start: Some(received_at), + handed_off: Instant::now(), + deferred_from: None, + }; + let _ = blockchain + .new_payload_attestation_message(message, arrival) + .inspect_err(|err| warn!(%err, "Failed to forward a payload attestation")); + } + Self::Aggregate { .. } + | Self::Attestation { .. } + | Self::Envelope(_) + | Self::PayloadAttestation(_) => {} } } } @@ -388,10 +438,12 @@ pub(crate) fn spawn_stateful_checks( object: Validated, ) { let permits = match &object { - Validated::Block { .. } | Validated::Column(_) => &server.gossip_validation_permits, - Validated::Aggregate { .. } | Validated::Attestation { .. } => { - &server.attestation_validation_permits + Validated::Block { .. } | Validated::Column(_) | Validated::Envelope(_) => { + &server.gossip_validation_permits } + Validated::Aggregate { .. } + | Validated::Attestation { .. } + | Validated::PayloadAttestation(_) => &server.attestation_validation_permits, }; let Ok(permit) = permits.clone().try_acquire_owned() else { let received_at = id.received_at; @@ -601,6 +653,13 @@ mod tests { ) -> Result<(), ActorError> { Ok(()) } + fn new_payload_attestation_message( + &self, + _message: ethlambda_types::beacon::containers::gloas::PayloadAttestationMessage, + _arrival: BlockArrival, + ) -> Result<(), ActorError> { + Ok(()) + } } #[tokio::test] @@ -804,6 +863,170 @@ mod tests { /// permit from is not the pool a block or a column draws from: exhausting /// one must leave the other untouched, or a burst on this topic could /// make a block or a column answer `Ignore(Overloaded)` too. + /// What `forward` handed the chain actor, by method. + #[derive(Debug, PartialEq)] + enum Sent { + Envelope, + PayloadAttestation, + } + + /// A chain actor stand-in that reports only the two gloas deliveries. + struct GloasRecorder(tokio::sync::mpsc::UnboundedSender); + + impl P2PToBlockChain for GloasRecorder { + fn new_block( + &self, + _block: SignedBeaconBlock, + _source: BlockSource, + _arrival: BlockArrival, + ) -> Result<(), ActorError> { + Ok(()) + } + fn new_attestation(&self, _attestation: SignedAttestation) -> Result<(), ActorError> { + Ok(()) + } + fn new_aggregated_attestation( + &self, + _attestation: SignedAggregatedAttestation, + ) -> Result<(), ActorError> { + Ok(()) + } + fn new_data_column_sidecars( + &self, + _sidecars: Vec, + ) -> Result<(), ActorError> { + Ok(()) + } + fn data_column_sidecars_awaiting_parent( + &self, + _sidecars: Vec, + ) -> Result<(), ActorError> { + Ok(()) + } + fn new_beacon_aggregate( + &self, + _aggregate: Box, + _attesting_indices: Vec, + _arrival: AggregateArrival, + ) -> Result<(), ActorError> { + Ok(()) + } + fn new_execution_payload_envelope( + &self, + _envelope: Box, + _arrival: BlockArrival, + ) -> Result<(), ActorError> { + let _ = self.0.send(Sent::Envelope); + Ok(()) + } + fn new_payload_attestation_message( + &self, + _message: PayloadAttestationMessage, + _arrival: BlockArrival, + ) -> Result<(), ActorError> { + let _ = self.0.send(Sent::PayloadAttestation); + Ok(()) + } + } + + pub(crate) fn payload_attestation(slot: u64, validator: u64) -> PayloadAttestationMessage { + let mut message = PayloadAttestationMessage { + validator_index: validator, + data: Default::default(), + signature: Default::default(), + }; + message.data.slot = slot; + message + } + + use crate::test_support::envelope; + + #[tokio::test] + async fn the_first_accept_for_an_envelope_key_stands_and_the_second_is_marked_seen() { + let mut server = unconnected_beacon_server(Config::mainnet(), 0).await; + let object = Validated::Envelope(Box::new(envelope(1, 3))); + + assert_eq!( + settle(&mut server, Outcome::Accept, &object), + Outcome::Accept + ); + assert_eq!( + settle(&mut server, Outcome::Accept, &object), + Outcome::Ignore(IgnoreReason::AlreadySeen) + ); + // A queued envelope records nothing, so a later accept still stands. + let other = Validated::Envelope(Box::new(envelope(2, 3))); + assert_eq!( + settle( + &mut server, + Outcome::Queue(QueueReason::BlockUnknown), + &other + ), + Outcome::Queue(QueueReason::BlockUnknown) + ); + assert_eq!( + settle(&mut server, Outcome::Accept, &other), + Outcome::Accept + ); + } + + #[tokio::test] + async fn the_first_accept_for_a_payload_attestation_key_stands_and_the_second_is_marked_seen() { + let mut server = unconnected_beacon_server(Config::mainnet(), 0).await; + let object = Validated::PayloadAttestation(payload_attestation(5, 1)); + + assert_eq!( + settle(&mut server, Outcome::Accept, &object), + Outcome::Accept + ); + assert_eq!( + settle(&mut server, Outcome::Accept, &object), + Outcome::Ignore(IgnoreReason::AlreadySeen) + ); + } + + /// An envelope reaches the chain actor on `Accept` and on `Queue` (the + /// actor holds it for its block), and on nothing else. + #[tokio::test] + async fn an_envelope_is_forwarded_on_accept_and_queue_only() { + let mut server = unconnected_beacon_server(Config::mainnet(), 0).await; + let (sender, mut received) = tokio::sync::mpsc::unbounded_channel(); + server.blockchain = Some(Arc::new(GloasRecorder(sender))); + + let forward = |outcome| { + Validated::Envelope(Box::new(envelope(1, 3))).forward(&server, Instant::now(), outcome) + }; + forward(Outcome::Accept); + assert_eq!(received.try_recv(), Ok(Sent::Envelope)); + forward(Outcome::Queue(QueueReason::BlockUnknown)); + assert_eq!(received.try_recv(), Ok(Sent::Envelope)); + forward(Outcome::Ignore(IgnoreReason::Overloaded)); + forward(Outcome::Reject(RejectReason::BadSignature)); + assert!(received.try_recv().is_err()); + } + + /// A payload attestation reaches the chain actor on `Accept` alone. + #[tokio::test] + async fn a_payload_attestation_is_forwarded_on_accept_only() { + let mut server = unconnected_beacon_server(Config::mainnet(), 0).await; + let (sender, mut received) = tokio::sync::mpsc::unbounded_channel(); + server.blockchain = Some(Arc::new(GloasRecorder(sender))); + + let forward = |outcome| { + Validated::PayloadAttestation(payload_attestation(5, 1)).forward( + &server, + Instant::now(), + outcome, + ) + }; + forward(Outcome::Ignore(IgnoreReason::UnknownBlock)); + forward(Outcome::Ignore(IgnoreReason::Overloaded)); + forward(Outcome::Reject(RejectReason::NotInPtc)); + assert!(received.try_recv().is_err()); + forward(Outcome::Accept); + assert_eq!(received.try_recv(), Ok(Sent::PayloadAttestation)); + } + #[tokio::test] async fn the_block_column_and_attestation_permit_pools_are_independent() { let server = unconnected_beacon_server(Config::mainnet(), 0).await; diff --git a/crates/net/p2p/src/gossipsub/handler.rs b/crates/net/p2p/src/gossipsub/handler.rs index 1328df4a1..e09073ca1 100644 --- a/crates/net/p2p/src/gossipsub/handler.rs +++ b/crates/net/p2p/src/gossipsub/handler.rs @@ -249,6 +249,10 @@ fn handle_beacon_gossip( triage_aggregate(server, wire, payload, id.received_at) } else if let Some(subnet_id) = beacon_topics::attestation_subnet(kind) { triage_attestation(server, topic_fork, payload, subnet_id) + } else if kind == beacon_topics::EXECUTION_PAYLOAD { + triage_envelope(server, payload) + } else if kind == beacon_topics::PAYLOAD_ATTESTATION_MESSAGE { + triage_payload_attestation(server, payload) } else { triage_other(wire, kind, payload) }; @@ -479,6 +483,65 @@ fn triage_attestation( }) } +/// Decode a gloas execution payload envelope and run its cheap gossip checks. +/// Same shape as [`triage_block`]. +fn triage_envelope(server: &P2PServer, payload: &[u8]) -> Dispatch { + const KIND: &str = beacon_topics::EXECUTION_PAYLOAD; + let envelope = match beacon_decode::decode_execution_payload_envelope(payload) { + Ok(envelope) => envelope, + Err(err) => { + metrics::inc_beacon_gossip(KIND, "decode_failed"); + debug!(kind = KIND, %err, bytes = payload.len(), "Beacon gossip decode failed"); + return Dispatch::Report(Outcome::Reject(RejectReason::Decode)); + } + }; + metrics::inc_beacon_gossip(KIND, "decoded"); + debug!( + slot = envelope.message.payload.slot_number, + builder_index = envelope.message.builder_index, + block_root = %ShortRoot(&envelope.message.beacon_block_root.0), + bytes = payload.len(), + "Beacon execution payload envelope decoded" + ); + if let Err(outcome) = + gossip::envelope::cheap_checks(&server.seen_envelopes, &server.store, &envelope) + { + return Dispatch::Report(outcome); + } + Dispatch::Validate(Validated::Envelope(Box::new(envelope))) +} + +/// Decode a gloas payload attestation message and run its cheap gossip +/// checks. Same shape as [`triage_block`]. +fn triage_payload_attestation(server: &P2PServer, payload: &[u8]) -> Dispatch { + const KIND: &str = beacon_topics::PAYLOAD_ATTESTATION_MESSAGE; + let message = match beacon_decode::decode_payload_attestation_message(payload) { + Ok(message) => message, + Err(err) => { + metrics::inc_beacon_gossip(KIND, "decode_failed"); + debug!(kind = KIND, %err, bytes = payload.len(), "Beacon gossip decode failed"); + return Dispatch::Report(Outcome::Reject(RejectReason::Decode)); + } + }; + metrics::inc_beacon_gossip(KIND, "decoded"); + // `trace` rather than `debug`: the committee votes in one burst per slot. + trace!( + slot = message.data.slot, + validator = message.validator_index, + block_root = %ShortRoot(&message.data.beacon_block_root.0), + "Beacon payload attestation message decoded" + ); + if let Err(outcome) = gossip::payload_attestation::cheap_checks( + &server.seen_payload_attestations, + &server.store, + &message, + unix_now_ms(), + ) { + return Dispatch::Report(outcome); + } + Dispatch::Validate(Validated::PayloadAttestation(message)) +} + /// Decode one of the five beacon topics with nothing particular to report, /// and count it. Ignored rather than validated: none of the five has a /// consumer, so this always answers `Dispatch::Report`. @@ -816,7 +879,7 @@ pub fn leave_expired_aggregator_subnets(server: &mut P2PServer) { #[cfg(test)] mod tests { use ethlambda_types::beacon::config::Config; - use ethlambda_types::beacon::containers::{AttestationData, electra, phase0, shared}; + use ethlambda_types::beacon::containers::{AttestationData, electra, gloas, phase0, shared}; use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::preset; use ethlambda_types::beacon::primitives::Slot; @@ -1238,4 +1301,78 @@ mod tests { Dispatch::Report(Outcome::Ignore(IgnoreReason::NoConsumer)) )); } + + /// A payload that does not decode as a gloas envelope is the sender's + /// fault, on either gloas topic. + #[tokio::test] + async fn an_undecodable_gloas_message_is_rejected() { + let server = unconnected_beacon_server(Config::mainnet(), 0).await; + + assert!(matches!( + triage_envelope(&server, &[0xff; 3]), + Dispatch::Report(Outcome::Reject(RejectReason::Decode)) + )); + assert!(matches!( + triage_payload_attestation(&server, &[0xff; 3]), + Dispatch::Report(Outcome::Reject(RejectReason::Decode)) + )); + } + + /// An envelope that clears the cheap checks goes on to the stateful ones, + /// and a second one for a key the seen cache holds is ignored before them. + #[tokio::test] + async fn an_envelope_passes_triage_unless_its_key_was_seen() { + let mut server = unconnected_beacon_server(Config::mainnet(), 0).await; + let envelope = crate::test_support::envelope(1, 3); + let payload = envelope.to_ssz(); + + assert!(matches!( + triage_envelope(&server, &payload), + Dispatch::Validate(Validated::Envelope(decoded)) if *decoded == envelope + )); + + server.seen_envelopes.record( + envelope.message.beacon_block_root, + envelope.message.builder_index, + ); + assert!(matches!( + triage_envelope(&server, &payload), + Dispatch::Report(Outcome::Ignore(IgnoreReason::AlreadySeen)) + )); + } + + /// A payload attestation for a slot before gloas is rejected by the cheap + /// checks; mainnet's gloas epoch is not zero, so slot zero is before it. + #[tokio::test] + async fn a_payload_attestation_before_gloas_is_rejected() { + let server = unconnected_beacon_server(Config::mainnet(), 0).await; + let message = gloas::PayloadAttestationMessage { + validator_index: 1, + data: Default::default(), + signature: Default::default(), + }; + + assert!(matches!( + triage_payload_attestation(&server, &message.to_ssz()), + Dispatch::Report(Outcome::Reject(RejectReason::PreGloasSlot)) + )); + } + + /// A payload attestation inside gloas but outside the current slot is + /// ignored, and one in the current slot goes on to the stateful checks. + #[tokio::test] + async fn a_payload_attestation_is_judged_on_the_clock() { + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 0); + let server = unconnected_beacon_server(config, 0).await; + let mut message = gloas::PayloadAttestationMessage { + validator_index: 1, + data: Default::default(), + signature: Default::default(), + }; + message.data.slot = 1_000_000; + assert!(matches!( + triage_payload_attestation(&server, &message.to_ssz()), + Dispatch::Report(Outcome::Ignore(IgnoreReason::NotCurrentSlot)) + )); + } } diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index d613ba15a..2547efd7e 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -52,7 +52,8 @@ use ethlambda_state_transition::beacon::aggregate::MAX_AGGREGATES_PER_SLOT; use ethlambda_state_transition::beacon::attestation_pool::SharedAttestationPool; use ethlambda_state_transition::beacon::gossip::{ SeenBlockColumns, SeenBlocks, SeenColumns, aggregate::SeenAggregates, - attestation::SeenAttestations, + attestation::SeenAttestations, envelope::SeenEnvelopes, + payload_attestation::SeenPayloadAttestations, }; use ethlambda_storage::{Chain, Store}; use ethlambda_types::beacon::preset::{MAX_VALIDATORS_PER_COMMITTEE, SLOTS_PER_EPOCH}; @@ -251,6 +252,17 @@ const SEEN_BLOCKS_CAPACITY: NonZeroUsize = NonZeroUsize::new(1024).expect("non-z /// Capacity of the first-valid-sidecar cache, keyed by `(slot, proposer, index)`. const SEEN_COLUMNS_CAPACITY: NonZeroUsize = NonZeroUsize::new(4096).expect("non-zero"); +/// Capacity of the first-valid-envelope cache, keyed by `(block root, +/// builder index)`. One block a slot carries one envelope, so this is sized +/// like [`SEEN_BLOCKS_CAPACITY`]. +const SEEN_ENVELOPES_CAPACITY: NonZeroUsize = NonZeroUsize::new(1024).expect("non-zero"); + +/// Capacity of the first-valid-payload-attestation cache, keyed by `(slot, +/// validator index)`. The rule only asks about the current slot, so one +/// committee's worth of votes for a few slots is generous headroom, in the +/// spirit of [`SEEN_COLUMNS_CAPACITY`]. +const SEEN_PAYLOAD_ATTESTATIONS_CAPACITY: NonZeroUsize = NonZeroUsize::new(4096).expect("non-zero"); + /// How many `(target_epoch, aggregator_index)` pairs the accepted-aggregate /// cache remembers, and how many `(hash_tree_root(data), committee_index)` /// bitfields alongside it (`SeenAggregates::new`'s two capacities, both sized @@ -1101,6 +1113,10 @@ impl P2P { seen_blocks: SeenBlocks::new(SEEN_BLOCKS_CAPACITY), seen_columns: SeenColumns::new(SEEN_COLUMNS_CAPACITY), seen_block_columns: SeenBlockColumns::new(SEEN_COLUMNS_CAPACITY), + seen_envelopes: SeenEnvelopes::new(SEEN_ENVELOPES_CAPACITY), + seen_payload_attestations: SeenPayloadAttestations::new( + SEEN_PAYLOAD_ATTESTATIONS_CAPACITY, + ), seen_aggregates: SeenAggregates::new( SEEN_AGGREGATES_CAPACITY, SEEN_AGGREGATES_CAPACITY, @@ -1218,6 +1234,12 @@ pub struct P2PServer { /// Gloas's counterpart of [`Self::seen_columns`]: the first valid sidecar /// per `(block root, index)`, since a gloas sidecar names no proposer. pub(crate) seen_block_columns: SeenBlockColumns, + /// The first valid `execution_payload` per `(block root, builder index)` + /// accepted from gossip. + pub(crate) seen_envelopes: SeenEnvelopes, + /// The first valid `payload_attestation_message` per `(slot, validator + /// index)` accepted from gossip. + pub(crate) seen_payload_attestations: SeenPayloadAttestations, /// Accepted `beacon_aggregate_and_proof`s, by `(target_epoch, /// aggregator_index)` and by `(hash_tree_root(data), committee_index)`. pub(crate) seen_aggregates: SeenAggregates, @@ -2481,6 +2503,51 @@ pub(crate) mod test_support { use crate::beacon::swarm::BeaconWireConfig; use crate::{P2PServer, SwarmConfig, WireConfig, build_swarm}; + /// A gloas envelope for `(root_byte, builder)` at slot `slot_number` + /// zero. Only what the cheap gossip checks and the seen cache read is + /// meaningful; nothing here is signature-valid. + pub(crate) fn envelope( + root_byte: u8, + builder_index: u64, + ) -> ethlambda_types::beacon::containers::gloas::SignedExecutionPayloadEnvelope { + use ethlambda_types::beacon::containers::{bellatrix, gloas}; + use ethlambda_types::beacon::primitives::{ + Bytes32, ExecutionAddress, ExecutionBlockHash, Uint256, + }; + let payload = gloas::ExecutionPayload { + parent_hash: ExecutionBlockHash::ZERO, + fee_recipient: ExecutionAddress::ZERO, + state_root: Bytes32::ZERO, + receipts_root: Bytes32::ZERO, + logs_bloom: bellatrix::LogsBloom::try_from(vec![0u8; preset::BYTES_PER_LOGS_BLOOM]) + .expect("built at exactly BYTES_PER_LOGS_BLOOM"), + prev_randao: Bytes32::ZERO, + block_number: 0, + gas_limit: 0, + gas_used: 0, + timestamp: 0, + extra_data: Default::default(), + base_fee_per_gas: Uint256::ZERO, + block_hash: ExecutionBlockHash::ZERO, + transactions: Default::default(), + withdrawals: Default::default(), + blob_gas_used: 0, + excess_blob_gas: 0, + block_access_list: Default::default(), + slot_number: 0, + }; + gloas::SignedExecutionPayloadEnvelope { + message: gloas::ExecutionPayloadEnvelope { + payload, + execution_requests: Default::default(), + builder_index, + beacon_block_root: Root::repeat_byte(root_byte), + parent_beacon_block_root: Root::ZERO, + }, + signature: Default::default(), + } + } + /// A real, unconnected beacon `P2PServer`, built the same way /// `req_resp::handlers::tests::unconnected_server` builds a lean one: /// port `0` throughout, so this cannot collide with a running node or a @@ -2584,6 +2651,13 @@ pub(crate) mod test_support { seen_block_columns: ethlambda_state_transition::beacon::gossip::SeenBlockColumns::new( crate::SEEN_COLUMNS_CAPACITY, ), + seen_envelopes: ethlambda_state_transition::beacon::gossip::envelope::SeenEnvelopes::new( + crate::SEEN_ENVELOPES_CAPACITY, + ), + seen_payload_attestations: + ethlambda_state_transition::beacon::gossip::payload_attestation::SeenPayloadAttestations::new( + crate::SEEN_PAYLOAD_ATTESTATIONS_CAPACITY, + ), seen_aggregates: ethlambda_state_transition::beacon::gossip::aggregate::SeenAggregates::new( crate::SEEN_AGGREGATES_CAPACITY, diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index d21514dfa..3baf54f4e 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -2651,6 +2651,13 @@ mod tests { seen_block_columns: ethlambda_state_transition::beacon::gossip::SeenBlockColumns::new( crate::SEEN_COLUMNS_CAPACITY, ), + seen_envelopes: ethlambda_state_transition::beacon::gossip::SeenEnvelopes::new( + crate::SEEN_ENVELOPES_CAPACITY, + ), + seen_payload_attestations: + ethlambda_state_transition::beacon::gossip::SeenPayloadAttestations::new( + crate::SEEN_PAYLOAD_ATTESTATIONS_CAPACITY, + ), seen_aggregates: ethlambda_state_transition::beacon::gossip::aggregate::SeenAggregates::new( crate::SEEN_AGGREGATES_CAPACITY, diff --git a/docs/beacon_wire.md b/docs/beacon_wire.md index 8a4e23e94..271c4f656 100644 --- a/docs/beacon_wire.md +++ b/docs/beacon_wire.md @@ -85,7 +85,14 @@ after a boundary read the stale digest and reject the record. Seven global topics, `/eth2/{digest}/{name}/ssz_snappy`, plus two subnet families this node's own node id selects a narrow slice of: the data column subnets it custodies and the attestation subnets it backbones, both described -below. +below. A gloas digest adds two more, `execution_payload` and +`payload_attestation_message`, which `BeaconTopics::for_fork` subscribes under +gloas digests and not under earlier ones. An envelope is validated like a block +(its stateful half on the blocking pool) and goes to the chain actor on `Accept` +and on `Queue`, since the actor holds an envelope whose block is not imported +yet. A payload attestation is validated on the attestation permit pool and goes +to the actor on `Accept` only: a vote for a block not imported here is dropped, +since it is valid only within its own slot. | Topic | Decoded as | | --- | --- | @@ -97,6 +104,8 @@ below. | `bls_to_execution_change` | `SignedBLSToExecutionChange` | | `sync_committee_contribution_and_proof` | `SignedContributionAndProof` | | `beacon_attestation_{subnet_id}` | `Attestation`, phase0 or electra | +| `execution_payload` | `SignedExecutionPayloadEnvelope`, gloas digests only | +| `payload_attestation_message` | `PayloadAttestationMessage`, gloas digests only | `beacon_attestation_{0..63}` is no longer wholly unsubscribed. This node holds `SUBNETS_PER_NODE` (2 on mainnet) long-lived subscriptions from that family, diff --git a/docs/metrics.md b/docs/metrics.md index d66d15944..8908096b3 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -363,8 +363,10 @@ own section. These are ethlambda-specific, not part of the leanMetrics spec. `kind` is the topic kind, with every `data_column_sidecar_{subnet}` sharing the label `data_column_sidecar` and every `beacon_attestation_{subnet_id}` sharing -`beacon_attestation`. `queue` means IGNORE to gossipsub while the chain actor -still receives the object and parks it; neither the aggregate nor the +`beacon_attestation`. The gloas topics `execution_payload` and +`payload_attestation_message` are labelled by their own name. `queue` means +IGNORE to gossipsub while the chain actor still receives the object and parks +it (an `execution_payload` whose block is unknown answers it); neither the aggregate nor the attestation topic ever answers `queue`, since the vote block's post-state is either cached or it is not (`IgnoreReason::UnknownBlock`/`StateUnavailable`), with nothing to hold the message for. **`verdict_expired_total` should stay at From 2bef861928df58e982bacdc49816afe4cb817a36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:43:42 -0300 Subject: [PATCH 064/132] fix(beacon): apply verified payload votes without re-verifying them The chain actor re-ran the whole wire path of on_payload_attestation_message for every gossip payload vote, a BLS verification each, serially on the actor, for votes ethlambda-p2p had already verified. Add fork_choice::apply_verified_payload_attestation, which keeps the store-state checks (known block state, current slot, committee seat) and skips the signature, and call it from the actor. Votes also get the catch-up tick blocks get, so a tick still owed for a new slot no longer drops every vote of that slot. Gossip validation answers Queue for an envelope whose block is unseen before it reaches the signature, so queued envelopes were never checked by p2p, contrary to two comments. Verify the signature where the actor parks an envelope for its columns (its block is known by then), so a forged copy carrying the honest payload cannot occupy the root's one slot; an invalid one is dropped and re-asked for if children wait. Also: factor the wire-arrival computation into BlockArrival::wire_ms, pick the stateful-check permit pool in one tested function, and restore test doc comments and metrics.md wording. --- crates/blockchain/src/beacon_envelope.rs | 77 +++++++- crates/blockchain/src/lib.rs | 42 ++--- .../src/beacon/fork_choice.rs | 164 +++++++++++++++++- crates/net/api/src/lib.rs | 15 ++ crates/net/p2p/src/beacon/transition.rs | 32 ++++ crates/net/p2p/src/beacon/verdict.rs | 65 ++++--- docs/metrics.md | 10 +- 7 files changed, 350 insertions(+), 55 deletions(-) diff --git a/crates/blockchain/src/beacon_envelope.rs b/crates/blockchain/src/beacon_envelope.rs index 7693e024a..d7568621c 100644 --- a/crates/blockchain/src/beacon_envelope.rs +++ b/crates/blockchain/src/beacon_envelope.rs @@ -46,6 +46,7 @@ use ethlambda_network_api::FetchRequest; use ethlambda_state_transition::beacon::fork::ForkName; use ethlambda_state_transition::beacon::fork_choice::{self, PayloadStatus, PayloadStatusEnum}; use ethlambda_state_transition::beacon::stf::ExecutionEngine; +use ethlambda_state_transition::beacon::stf::gloas as stf_gloas; use ethlambda_types::ShortRoot; use ethlambda_types::beacon::containers::{DataColumnSidecar, SignedBeaconBlock, gloas}; use ethlambda_types::primitives::{H256, HashTreeRoot as _}; @@ -272,7 +273,10 @@ impl BlockChainServer { // The cheap half of verification, before anything is held: only an // envelope that matches the bid its block committed to may occupy a - // hold or cost a fetch. The signature is p2p's gossip check. + // hold or cost a fetch. The signature is checked later, where an + // envelope is parked (see `Self::hold_envelope_awaiting_columns`), or + // by `on_execution_payload_envelope` when it is applied: gossip + // validation does not reach it for an envelope it queued. if !envelope_matches_bid(&held.envelope.message, bid, slot) { warn!( %slot, @@ -422,6 +426,20 @@ impl BlockChainServer { self.publish_envelope_queues(); } + /// Whether `envelope`'s builder signature holds against the post-state of + /// the block `root`, which must be imported. A state that cannot be read, + /// or a builder the state does not know, reads as an invalid signature. + fn envelope_signature_holds( + &self, + root: H256, + envelope: &gloas::SignedExecutionPayloadEnvelope, + ) -> bool { + let Some(state) = self.store.get_state(&root).ok().flatten() else { + return false; + }; + stf_gloas::verify_execution_payload_envelope_signature(&state, envelope).unwrap_or(false) + } + fn hold_envelope_awaiting_columns( &mut self, root: H256, @@ -434,6 +452,21 @@ impl BlockChainServer { if self.envelopes.awaiting_columns.contains_key(&root) { return; } + // Gossip validation answered `Queue` for an envelope whose block it + // had not seen, before it reached the signature, so nothing has + // checked this one yet. A forged copy carrying the honest payload and + // a bad signature would otherwise take the root's one slot here and + // shut the honest envelope out. One BLS verification per root, against + // the state of the block it names. + if !self.envelope_signature_holds(root, &held.envelope) { + warn!( + %slot, + block_root = %ShortRoot(&root.0), + "Dropping an envelope with an invalid signature" + ); + self.refetch_envelope_if_children_wait(root); + return; + } info!( %slot, block_root = %ShortRoot(&root.0), @@ -550,7 +583,7 @@ impl BlockChainServer { // duplicates of it. One parked for columns is enough too: it // matched the bid, whose block hash commits to the whole // payload, so a second bid-consistent candidate differs only - // in a signature p2p already checked. + // in a signature, which a parked one has had checked. for held in self.take_ready_envelopes(root) { if self.apply_or_hold_envelope(held) { break; @@ -1059,6 +1092,46 @@ mod tests { ); } + /// Gossip validation queues an envelope for an unseen block before it + /// reaches the signature, so the actor checks it where it parks one: a + /// copy with the honest payload and a bad signature, arriving first, must + /// not take the root's one slot from the honest envelope. + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn a_bad_signature_copy_does_not_take_the_parking_slot() { + let case = Case::new("get_head", TIEBREAK); + let mut server = case.server(12); + server + .on_block(case.block(PARENT), ImportTimings::default()) + .await; + let honest = case.envelope(PARENT_ENVELOPE); + let mut forged = honest.clone(); + forged.signature = Default::default(); + assert_ne!(forged.signature, honest.signature); + + server.hold_envelope_awaiting_columns( + root(PARENT), + 1, + HeldEnvelope::new(forged, 1, 0), + vec![], + ); + assert!(server.envelopes.awaiting_columns.is_empty()); + + server.hold_envelope_awaiting_columns( + root(PARENT), + 1, + HeldEnvelope::new(honest.clone(), 1, 0), + vec![], + ); + assert_eq!( + server.envelopes.awaiting_columns[&root(PARENT)].envelope, + honest + ); + } + #[tokio::test] #[cfg_attr( not(feature = "beacon-spec-tests"), diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index b4bf4ac9f..da7861c1f 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -3546,23 +3546,19 @@ impl BlockChainServer { /// and only the arrival says which slot that is. /// /// An error is logged at `debug` and the vote dropped. The common cause - /// is a vote for a block not imported here yet, which per the - /// specification is only valid within its slot, so holding it would - /// serve nothing. + /// is a vote for a block not imported here yet. The vote is only valid + /// within its own slot and the specification makes queueing it optional, + /// so it is not held. + /// + /// The message passed gossip validation in `ethlambda-p2p`, which + /// verified its signature and the sender's committee seat, so only the + /// store-state checks run here. fn apply_payload_attestation_message( &mut self, message: &gloas::PayloadAttestationMessage, arrival: &BlockArrival, ) { - let arrival_ms = unix_now_ms(); - let received = arrival.decode_start.unwrap_or(arrival.handed_off); - let in_flight_ms = u64::try_from( - Instant::now() - .saturating_duration_since(received) - .as_millis(), - ) - .unwrap_or(u64::MAX); - self.advance_beacon_clock_to(arrival_ms.saturating_sub(in_flight_ms)); + self.advance_beacon_clock_to(arrival.wire_ms(Instant::now(), unix_now_ms())); let config = self.store.config(); let slot = message.data.slot; let validator = message.validator_index; @@ -3809,10 +3805,7 @@ impl Handler for BlockChainServer { // the attestation and payload timeliness deadlines. The arrival is the // wire receipt (`decode_start`), which predates validation and the // mailbox wait; `arrival_ms` was sampled at pickup. - let received = msg.arrival.decode_start.unwrap_or(msg.arrival.handed_off); - let in_flight_ms = u64::try_from(picked_up.saturating_duration_since(received).as_millis()) - .unwrap_or(u64::MAX); - self.advance_beacon_clock_to(arrival_ms.saturating_sub(in_flight_ms)); + self.advance_beacon_clock_to(msg.arrival.wire_ms(picked_up, arrival_ms)); self.on_block(msg.block, timings).await; // A gloas envelope that arrived ahead of this block is judgeable now. self.settle_envelopes().await; @@ -3827,20 +3820,27 @@ impl Handler for BlockChainServer { }; let arrival_ms = unix_now_ms(); let picked_up = Instant::now(); - let received = msg.arrival.decode_start.unwrap_or(msg.arrival.handed_off); - let in_flight_ms = u64::try_from(picked_up.saturating_duration_since(received).as_millis()) - .unwrap_or(u64::MAX); - self.receive_envelope(*msg.envelope, arrival_ms.saturating_sub(in_flight_ms)); + self.receive_envelope(*msg.envelope, msg.arrival.wire_ms(picked_up, arrival_ms)); self.settle_envelopes().await; } } impl Handler for BlockChainServer { - async fn handle(&mut self, msg: NewPayloadAttestationMessage, _ctx: &Context) { + async fn handle(&mut self, msg: NewPayloadAttestationMessage, ctx: &Context) { // Beacon-only: lean has no payload timeliness committee. let ChainDuties::Beacon = &self.duties else { return; }; + // The catch-up tick a block gets: a tick still owed for the vote's + // slot would otherwise leave the store a slot behind and drop every + // vote of the new slot as not current. + let slot = msg.message.data.slot; + let config = self.store.config(); + if self.ms_until_slot_start(slot) == 0 + && fork_choice::get_current_slot(&self.store, &config) < slot + { + self.on_tick(unix_now_ms(), ctx).await; + } self.apply_payload_attestation_message(&msg.message, &msg.arrival); } } diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 03b473d9a..b69973daf 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -4719,7 +4719,13 @@ pub fn notify_ptc_messages( data: payload_attestation.data, signature: Default::default(), }; - apply_payload_attestation_message(store, &attested_state, &message, true, config)?; + apply_payload_attestation_message( + store, + &attested_state, + &message, + PtcMessageSource::Block, + config, + )?; } } Ok(()) @@ -4751,7 +4757,56 @@ pub fn on_payload_attestation_message( .ok_or(Error::SpecAssert( "data.beacon_block_root in store.block_states", ))?; - apply_payload_attestation_message(store, &state, ptc_message, is_from_block, config) + let source = if is_from_block { + PtcMessageSource::Block + } else { + PtcMessageSource::Wire + }; + apply_payload_attestation_message(store, &state, ptc_message, source, config) +} + +/// [`on_payload_attestation_message`] for a message that +/// `ethlambda-p2p`'s `payload_attestation_message` gossip validation already +/// accepted: the same store-state checks (the block's state is known, the +/// vote is for the current slot, the validator holds a seat in the block's +/// committee, which also locates the seats to write), without the signature +/// check and the indexed-attestation validation behind it, which cost a BLS +/// verification per vote on the chain actor. +/// +/// Only a gossip-verified message may call this. A message from anywhere else +/// (a block's payload attestations go through [`notify_ptc_messages`], which +/// has its own verified path) would be applied unauthenticated. +pub fn apply_verified_payload_attestation( + store: &mut Store, + ptc_message: &gloas::PayloadAttestationMessage, + config: &Config, +) -> Result<()> { + let state = store + .get_state(&ptc_message.data.beacon_block_root) + .expect("get") + .ok_or(Error::SpecAssert( + "data.beacon_block_root in store.block_states", + ))?; + apply_payload_attestation_message( + store, + &state, + ptc_message, + PtcMessageSource::VerifiedWire, + config, + ) +} + +/// Where a payload attestation message came from, which decides what +/// [`apply_payload_attestation_message`] still has to check. +#[derive(Clone, Copy, PartialEq, Eq)] +enum PtcMessageSource { + /// Inside a block, already verified with it: no slot or signature check. + Block, + /// Straight off the wire: current slot, then the signature. + Wire, + /// Off the wire and already signature-verified by gossip validation: + /// current slot only. + VerifiedWire, } /// The body of [`on_payload_attestation_message`] after its state read, given @@ -4762,7 +4817,7 @@ fn apply_payload_attestation_message( store: &mut Store, state: &BeaconState, ptc_message: &gloas::PayloadAttestationMessage, - is_from_block: bool, + source: PtcMessageSource, config: &Config, ) -> Result<()> { let data = ptc_message.data; @@ -4787,11 +4842,13 @@ fn apply_payload_attestation_message( // Verify the signature and check that it is for the current slot if it is // coming from the wire. - if !is_from_block { + if source != PtcMessageSource::Block { verify( data.slot == get_current_slot(store, config), "data.slot == get_current_slot(store)", )?; + } + if source == PtcMessageSource::Wire { let indexed = gloas::IndexedPayloadAttestation { attesting_indices: gloas::PayloadTimelinessCommitteeIndices::try_from(vec![ ptc_message.validator_index, @@ -7679,10 +7736,10 @@ mod tests { } } - #[test] - fn on_payload_attestation_message_writes_every_seat_the_validator_holds() { - let config = Config::active().with_fork_epoch(ForkName::Gloas, 0); - let root = Root::repeat_byte(0xb1); + /// A store holding a gloas block at slot 1 whose state seats validator 3 + /// twice in the slot's committee (every other seat is validator 0), with + /// empty vote vectors. + fn payload_attestation_store(root: Root) -> Store { let mut store = store_anchored_at(root); store .insert_signed_block( @@ -7712,6 +7769,14 @@ mod tests { store.insert_state(root, state).unwrap(); store.set_payload_timeliness_vote(root, vec![None; preset::PTC_SIZE]); store.set_payload_data_availability_vote(root, vec![None; preset::PTC_SIZE]); + store + } + + #[test] + fn on_payload_attestation_message_writes_every_seat_the_validator_holds() { + let config = Config::active().with_fork_epoch(ForkName::Gloas, 0); + let root = Root::repeat_byte(0xb1); + let mut store = payload_attestation_store(root); let message = |validator_index: u64, slot: Slot| gloas::PayloadAttestationMessage { validator_index, @@ -7759,6 +7824,89 @@ mod tests { ); } + fn payload_attestation_message( + root: Root, + validator_index: u64, + slot: Slot, + ) -> gloas::PayloadAttestationMessage { + gloas::PayloadAttestationMessage { + validator_index, + data: gloas::PayloadAttestationData { + beacon_block_root: root, + slot, + payload_present: true, + blob_data_available: true, + }, + signature: Default::default(), + } + } + + /// A gossip-verified vote is applied to every seat of its validator + /// without a signature check (the zero signature here would fail one), + /// but still only in the slot it is for, and only from a committee member. + #[test] + fn apply_verified_payload_attestation_skips_the_signature_but_not_the_slot() { + let config = Config::active().with_fork_epoch(ForkName::Gloas, 0); + let root = Root::repeat_byte(0xb1); + let mut store = payload_attestation_store(root); + let seated = |store: &Store| -> Vec { + store + .payload_timeliness_vote(&root) + .unwrap() + .iter() + .enumerate() + .filter_map(|(seat, vote)| vote.map(|_| seat)) + .collect() + }; + + // The store clock reads slot 0: a vote for slot 1 is not current yet. + assert_eq!( + failed_assertion(apply_verified_payload_attestation( + &mut store, + &payload_attestation_message(root, 3, 1), + &config + )), + Some("data.slot == get_current_slot(store)") + ); + assert!(seated(&store).is_empty()); + + store + .set_time_ms(config.slot_duration_ms) + .expect("set the clock to slot 1"); + // A validator outside the committee still writes nothing. + assert_eq!( + failed_assertion(apply_verified_payload_attestation( + &mut store, + &payload_attestation_message(root, 2, 1), + &config + )), + Some("len(ptc_indices) > 0") + ); + apply_verified_payload_attestation( + &mut store, + &payload_attestation_message(root, 3, 1), + &config, + ) + .unwrap(); + assert_eq!(seated(&store), vec![2, 7]); + + // The unverified entry point refuses the same message: its signature + // is not valid. + let mut fresh = payload_attestation_store(root); + fresh + .set_time_ms(config.slot_duration_ms) + .expect("set the clock to slot 1"); + assert!( + on_payload_attestation_message( + &mut fresh, + &payload_attestation_message(root, 3, 1), + false, + &config + ) + .is_err() + ); + } + /// A restarted follower holds a verified payload (restored from its table) /// but no payload-committee votes, which are never persisted. Both the /// head walk (`payload_timeliness` on the tiebreaker path) and the first diff --git a/crates/net/api/src/lib.rs b/crates/net/api/src/lib.rs index a9332f578..705e98f8e 100644 --- a/crates/net/api/src/lib.rs +++ b/crates/net/api/src/lib.rs @@ -162,6 +162,21 @@ pub struct DeferredFrom { } impl BlockArrival { + /// The Unix millisecond this delivery came off the wire, given that the + /// actor picked it up at `picked_up` (an [`Instant`]) at `picked_up_ms` + /// (Unix milliseconds). + /// + /// The wire receipt predates validation and the mailbox wait, and it is + /// what a sub-slot deadline should be judged against. Meant for every + /// delivery carrying a [`BlockArrival`], blocks, envelopes and payload + /// attestations alike. + pub fn wire_ms(&self, picked_up: Instant, picked_up_ms: u64) -> u64 { + let received = self.decode_start.unwrap_or(self.handed_off); + let in_flight_ms = u64::try_from(picked_up.saturating_duration_since(received).as_millis()) + .unwrap_or(u64::MAX); + picked_up_ms.saturating_sub(in_flight_ms) + } + /// An arrival whose earliest knowable moment is now. /// /// For producers that did not decode the block themselves, so have no diff --git a/crates/net/p2p/src/beacon/transition.rs b/crates/net/p2p/src/beacon/transition.rs index 4447984ad..59016d2e8 100644 --- a/crates/net/p2p/src/beacon/transition.rs +++ b/crates/net/p2p/src/beacon/transition.rs @@ -363,6 +363,38 @@ mod tests { assert_eq!(window, vec![schedule.digest_at(GLOAS - 1)]); } + /// While the rollover window holds a fulu and a gloas digest, only the + /// gloas one carries the two topics gloas adds, whether the gloas digest + /// is joined ahead of the fork or is current. + #[tokio::test] + async fn only_the_gloas_digest_holds_the_gloas_topics() { + let has_gloas_kinds = |held: &BeaconTopics| { + topics::GLOAS_TOPIC_KINDS.iter().all(|kind| { + let name = topics::topic_name(held.fork_digest, kind); + held.topics.iter().any(|topic| topic.to_string() == name) + }) + }; + let has_no_gloas_kinds = |held: &BeaconTopics| { + topics::GLOAS_TOPIC_KINDS.iter().all(|kind| { + let name = topics::topic_name(held.fork_digest, kind); + held.topics.iter().all(|topic| topic.to_string() != name) + }) + }; + + let mut server = unconnected_beacon_server(config(), 0).await; + // Ahead of the fork: fulu is current, gloas is joined in the window. + apply(&mut server, GLOAS - 1); + let wire = server.wire.beacon().unwrap(); + assert!(has_no_gloas_kinds(&wire.topics)); + assert!(has_gloas_kinds(&wire.window_topics[0])); + + // After it: gloas is current, fulu is still held. + apply(&mut server, GLOAS); + let wire = server.wire.beacon().unwrap(); + assert!(has_gloas_kinds(&wire.topics)); + assert!(has_no_gloas_kinds(&wire.window_topics[0])); + } + #[tokio::test] async fn a_message_takes_its_fork_from_its_topic() { let config = config(); diff --git a/crates/net/p2p/src/beacon/verdict.rs b/crates/net/p2p/src/beacon/verdict.rs index af995e22b..ebebdef92 100644 --- a/crates/net/p2p/src/beacon/verdict.rs +++ b/crates/net/p2p/src/beacon/verdict.rs @@ -173,10 +173,10 @@ impl Validated { /// An envelope goes on when accepted, and also when queued: the chain /// actor holds an envelope whose block it has not imported yet, which is /// exactly what `Queue` means, and its own checks judge it again once the - /// block is there. Any other outcome keeps it away, since the actor trusts - /// the signature gossip verified. A payload attestation goes on only when - /// accepted: a vote for a block not imported here is only valid within its - /// own slot, so there is nothing to hold it for. + /// block is there; a queued one has not had its signature checked, so the + /// actor checks it before it parks one. Any other outcome keeps it away. A + /// payload attestation goes on only when accepted: it is only valid within + /// its own slot, and the specification makes queueing one optional. /// /// What an accepted subnet attestation does feed is the attestation pool, /// when its subnet is one a validator client's aggregator had this node @@ -412,6 +412,23 @@ pub(crate) fn report(server: &P2PServer, id: GossipId, outcome: Outcome) -> bool forward } +/// The permit pool `object`'s stateful checks draw from: blocks, columns and +/// envelopes from the gossip pool, everything the attesters send from the +/// attestation pool, so neither burst starves the other. +fn permits_for<'a>( + server: &'a P2PServer, + object: &Validated, +) -> &'a std::sync::Arc { + match object { + Validated::Block { .. } | Validated::Column(_) | Validated::Envelope(_) => { + &server.gossip_validation_permits + } + Validated::Aggregate { .. } + | Validated::Attestation { .. } + | Validated::PayloadAttestation(_) => &server.attestation_validation_permits, + } +} + /// Run `object`'s stateful checks on a blocking thread. The verdict comes back /// to the actor as a [`GossipVerdict`]. /// @@ -437,15 +454,7 @@ pub(crate) fn spawn_stateful_checks( id: GossipId, object: Validated, ) { - let permits = match &object { - Validated::Block { .. } | Validated::Column(_) | Validated::Envelope(_) => { - &server.gossip_validation_permits - } - Validated::Aggregate { .. } - | Validated::Attestation { .. } - | Validated::PayloadAttestation(_) => &server.attestation_validation_permits, - }; - let Ok(permit) = permits.clone().try_acquire_owned() else { + let Ok(permit) = permits_for(server, &object).clone().try_acquire_owned() else { let received_at = id.received_at; let outcome = Outcome::Ignore(IgnoreReason::Overloaded); report(server, id, outcome); @@ -502,7 +511,7 @@ mod tests { use spawned_concurrency::error::ActorError; use super::*; - use crate::test_support::{unconnected_beacon_server, valid_shaped_sidecar}; + use crate::test_support::{envelope, unconnected_beacon_server, valid_shaped_sidecar}; /// A minimal fulu block for a given `(slot, proposer)`: `settle` and /// `record_seen` only ever read those two fields plus the root passed @@ -859,10 +868,6 @@ mod tests { assert!(!chain.0.load(Ordering::SeqCst)); } - /// The pool an aggregate or a subnet attestation draws its stateful-check - /// permit from is not the pool a block or a column draws from: exhausting - /// one must leave the other untouched, or a burst on this topic could - /// make a block or a column answer `Ignore(Overloaded)` too. /// What `forward` handed the chain actor, by method. #[derive(Debug, PartialEq)] enum Sent { @@ -939,8 +944,6 @@ mod tests { message } - use crate::test_support::envelope; - #[tokio::test] async fn the_first_accept_for_an_envelope_key_stands_and_the_second_is_marked_seen() { let mut server = unconnected_beacon_server(Config::mainnet(), 0).await; @@ -1027,11 +1030,33 @@ mod tests { assert_eq!(received.try_recv(), Ok(Sent::PayloadAttestation)); } + /// The pool an aggregate or a subnet attestation draws its stateful-check + /// permit from is not the pool a block or a column draws from: exhausting + /// one must leave the other untouched, or a burst on this topic could + /// make a block or a column answer `Ignore(Overloaded)` too. + /// + /// An envelope shares the block pool and a payload attestation the + /// attestation pool, for the same reason: a slot's committee votes arrive + /// in a burst that must not starve block validation. #[tokio::test] async fn the_block_column_and_attestation_permit_pools_are_independent() { let server = unconnected_beacon_server(Config::mainnet(), 0).await; let attestation_permits_before = server.attestation_validation_permits.available_permits(); + let same_pool = |a: &std::sync::Arc, b: &std::sync::Arc<_>| { + std::sync::Arc::ptr_eq(a, b) + }; + let envelope = Validated::Envelope(Box::new(envelope(1, 3))); + let vote = Validated::PayloadAttestation(payload_attestation(5, 1)); + assert!(same_pool( + permits_for(&server, &envelope), + &server.gossip_validation_permits + )); + assert!(same_pool( + permits_for(&server, &vote), + &server.attestation_validation_permits + )); + let mut held = Vec::new(); while let Ok(permit) = server.gossip_validation_permits.clone().try_acquire_owned() { held.push(permit); diff --git a/docs/metrics.md b/docs/metrics.md index 8908096b3..fcd7c9315 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -366,10 +366,12 @@ label `data_column_sidecar` and every `beacon_attestation_{subnet_id}` sharing `beacon_attestation`. The gloas topics `execution_payload` and `payload_attestation_message` are labelled by their own name. `queue` means IGNORE to gossipsub while the chain actor still receives the object and parks -it (an `execution_payload` whose block is unknown answers it); neither the aggregate nor the -attestation topic ever answers `queue`, since the vote block's post-state is -either cached or it is not (`IgnoreReason::UnknownBlock`/`StateUnavailable`), -with nothing to hold the message for. **`verdict_expired_total` should stay at +it; of the gloas topics only `execution_payload` answers it, for an envelope +whose block is not known yet. The aggregate, attestation and +`payload_attestation_message` topics never answer `queue`, since the vote +block's post-state is either cached or it is not +(`IgnoreReason::UnknownBlock`/`StateUnavailable`), with nothing to hold the +message for. **`verdict_expired_total` should stay at zero**: a rising count means validation is too slow for gossipsub's message cache. From 071cd527a0225b96b5839f5ae0f515776ec866ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:34:55 -0300 Subject: [PATCH 065/132] feat(beacon): follow gloas ForkName::Gloas.is_followed() is now true. The chain actor imports gloas blocks and envelopes and the network side delivers them, so the node no longer needs to refuse a gloas anchor at startup, report syncing once the clock reaches the fork, or ignore an undecodable gloas block as an unsupported fork. Fork choice now records the head node's payload status next to the head root (Store::head_payload_status, in memory; unknown until the first head computation after a restart). The envelope by-range server serves the head's envelope only when that node is FULL, which removes the matching entry from spec_deviations.md. The node is still only a follower, so the validator duty, attestation data and aggregate endpoints keep refusing a gloas epoch. That refusal used to ride on is_followed; it is now an explicit gloas check (refuse_validator_duties_from_gloas), since following a fork and serving its validator duties are different questions. --- bin/ethlambda/src/beacon.rs | 8 +- bin/ethlambda/src/main.rs | 79 ++++++--------- bin/ethlambda/src/network/built_in.rs | 12 +-- bin/ethlambda/src/network/mod.rs | 5 +- crates/blockchain/src/lib.rs | 58 +++++------ .../src/beacon/fork_choice.rs | 7 ++ crates/common/types/src/beacon/fork.rs | 22 ++--- crates/net/p2p/src/req_resp/envelopes.rs | 2 + crates/net/rpc/src/beacon/mod.rs | 48 +++++++--- crates/net/rpc/src/beacon/pool.rs | 4 +- crates/net/rpc/src/beacon/validator.rs | 8 +- crates/storage/src/store.rs | 95 +++++++++++++++++-- docs/beacon_stf.md | 15 +-- docs/beacon_wire.md | 7 +- docs/checkpoint_sync.md | 2 +- docs/rpc.md | 4 +- docs/spec_deviations.md | 21 ---- 17 files changed, 234 insertions(+), 163 deletions(-) diff --git a/bin/ethlambda/src/beacon.rs b/bin/ethlambda/src/beacon.rs index 2965a834b..4af0cb8e0 100644 --- a/bin/ethlambda/src/beacon.rs +++ b/bin/ethlambda/src/beacon.rs @@ -181,10 +181,10 @@ pub fn wire_params( // happens at it, and the old topics are left after it, per // `SUBSCRIBE_LEAD_EPOCHS` and `UNSUBSCRIBE_LAG_EPOCHS`. Say what is coming. // - // The gloas boundary keeps its own warning: crossing its digest works like - // any other, but the network side delivers neither gloas blocks nor - // payload envelopes or payload attestations to the chain actor yet, so - // the node stops tracking the chain there regardless. + // A fork the node does not follow keeps its own warning: crossing its + // digest works like any other, but the node stops tracking the chain + // there regardless. No fork is in that state today, since gloas is + // followed, so the warning is the guard for the next one. let schedule = ForkSchedule::new(&chain, genesis.genesis_validators_root); match schedule.next_boundary_after(epoch) { Some(next) if !next.fork.is_followed() => warn!( diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index 86325e970..ec47f71ac 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -1693,19 +1693,16 @@ fn first_config_difference(persisted: &Config, supplied: &Config) -> Option = Arc::new(InMemoryBackend::new()); - // `Store` is not `Debug`, so take the error by pattern. - let Err(err) = fetch_initial_beacon_state(&[], backend.clone(), &source).await else { - panic!("a gloas genesis must not become an anchor"); - }; - assert!( - matches!( - err, - checkpoint_sync::CheckpointSyncError::UnsupportedFork { - fork: ForkName::Gloas - } - ), - "the refusal must name the fork: {err}" - ); - assert!( - Store::from_db_state(backend).unwrap().is_none(), - "a refused anchor leaves the directory empty" - ); + let store = fetch_initial_beacon_state(&[], backend, &source) + .await + .expect("a gloas genesis anchors"); + let (head_slot, _) = store + .beacon_head() + .expect("an anchored directory has a head"); + assert_eq!(head_slot, 0, "a genesis anchor is at slot 0"); } /// A changed fork epoch leaves genesis time and the validators root diff --git a/bin/ethlambda/src/network/built_in.rs b/bin/ethlambda/src/network/built_in.rs index d3cf05f8f..fddf69356 100644 --- a/bin/ethlambda/src/network/built_in.rs +++ b/bin/ethlambda/src/network/built_in.rs @@ -67,13 +67,11 @@ impl BuiltInNetwork { .wrap_err_with(|| format!("the built-in {} config.yaml did not parse", self.name()))?; super::check_preset(parsed.config.preset_base.as_str())?; super::check_constants(&parsed.config)?; - // A built-in config can carry a fork whose *keys* this build claims - // but whose state transition it does not implement yet (gloas), or - // one whose keys it does not even claim (heze). The ignored-keys - // warning below only ever said the second half of that; it used to - // say both, back when GLOAS_* were unclaimed too. `warn_if_unfollowed_fork_scheduled` - // says the first half explicitly instead of leaving it to be - // discovered as a stall. + // A built-in config can carry a fork whose keys this build does not + // even claim (heze), which the ignored-keys warning below says. A fork + // whose keys it claims but whose rules it does not follow is + // `warn_if_unfollowed_fork_scheduled`'s to say; none exists today, + // since gloas is followed. parsed.warn_about_ignored_keys(); let genesis = chain.genesis(); diff --git a/bin/ethlambda/src/network/mod.rs b/bin/ethlambda/src/network/mod.rs index 594564e95..90d4c8d49 100644 --- a/bin/ethlambda/src/network/mod.rs +++ b/bin/ethlambda/src/network/mod.rs @@ -247,8 +247,9 @@ fn derive_genesis_fields(config: &mut Config, genesis_time: u64) { /// config parsed. A loaded network reaches this the same as a built-in one, /// and can even schedule such a fork at epoch 0, in which case its own genesis /// state already decodes as that fork and startup refuses it outright -/// (`refuse_unfollowable_fork` in `main.rs`, since fork choice itself accepts -/// a gloas anchor); this warning fires first either way. +/// (`refuse_unfollowable_fork` in `main.rs`); this warning fires first either +/// way. Every fork `ForkName` models is followed today, gloas included, so +/// this is the guard for the next fork the node cannot follow. fn warn_if_unfollowed_fork_scheduled(network: &str, config: &Config) { let unfollowed = ethlambda_types::beacon::fork::ForkName::ALL .into_iter() diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index da7861c1f..8a7b9a4c9 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -3488,6 +3488,9 @@ impl BlockChainServer { /// Whether the clock is at a fork this node does not follow while the /// node's own head is still from one it does. /// + /// Every fork is followed today, so this is false; it stays as the guard + /// for the next fork the node cannot follow. + /// /// Blocks past the fork do not reach the live chain from the network, so the freshest block /// the store knows stays at the head and the sync tracker would read the /// network as stalled `NETWORK_STALL_THRESHOLD` slots after the fork and @@ -5136,17 +5139,17 @@ mod tests { } #[test] - fn the_node_reports_syncing_once_the_clock_passes_the_fork_it_does_not_follow() { - // No block arrives at all: gossip blocks past the fork are ignored in - // p2p and peers' Status messages are dropped, so the store's freshest - // block stays at the head. The clock alone must keep the node from - // reading the network as stalled. + fn the_clock_reaching_a_followed_fork_does_not_force_syncing() { + // Gloas is followed, so blocks past the fork do reach the live chain + // and the tracker's own stall detection applies as before: with no + // block at all it reads a stalled network as synced, the same as when + // no fork is scheduled. let mut server = syncing_server(Some(1)); let past_the_fork = preset::SLOTS_PER_EPOCH + 8; server.update_sync_status(past_the_fork); assert_eq!( server.sync_status_controller.get(), - crate::metrics::SyncStatus::Syncing + crate::metrics::SyncStatus::Synced ); } @@ -5168,33 +5171,24 @@ mod tests { } #[test] - fn the_clock_rule_holds_for_a_followed_head_once_the_clock_is_at_gloas() { + fn the_clock_rule_never_holds_while_every_fork_is_followed() { + // The rule is a guard for a fork the node cannot follow; with gloas + // followed there is none, whatever the clock and head say. let first_gloas_slot = preset::SLOTS_PER_EPOCH; - assert!(BlockChainServer::clock_is_past_followed_forks( - &gloas_at_epoch_one(), - first_gloas_slot + 8, - 0 - )); - } - - #[test] - fn the_clock_rule_does_not_hold_before_the_clock_reaches_gloas() { - let first_gloas_slot = preset::SLOTS_PER_EPOCH; - assert!(!BlockChainServer::clock_is_past_followed_forks( - &gloas_at_epoch_one(), - first_gloas_slot - 1, - 0 - )); - } - - #[test] - fn the_clock_rule_does_not_hold_for_a_head_already_past_the_fork() { - let first_gloas_slot = preset::SLOTS_PER_EPOCH; - assert!(!BlockChainServer::clock_is_past_followed_forks( - &gloas_at_epoch_one(), - first_gloas_slot + 8, - first_gloas_slot - )); + for (current_slot, head_slot) in [ + (first_gloas_slot + 8, 0), + (first_gloas_slot - 1, 0), + (first_gloas_slot + 8, first_gloas_slot), + ] { + assert!( + !BlockChainServer::clock_is_past_followed_forks( + &gloas_at_epoch_one(), + current_slot, + head_slot + ), + "clock {current_slot}, head {head_slot}" + ); + } } #[tokio::test] diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index b69973daf..24b1b1618 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -2160,6 +2160,9 @@ pub fn get_head_node(store: &Store, config: &Config) -> Result { /// on its first tick. That writer also keeps the canonical `BlockRoots` index /// in step with the branch fork choice just picked. /// +/// The head node's payload status is recorded too ([`Store::head_payload_status`]): +/// the root alone cannot say which payload branch of a gloas block was picked. +/// /// Written unconditionally on every call, not only when the head changes: a /// value written once and then left alone is a second source of truth a bug /// can let drift, and the write is one small metadata row plus an index diff @@ -2167,6 +2170,9 @@ pub fn get_head_node(store: &Store, config: &Config) -> Result { /// tree walk. pub fn get_head(store: &mut Store, config: &Config) -> Result { let head = get_head_node(store, config)?; + // Ahead of the head root, so a reader never pairs the new root with the + // old node's status (see `Store::set_head_payload_status`). + store.set_head_payload_status(head.payload_status); store .update_checkpoints(ForkCheckpoints::head_only(head.root)) .expect("record beacon head"); @@ -7979,6 +7985,7 @@ mod tests { .expect("the live head walk must succeed on a resumed store"); assert_eq!(head.root, walked.root); assert_eq!(get_head(&mut store, &config).unwrap(), walked.root); + assert_eq!(store.head_payload_status(), Some(walked.payload_status)); let message = gloas::PayloadAttestationMessage { validator_index: 3, diff --git a/crates/common/types/src/beacon/fork.rs b/crates/common/types/src/beacon/fork.rs index 6347e0d60..8f0c4c918 100644 --- a/crates/common/types/src/beacon/fork.rs +++ b/crates/common/types/src/beacon/fork.rs @@ -112,13 +112,13 @@ impl ForkName { /// Whether this node follows a chain that has reached this fork. /// - /// Phase0 through fulu are followed. Gloas is not yet: the chain actor - /// imports gloas blocks and payload envelopes, but the network side does - /// not deliver envelopes or payload attestations, so the node still - /// refuses gloas anchors, ignores gloas gossip and reports syncing once - /// the clock reaches the fork. This is the one place to - /// change when the node starts following gloas; every site whose rule is - /// "does this node follow the fork" calls it. + /// Every beacon fork is followed. Gloas needs the payload envelopes and + /// payload attestations that phase0 through fulu do not: the chain actor + /// imports the envelopes, the network side delivers them and the two + /// gloas gossip topics, and fork choice keeps the head's payload status. + /// The question stays a function so a fork this node cannot follow has + /// one place to say so; every site whose rule is "does this node follow + /// the fork" calls it. /// /// # Panics /// @@ -132,8 +132,8 @@ impl ForkName { | ForkName::Capella | ForkName::Deneb | ForkName::Electra - | ForkName::Fulu => true, - ForkName::Gloas => false, + | ForkName::Fulu + | ForkName::Gloas => true, ForkName::Lean => super::lean_fork_unreachable("ForkName::is_followed"), } } @@ -301,9 +301,9 @@ mod tests { } #[test] - fn every_beacon_fork_through_fulu_is_followed_and_gloas_is_not() { + fn every_beacon_fork_through_gloas_is_followed() { for fork in ForkName::ALL { - assert_eq!(fork.is_followed(), fork != ForkName::Gloas, "{fork:?}"); + assert!(fork.is_followed(), "{fork:?}"); } } diff --git a/crates/net/p2p/src/req_resp/envelopes.rs b/crates/net/p2p/src/req_resp/envelopes.rs index 8b1a22c44..1e56fb5d7 100644 --- a/crates/net/p2p/src/req_resp/envelopes.rs +++ b/crates/net/p2p/src/req_resp/envelopes.rs @@ -173,6 +173,7 @@ mod tests { use ethlambda_storage::{ForkCheckpoints, backend::InMemoryBackend}; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::SignedBeaconBlock; + use ethlambda_types::beacon::fork_choice::PayloadStatus; use ethlambda_types::checkpoint::Checkpoint; use std::sync::Arc; @@ -216,6 +217,7 @@ mod tests { fn extend_chain(store: &mut Store, block: SignedBeaconBlock) -> H256 { let root = block.message_hash_tree_root(); store.insert_signed_block(root, block).expect("insert"); + store.set_head_payload_status(PayloadStatus::Full); store .update_checkpoints(ForkCheckpoints::head_only(root)) .expect("advance head"); diff --git a/crates/net/rpc/src/beacon/mod.rs b/crates/net/rpc/src/beacon/mod.rs index 8fa54f6c9..7498d5fad 100644 --- a/crates/net/rpc/src/beacon/mod.rs +++ b/crates/net/rpc/src/beacon/mod.rs @@ -11,7 +11,7 @@ use axum::{ response::{IntoResponse, Response}, }; use ethlambda_storage::Store; -use ethlambda_types::beacon::{config::Config, primitives::Epoch}; +use ethlambda_types::beacon::{config::Config, fork::ForkName, primitives::Epoch}; use serde::Serialize; use crate::shared::block_id::IdError; @@ -60,23 +60,33 @@ pub(crate) enum ApiError { ServiceUnavailable(&'static str), } -/// Refuses `epoch` when the node's schedule places it at a fork the node does -/// not follow (gloas, see -/// [`ethlambda_types::beacon::fork::ForkName::is_followed`]). +/// Refuses `epoch` when the node's schedule places it at gloas: validator +/// duties are not supported from that fork. /// -/// The head is a fulu state, so an answer for a gloas epoch would carry fulu's -/// shapes under gloas's fork (a vote a validator client would sign as valid, -/// an aggregate of the wrong shape). Refused by name, as the submit endpoints -/// refuse the gloas header. -pub(crate) fn refuse_unfollowed_epoch( +/// The node follows gloas but only as a follower. It has no builder bid +/// handling, no payload-timeliness committee duty and no gloas proposer +/// flow, and its attestation pool holds electra-shaped votes. An answer for a +/// gloas epoch would carry fulu's shapes under gloas's fork (a vote a +/// validator client would sign as valid, an aggregate of the wrong shape), so +/// it is refused by name, as the submit endpoints refuse the gloas header. +/// Deliberately not [`ethlambda_types::beacon::fork::ForkName::is_followed`]: +/// following a fork and serving its validator duties are different questions. +pub(crate) fn refuse_validator_duties_from_gloas( config: &Config, epoch: Epoch, message: &'static str, ) -> Result<(), ApiError> { - if config.fork_at_epoch(epoch).is_followed() { - Ok(()) - } else { - Err(ApiError::BadRequest(message)) + match config.fork_at_epoch(epoch) { + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Fulu => Ok(()), + // `Config::fork_at_epoch` never returns Lean; refused like gloas + // rather than answered. + ForkName::Gloas | ForkName::Lean => Err(ApiError::BadRequest(message)), } } @@ -140,6 +150,18 @@ mod tests { assert_eq!(json["message"], "block not found"); } + #[test] + fn validator_duties_are_refused_from_the_gloas_epoch_even_though_gloas_is_followed() { + let mut config = Config::mainnet(); + config.gloas_fork_epoch = 10; + assert!(ForkName::Gloas.is_followed()); + let refuse = |epoch| refuse_validator_duties_from_gloas(&config, epoch, "refused"); + assert!(refuse(0).is_ok()); + assert!(refuse(9).is_ok()); + assert!(matches!(refuse(10), Err(ApiError::BadRequest("refused")))); + assert!(matches!(refuse(11), Err(ApiError::BadRequest("refused")))); + } + #[test] fn a_malformed_id_is_a_400_and_an_absent_one_a_404() { assert_eq!( diff --git a/crates/net/rpc/src/beacon/pool.rs b/crates/net/rpc/src/beacon/pool.rs index dfaffaafa..9ce9ecb04 100644 --- a/crates/net/rpc/src/beacon/pool.rs +++ b/crates/net/rpc/src/beacon/pool.rs @@ -49,7 +49,7 @@ use ethlambda_types::{ use serde::{Deserialize, Serialize}; use tracing::{debug, warn}; -use crate::beacon::{ApiError, refuse_unfollowed_epoch, validator::head}; +use crate::beacon::{ApiError, refuse_validator_duties_from_gloas, validator::head}; pub(crate) fn routes() -> Router { Router::new() @@ -364,7 +364,7 @@ async fn get_aggregate_attestation( // The pool holds electra-shaped votes, and the response is labelled with // the slot's fork, so a gloas slot would be served as gloas with a body of // the wrong shape. - if let Err(err) = refuse_unfollowed_epoch( + if let Err(err) = refuse_validator_duties_from_gloas( &store.config(), compute_epoch_at_slot(query.slot), "gloas aggregates are not supported", diff --git a/crates/net/rpc/src/beacon/validator.rs b/crates/net/rpc/src/beacon/validator.rs index bfc3514de..2d827de73 100644 --- a/crates/net/rpc/src/beacon/validator.rs +++ b/crates/net/rpc/src/beacon/validator.rs @@ -36,7 +36,7 @@ use ethlambda_state_transition::beacon::{ helpers::accessors::{CommitteeCacheExt as _, get_block_root_at_slot}, }; -use crate::beacon::{ApiError, refuse_unfollowed_epoch}; +use crate::beacon::{ApiError, refuse_validator_duties_from_gloas}; pub(crate) fn routes() -> Router { Router::new() @@ -206,7 +206,7 @@ async fn get_proposer_duties(Path(epoch): Path, State(store): State Result { let epoch = parse_epoch(epoch)?; - refuse_unfollowed_epoch( + refuse_validator_duties_from_gloas( &store.config(), epoch, "gloas proposer duties are not supported", @@ -308,7 +308,7 @@ fn attester_duties( .map(|index| index.parse::()) .collect::, _>>() .map_err(|_| ApiError::BadRequest("invalid validator index"))?; - refuse_unfollowed_epoch( + refuse_validator_duties_from_gloas( &store.config(), epoch, "gloas attester duties are not supported", @@ -399,7 +399,7 @@ async fn get_attestation_data( } fn attestation_data(store: &Store, slot: Slot) -> Result { - refuse_unfollowed_epoch( + refuse_validator_duties_from_gloas( &store.config(), compute_epoch_at_slot(slot), "gloas attestation data is not supported", diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 30967afc5..288a3d4e9 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -24,7 +24,8 @@ use ethlambda_types::{ }, fork::ForkName, fork_choice::{ - BlockPayloadLink, LatestMessage, PayloadStatusEnum, PayloadStatusV1, PowBlock, + BlockPayloadLink, LatestMessage, PayloadStatus, PayloadStatusEnum, PayloadStatusV1, + PowBlock, }, preset::{PTC_SIZE, Preset, SLOTS_PER_EPOCH}, primitives::ExecutionBlockHash, @@ -723,6 +724,16 @@ pub(crate) struct BeaconScratch { /// finality. A pruned link is derived again by decoding if anything asks /// for it. pub(crate) payload_links: HashMap, + /// The payload branch of the head fork choice last picked: `Full` when + /// the head node is its block with the payload revealed, `Empty` when it + /// is the block alone. A pre-gloas head is `Full` (see + /// `fork_choice::get_head_node`). + /// + /// `None` until the first head computation of this process: nothing + /// persists it, and a resumed directory recomputes it on the first tick + /// rather than storing a second source of truth for what the head walk + /// derives. Readers treat `None` as "not known to be `Full`". + pub(crate) head_payload_status: Option, /// Gloas: the payload timeliness committee's per-member vote on whether /// the block's payload showed up on time, one slot per `PTC_SIZE` index, /// `None` until that member votes. Keyed by beacon block root, like @@ -2427,6 +2438,23 @@ impl Store { // ============ Beacon Head ============ + /// The payload status of the head node fork choice last picked, or `None` + /// before this process has computed a head. See + /// [`BeaconScratch::head_payload_status`]. + pub fn head_payload_status(&self) -> Option { + self.beacon.lock().unwrap().head_payload_status + } + + /// Records the payload status of the head node fork choice just picked. + /// + /// Call it **before** [`Store::update_checkpoints`] moves the head root: + /// a reader that sees the new status against the old head merely withholds + /// an envelope for a moment, where the other order could serve the new + /// head's envelope while its node is still the empty one. + pub fn set_head_payload_status(&self, status: PayloadStatus) { + self.beacon.lock().unwrap().head_payload_status = Some(status); + } + /// The beacon fork-choice head as `(slot, root)`, or `None` if the head /// row names a block this store has no header for. /// @@ -3405,11 +3433,10 @@ impl Store { /// chain when the next canonical block builds on it: that block's bid /// `parent_block_hash` equals the payload's `block_hash`. A block whose /// successor builds on its empty branch has its envelope left out, even - /// when this node holds one. The head has no successor and fork choice does - /// not persist which payload branch it picked, so its envelope is served - /// when held: only a verified envelope is ever stored. See - /// `docs/spec_deviations.md`, "The head's payload envelope is served - /// whenever it is held". + /// when this node holds one. The head has no successor, so its envelope is + /// served only when the head node fork choice picked is `Full` + /// ([`Self::head_payload_status`]); before this process has computed a head + /// the status is unknown and the head's envelope is withheld. /// /// Canonical blocks come from `BlockRoots` and the whole answer is read /// from one view, like [`get_signed_blocks_by_slot_range`](Self::get_signed_blocks_by_slot_range). @@ -3424,6 +3451,10 @@ impl Store { if self.chain != Chain::Beacon { Self::lean_only("canonical_execution_payload_envelopes"); } + // Read ahead of the view, and written ahead of the head by + // `set_head_payload_status`'s contract, so a head move partway through + // errs toward withholding. + let head_is_full = self.head_payload_status() == Some(PayloadStatus::Full); let view = self.backend.begin_read().expect("read view"); let canonical_root = |slot: u64| -> Option { view.get(Table::BlockRoots, &encode_block_root_key(slot)) @@ -3470,7 +3501,9 @@ impl Store { let envelope = gloas::SignedExecutionPayloadEnvelope::from_ssz_bytes(&bytes) .expect("a stored envelope decodes"); let builds_on_it = match window.get(index + 1) { - None => true, + // No successor: this is the head, whose envelope is part of + // the chain only on the FULL node. + None => head_is_full, Some((_, next_root)) => { let next = view .get(Table::BlockHeaders, &next_root.to_ssz()) @@ -7537,6 +7570,54 @@ mod tests { }) } + /// The head's envelope belongs to the served chain only on the FULL node: + /// an EMPTY head, or a head whose status this process has not computed, + /// withholds it. + #[test] + fn the_heads_envelope_is_served_only_when_the_head_node_is_full() { + let mut store = Store::init_beacon( + Arc::new(InMemoryBackend::new()), + 0, + Config::mainnet(), + H256::ZERO, + Store::beacon_checkpoint_as_stored(BeaconCheckpoint::default()), + 0, + ); + store + .insert_signed_block(H256::ZERO, gloas_test_block(0, H256::ZERO)) + .expect("insert anchor"); + let block = gloas_test_block(1, H256::ZERO); + let root = block.message_hash_tree_root(); + store.insert_signed_block(root, block).expect("insert head"); + store + .update_checkpoints(ForkCheckpoints::head_only(root)) + .expect("advance head"); + let envelope = test_envelope(root); + store.insert_verified_payload(1, &envelope); + + assert_eq!(store.head_payload_status(), None); + assert!( + store + .canonical_execution_payload_envelopes(1, 1) + .unwrap() + .is_empty() + ); + + store.set_head_payload_status(PayloadStatus::Empty); + assert!( + store + .canonical_execution_payload_envelopes(1, 1) + .unwrap() + .is_empty() + ); + + store.set_head_payload_status(PayloadStatus::Full); + assert_eq!( + store.canonical_execution_payload_envelopes(1, 1).unwrap(), + vec![envelope] + ); + } + #[test] fn a_verified_payload_survives_reopen() { let root = H256::from([1u8; 32]); diff --git a/docs/beacon_stf.md b/docs/beacon_stf.md index 2e44d0931..0b5bcbf18 100644 --- a/docs/beacon_stf.md +++ b/docs/beacon_stf.md @@ -316,12 +316,15 @@ wholesale, so its code sits beside the pre-gloas code rather than inside it: `st section of `fork_choice.rs` (fork choice), and `containers/gloas.rs` in `ethlambda-types`. Every gloas case that is not ignored passes on both presets. -The live follower does not follow gloas yet. `process_or_pend_block` refuses a -gloas block and `refuse_unfollowable_fork` refuses a gloas anchor at startup, -because nothing delivers payload envelopes or payload attestations to the chain -actor. Gloas blocks and data columns are validated by their own gossip rules; -the other gloas topics are answered with `Ignore(UnsupportedFork)` rather than -scored, so an honest peer past the fork epoch is not penalized. +The live follower follows gloas (`ForkName::Gloas.is_followed()`): the chain +actor imports gloas blocks and their payload envelopes, a gloas checkpoint +anchor is accepted (it starts EMPTY, with no payload known), and the clock +passing the fork epoch no longer forces the sync status to syncing. Fork choice +records the head node's payload status next to the head root +(`Store::head_payload_status`, in memory, recomputed by the first head +computation after a restart), which the envelope by-range server reads to +withhold the head's envelope while the head is the EMPTY node. The node is +still only a follower: the validator-client endpoints refuse a gloas epoch. ### The deferred payload diff --git a/docs/beacon_wire.md b/docs/beacon_wire.md index 271c4f656..2f4a62f85 100644 --- a/docs/beacon_wire.md +++ b/docs/beacon_wire.md @@ -180,9 +180,10 @@ but its own progressive `Attestation`, so it decodes to its own attestation is electra's `SingleAttestation` unchanged. In both, `data.index` is the payload flag (0 or 1), and a vote for the full payload (1) is IGNOREd until the block's envelope has been seen and its payload validated -(`verify_attestation_payload_status`). A block at a gloas slot that fails to -decode is IGNOREd as `unsupported_fork` rather than scored against the peer, -since the container this build models may trail the network's. +(`verify_attestation_payload_status`). Gloas is a followed fork, so a block at +a gloas slot that fails to decode is REJECTed like one at any other followed +fork; `unsupported_fork` is kept for a fork the node does not follow +(`ForkName::is_followed`), none today. The remaining five global topics are decoded, logged at `debug`, and IGNOREd, since nothing consumes them; an undecodable payload on any topic is REJECTed. diff --git a/docs/checkpoint_sync.md b/docs/checkpoint_sync.md index 5c9e34b7b..2316ebd7a 100644 --- a/docs/checkpoint_sync.md +++ b/docs/checkpoint_sync.md @@ -145,7 +145,7 @@ All checks are performed before a downloaded checkpoint anchor is accepted. The | Justified epoch >= finalized epoch | Justified must be at or after finalized | | Block header slot <= state slot | Block header cannot be ahead of the state | | Block's fork matches state's fork | The fetched block must decode as the same fork the state resolved to | -| Anchor is not gloas (`refuse_unfollowable_fork`) | Nothing delivers payload envelopes or payload attestations to the chain actor yet, so a gloas anchor could not be followed. Runs on both anchor sources, the checkpoint provider's state and a loaded network's own genesis state | +| Anchor's fork is followed (`refuse_unfollowable_fork`) | A fork the node cannot follow is refused with `UnsupportedFork` before any store is built. Every fork is followed today, gloas included, so this refuses nothing; a gloas anchor starts with no payload known, so its head is EMPTY and its first FULL child is held until the parent's envelope is fetched. Runs on both anchor sources, the checkpoint provider's state and a loaded network's own genesis state | | Anchor pairing | See [Anchor Pairing](#anchor-pairing) below | Beacon has no same-slot-checkpoints-matching-roots check: nothing here computes two separate checkpoint roots to compare, since a trusted checkpoint-synced anchor is finalized by fiat, and the beacon spec's own construction applies that single checkpoint to all four of the store's justified/finalized slots at once. diff --git a/docs/rpc.md b/docs/rpc.md index 6e50f0bd8..ccb6fff70 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -261,7 +261,9 @@ the chain actor writes, so no request waits on the actor. `dependent_root` follows each endpoint's v1 definition. Attester duties walk every committee of the epoch, a full shuffle per request on mainnet. An epoch the node's schedule places at gloas is a `400` on both duty endpoints: - the head is a fulu state, and this node does not follow gloas. + the head is a fulu state, and the node serves no validator duties from gloas + (it follows the fork but has no builder, payload-timeliness or gloas proposer + support). - **`attestation_data`** follows phase0's `validator.md`: the head block, the epoch's boundary block as target, and as source the current justified checkpoint of the head state advanced to the slot's epoch (through fork diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 776364f86..6a85d7372 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -216,27 +216,6 @@ the root of the state's latest block header. shapes give the same root. - **Equivalence:** the same check, on both kinds of state. -## The head's payload envelope is served whenever it is held - -`ExecutionPayloadEnvelopesByRange` (gloas `p2p-interface.md`) says clients -MUST NOT include the envelope of a block whose `ForkChoiceNode` in the -served chain has `payload_status` equal to `PAYLOAD_STATUS_EMPTY`. - -- **The specification:** for the head block, the envelope is part of the chain - only when the head node's payload status is FULL. -- **ethlambda:** for every block but the head, the envelope is served when the - next canonical block's bid `parent_block_hash` equals the payload's - `block_hash`, which is the specification's rule. For the head there is no - next block, and the head node's payload status is not yet exposed by the - store (`KEY_HEAD` holds a root only), so the envelope is served whenever it - is held. Only verified envelopes are ever stored. -- **Effect:** when fork choice picked the head's EMPTY branch despite a - verified envelope being held, a peer receives an envelope fork choice - currently does not build on. The next block resolves it, since that block's - bid then decides the envelope's membership like any other. -- **Fixed when:** the head node's payload status is stored. - `Store::canonical_execution_payload_envelopes` is the one place to change. - ## The attestation deadline takes the epoch that picks the fork's rule - **The specification:** `get_attestation_due_ms()` takes no argument, and From 4e831ae3406075a379108452118f9c6ac9fe1b52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:41:40 -0300 Subject: [PATCH 066/132] fix(beacon): pair the head's payload status with its root The head root lives in the backend and its payload status in memory, so no write order keeps a reader from seeing the new root with the old status: the envelope server could serve an EMPTY head's envelope. Record the status together with the root of the node it describes, and let readers use it only when that root is the head they see. A mismatch reads as unknown, which withholds the head's envelope. Also drop two stale mentions of gloas as an unfollowed fork (a doc comment and a test name). --- crates/blockchain/src/beacon_columns.rs | 6 +- .../src/beacon/fork_choice.rs | 6 +- crates/net/p2p/src/beacon/decode.rs | 4 +- crates/net/p2p/src/req_resp/envelopes.rs | 2 +- crates/storage/src/store.rs | 80 +++++++++++++------ 5 files changed, 64 insertions(+), 34 deletions(-) diff --git a/crates/blockchain/src/beacon_columns.rs b/crates/blockchain/src/beacon_columns.rs index b837a12b0..88d035a2a 100644 --- a/crates/blockchain/src/beacon_columns.rs +++ b/crates/blockchain/src/beacon_columns.rs @@ -614,10 +614,8 @@ mod tests { } #[test] - fn a_gloas_sidecar_parks_although_gloas_is_not_in_the_followed_set() { - // Gloas blocks import now, so a parked row has a block to drain it; - // `ForkName::is_followed` belongs to the anchor, RPC and gossip side - // and no longer decides whether the parking lot keeps a column. + fn a_gloas_sidecar_is_parked_and_counted_awaiting_its_parent() { + // Gloas blocks import, so a parked row has a block to drain it. let (mut server, p2p) = gloas_server(); let block_root = H256::repeat_byte(5); diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index 24b1b1618..d23523a14 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -2170,9 +2170,9 @@ pub fn get_head_node(store: &Store, config: &Config) -> Result { /// tree walk. pub fn get_head(store: &mut Store, config: &Config) -> Result { let head = get_head_node(store, config)?; - // Ahead of the head root, so a reader never pairs the new root with the - // old node's status (see `Store::set_head_payload_status`). - store.set_head_payload_status(head.payload_status); + // Recorded with its root, so a reader can tell which head the status is + // for whichever of the two writes it sees first. + store.set_head_payload_status(head.root, head.payload_status); store .update_checkpoints(ForkCheckpoints::head_only(head.root)) .expect("record beacon head"); diff --git a/crates/net/p2p/src/beacon/decode.rs b/crates/net/p2p/src/beacon/decode.rs index 57736b4f5..7a5c31d4f 100644 --- a/crates/net/p2p/src/beacon/decode.rs +++ b/crates/net/p2p/src/beacon/decode.rs @@ -190,8 +190,8 @@ pub fn fork_at_slot(config: &Config, slot: Slot) -> ForkName { /// detail are dispatched by name, and a handler that already knows it is /// holding a block should not have to unwrap a [`BeaconGossip`] to find one. /// -/// A block whose slot is at a fork this node does not follow (gloas) and that -/// fails to decode is `UnsupportedFork`, not `Ssz`: the container this build +/// A block whose slot is at a fork this node does not follow (none today) and +/// that fails to decode is `UnsupportedFork`, not `Ssz`: the container this build /// models for that fork is the one at the pinned specification release, and /// an honest peer's block may follow a later one. Bytes too short to name a /// slot, and a block at a followed fork, stay `Ssz` or `Truncated`. diff --git a/crates/net/p2p/src/req_resp/envelopes.rs b/crates/net/p2p/src/req_resp/envelopes.rs index 1e56fb5d7..c7f2249c9 100644 --- a/crates/net/p2p/src/req_resp/envelopes.rs +++ b/crates/net/p2p/src/req_resp/envelopes.rs @@ -217,7 +217,7 @@ mod tests { fn extend_chain(store: &mut Store, block: SignedBeaconBlock) -> H256 { let root = block.message_hash_tree_root(); store.insert_signed_block(root, block).expect("insert"); - store.set_head_payload_status(PayloadStatus::Full); + store.set_head_payload_status(root, PayloadStatus::Full); store .update_checkpoints(ForkCheckpoints::head_only(root)) .expect("advance head"); diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 288a3d4e9..c0df581e4 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -724,16 +724,22 @@ pub(crate) struct BeaconScratch { /// finality. A pruned link is derived again by decoding if anything asks /// for it. pub(crate) payload_links: HashMap, - /// The payload branch of the head fork choice last picked: `Full` when - /// the head node is its block with the payload revealed, `Empty` when it - /// is the block alone. A pre-gloas head is `Full` (see + /// The head node fork choice last picked, as its block root and payload + /// branch: `Full` when the node is the block with the payload revealed, + /// `Empty` when it is the block alone. A pre-gloas head is `Full` (see /// `fork_choice::get_head_node`). /// + /// The root is stored with the status so a reader can tell whether the + /// status belongs to the head it sees: the head root lives in the backend + /// and this does not, so the two cannot be written atomically, and a + /// status alone could be paired with the wrong root. Readers compare roots + /// and treat a mismatch as unknown. + /// /// `None` until the first head computation of this process: nothing /// persists it, and a resumed directory recomputes it on the first tick /// rather than storing a second source of truth for what the head walk /// derives. Readers treat `None` as "not known to be `Full`". - pub(crate) head_payload_status: Option, + pub(crate) head_node_payload_status: Option<(H256, PayloadStatus)>, /// Gloas: the payload timeliness committee's per-member vote on whether /// the block's payload showed up on time, one slot per `PTC_SIZE` index, /// `None` until that member votes. Keyed by beacon block root, like @@ -2438,21 +2444,32 @@ impl Store { // ============ Beacon Head ============ - /// The payload status of the head node fork choice last picked, or `None` - /// before this process has computed a head. See - /// [`BeaconScratch::head_payload_status`]. + /// The payload status of the head node fork choice last picked, for the + /// store's *current* head root only: `None` before this process has + /// computed a head, and also while the recorded node is not the current + /// head (a head move whose status is not recorded yet). Never a status + /// belonging to another root. pub fn head_payload_status(&self) -> Option { - self.beacon.lock().unwrap().head_payload_status + let head = self.head().ok()?; + self.payload_status_of_head_node(head) } - /// Records the payload status of the head node fork choice just picked. - /// - /// Call it **before** [`Store::update_checkpoints`] moves the head root: - /// a reader that sees the new status against the old head merely withholds - /// an envelope for a moment, where the other order could serve the new - /// head's envelope while its node is still the empty one. - pub fn set_head_payload_status(&self, status: PayloadStatus) { - self.beacon.lock().unwrap().head_payload_status = Some(status); + /// The recorded status, if it was recorded for `head`. + fn payload_status_of_head_node(&self, head: H256) -> Option { + self.beacon + .lock() + .unwrap() + .head_node_payload_status + .filter(|(root, _)| *root == head) + .map(|(_, status)| status) + } + + /// Records the head node fork choice just picked: its block `root` and + /// payload `status`. Order against [`Store::update_checkpoints`] does not + /// matter, since readers only use the status for a head whose root equals + /// the recorded one. + pub fn set_head_payload_status(&self, root: H256, status: PayloadStatus) { + self.beacon.lock().unwrap().head_node_payload_status = Some((root, status)); } /// The beacon fork-choice head as `(slot, root)`, or `None` if the head @@ -3451,10 +3468,7 @@ impl Store { if self.chain != Chain::Beacon { Self::lean_only("canonical_execution_payload_envelopes"); } - // Read ahead of the view, and written ahead of the head by - // `set_head_payload_status`'s contract, so a head move partway through - // errs toward withholding. - let head_is_full = self.head_payload_status() == Some(PayloadStatus::Full); + let recorded_head = self.beacon.lock().unwrap().head_node_payload_status; let view = self.backend.begin_read().expect("read view"); let canonical_root = |slot: u64| -> Option { view.get(Table::BlockRoots, &encode_block_root_key(slot)) @@ -3475,10 +3489,16 @@ impl Store { .collect(); // Head and its slot come out of the same view as the rest, so a head // move partway through cannot pair one branch's index with another's. - let head_slot = view + let head_root = view .get(Table::Metadata, KEY_HEAD) .expect("get head") - .map(|bytes| H256::from_ssz_bytes(&bytes).expect("valid head root")) + .map(|bytes| H256::from_ssz_bytes(&bytes).expect("valid head root")); + // The head's envelope is served only when the recorded node is this + // view's head and FULL; a status recorded for another root says + // nothing about this one. + let head_is_full = head_root.is_some() + && recorded_head == head_root.map(|root| (root, PayloadStatus::Full)); + let head_slot = head_root .and_then(|head| view.get(Table::BlockHeaders, &head.to_ssz()).expect("get")) .map(|bytes| decode_beacon_block_value(&bytes).slot()); if let Some(&(last_slot, last_root)) = window.last() @@ -7603,7 +7623,19 @@ mod tests { .is_empty() ); - store.set_head_payload_status(PayloadStatus::Empty); + store.set_head_payload_status(root, PayloadStatus::Empty); + assert_eq!(store.head_payload_status(), Some(PayloadStatus::Empty)); + assert!( + store + .canonical_execution_payload_envelopes(1, 1) + .unwrap() + .is_empty() + ); + + // A FULL status recorded for another root (the previous head, say) + // must not release this head's envelope. + store.set_head_payload_status(H256::repeat_byte(0xee), PayloadStatus::Full); + assert_eq!(store.head_payload_status(), None); assert!( store .canonical_execution_payload_envelopes(1, 1) @@ -7611,7 +7643,7 @@ mod tests { .is_empty() ); - store.set_head_payload_status(PayloadStatus::Full); + store.set_head_payload_status(root, PayloadStatus::Full); assert_eq!( store.canonical_execution_payload_envelopes(1, 1).unwrap(), vec![envelope] From 462ed1b6e33b3c54b1b735b3d6e8b5ca7ab984ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:34:54 -0300 Subject: [PATCH 067/132] feat(p2p): fetch execution payload envelopes by root and by range The chain actor already asks for a missing envelope with FetchRequest.needs_envelope, but p2p only logged it, so a gloas follower could never obtain the payload a child block waits on. By root: a lookup per block root with the block fetch's dedup, peer choice (now shared in choose_fetch_peer) and retry ladder (retire_root_attempt). Answers keep only the requested root, an empty one retries, and what is left goes through the gossip envelope rules before reaching the actor; Queue verdicts are forwarded so the actor can hold an envelope whose block has not imported. By range: once a BeaconBlocksByRange answer has been handed to the actor, ask the same peer for the span's envelopes (only where the span reaches gloas). Sending after the blocks, rather than beside them, keeps the actor's FIFO mailbox seeing blocks first without racing two streams, and never holds block delivery on the envelope answer. A failed or short answer is recovered by the actor's per-slot by-root re-ask. Test swarm harnesses now drain the event stream: the swarm task exits on its first event otherwise, which made send_request race a dead task. --- crates/net/p2p/src/beacon/envelope_checks.rs | 226 ++++++ crates/net/p2p/src/beacon/mod.rs | 1 + crates/net/p2p/src/lib.rs | 76 +- crates/net/p2p/src/req_resp/handlers.rs | 701 +++++++++++++++++-- crates/net/p2p/src/req_resp/mod.rs | 4 +- 5 files changed, 944 insertions(+), 64 deletions(-) create mode 100644 crates/net/p2p/src/beacon/envelope_checks.rs diff --git a/crates/net/p2p/src/beacon/envelope_checks.rs b/crates/net/p2p/src/beacon/envelope_checks.rs new file mode 100644 index 000000000..01c367a99 --- /dev/null +++ b/crates/net/p2p/src/beacon/envelope_checks.rs @@ -0,0 +1,226 @@ +//! The gossip rules for an execution payload envelope, run on every envelope a +//! peer sent in answer to a request, before the chain actor gets it. +//! +//! The specification treats an envelope obtained by any other means as if it +//! had arrived on gossip, so a fetched one passes the same rules +//! (`gossip::envelope::{cheap_checks, stateful_checks}`) minus the seen cache, +//! which only means something on a topic: the chain actor already drops an +//! envelope for a block it holds one for. Verdicts map onto the actor like so: +//! +//! - `Accept`: forwarded. +//! - `Queue`: forwarded too. The envelope's block (or its post-state) is not +//! here yet, and the actor holds such an envelope until the block imports, +//! which is exactly what a range answer needs, since its envelopes arrive +//! while the blocks they belong to are still queued in the actor's mailbox. +//! - `Ignore` and `Reject`: dropped. +//! +//! Envelopes go to the actor in the order given. The caller sorts them by +//! slot, and the checks run concurrently but are awaited in order, so a slower +//! signature check does not reorder the batch. + +use std::num::NonZeroUsize; + +use ethlambda_network_api::BlockArrival; +use ethlambda_state_transition::beacon::gossip::Outcome; +use ethlambda_state_transition::beacon::gossip::envelope::{ + SeenEnvelopes, cheap_checks, stateful_checks, +}; +use ethlambda_storage::Store; +use ethlambda_types::beacon::containers::gloas::SignedExecutionPayloadEnvelope; +use tracing::{debug, error, warn}; + +use crate::P2PServer; + +/// Whether a fetched envelope goes to the chain actor. +pub(crate) fn forwards(outcome: &Outcome) -> bool { + match outcome { + Outcome::Accept | Outcome::Queue(_) => true, + Outcome::Ignore(_) | Outcome::Reject(_) => false, + } +} + +/// Both halves of the gossip rule for one envelope. Blocking: the second half +/// verifies a signature against the block's post-state. +fn judge(store: &Store, envelope: &SignedExecutionPayloadEnvelope) -> Outcome { + // A fresh, empty cache: a fetched envelope is not a gossip duplicate of + // anything, so the cache's one rule never fires. + let seen = SeenEnvelopes::new(NonZeroUsize::MIN); + match cheap_checks(&seen, store, envelope) { + Ok(()) => stateful_checks(store, envelope), + Err(outcome) => outcome, + } +} + +/// Run the gossip rules on `envelopes` and send the chain actor the ones that +/// pass, in order. +/// +/// Returns at once: the checks run on a task of their own, bounded by +/// [`P2PServer::column_check_permits`] like the column checks are, since a +/// range answer can carry a full request's worth of signatures. +pub(crate) fn check_and_forward( + server: &P2PServer, + envelopes: Vec, +) { + if envelopes.is_empty() { + return; + } + let Some(blockchain) = server.blockchain.clone() else { + return; + }; + let store = server.store.clone(); + let permits = server.column_check_permits.clone(); + tokio::spawn(async move { + let mut checks = Vec::with_capacity(envelopes.len()); + for envelope in envelopes { + // The semaphore is never closed, so this only returns once a + // permit is free. + let Ok(permit) = permits.clone().acquire_owned().await else { + return; + }; + let store = store.clone(); + checks.push(tokio::task::spawn_blocking(move || { + let _permit = permit; + let outcome = judge(&store, &envelope); + (envelope, outcome) + })); + } + + for check in checks { + let (envelope, outcome) = match check.await { + Ok(checked) => checked, + Err(err) => { + error!(%err, "An envelope check panicked; dropping the envelope"); + continue; + } + }; + if !forwards(&outcome) { + let block_root = envelope.message.beacon_block_root; + let (_, reason) = outcome.labels(); + debug!( + block_root = %ethlambda_types::ShortRoot(&block_root.0), + reason, + "Dropping a fetched execution payload envelope" + ); + continue; + } + let _ = blockchain + .new_execution_payload_envelope(Box::new(envelope), BlockArrival::now()) + .inspect_err(|err| warn!(%err, "Failed to forward a fetched envelope")); + } + }); +} + +#[cfg(test)] +pub(crate) mod tests { + use std::sync::Arc; + + use ethlambda_network_api::{AggregateArrival, BlockSource, P2PToBlockChain}; + use ethlambda_state_transition::beacon::gossip::{IgnoreReason, QueueReason, RejectReason}; + use ethlambda_types::attestation::{SignedAggregatedAttestation, SignedAttestation}; + use ethlambda_types::beacon::config::Config; + use ethlambda_types::beacon::containers::{ + DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock, + }; + use ethlambda_types::beacon::primitives::ValidatorIndex; + use ethlambda_types::primitives::H256; + use spawned_concurrency::error::ActorError; + use tokio::sync::mpsc; + + use super::*; + use crate::test_support::unconnected_beacon_server; + + /// What a [`Recorder`] saw the chain actor be sent, in order. + #[derive(Debug, PartialEq)] + pub(crate) enum Seen { + Block(u64), + Envelope(H256, u64), + } + + /// A chain actor stand-in that reports every block and envelope it gets, + /// in one stream so their relative order is observable. + pub(crate) struct Recorder(pub mpsc::UnboundedSender); + + impl P2PToBlockChain for Recorder { + fn new_block( + &self, + block: SignedBeaconBlock, + _: BlockSource, + _: BlockArrival, + ) -> Result<(), ActorError> { + let _ = self.0.send(Seen::Block(block.slot())); + Ok(()) + } + fn new_attestation(&self, _: SignedAttestation) -> Result<(), ActorError> { + Ok(()) + } + fn new_aggregated_attestation( + &self, + _: SignedAggregatedAttestation, + ) -> Result<(), ActorError> { + Ok(()) + } + fn new_data_column_sidecars(&self, _: Vec) -> Result<(), ActorError> { + Ok(()) + } + fn data_column_sidecars_awaiting_parent( + &self, + _: Vec, + ) -> Result<(), ActorError> { + Ok(()) + } + fn new_beacon_aggregate( + &self, + _: Box, + _: Vec, + _: AggregateArrival, + ) -> Result<(), ActorError> { + Ok(()) + } + fn new_execution_payload_envelope( + &self, + envelope: Box, + _: BlockArrival, + ) -> Result<(), ActorError> { + let _ = self.0.send(Seen::Envelope( + envelope.message.beacon_block_root, + envelope.message.payload.slot_number, + )); + Ok(()) + } + } + + pub(crate) fn envelope(slot: u64, byte: u8) -> SignedExecutionPayloadEnvelope { + crate::beacon::encoding::test_support::envelope(H256::repeat_byte(byte), slot, H256::ZERO) + } + + #[test] + fn only_accept_and_queue_reach_the_chain_actor() { + assert!(forwards(&Outcome::Accept)); + assert!(forwards(&Outcome::Queue(QueueReason::BlockUnknown))); + assert!(!forwards(&Outcome::Ignore(IgnoreReason::Finalized))); + assert!(!forwards(&Outcome::Reject(RejectReason::BadSignature))); + } + + /// An envelope whose block this node has not seen is queued, so it reaches + /// the actor, and a batch keeps the order it was given. + #[tokio::test] + async fn envelopes_for_unknown_blocks_are_forwarded_in_order() { + let mut server = unconnected_beacon_server(Config::mainnet(), 0).await; + let (sender, mut received) = mpsc::unbounded_channel(); + server.blockchain = Some(Arc::new(Recorder(sender))); + + check_and_forward( + &server, + vec![envelope(7, 1), envelope(8, 2), envelope(9, 3)], + ); + + for (slot, byte) in [(7, 1), (8, 2), (9, 3)] { + assert_eq!( + received.recv().await, + Some(Seen::Envelope(H256::repeat_byte(byte), slot)) + ); + } + drop(server); + assert_eq!(received.recv().await, None); + } +} diff --git a/crates/net/p2p/src/beacon/mod.rs b/crates/net/p2p/src/beacon/mod.rs index cab72149d..8ebc3721e 100644 --- a/crates/net/p2p/src/beacon/mod.rs +++ b/crates/net/p2p/src/beacon/mod.rs @@ -16,6 +16,7 @@ pub mod column_checks; pub mod decode; pub mod encoding; +pub mod envelope_checks; pub mod fork_schedule; pub mod handler; pub mod messages; diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index 2547efd7e..c46157b90 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -94,7 +94,7 @@ use crate::{ lean::protocols::MAX_REQUEST_BLOCKS, req_resp::{ Codec, MAX_COMPRESSED_PAYLOAD_SIZE, ReqResp, ReqRespEvent, Request, build_status, - fetch_block_from_peer, fetch_data_columns_from_peer, + fetch_block_from_peer, fetch_data_columns_from_peer, fetch_envelope_from_peer, handlers::{columns_custodied_by, resume_range_batch_held_for_custody}, }, swarm_adapter::SwarmHandle, @@ -366,6 +366,20 @@ pub(crate) enum PendingRequestKind { start_slot: u64, end_slot: u64, }, + /// An `ExecutionPayloadEnvelopesByRoot` lookup for this block root's + /// envelope. Its attempts and failed peers live in + /// `pending_envelope_requests`, keyed by the same root. + EnvelopeRoot(H256), + /// An `ExecutionPayloadEnvelopesByRange` sweep for a range sync batch, + /// asked for the slots a `BeaconBlocksByRange` answer just delivered. + /// + /// Like [`Self::ColumnRange`] it carries no per-root bookkeeping: a short + /// or empty answer is not a failure to retry, since the chain actor asks + /// again by root for every parent still missing its envelope. + EnvelopeRange { + start_slot: u64, + end_slot: u64, + }, } /// Which single-protocol field of [`ReqResp`] an outbound request travels @@ -1103,6 +1117,7 @@ impl P2P { connected_peers: HashMap::new(), peer_custody: HashMap::new(), pending_root_requests: HashMap::new(), + pending_envelope_requests: HashMap::new(), pending_column_requests: HashMap::new(), outbound_requests: HashMap::new(), range_sync_state: None, @@ -1208,6 +1223,11 @@ pub struct P2PServer { pub(crate) peer_custody: HashMap>, pub(crate) pending_root_requests: HashMap, /// One entry per block root with an in-flight or backed-off + /// `ExecutionPayloadEnvelopesByRoot` lookup: `pending_root_requests`'s + /// counterpart for envelopes, deduplicating a repeated ask and carrying + /// the retry ladder. + pub(crate) pending_envelope_requests: HashMap, + /// One entry per block root with an in-flight or backed-off /// `DataColumnsByRoot` lookup. Mirrors `pending_root_requests`'s role for /// the block path: `fetch_missing_columns` dedupes against it, and /// `handle_column_fetch_failure` is the only place an entry is retired. @@ -1346,6 +1366,8 @@ pub(crate) trait P2PProtocol: Send + Sync { #[allow(dead_code)] // invoked via send_after, not called directly fn retry_data_column_fetch(&self, block_root: H256) -> Result<(), ActorError>; #[allow(dead_code)] // invoked via send_after, not called directly + fn retry_envelope_fetch(&self, block_root: H256) -> Result<(), ActorError>; + #[allow(dead_code)] // invoked via send_after, not called directly fn retry_peer_redial(&self, peer_id: PeerId) -> Result<(), ActorError>; #[allow(dead_code)] // invoked via send_after, not called directly fn discover_peers(&self) -> Result<(), ActorError>; @@ -1382,6 +1404,26 @@ impl P2PServer { } } + #[send_handler] + async fn handle_retry_envelope_fetch( + &mut self, + msg: p2p_protocol::RetryEnvelopeFetch, + _ctx: &Context, + ) { + let block_root = msg.block_root; + // Same "might have completed during backoff" guard as + // `handle_retry_block_fetch`. + if !self.pending_envelope_requests.contains_key(&block_root) { + trace!(%block_root, "Envelope fetch completed during backoff, skipping retry"); + return; + } + trace!(%block_root, "Retrying envelope fetch after backoff"); + if !fetch_envelope_from_peer(self, block_root).await { + tracing::error!(%block_root, "Failed to retry envelope fetch, giving up"); + self.pending_envelope_requests.remove(&block_root); + } + } + #[send_handler] async fn handle_retry_data_column_fetch( &mut self, @@ -1574,7 +1616,7 @@ async fn fetch_missing(server: &mut P2PServer, request: FetchRequest) { columns, } = request; if needs_envelope { - debug!(%block_root, "Envelope fetch by root is not served yet; ignoring"); + fetch_missing_envelope(server, block_root).await; } if needs_block { fetch_missing_block(server, block_root).await; @@ -1594,6 +1636,17 @@ async fn fetch_missing_block(server: &mut P2PServer, root: H256) { fetch_block_from_peer(server, root).await; } +/// The by-root envelope half of a [`FetchRequest`]. +async fn fetch_missing_envelope(server: &mut P2PServer, block_root: H256) { + // The chain actor re-asks once per slot for every parent still missing its + // envelope, so a root already in flight is the normal case, not a fault. + if server.pending_envelope_requests.contains_key(&block_root) { + trace!(%block_root, "Envelope fetch already in progress, ignoring duplicate"); + return; + } + fetch_envelope_from_peer(server, block_root).await; +} + /// The by-root column half of a [`FetchRequest`]. async fn fetch_missing_columns(server: &mut P2PServer, block_root: H256, columns: Vec) { // Same one-lookup-per-root rule as the block half, but merged rather @@ -2485,6 +2538,21 @@ fn compute_message_id(message: &libp2p::gossipsub::Message) -> libp2p::gossipsub /// keeps its own lean-flavored copy, since that one builds a different wire. #[cfg(test)] pub(crate) mod test_support { + /// Keep a test swarm's event stream consumed. The swarm task exits on + /// its first event once nothing receives them, which takes the command + /// channel down with it and leaves `send_request` racing a dead task. + pub(crate) fn drain_swarm_events( + stream: impl futures::Stream> + + Send + + 'static, + ) { + use futures::StreamExt as _; + tokio::spawn(async move { + let mut stream = Box::pin(stream); + while stream.next().await.is_some() {} + }); + } + use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, Ipv4Addr}; use std::sync::Arc; @@ -2578,8 +2646,9 @@ pub(crate) mod test_support { }) .expect("swarm builds"); - let (_swarm_stream, swarm_handle) = + let (swarm_stream, swarm_handle) = crate::swarm_adapter::start_swarm_adapter(built.swarm, HashMap::new()); + drain_swarm_events(swarm_stream); let discovery = crate::discovery::spawn_discovery(crate::discovery::DiscoverySpawnConfig { node_key: secp256k1::SecretKey::new(&mut rand::rngs::OsRng) @@ -2632,6 +2701,7 @@ pub(crate) mod test_support { connected_peers: HashMap::new(), peer_custody: HashMap::new(), pending_root_requests: HashMap::new(), + pending_envelope_requests: HashMap::new(), pending_column_requests: HashMap::new(), outbound_requests: HashMap::new(), range_sync_state: None, diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index 3baf54f4e..773727ff6 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -24,6 +24,7 @@ use ethlambda_state_transition::beacon::das; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::constants; use ethlambda_types::beacon::containers::fulu::{ColumnIndices, DataColumnsByRootIdentifier}; +use ethlambda_types::beacon::containers::gloas::SignedExecutionPayloadEnvelope; use ethlambda_types::beacon::containers::{DataColumnSidecar, SignedBeaconBlock}; use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::checkpoint::Checkpoint; @@ -37,12 +38,14 @@ use super::{ use crate::beacon::BeaconWire; use crate::beacon::column_checks; use crate::beacon::decode::fork_at_slot; +use crate::beacon::envelope_checks; use crate::beacon::handler::{self as beacon_handler, StatusVersion}; use crate::beacon::messages::{ - BeaconMetaData, BeaconStatus, DataColumnsByRangeRequest, Goodbye, Ping, + BeaconMetaData, BeaconStatus, DataColumnsByRangeRequest, + ExecutionPayloadEnvelopesByRangeRequest, Goodbye, Ping, }; use crate::beacon::protocols::{ - MAX_REQUEST_BLOCKS as MAX_BEACON_REQUEST_BLOCKS, MAX_REQUEST_BLOCKS_DENEB, + MAX_REQUEST_BLOCKS as MAX_BEACON_REQUEST_BLOCKS, MAX_REQUEST_BLOCKS_DENEB, MAX_REQUEST_PAYLOADS, }; use crate::beacon::transition; use crate::discovery::enr::node_id_from_peer_id; @@ -268,7 +271,10 @@ pub async fn handle_req_resp_message( // exactly the way #608 fixed, since no further // event will ever name this id again. Some( - PendingRequestKind::Root(_) | PendingRequestKind::Range { .. }, + PendingRequestKind::Root(_) + | PendingRequestKind::Range { .. } + | PendingRequestKind::EnvelopeRoot(_) + | PendingRequestKind::EnvelopeRange { .. }, ) => { error!( %peer, @@ -292,15 +298,48 @@ pub async fn handle_req_resp_message( kind = "execution_payload_envelopes_response", peer_count, "P2P message received" ); - // Nothing requests envelopes yet, so no id can - // legitimately be waiting for this answer. - server.outbound_requests.remove(&request_id); - debug!( - %peer, - ?request_id, - count = envelopes.len(), - "Received an execution payload envelopes response nothing asked for" - ); + // Removed here, like the other terminal answers: this + // response is the whole exchange for the id. + match server.outbound_requests.remove(&request_id) { + Some(PendingRequestKind::EnvelopeRoot(block_root)) => { + handle_envelopes_by_root_response( + server, peer, block_root, envelopes, ctx, + ) + .await; + } + Some(PendingRequestKind::EnvelopeRange { + start_slot, + end_slot, + }) => { + handle_envelopes_by_range_response( + server, peer, start_slot, end_slot, envelopes, + ); + } + // Unreachable by construction, as for the + // column payload above: only the two envelope + // kinds are sent on the envelope protocols. + Some( + PendingRequestKind::Root(_) + | PendingRequestKind::Range { .. } + | PendingRequestKind::Columns(..) + | PendingRequestKind::ColumnRange { .. }, + ) => { + error!( + %peer, + ?request_id, + count = envelopes.len(), + "Envelopes response answered a non-envelope request id" + ); + } + None => { + debug!( + %peer, + ?request_id, + count = envelopes.len(), + "Received envelopes response for unknown request_id" + ); + } + } } ResponsePayload::Blocks(blocks) => { trace!(kind = "blocks_response", peer_count, "P2P message received"); @@ -341,11 +380,13 @@ pub async fn handle_req_resp_message( // `DataColumnSidecars`, never with `Blocks`. Some( PendingRequestKind::Columns(..) - | PendingRequestKind::ColumnRange { .. }, + | PendingRequestKind::ColumnRange { .. } + | PendingRequestKind::EnvelopeRoot(_) + | PendingRequestKind::EnvelopeRange { .. }, ) => { error!( %peer, - "Blocks response answered a data column request id" + "Blocks response answered a non-block request id" ); } None => { @@ -381,6 +422,13 @@ pub async fn handle_req_resp_message( // just means these columns come from gossip or // from the by-root path instead. Some(PendingRequestKind::ColumnRange { .. }) => {} + Some(PendingRequestKind::EnvelopeRoot(block_root)) => { + // Same reasoning as the `Root` arm above. + handle_envelope_fetch_failure(server, block_root, peer, ctx).await; + } + // As for a column range: nothing waits on it, and + // the chain actor re-asks by root. + Some(PendingRequestKind::EnvelopeRange { .. }) => {} None => {} } } @@ -436,6 +484,20 @@ pub async fn handle_req_resp_message( "DataColumnsByRange request failed; columns fall back to the by-root path" ); } + Some(PendingRequestKind::EnvelopeRoot(block_root)) => { + handle_envelope_fetch_failure(server, block_root, peer, ctx).await; + } + Some(PendingRequestKind::EnvelopeRange { + start_slot, + end_slot, + }) => { + debug!( + %peer, + start_slot, + end_slot, + "ExecutionPayloadEnvelopesByRange request failed; envelopes fall back to the by-root path" + ); + } // The handshake is the only *tracked* request kind absent // here: every other outcome for a `Root`, `Range` or `Columns` // id is handled above, so reaching `None` means either the @@ -824,44 +886,17 @@ pub async fn fetch_block_from_peer(server: &mut P2PServer, root: H256) -> bool { return false; } - // Exclude peers that already returned empty responses for this root - let failed = server - .pending_root_requests - .get(&root) - .map(|p| &p.failed_peers); - let pool: Vec<_> = if failed.is_none_or(|f| f.is_empty()) { - server.connected_peers.keys().copied().collect() - } else { - let failed = failed.unwrap(); + let Some((peer, excluded)) = choose_fetch_peer( + &server.connected_peers, server - .connected_peers - .keys() - .copied() - .filter(|p| !failed.contains(p)) - .collect() - }; - - // Fall back to full set if all peers have failed (new peers may have connected, - // or previously-failing peers may have caught up). Clear failed_peers so subsequent - // retries start a fresh round of elimination. - let pool = if pool.is_empty() { - debug!(%root, "All peers failed for this block, retrying with full peer set"); - if let Some(pending) = server.pending_root_requests.get_mut(&root) { - pending.failed_peers.clear(); - } - server.connected_peers.keys().copied().collect() - } else { - pool - }; - - let peer = match pool.choose(&mut rand::thread_rng()) { - Some(&p) => p, - None => { - debug!(%root, "Failed to select random peer"); - return false; - } + .pending_root_requests + .get_mut(&root) + .map(|pending| &mut pending.failed_peers), + &root, + ) else { + debug!(%root, "Failed to select random peer"); + return false; }; - let excluded = server.connected_peers.len() - pool.len(); let sent = if server.wire.is_beacon() { trace!(%peer, %root, excluded, "Sending BeaconBlocksByRoot request for missing block"); @@ -917,6 +952,39 @@ pub async fn fetch_block_from_peer(server: &mut P2PServer, root: H256) -> bool { true } +/// Pick a random connected peer that has not already failed this lookup. +/// +/// Returns the peer and how many were excluded. When every peer has failed it +/// falls back to the full set (new peers may have connected, or failing ones +/// may have caught up) and clears `failed`, so the next retries start a fresh +/// round of elimination. Shared by the block and envelope by-root fetches, +/// whose peer choice has nothing to do with what is being fetched. +fn choose_fetch_peer( + connected: &HashMap, + failed: Option<&mut HashSet>, + root: &H256, +) -> Option<(PeerId, usize)> { + let pool: Vec = match failed.as_deref() { + Some(failed) if !failed.is_empty() => connected + .keys() + .copied() + .filter(|peer| !failed.contains(peer)) + .collect(), + _ => connected.keys().copied().collect(), + }; + let pool = if pool.is_empty() { + debug!(%root, "All peers failed for this lookup, retrying with full peer set"); + if let Some(failed) = failed { + failed.clear(); + } + connected.keys().copied().collect() + } else { + pool + }; + let peer = *pool.choose(&mut rand::thread_rng())?; + Some((peer, connected.len() - pool.len())) +} + /// Record what columns `peer` custodies, given the custody group count it /// advertised. /// @@ -1438,23 +1506,67 @@ async fn handle_fetch_failure( return; }; - pending.failed_peers.insert(peer); - - if pending.attempts >= MAX_FETCH_RETRIES { - error!(%root, %peer, attempts=%pending.attempts, + let attempts = pending.attempts; + let Some(backoff) = retire_root_attempt(pending, peer) else { + error!(%root, %peer, attempts, "Block fetch failed after max retries, giving up"); server.pending_root_requests.remove(&root); return; + }; + + debug!(%root, %peer, attempts, ?backoff, "Block fetch failed, scheduling retry"); + + send_after(backoff, ctx.clone(), p2p_protocol::RetryBlockFetch { root }); +} + +/// Charge `peer` with a failed attempt and say how long to wait before the +/// next one: `None` once the ladder is exhausted. The step shared by the block +/// and envelope by-root lookups, whose bookkeeping is the same +/// [`PendingRequest`]. +fn retire_root_attempt(pending: &mut PendingRequest, peer: PeerId) -> Option { + pending.failed_peers.insert(peer); + + if pending.attempts >= MAX_FETCH_RETRIES { + return None; } let backoff_ms = INITIAL_BACKOFF_MS * BACKOFF_MULTIPLIER.pow(pending.attempts - 1); - let backoff = Duration::from_millis(backoff_ms); + pending.attempts += 1; + Some(Duration::from_millis(backoff_ms)) +} - debug!(%root, %peer, attempts=%pending.attempts, ?backoff, "Block fetch failed, scheduling retry"); +/// Retire one failed attempt at fetching `block_root`'s envelope. +/// +/// [`handle_fetch_failure`]'s counterpart, with the same invariant: every path +/// that ends an attempt comes through here, or the root stays in +/// `pending_envelope_requests` and deduplicates every later ask for it. A +/// lookup that gives up is not lost for good: the chain actor asks again each +/// slot for every parent still missing its envelope. +async fn handle_envelope_fetch_failure( + server: &mut P2PServer, + block_root: H256, + peer: PeerId, + ctx: &Context, +) { + let Some(pending) = server.pending_envelope_requests.get_mut(&block_root) else { + return; + }; - pending.attempts += 1; + let attempts = pending.attempts; + let Some(backoff) = retire_root_attempt(pending, peer) else { + error!(%block_root, %peer, attempts, + "Envelope fetch failed after max retries, giving up"); + server.pending_envelope_requests.remove(&block_root); + return; + }; - send_after(backoff, ctx.clone(), p2p_protocol::RetryBlockFetch { root }); + debug!(%block_root, %peer, attempts, ?backoff, "Envelope fetch failed, scheduling retry"); + + send_after( + backoff, + ctx.clone(), + p2p_protocol::RetryEnvelopeFetch { block_root }, + ); } /// Retire one failed attempt at fetching `block_root`'s missing columns. @@ -2333,6 +2445,235 @@ pub async fn request_beacon_block_by_root( Some(request_id) } +/// Fetch the envelope of `block_root` from a random connected peer. +/// +/// [`fetch_block_from_peer`]'s counterpart for envelopes: same peer choice, and +/// the `pending_envelope_requests` entry is what dedupes a repeated ask and +/// what the retry path reads. Beacon-only, since the lean wire has no +/// envelopes. Returns whether a request went out. +pub async fn fetch_envelope_from_peer(server: &mut P2PServer, block_root: H256) -> bool { + if !server.wire.is_beacon() { + debug!(%block_root, "Cannot fetch an envelope on the lean wire"); + return false; + } + if server.connected_peers.is_empty() { + debug!(%block_root, "Cannot fetch envelope: no connected peers"); + return false; + } + + let Some((peer, excluded)) = choose_fetch_peer( + &server.connected_peers, + server + .pending_envelope_requests + .get_mut(&block_root) + .map(|pending| &mut pending.failed_peers), + &block_root, + ) else { + debug!(%block_root, "Failed to select random peer"); + return false; + }; + + trace!(%peer, %block_root, excluded, "Sending ExecutionPayloadEnvelopesByRoot request"); + let Some(request_id) = server + .swarm_handle + .send_request( + peer, + Request::ExecutionPayloadEnvelopesByRoot(vec![block_root]), + ReqRespProtocol::ExecutionPayloadEnvelopesByRoot, + ) + .await + else { + debug!(%block_root, "Failed to send ExecutionPayloadEnvelopesByRoot request (swarm adapter closed)"); + return false; + }; + server + .outbound_requests + .insert(request_id, PendingRequestKind::EnvelopeRoot(block_root)); + server + .pending_envelope_requests + .entry(block_root) + .or_insert(PendingRequest { + attempts: 1, + failed_peers: HashSet::new(), + }); + true +} + +/// Keep the envelopes of `envelopes` that answer a by-root request for +/// `block_root`. Returns them with how many were dropped. +/// +/// Mirrors [`retain_requested_columns`]: nothing downstream asks whether an +/// envelope was wanted, so an unsolicited one would otherwise be checked and +/// handed to the chain actor on a peer's say-so. +fn retain_requested_envelopes( + envelopes: Vec, + block_root: H256, +) -> (Vec, usize) { + let received = envelopes.len(); + let kept: Vec<_> = envelopes + .into_iter() + .filter(|envelope| envelope.message.beacon_block_root == block_root) + .collect(); + let dropped = received - kept.len(); + (kept, dropped) +} + +/// Take delivery of an `ExecutionPayloadEnvelopesByRoot` answer. +/// +/// An answer with no envelope for the requested root, an empty one included, is +/// a failed attempt and goes through [`handle_envelope_fetch_failure`], the +/// same way a block-by-root answer does. What does match goes through the +/// envelope checks (`beacon::envelope_checks`), which hand the chain actor the +/// ones that pass. +async fn handle_envelopes_by_root_response( + server: &mut P2PServer, + peer: PeerId, + block_root: H256, + envelopes: Vec, + ctx: &Context, +) { + let received = envelopes.len(); + trace!(%peer, %block_root, received, "Received ExecutionPayloadEnvelopesByRoot response"); + + let (envelopes, unrequested) = retain_requested_envelopes(envelopes, block_root); + if unrequested > 0 { + debug!( + %peer, + %block_root, + unrequested, + "Dropping envelopes the ExecutionPayloadEnvelopesByRoot request did not ask for" + ); + } + + if envelopes.is_empty() { + debug!(%peer, %block_root, "ExecutionPayloadEnvelopesByRoot response carried no matching envelope"); + handle_envelope_fetch_failure(server, block_root, peer, ctx).await; + return; + } + + server.pending_envelope_requests.remove(&block_root); + + if server.blockchain.is_none() { + debug!(%peer, %block_root, "No blockchain handler available"); + return; + } + + envelope_checks::check_and_forward(server, envelopes); +} + +/// Whether a range sync batch over `span` can contain a gloas block, and so an +/// envelope worth asking for. Earlier forks have no envelopes, and the server +/// answers such a request empty at best. +fn range_reaches_gloas(config: &Config, span: &std::ops::RangeInclusive) -> bool { + fork_at_slot(config, *span.end()) >= ForkName::Gloas +} + +/// Ask `peer` for the envelopes in `[start_slot, start_slot + count)`, the span +/// of a `BeaconBlocksByRange` answer that was just handed to the chain actor. +/// +/// Sent after that answer rather than beside the blocks request, so the +/// ordering the chain actor needs does not depend on which of two streams +/// finishes first: the blocks are already in its mailbox when this request +/// goes out, and its answer is forwarded by a task spawned later still, so +/// the actor sees every envelope after the blocks of the batch (its mailbox is +/// FIFO). It also never holds the batch's block delivery on this answer. +/// +/// Best effort: nothing waits on the answer, and an envelope that does not +/// arrive is asked for by root once its block is a parent missing one. The +/// return value says whether a request went out. Does nothing for a span that +/// ends before gloas. +pub(crate) async fn request_beacon_envelopes_by_range( + server: &mut P2PServer, + peer: PeerId, + start_slot: u64, + count: u64, +) -> bool { + let Some(wire) = server.wire.beacon() else { + return false; + }; + // The ceiling the server truncates to, so the span recorded below is the + // one the peer actually answers for. + let count = count.min(MAX_REQUEST_PAYLOADS); + if count == 0 { + return false; + } + let end_slot = start_slot.saturating_add(count - 1); + if !range_reaches_gloas(&wire.config, &(start_slot..=end_slot)) { + return false; + } + + trace!(%peer, start_slot, count, "Sending ExecutionPayloadEnvelopesByRange request"); + let request = ExecutionPayloadEnvelopesByRangeRequest { start_slot, count }; + let Some(request_id) = server + .swarm_handle + .send_request( + peer, + Request::ExecutionPayloadEnvelopesByRange(request), + ReqRespProtocol::ExecutionPayloadEnvelopesByRange, + ) + .await + else { + debug!(%peer, "Failed to send ExecutionPayloadEnvelopesByRange request (swarm adapter closed)"); + return false; + }; + server.outbound_requests.insert( + request_id, + PendingRequestKind::EnvelopeRange { + start_slot, + end_slot, + }, + ); + true +} + +/// Keep the envelopes of a range answer that fall in `[start_slot, end_slot]`, +/// in slot order, with how many were dropped. An envelope outside the span is +/// dropped on its own rather than failing the answer, the way a block outside +/// its range is. The sort is what lets the chain actor see a parent's envelope +/// before its child's. +fn retain_envelopes_in_range( + envelopes: Vec, + start_slot: u64, + end_slot: u64, +) -> (Vec, usize) { + let received = envelopes.len(); + let mut kept: Vec<_> = envelopes + .into_iter() + .filter(|envelope| (start_slot..=end_slot).contains(&envelope.message.payload.slot_number)) + .collect(); + kept.sort_by_key(|envelope| envelope.message.payload.slot_number); + let dropped = received - kept.len(); + (kept, dropped) +} + +/// Take delivery of an `ExecutionPayloadEnvelopesByRange` answer. +/// +/// No failure handling, unlike the by-root path: nothing waits on this answer, +/// so a short or empty one is not an attempt spent. The envelopes it lacks are +/// asked for by root, since the chain actor re-asks every slot for each parent +/// still missing one. +fn handle_envelopes_by_range_response( + server: &mut P2PServer, + peer: PeerId, + start_slot: u64, + end_slot: u64, + envelopes: Vec, +) { + let received = envelopes.len(); + trace!(%peer, start_slot, end_slot, received, "Received ExecutionPayloadEnvelopesByRange response"); + + if server.blockchain.is_none() { + debug!(%peer, "No blockchain handler available"); + return; + } + + let (envelopes, dropped) = retain_envelopes_in_range(envelopes, start_slot, end_slot); + if dropped > 0 { + debug!(%peer, start_slot, end_slot, dropped, "Dropping out-of-range envelopes"); + } + envelope_checks::check_and_forward(server, envelopes); +} + /// Take delivery of a `DataColumnsByRoot` response and run every sidecar it /// carried through the chain checks (`beacon::column_checks`), which send the /// chain actor the ones that pass. The specification requires a sidecar @@ -2531,6 +2872,16 @@ async fn handle_beacon_blocks_by_range_response( debug!(%peer, received, accepted, "Beacon blocks received"); + // The batch's blocks are all in the actor's mailbox now; ask for the + // envelopes of the same span only after that (see the function's doc). + request_beacon_envelopes_by_range( + server, + peer, + start_slot, + end_slot.saturating_sub(start_slot).saturating_add(1), + ) + .await; + // Highest slot *handed to* the actor, not the highest imported: see // [`beacon_sync_target`] for why range sync reads this beside the store's // head rather than the head alone. @@ -2590,8 +2941,9 @@ mod tests { }) .expect("swarm builds"); - let (_swarm_stream, swarm_handle) = + let (swarm_stream, swarm_handle) = crate::swarm_adapter::start_swarm_adapter(built.swarm, HashMap::new()); + crate::test_support::drain_swarm_events(swarm_stream); let discovery = crate::discovery::spawn_discovery(crate::discovery::DiscoverySpawnConfig { node_key: secp256k1::SecretKey::new(&mut rand::rngs::OsRng) @@ -2632,6 +2984,7 @@ mod tests { connected_peers: HashMap::new(), peer_custody: HashMap::new(), pending_root_requests: HashMap::new(), + pending_envelope_requests: HashMap::new(), pending_column_requests: HashMap::new(), outbound_requests: HashMap::new(), range_sync_state: None, @@ -3078,4 +3431,234 @@ mod tests { assert_eq!(kept, vec![gloas_column(asked, 3)]); assert_eq!(dropped, 2); } + fn envelope_for(slot: u64, block_root: H256) -> SignedExecutionPayloadEnvelope { + crate::beacon::encoding::test_support::envelope(block_root, slot, H256::ZERO) + } + + fn fulu_block_at(slot: u64) -> SignedBeaconBlock { + use ethlambda_types::beacon::containers::electra; + SignedBeaconBlock::Fulu(electra::SignedBeaconBlock { + message: electra::BeaconBlock { + slot, + proposer_index: 0, + parent_root: H256::ZERO, + state_root: H256::ZERO, + body: electra::BeaconBlockBody::empty(), + }, + signature: Default::default(), + }) + } + + #[test] + fn a_by_root_envelope_answer_keeps_only_the_requested_root() { + let asked = H256::repeat_byte(1); + let other = H256::repeat_byte(2); + let answer = vec![envelope_for(5, other), envelope_for(5, asked)]; + + let (kept, dropped) = retain_requested_envelopes(answer, asked); + + assert_eq!(kept, vec![envelope_for(5, asked)]); + assert_eq!(dropped, 1); + } + + #[test] + fn a_range_envelope_answer_keeps_the_span_in_slot_order() { + let root = H256::repeat_byte(1); + let answer = vec![ + envelope_for(12, root), + envelope_for(9, root), + envelope_for(10, root), + envelope_for(11, root), + ]; + + let (kept, dropped) = retain_envelopes_in_range(answer, 10, 11); + + let slots: Vec = kept + .iter() + .map(|envelope| envelope.message.payload.slot_number) + .collect(); + assert_eq!(slots, vec![10, 11]); + assert_eq!(dropped, 2); + } + + #[test] + fn only_a_span_reaching_gloas_asks_for_envelopes() { + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 10); + let first_gloas_slot = 10 * ethlambda_types::beacon::preset::SLOTS_PER_EPOCH; + + assert!(!range_reaches_gloas(&config, &(0..=first_gloas_slot - 1))); + // A span that starts before gloas and ends inside it overlaps. + assert!(range_reaches_gloas( + &config, + &(first_gloas_slot - 3..=first_gloas_slot) + )); + assert!(range_reaches_gloas( + &config, + &(first_gloas_slot + 5..=first_gloas_slot + 9) + )); + // A chain with no gloas scheduled never does. + assert!(!range_reaches_gloas( + &Config::mainnet(), + &(0..=u64::MAX / 2) + )); + } + + #[test] + fn an_envelope_lookup_backs_off_by_doubling_then_gives_up() { + let mut pending = PendingRequest { + attempts: 1, + failed_peers: HashSet::new(), + }; + let peer = PeerId::random(); + + let first = retire_root_attempt(&mut pending, peer).expect("first retry"); + let second = retire_root_attempt(&mut pending, peer).expect("second retry"); + + assert_eq!(first, Duration::from_millis(INITIAL_BACKOFF_MS)); + assert_eq!(second, first * BACKOFF_MULTIPLIER as u32); + assert!(pending.failed_peers.contains(&peer)); + assert_eq!(pending.attempts, 3); + + pending.attempts = MAX_FETCH_RETRIES; + assert_eq!(retire_root_attempt(&mut pending, peer), None); + } + + #[test] + fn a_failed_peer_is_skipped_until_every_peer_has_failed() { + let root = H256::repeat_byte(1); + let (a, b) = (PeerId::random(), PeerId::random()); + let connected: HashMap = [(a, ()), (b, ())].into(); + let mut failed: HashSet = [a].into(); + + for _ in 0..16 { + assert_eq!( + choose_fetch_peer(&connected, Some(&mut failed), &root), + Some((b, 1)) + ); + } + + // With every peer failed the set is cleared, so a new round starts. + failed.insert(b); + let (_, excluded) = choose_fetch_peer(&connected, Some(&mut failed), &root).unwrap(); + assert_eq!(excluded, 0); + assert!(failed.is_empty()); + + assert_eq!(choose_fetch_peer::<()>(&HashMap::new(), None, &root), None); + } + + #[tokio::test] + async fn an_envelope_fetch_with_nobody_to_ask_leaves_nothing_pending() { + let mut server = crate::test_support::unconnected_beacon_server(Config::mainnet(), 0).await; + + let sent = fetch_envelope_from_peer(&mut server, H256::repeat_byte(1)).await; + + assert!(!sent); + assert!(server.pending_envelope_requests.is_empty()); + assert!(server.outbound_requests.is_empty()); + } + + #[tokio::test] + async fn an_envelope_fetch_on_the_lean_wire_sends_nothing() { + let mut server = unconnected_server().await; + server + .connected_peers + .insert(PeerId::random(), ConnectionDirection::Inbound); + + assert!(!fetch_envelope_from_peer(&mut server, H256::repeat_byte(1)).await); + assert!(server.pending_envelope_requests.is_empty()); + } + + /// One request goes out per root however often the chain actor asks, and + /// it names the root alone. + #[tokio::test] + async fn an_envelope_fetch_is_tracked_and_deduplicated() { + let mut server = crate::test_support::unconnected_beacon_server(Config::mainnet(), 0).await; + server + .connected_peers + .insert(PeerId::random(), ConnectionDirection::Outbound); + let root = H256::repeat_byte(7); + + crate::fetch_missing_envelope(&mut server, root).await; + crate::fetch_missing_envelope(&mut server, root).await; + + assert_eq!(server.pending_envelope_requests[&root].attempts, 1); + let asked: Vec<_> = server.outbound_requests.iter().collect(); + assert_eq!( + asked.len(), + 1, + "a duplicate ask must not send a second request" + ); + let (id, kind) = asked[0]; + assert_eq!( + id.protocol, + ReqRespProtocol::ExecutionPayloadEnvelopesByRoot + ); + assert!(matches!(kind, PendingRequestKind::EnvelopeRoot(r) if *r == root)); + } + + #[tokio::test] + async fn a_range_batch_asks_for_envelopes_only_where_gloas_overlaps() { + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 10); + let first_gloas_slot = 10 * ethlambda_types::beacon::preset::SLOTS_PER_EPOCH; + let mut server = crate::test_support::unconnected_beacon_server(config, 0).await; + let peer = PeerId::random(); + server + .connected_peers + .insert(peer, ConnectionDirection::Outbound); + + assert!(!request_beacon_envelopes_by_range(&mut server, peer, 0, 64).await); + assert!(server.outbound_requests.is_empty()); + + // The span is clamped to what the peer will answer for. + assert!( + request_beacon_envelopes_by_range(&mut server, peer, first_gloas_slot, 1_000).await + ); + let (id, kind) = server.outbound_requests.iter().next().unwrap(); + assert_eq!( + id.protocol, + ReqRespProtocol::ExecutionPayloadEnvelopesByRange + ); + assert!(matches!( + kind, + PendingRequestKind::EnvelopeRange { start_slot, end_slot } + if *start_slot == first_gloas_slot + && *end_slot == first_gloas_slot + MAX_REQUEST_PAYLOADS - 1 + )); + } + + /// The range answer reaches the chain actor after the blocks the batch + /// handed it, and in slot order, whichever order the peer listed them. + #[tokio::test] + async fn range_envelopes_are_forwarded_after_the_batch_blocks_in_slot_order() { + use crate::beacon::envelope_checks::tests::{Recorder, Seen}; + + let mut server = crate::test_support::unconnected_beacon_server(Config::mainnet(), 0).await; + let (sender, mut received) = tokio::sync::mpsc::unbounded_channel(); + let chain = Arc::new(Recorder(sender)); + server.blockchain = Some(chain.clone()); + let root = H256::repeat_byte(3); + + // What `handle_beacon_blocks_by_range_response` does before it asks + // for the span's envelopes. + for slot in [20, 21] { + let _ = ethlambda_network_api::P2PToBlockChain::new_block( + chain.as_ref(), + fulu_block_at(slot), + BlockSource::Sync, + BlockArrival::now(), + ); + } + handle_envelopes_by_range_response( + &mut server, + PeerId::random(), + 20, + 21, + vec![envelope_for(21, root), envelope_for(20, root)], + ); + + assert_eq!(received.recv().await, Some(Seen::Block(20))); + assert_eq!(received.recv().await, Some(Seen::Block(21))); + assert_eq!(received.recv().await, Some(Seen::Envelope(root, 20))); + assert_eq!(received.recv().await, Some(Seen::Envelope(root, 21))); + } } diff --git a/crates/net/p2p/src/req_resp/mod.rs b/crates/net/p2p/src/req_resp/mod.rs index d8e256645..25f518d5f 100644 --- a/crates/net/p2p/src/req_resp/mod.rs +++ b/crates/net/p2p/src/req_resp/mod.rs @@ -9,7 +9,7 @@ pub(crate) use behaviour::{ReqResp, ReqRespEvent}; pub use codec::Codec; pub use encoding::{MAX_COMPRESSED_PAYLOAD_SIZE, MAX_PAYLOAD_SIZE}; pub use handlers::{ - build_status, fetch_block_from_peer, fetch_data_columns_from_peer, handle_req_resp_message, - request_beacon_block_by_root, request_beacon_blocks_by_range, + build_status, fetch_block_from_peer, fetch_data_columns_from_peer, fetch_envelope_from_peer, + handle_req_resp_message, request_beacon_block_by_root, request_beacon_blocks_by_range, }; pub use messages::{Request, Response, ResponsePayload}; From 95e6b68eb1a3b4391e55ee31b1a618b915c4bc72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:42:46 -0300 Subject: [PATCH 068/132] refactor(p2p): move the envelope fetch client into its own module handlers.rs had grown past what is easy to navigate. The envelope client (by-root fetch, response handling, range request and response) and its tests move to req_resp/envelope_client.rs, a sibling of the server side so that module stays untouched. No behavior change; choose_fetch_peer and retire_root_attempt become pub(super) since both fetch ladders share them. --- .../net/p2p/src/req_resp/envelope_client.rs | 531 ++++++++++++++++++ crates/net/p2p/src/req_resp/handlers.rs | 512 +---------------- crates/net/p2p/src/req_resp/mod.rs | 6 +- 3 files changed, 545 insertions(+), 504 deletions(-) create mode 100644 crates/net/p2p/src/req_resp/envelope_client.rs diff --git a/crates/net/p2p/src/req_resp/envelope_client.rs b/crates/net/p2p/src/req_resp/envelope_client.rs new file mode 100644 index 000000000..315dc3c35 --- /dev/null +++ b/crates/net/p2p/src/req_resp/envelope_client.rs @@ -0,0 +1,531 @@ +//! The client side of the gloas execution payload envelope protocols: asking +//! peers for envelopes by root, and alongside range sync batches by range. +//! +//! The server side is in [`super::envelopes`]. Everything fetched here goes +//! through `beacon::envelope_checks` before the chain actor sees it. + +use std::collections::HashSet; + +use ethlambda_types::beacon::config::Config; +use ethlambda_types::beacon::containers::gloas::SignedExecutionPayloadEnvelope; +use ethlambda_types::beacon::fork::ForkName; +use ethlambda_types::primitives::H256; +use libp2p::PeerId; +use spawned_concurrency::tasks::{Context, send_after}; +use tracing::{debug, error, trace}; + +use super::Request; +use super::handlers::{choose_fetch_peer, retire_root_attempt}; +use crate::beacon::decode::fork_at_slot; +use crate::beacon::envelope_checks; +use crate::beacon::messages::ExecutionPayloadEnvelopesByRangeRequest; +use crate::beacon::protocols::MAX_REQUEST_PAYLOADS; +use crate::{P2PServer, PendingRequest, PendingRequestKind, ReqRespProtocol, p2p_protocol}; + +/// Retire one failed attempt at fetching `block_root`'s envelope. +/// +/// [`handle_fetch_failure`]'s counterpart, with the same invariant: every path +/// that ends an attempt comes through here, or the root stays in +/// `pending_envelope_requests` and deduplicates every later ask for it. A +/// lookup that gives up is not lost for good: the chain actor asks again each +/// slot for every parent still missing its envelope. +pub(super) async fn handle_envelope_fetch_failure( + server: &mut P2PServer, + block_root: H256, + peer: PeerId, + ctx: &Context, +) { + let Some(pending) = server.pending_envelope_requests.get_mut(&block_root) else { + return; + }; + + let attempts = pending.attempts; + let Some(backoff) = retire_root_attempt(pending, peer) else { + error!(%block_root, %peer, attempts, + "Envelope fetch failed after max retries, giving up"); + server.pending_envelope_requests.remove(&block_root); + return; + }; + + debug!(%block_root, %peer, attempts, ?backoff, "Envelope fetch failed, scheduling retry"); + + send_after( + backoff, + ctx.clone(), + p2p_protocol::RetryEnvelopeFetch { block_root }, + ); +} + +/// Fetch the envelope of `block_root` from a random connected peer. +/// +/// [`fetch_block_from_peer`]'s counterpart for envelopes: same peer choice, and +/// the `pending_envelope_requests` entry is what dedupes a repeated ask and +/// what the retry path reads. Beacon-only, since the lean wire has no +/// envelopes. Returns whether a request went out. +pub async fn fetch_envelope_from_peer(server: &mut P2PServer, block_root: H256) -> bool { + if !server.wire.is_beacon() { + debug!(%block_root, "Cannot fetch an envelope on the lean wire"); + return false; + } + if server.connected_peers.is_empty() { + debug!(%block_root, "Cannot fetch envelope: no connected peers"); + return false; + } + + let Some((peer, excluded)) = choose_fetch_peer( + &server.connected_peers, + server + .pending_envelope_requests + .get_mut(&block_root) + .map(|pending| &mut pending.failed_peers), + &block_root, + ) else { + debug!(%block_root, "Failed to select random peer"); + return false; + }; + + trace!(%peer, %block_root, excluded, "Sending ExecutionPayloadEnvelopesByRoot request"); + let Some(request_id) = server + .swarm_handle + .send_request( + peer, + Request::ExecutionPayloadEnvelopesByRoot(vec![block_root]), + ReqRespProtocol::ExecutionPayloadEnvelopesByRoot, + ) + .await + else { + debug!(%block_root, "Failed to send ExecutionPayloadEnvelopesByRoot request (swarm adapter closed)"); + return false; + }; + server + .outbound_requests + .insert(request_id, PendingRequestKind::EnvelopeRoot(block_root)); + server + .pending_envelope_requests + .entry(block_root) + .or_insert(PendingRequest { + attempts: 1, + failed_peers: HashSet::new(), + }); + true +} + +/// Keep the envelopes of `envelopes` that answer a by-root request for +/// `block_root`. Returns them with how many were dropped. +/// +/// Mirrors [`retain_requested_columns`]: nothing downstream asks whether an +/// envelope was wanted, so an unsolicited one would otherwise be checked and +/// handed to the chain actor on a peer's say-so. +fn retain_requested_envelopes( + envelopes: Vec, + block_root: H256, +) -> (Vec, usize) { + let received = envelopes.len(); + let kept: Vec<_> = envelopes + .into_iter() + .filter(|envelope| envelope.message.beacon_block_root == block_root) + .collect(); + let dropped = received - kept.len(); + (kept, dropped) +} + +/// Take delivery of an `ExecutionPayloadEnvelopesByRoot` answer. +/// +/// An answer with no envelope for the requested root, an empty one included, is +/// a failed attempt and goes through [`handle_envelope_fetch_failure`], the +/// same way a block-by-root answer does. What does match goes through the +/// envelope checks (`beacon::envelope_checks`), which hand the chain actor the +/// ones that pass. +pub(super) async fn handle_envelopes_by_root_response( + server: &mut P2PServer, + peer: PeerId, + block_root: H256, + envelopes: Vec, + ctx: &Context, +) { + let received = envelopes.len(); + trace!(%peer, %block_root, received, "Received ExecutionPayloadEnvelopesByRoot response"); + + let (envelopes, unrequested) = retain_requested_envelopes(envelopes, block_root); + if unrequested > 0 { + debug!( + %peer, + %block_root, + unrequested, + "Dropping envelopes the ExecutionPayloadEnvelopesByRoot request did not ask for" + ); + } + + if envelopes.is_empty() { + debug!(%peer, %block_root, "ExecutionPayloadEnvelopesByRoot response carried no matching envelope"); + handle_envelope_fetch_failure(server, block_root, peer, ctx).await; + return; + } + + server.pending_envelope_requests.remove(&block_root); + + if server.blockchain.is_none() { + debug!(%peer, %block_root, "No blockchain handler available"); + return; + } + + envelope_checks::check_and_forward(server, envelopes); +} + +/// Whether a range sync batch over `span` can contain a gloas block, and so an +/// envelope worth asking for. Earlier forks have no envelopes, and the server +/// answers such a request empty at best. +fn range_reaches_gloas(config: &Config, span: &std::ops::RangeInclusive) -> bool { + fork_at_slot(config, *span.end()) >= ForkName::Gloas +} + +/// Ask `peer` for the envelopes in `[start_slot, start_slot + count)`, the span +/// of a `BeaconBlocksByRange` answer that was just handed to the chain actor. +/// +/// Sent after that answer rather than beside the blocks request, so the +/// ordering the chain actor needs does not depend on which of two streams +/// finishes first: the blocks are already in its mailbox when this request +/// goes out, and its answer is forwarded by a task spawned later still, so +/// the actor sees every envelope after the blocks of the batch (its mailbox is +/// FIFO). It also never holds the batch's block delivery on this answer. +/// +/// Best effort: nothing waits on the answer, and an envelope that does not +/// arrive is asked for by root once its block is a parent missing one. The +/// return value says whether a request went out. Does nothing for a span that +/// ends before gloas. +pub(crate) async fn request_beacon_envelopes_by_range( + server: &mut P2PServer, + peer: PeerId, + start_slot: u64, + count: u64, +) -> bool { + let Some(wire) = server.wire.beacon() else { + return false; + }; + // The ceiling the server truncates to, so the span recorded below is the + // one the peer actually answers for. + let count = count.min(MAX_REQUEST_PAYLOADS); + if count == 0 { + return false; + } + let end_slot = start_slot.saturating_add(count - 1); + if !range_reaches_gloas(&wire.config, &(start_slot..=end_slot)) { + return false; + } + + trace!(%peer, start_slot, count, "Sending ExecutionPayloadEnvelopesByRange request"); + let request = ExecutionPayloadEnvelopesByRangeRequest { start_slot, count }; + let Some(request_id) = server + .swarm_handle + .send_request( + peer, + Request::ExecutionPayloadEnvelopesByRange(request), + ReqRespProtocol::ExecutionPayloadEnvelopesByRange, + ) + .await + else { + debug!(%peer, "Failed to send ExecutionPayloadEnvelopesByRange request (swarm adapter closed)"); + return false; + }; + server.outbound_requests.insert( + request_id, + PendingRequestKind::EnvelopeRange { + start_slot, + end_slot, + }, + ); + true +} + +/// Keep the envelopes of a range answer that fall in `[start_slot, end_slot]`, +/// in slot order, with how many were dropped. An envelope outside the span is +/// dropped on its own rather than failing the answer, the way a block outside +/// its range is. The sort is what lets the chain actor see a parent's envelope +/// before its child's. +fn retain_envelopes_in_range( + envelopes: Vec, + start_slot: u64, + end_slot: u64, +) -> (Vec, usize) { + let received = envelopes.len(); + let mut kept: Vec<_> = envelopes + .into_iter() + .filter(|envelope| (start_slot..=end_slot).contains(&envelope.message.payload.slot_number)) + .collect(); + kept.sort_by_key(|envelope| envelope.message.payload.slot_number); + let dropped = received - kept.len(); + (kept, dropped) +} + +/// Take delivery of an `ExecutionPayloadEnvelopesByRange` answer. +/// +/// No failure handling, unlike the by-root path: nothing waits on this answer, +/// so a short or empty one is not an attempt spent. The envelopes it lacks are +/// asked for by root, since the chain actor re-asks every slot for each parent +/// still missing one. +pub(super) fn handle_envelopes_by_range_response( + server: &mut P2PServer, + peer: PeerId, + start_slot: u64, + end_slot: u64, + envelopes: Vec, +) { + let received = envelopes.len(); + trace!(%peer, start_slot, end_slot, received, "Received ExecutionPayloadEnvelopesByRange response"); + + if server.blockchain.is_none() { + debug!(%peer, "No blockchain handler available"); + return; + } + + let (envelopes, dropped) = retain_envelopes_in_range(envelopes, start_slot, end_slot); + if dropped > 0 { + debug!(%peer, start_slot, end_slot, dropped, "Dropping out-of-range envelopes"); + } + envelope_checks::check_and_forward(server, envelopes); +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + use std::time::Duration; + + use ethlambda_network_api::{BlockArrival, BlockSource}; + use ethlambda_types::beacon::containers::SignedBeaconBlock; + + use super::*; + use crate::req_resp::handlers::tests::unconnected_server; + use crate::{BACKOFF_MULTIPLIER, ConnectionDirection, INITIAL_BACKOFF_MS, MAX_FETCH_RETRIES}; + + fn envelope_for(slot: u64, block_root: H256) -> SignedExecutionPayloadEnvelope { + crate::beacon::encoding::test_support::envelope(block_root, slot, H256::ZERO) + } + + fn fulu_block_at(slot: u64) -> SignedBeaconBlock { + use ethlambda_types::beacon::containers::electra; + SignedBeaconBlock::Fulu(electra::SignedBeaconBlock { + message: electra::BeaconBlock { + slot, + proposer_index: 0, + parent_root: H256::ZERO, + state_root: H256::ZERO, + body: electra::BeaconBlockBody::empty(), + }, + signature: Default::default(), + }) + } + + #[test] + fn a_by_root_envelope_answer_keeps_only_the_requested_root() { + let asked = H256::repeat_byte(1); + let other = H256::repeat_byte(2); + let answer = vec![envelope_for(5, other), envelope_for(5, asked)]; + + let (kept, dropped) = retain_requested_envelopes(answer, asked); + + assert_eq!(kept, vec![envelope_for(5, asked)]); + assert_eq!(dropped, 1); + } + + #[test] + fn a_range_envelope_answer_keeps_the_span_in_slot_order() { + let root = H256::repeat_byte(1); + let answer = vec![ + envelope_for(12, root), + envelope_for(9, root), + envelope_for(10, root), + envelope_for(11, root), + ]; + + let (kept, dropped) = retain_envelopes_in_range(answer, 10, 11); + + let slots: Vec = kept + .iter() + .map(|envelope| envelope.message.payload.slot_number) + .collect(); + assert_eq!(slots, vec![10, 11]); + assert_eq!(dropped, 2); + } + + #[test] + fn only_a_span_reaching_gloas_asks_for_envelopes() { + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 10); + let first_gloas_slot = 10 * ethlambda_types::beacon::preset::SLOTS_PER_EPOCH; + + assert!(!range_reaches_gloas(&config, &(0..=first_gloas_slot - 1))); + // A span that starts before gloas and ends inside it overlaps. + assert!(range_reaches_gloas( + &config, + &(first_gloas_slot - 3..=first_gloas_slot) + )); + assert!(range_reaches_gloas( + &config, + &(first_gloas_slot + 5..=first_gloas_slot + 9) + )); + // A chain with no gloas scheduled never does. + assert!(!range_reaches_gloas( + &Config::mainnet(), + &(0..=u64::MAX / 2) + )); + } + + #[test] + fn an_envelope_lookup_backs_off_by_doubling_then_gives_up() { + let mut pending = PendingRequest { + attempts: 1, + failed_peers: HashSet::new(), + }; + let peer = PeerId::random(); + + let first = retire_root_attempt(&mut pending, peer).expect("first retry"); + let second = retire_root_attempt(&mut pending, peer).expect("second retry"); + + assert_eq!(first, Duration::from_millis(INITIAL_BACKOFF_MS)); + assert_eq!(second, first * BACKOFF_MULTIPLIER as u32); + assert!(pending.failed_peers.contains(&peer)); + assert_eq!(pending.attempts, 3); + + pending.attempts = MAX_FETCH_RETRIES; + assert_eq!(retire_root_attempt(&mut pending, peer), None); + } + + #[test] + fn a_failed_peer_is_skipped_until_every_peer_has_failed() { + let root = H256::repeat_byte(1); + let (a, b) = (PeerId::random(), PeerId::random()); + let connected: HashMap = [(a, ()), (b, ())].into(); + let mut failed: HashSet = [a].into(); + + for _ in 0..16 { + assert_eq!( + choose_fetch_peer(&connected, Some(&mut failed), &root), + Some((b, 1)) + ); + } + + // With every peer failed the set is cleared, so a new round starts. + failed.insert(b); + let (_, excluded) = choose_fetch_peer(&connected, Some(&mut failed), &root).unwrap(); + assert_eq!(excluded, 0); + assert!(failed.is_empty()); + + assert_eq!(choose_fetch_peer::<()>(&HashMap::new(), None, &root), None); + } + + #[tokio::test] + async fn an_envelope_fetch_with_nobody_to_ask_leaves_nothing_pending() { + let mut server = crate::test_support::unconnected_beacon_server(Config::mainnet(), 0).await; + + let sent = fetch_envelope_from_peer(&mut server, H256::repeat_byte(1)).await; + + assert!(!sent); + assert!(server.pending_envelope_requests.is_empty()); + assert!(server.outbound_requests.is_empty()); + } + + #[tokio::test] + async fn an_envelope_fetch_on_the_lean_wire_sends_nothing() { + let mut server = unconnected_server().await; + server + .connected_peers + .insert(PeerId::random(), ConnectionDirection::Inbound); + + assert!(!fetch_envelope_from_peer(&mut server, H256::repeat_byte(1)).await); + assert!(server.pending_envelope_requests.is_empty()); + } + + /// One request goes out per root however often the chain actor asks, and + /// it names the root alone. + #[tokio::test] + async fn an_envelope_fetch_is_tracked_and_deduplicated() { + let mut server = crate::test_support::unconnected_beacon_server(Config::mainnet(), 0).await; + server + .connected_peers + .insert(PeerId::random(), ConnectionDirection::Outbound); + let root = H256::repeat_byte(7); + + crate::fetch_missing_envelope(&mut server, root).await; + crate::fetch_missing_envelope(&mut server, root).await; + + assert_eq!(server.pending_envelope_requests[&root].attempts, 1); + let asked: Vec<_> = server.outbound_requests.iter().collect(); + assert_eq!( + asked.len(), + 1, + "a duplicate ask must not send a second request" + ); + let (id, kind) = asked[0]; + assert_eq!( + id.protocol, + ReqRespProtocol::ExecutionPayloadEnvelopesByRoot + ); + assert!(matches!(kind, PendingRequestKind::EnvelopeRoot(r) if *r == root)); + } + + #[tokio::test] + async fn a_range_batch_asks_for_envelopes_only_where_gloas_overlaps() { + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 10); + let first_gloas_slot = 10 * ethlambda_types::beacon::preset::SLOTS_PER_EPOCH; + let mut server = crate::test_support::unconnected_beacon_server(config, 0).await; + let peer = PeerId::random(); + server + .connected_peers + .insert(peer, ConnectionDirection::Outbound); + + assert!(!request_beacon_envelopes_by_range(&mut server, peer, 0, 64).await); + assert!(server.outbound_requests.is_empty()); + + // The span is clamped to what the peer will answer for. + assert!( + request_beacon_envelopes_by_range(&mut server, peer, first_gloas_slot, 1_000).await + ); + let (id, kind) = server.outbound_requests.iter().next().unwrap(); + assert_eq!( + id.protocol, + ReqRespProtocol::ExecutionPayloadEnvelopesByRange + ); + assert!(matches!( + kind, + PendingRequestKind::EnvelopeRange { start_slot, end_slot } + if *start_slot == first_gloas_slot + && *end_slot == first_gloas_slot + MAX_REQUEST_PAYLOADS - 1 + )); + } + + /// The range answer reaches the chain actor after the blocks the batch + /// handed it, and in slot order, whichever order the peer listed them. + #[tokio::test] + async fn range_envelopes_are_forwarded_after_the_batch_blocks_in_slot_order() { + use crate::beacon::envelope_checks::tests::{Recorder, Seen}; + + let mut server = crate::test_support::unconnected_beacon_server(Config::mainnet(), 0).await; + let (sender, mut received) = tokio::sync::mpsc::unbounded_channel(); + let chain = Arc::new(Recorder(sender)); + server.blockchain = Some(chain.clone()); + let root = H256::repeat_byte(3); + + // What `handle_beacon_blocks_by_range_response` does before it asks + // for the span's envelopes. + for slot in [20, 21] { + let _ = ethlambda_network_api::P2PToBlockChain::new_block( + chain.as_ref(), + fulu_block_at(slot), + BlockSource::Sync, + BlockArrival::now(), + ); + } + handle_envelopes_by_range_response( + &mut server, + PeerId::random(), + 20, + 21, + vec![envelope_for(21, root), envelope_for(20, root)], + ); + + assert_eq!(received.recv().await, Some(Seen::Block(20))); + assert_eq!(received.recv().await, Some(Seen::Block(21))); + assert_eq!(received.recv().await, Some(Seen::Envelope(root, 20))); + assert_eq!(received.recv().await, Some(Seen::Envelope(root, 21))); + } +} diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index 773727ff6..5a1abfaea 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -24,13 +24,16 @@ use ethlambda_state_transition::beacon::das; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::constants; use ethlambda_types::beacon::containers::fulu::{ColumnIndices, DataColumnsByRootIdentifier}; -use ethlambda_types::beacon::containers::gloas::SignedExecutionPayloadEnvelope; use ethlambda_types::beacon::containers::{DataColumnSidecar, SignedBeaconBlock}; use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::checkpoint::Checkpoint; use ethlambda_types::primitives::HashTreeRoot as _; use ethlambda_types::{block::SignedBlock, primitives::H256}; +use super::envelope_client::{ + handle_envelope_fetch_failure, handle_envelopes_by_range_response, + handle_envelopes_by_root_response, request_beacon_envelopes_by_range, +}; use super::{ Request, Response, ResponsePayload, envelopes, messages::{ResponseCode, error_message}, @@ -38,14 +41,12 @@ use super::{ use crate::beacon::BeaconWire; use crate::beacon::column_checks; use crate::beacon::decode::fork_at_slot; -use crate::beacon::envelope_checks; use crate::beacon::handler::{self as beacon_handler, StatusVersion}; use crate::beacon::messages::{ - BeaconMetaData, BeaconStatus, DataColumnsByRangeRequest, - ExecutionPayloadEnvelopesByRangeRequest, Goodbye, Ping, + BeaconMetaData, BeaconStatus, DataColumnsByRangeRequest, Goodbye, Ping, }; use crate::beacon::protocols::{ - MAX_REQUEST_BLOCKS as MAX_BEACON_REQUEST_BLOCKS, MAX_REQUEST_BLOCKS_DENEB, MAX_REQUEST_PAYLOADS, + MAX_REQUEST_BLOCKS as MAX_BEACON_REQUEST_BLOCKS, MAX_REQUEST_BLOCKS_DENEB, }; use crate::beacon::transition; use crate::discovery::enr::node_id_from_peer_id; @@ -959,7 +960,7 @@ pub async fn fetch_block_from_peer(server: &mut P2PServer, root: H256) -> bool { /// may have caught up) and clears `failed`, so the next retries start a fresh /// round of elimination. Shared by the block and envelope by-root fetches, /// whose peer choice has nothing to do with what is being fetched. -fn choose_fetch_peer( +pub(super) fn choose_fetch_peer( connected: &HashMap, failed: Option<&mut HashSet>, root: &H256, @@ -1523,7 +1524,7 @@ async fn handle_fetch_failure( /// next one: `None` once the ladder is exhausted. The step shared by the block /// and envelope by-root lookups, whose bookkeeping is the same /// [`PendingRequest`]. -fn retire_root_attempt(pending: &mut PendingRequest, peer: PeerId) -> Option { +pub(super) fn retire_root_attempt(pending: &mut PendingRequest, peer: PeerId) -> Option { pending.failed_peers.insert(peer); if pending.attempts >= MAX_FETCH_RETRIES { @@ -1535,40 +1536,6 @@ fn retire_root_attempt(pending: &mut PendingRequest, peer: PeerId) -> Option, -) { - let Some(pending) = server.pending_envelope_requests.get_mut(&block_root) else { - return; - }; - - let attempts = pending.attempts; - let Some(backoff) = retire_root_attempt(pending, peer) else { - error!(%block_root, %peer, attempts, - "Envelope fetch failed after max retries, giving up"); - server.pending_envelope_requests.remove(&block_root); - return; - }; - - debug!(%block_root, %peer, attempts, ?backoff, "Envelope fetch failed, scheduling retry"); - - send_after( - backoff, - ctx.clone(), - p2p_protocol::RetryEnvelopeFetch { block_root }, - ); -} - /// Retire one failed attempt at fetching `block_root`'s missing columns. /// /// Mirrors [`handle_fetch_failure`]'s funnel and invariant: every path that @@ -2445,235 +2412,6 @@ pub async fn request_beacon_block_by_root( Some(request_id) } -/// Fetch the envelope of `block_root` from a random connected peer. -/// -/// [`fetch_block_from_peer`]'s counterpart for envelopes: same peer choice, and -/// the `pending_envelope_requests` entry is what dedupes a repeated ask and -/// what the retry path reads. Beacon-only, since the lean wire has no -/// envelopes. Returns whether a request went out. -pub async fn fetch_envelope_from_peer(server: &mut P2PServer, block_root: H256) -> bool { - if !server.wire.is_beacon() { - debug!(%block_root, "Cannot fetch an envelope on the lean wire"); - return false; - } - if server.connected_peers.is_empty() { - debug!(%block_root, "Cannot fetch envelope: no connected peers"); - return false; - } - - let Some((peer, excluded)) = choose_fetch_peer( - &server.connected_peers, - server - .pending_envelope_requests - .get_mut(&block_root) - .map(|pending| &mut pending.failed_peers), - &block_root, - ) else { - debug!(%block_root, "Failed to select random peer"); - return false; - }; - - trace!(%peer, %block_root, excluded, "Sending ExecutionPayloadEnvelopesByRoot request"); - let Some(request_id) = server - .swarm_handle - .send_request( - peer, - Request::ExecutionPayloadEnvelopesByRoot(vec![block_root]), - ReqRespProtocol::ExecutionPayloadEnvelopesByRoot, - ) - .await - else { - debug!(%block_root, "Failed to send ExecutionPayloadEnvelopesByRoot request (swarm adapter closed)"); - return false; - }; - server - .outbound_requests - .insert(request_id, PendingRequestKind::EnvelopeRoot(block_root)); - server - .pending_envelope_requests - .entry(block_root) - .or_insert(PendingRequest { - attempts: 1, - failed_peers: HashSet::new(), - }); - true -} - -/// Keep the envelopes of `envelopes` that answer a by-root request for -/// `block_root`. Returns them with how many were dropped. -/// -/// Mirrors [`retain_requested_columns`]: nothing downstream asks whether an -/// envelope was wanted, so an unsolicited one would otherwise be checked and -/// handed to the chain actor on a peer's say-so. -fn retain_requested_envelopes( - envelopes: Vec, - block_root: H256, -) -> (Vec, usize) { - let received = envelopes.len(); - let kept: Vec<_> = envelopes - .into_iter() - .filter(|envelope| envelope.message.beacon_block_root == block_root) - .collect(); - let dropped = received - kept.len(); - (kept, dropped) -} - -/// Take delivery of an `ExecutionPayloadEnvelopesByRoot` answer. -/// -/// An answer with no envelope for the requested root, an empty one included, is -/// a failed attempt and goes through [`handle_envelope_fetch_failure`], the -/// same way a block-by-root answer does. What does match goes through the -/// envelope checks (`beacon::envelope_checks`), which hand the chain actor the -/// ones that pass. -async fn handle_envelopes_by_root_response( - server: &mut P2PServer, - peer: PeerId, - block_root: H256, - envelopes: Vec, - ctx: &Context, -) { - let received = envelopes.len(); - trace!(%peer, %block_root, received, "Received ExecutionPayloadEnvelopesByRoot response"); - - let (envelopes, unrequested) = retain_requested_envelopes(envelopes, block_root); - if unrequested > 0 { - debug!( - %peer, - %block_root, - unrequested, - "Dropping envelopes the ExecutionPayloadEnvelopesByRoot request did not ask for" - ); - } - - if envelopes.is_empty() { - debug!(%peer, %block_root, "ExecutionPayloadEnvelopesByRoot response carried no matching envelope"); - handle_envelope_fetch_failure(server, block_root, peer, ctx).await; - return; - } - - server.pending_envelope_requests.remove(&block_root); - - if server.blockchain.is_none() { - debug!(%peer, %block_root, "No blockchain handler available"); - return; - } - - envelope_checks::check_and_forward(server, envelopes); -} - -/// Whether a range sync batch over `span` can contain a gloas block, and so an -/// envelope worth asking for. Earlier forks have no envelopes, and the server -/// answers such a request empty at best. -fn range_reaches_gloas(config: &Config, span: &std::ops::RangeInclusive) -> bool { - fork_at_slot(config, *span.end()) >= ForkName::Gloas -} - -/// Ask `peer` for the envelopes in `[start_slot, start_slot + count)`, the span -/// of a `BeaconBlocksByRange` answer that was just handed to the chain actor. -/// -/// Sent after that answer rather than beside the blocks request, so the -/// ordering the chain actor needs does not depend on which of two streams -/// finishes first: the blocks are already in its mailbox when this request -/// goes out, and its answer is forwarded by a task spawned later still, so -/// the actor sees every envelope after the blocks of the batch (its mailbox is -/// FIFO). It also never holds the batch's block delivery on this answer. -/// -/// Best effort: nothing waits on the answer, and an envelope that does not -/// arrive is asked for by root once its block is a parent missing one. The -/// return value says whether a request went out. Does nothing for a span that -/// ends before gloas. -pub(crate) async fn request_beacon_envelopes_by_range( - server: &mut P2PServer, - peer: PeerId, - start_slot: u64, - count: u64, -) -> bool { - let Some(wire) = server.wire.beacon() else { - return false; - }; - // The ceiling the server truncates to, so the span recorded below is the - // one the peer actually answers for. - let count = count.min(MAX_REQUEST_PAYLOADS); - if count == 0 { - return false; - } - let end_slot = start_slot.saturating_add(count - 1); - if !range_reaches_gloas(&wire.config, &(start_slot..=end_slot)) { - return false; - } - - trace!(%peer, start_slot, count, "Sending ExecutionPayloadEnvelopesByRange request"); - let request = ExecutionPayloadEnvelopesByRangeRequest { start_slot, count }; - let Some(request_id) = server - .swarm_handle - .send_request( - peer, - Request::ExecutionPayloadEnvelopesByRange(request), - ReqRespProtocol::ExecutionPayloadEnvelopesByRange, - ) - .await - else { - debug!(%peer, "Failed to send ExecutionPayloadEnvelopesByRange request (swarm adapter closed)"); - return false; - }; - server.outbound_requests.insert( - request_id, - PendingRequestKind::EnvelopeRange { - start_slot, - end_slot, - }, - ); - true -} - -/// Keep the envelopes of a range answer that fall in `[start_slot, end_slot]`, -/// in slot order, with how many were dropped. An envelope outside the span is -/// dropped on its own rather than failing the answer, the way a block outside -/// its range is. The sort is what lets the chain actor see a parent's envelope -/// before its child's. -fn retain_envelopes_in_range( - envelopes: Vec, - start_slot: u64, - end_slot: u64, -) -> (Vec, usize) { - let received = envelopes.len(); - let mut kept: Vec<_> = envelopes - .into_iter() - .filter(|envelope| (start_slot..=end_slot).contains(&envelope.message.payload.slot_number)) - .collect(); - kept.sort_by_key(|envelope| envelope.message.payload.slot_number); - let dropped = received - kept.len(); - (kept, dropped) -} - -/// Take delivery of an `ExecutionPayloadEnvelopesByRange` answer. -/// -/// No failure handling, unlike the by-root path: nothing waits on this answer, -/// so a short or empty one is not an attempt spent. The envelopes it lacks are -/// asked for by root, since the chain actor re-asks every slot for each parent -/// still missing one. -fn handle_envelopes_by_range_response( - server: &mut P2PServer, - peer: PeerId, - start_slot: u64, - end_slot: u64, - envelopes: Vec, -) { - let received = envelopes.len(); - trace!(%peer, start_slot, end_slot, received, "Received ExecutionPayloadEnvelopesByRange response"); - - if server.blockchain.is_none() { - debug!(%peer, "No blockchain handler available"); - return; - } - - let (envelopes, dropped) = retain_envelopes_in_range(envelopes, start_slot, end_slot); - if dropped > 0 { - debug!(%peer, start_slot, end_slot, dropped, "Dropping out-of-range envelopes"); - } - envelope_checks::check_and_forward(server, envelopes); -} - /// Take delivery of a `DataColumnsByRoot` response and run every sidecar it /// carried through the chain checks (`beacon::column_checks`), which send the /// chain actor the ones that pass. The specification requires a sidecar @@ -2899,7 +2637,7 @@ async fn handle_beacon_blocks_by_range_response( } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; use crate::ConnectionDirection; use ethlambda_storage::{ForkCheckpoints, backend::InMemoryBackend}; @@ -2925,7 +2663,7 @@ mod tests { /// had a harness for; see 64a6bce9), the fetch path only touches /// `&mut P2PServer`, so building one real server is the whole cost of /// testing it. - async fn unconnected_server() -> P2PServer { + pub(crate) async fn unconnected_server() -> P2PServer { let built = crate::build_swarm(crate::SwarmConfig { node_key: vec![3u8; 32], bootnodes: Vec::new(), @@ -3431,234 +3169,4 @@ mod tests { assert_eq!(kept, vec![gloas_column(asked, 3)]); assert_eq!(dropped, 2); } - fn envelope_for(slot: u64, block_root: H256) -> SignedExecutionPayloadEnvelope { - crate::beacon::encoding::test_support::envelope(block_root, slot, H256::ZERO) - } - - fn fulu_block_at(slot: u64) -> SignedBeaconBlock { - use ethlambda_types::beacon::containers::electra; - SignedBeaconBlock::Fulu(electra::SignedBeaconBlock { - message: electra::BeaconBlock { - slot, - proposer_index: 0, - parent_root: H256::ZERO, - state_root: H256::ZERO, - body: electra::BeaconBlockBody::empty(), - }, - signature: Default::default(), - }) - } - - #[test] - fn a_by_root_envelope_answer_keeps_only_the_requested_root() { - let asked = H256::repeat_byte(1); - let other = H256::repeat_byte(2); - let answer = vec![envelope_for(5, other), envelope_for(5, asked)]; - - let (kept, dropped) = retain_requested_envelopes(answer, asked); - - assert_eq!(kept, vec![envelope_for(5, asked)]); - assert_eq!(dropped, 1); - } - - #[test] - fn a_range_envelope_answer_keeps_the_span_in_slot_order() { - let root = H256::repeat_byte(1); - let answer = vec![ - envelope_for(12, root), - envelope_for(9, root), - envelope_for(10, root), - envelope_for(11, root), - ]; - - let (kept, dropped) = retain_envelopes_in_range(answer, 10, 11); - - let slots: Vec = kept - .iter() - .map(|envelope| envelope.message.payload.slot_number) - .collect(); - assert_eq!(slots, vec![10, 11]); - assert_eq!(dropped, 2); - } - - #[test] - fn only_a_span_reaching_gloas_asks_for_envelopes() { - let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 10); - let first_gloas_slot = 10 * ethlambda_types::beacon::preset::SLOTS_PER_EPOCH; - - assert!(!range_reaches_gloas(&config, &(0..=first_gloas_slot - 1))); - // A span that starts before gloas and ends inside it overlaps. - assert!(range_reaches_gloas( - &config, - &(first_gloas_slot - 3..=first_gloas_slot) - )); - assert!(range_reaches_gloas( - &config, - &(first_gloas_slot + 5..=first_gloas_slot + 9) - )); - // A chain with no gloas scheduled never does. - assert!(!range_reaches_gloas( - &Config::mainnet(), - &(0..=u64::MAX / 2) - )); - } - - #[test] - fn an_envelope_lookup_backs_off_by_doubling_then_gives_up() { - let mut pending = PendingRequest { - attempts: 1, - failed_peers: HashSet::new(), - }; - let peer = PeerId::random(); - - let first = retire_root_attempt(&mut pending, peer).expect("first retry"); - let second = retire_root_attempt(&mut pending, peer).expect("second retry"); - - assert_eq!(first, Duration::from_millis(INITIAL_BACKOFF_MS)); - assert_eq!(second, first * BACKOFF_MULTIPLIER as u32); - assert!(pending.failed_peers.contains(&peer)); - assert_eq!(pending.attempts, 3); - - pending.attempts = MAX_FETCH_RETRIES; - assert_eq!(retire_root_attempt(&mut pending, peer), None); - } - - #[test] - fn a_failed_peer_is_skipped_until_every_peer_has_failed() { - let root = H256::repeat_byte(1); - let (a, b) = (PeerId::random(), PeerId::random()); - let connected: HashMap = [(a, ()), (b, ())].into(); - let mut failed: HashSet = [a].into(); - - for _ in 0..16 { - assert_eq!( - choose_fetch_peer(&connected, Some(&mut failed), &root), - Some((b, 1)) - ); - } - - // With every peer failed the set is cleared, so a new round starts. - failed.insert(b); - let (_, excluded) = choose_fetch_peer(&connected, Some(&mut failed), &root).unwrap(); - assert_eq!(excluded, 0); - assert!(failed.is_empty()); - - assert_eq!(choose_fetch_peer::<()>(&HashMap::new(), None, &root), None); - } - - #[tokio::test] - async fn an_envelope_fetch_with_nobody_to_ask_leaves_nothing_pending() { - let mut server = crate::test_support::unconnected_beacon_server(Config::mainnet(), 0).await; - - let sent = fetch_envelope_from_peer(&mut server, H256::repeat_byte(1)).await; - - assert!(!sent); - assert!(server.pending_envelope_requests.is_empty()); - assert!(server.outbound_requests.is_empty()); - } - - #[tokio::test] - async fn an_envelope_fetch_on_the_lean_wire_sends_nothing() { - let mut server = unconnected_server().await; - server - .connected_peers - .insert(PeerId::random(), ConnectionDirection::Inbound); - - assert!(!fetch_envelope_from_peer(&mut server, H256::repeat_byte(1)).await); - assert!(server.pending_envelope_requests.is_empty()); - } - - /// One request goes out per root however often the chain actor asks, and - /// it names the root alone. - #[tokio::test] - async fn an_envelope_fetch_is_tracked_and_deduplicated() { - let mut server = crate::test_support::unconnected_beacon_server(Config::mainnet(), 0).await; - server - .connected_peers - .insert(PeerId::random(), ConnectionDirection::Outbound); - let root = H256::repeat_byte(7); - - crate::fetch_missing_envelope(&mut server, root).await; - crate::fetch_missing_envelope(&mut server, root).await; - - assert_eq!(server.pending_envelope_requests[&root].attempts, 1); - let asked: Vec<_> = server.outbound_requests.iter().collect(); - assert_eq!( - asked.len(), - 1, - "a duplicate ask must not send a second request" - ); - let (id, kind) = asked[0]; - assert_eq!( - id.protocol, - ReqRespProtocol::ExecutionPayloadEnvelopesByRoot - ); - assert!(matches!(kind, PendingRequestKind::EnvelopeRoot(r) if *r == root)); - } - - #[tokio::test] - async fn a_range_batch_asks_for_envelopes_only_where_gloas_overlaps() { - let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 10); - let first_gloas_slot = 10 * ethlambda_types::beacon::preset::SLOTS_PER_EPOCH; - let mut server = crate::test_support::unconnected_beacon_server(config, 0).await; - let peer = PeerId::random(); - server - .connected_peers - .insert(peer, ConnectionDirection::Outbound); - - assert!(!request_beacon_envelopes_by_range(&mut server, peer, 0, 64).await); - assert!(server.outbound_requests.is_empty()); - - // The span is clamped to what the peer will answer for. - assert!( - request_beacon_envelopes_by_range(&mut server, peer, first_gloas_slot, 1_000).await - ); - let (id, kind) = server.outbound_requests.iter().next().unwrap(); - assert_eq!( - id.protocol, - ReqRespProtocol::ExecutionPayloadEnvelopesByRange - ); - assert!(matches!( - kind, - PendingRequestKind::EnvelopeRange { start_slot, end_slot } - if *start_slot == first_gloas_slot - && *end_slot == first_gloas_slot + MAX_REQUEST_PAYLOADS - 1 - )); - } - - /// The range answer reaches the chain actor after the blocks the batch - /// handed it, and in slot order, whichever order the peer listed them. - #[tokio::test] - async fn range_envelopes_are_forwarded_after_the_batch_blocks_in_slot_order() { - use crate::beacon::envelope_checks::tests::{Recorder, Seen}; - - let mut server = crate::test_support::unconnected_beacon_server(Config::mainnet(), 0).await; - let (sender, mut received) = tokio::sync::mpsc::unbounded_channel(); - let chain = Arc::new(Recorder(sender)); - server.blockchain = Some(chain.clone()); - let root = H256::repeat_byte(3); - - // What `handle_beacon_blocks_by_range_response` does before it asks - // for the span's envelopes. - for slot in [20, 21] { - let _ = ethlambda_network_api::P2PToBlockChain::new_block( - chain.as_ref(), - fulu_block_at(slot), - BlockSource::Sync, - BlockArrival::now(), - ); - } - handle_envelopes_by_range_response( - &mut server, - PeerId::random(), - 20, - 21, - vec![envelope_for(21, root), envelope_for(20, root)], - ); - - assert_eq!(received.recv().await, Some(Seen::Block(20))); - assert_eq!(received.recv().await, Some(Seen::Block(21))); - assert_eq!(received.recv().await, Some(Seen::Envelope(root, 20))); - assert_eq!(received.recv().await, Some(Seen::Envelope(root, 21))); - } } diff --git a/crates/net/p2p/src/req_resp/mod.rs b/crates/net/p2p/src/req_resp/mod.rs index 25f518d5f..9fd25569f 100644 --- a/crates/net/p2p/src/req_resp/mod.rs +++ b/crates/net/p2p/src/req_resp/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod behaviour; pub(crate) mod codec; pub(crate) mod encoding; +mod envelope_client; mod envelopes; pub mod handlers; pub(crate) mod messages; @@ -8,8 +9,9 @@ pub(crate) mod messages; pub(crate) use behaviour::{ReqResp, ReqRespEvent}; pub use codec::Codec; pub use encoding::{MAX_COMPRESSED_PAYLOAD_SIZE, MAX_PAYLOAD_SIZE}; +pub use envelope_client::fetch_envelope_from_peer; pub use handlers::{ - build_status, fetch_block_from_peer, fetch_data_columns_from_peer, fetch_envelope_from_peer, - handle_req_resp_message, request_beacon_block_by_root, request_beacon_blocks_by_range, + build_status, fetch_block_from_peer, fetch_data_columns_from_peer, handle_req_resp_message, + request_beacon_block_by_root, request_beacon_blocks_by_range, }; pub use messages::{Request, Response, ResponsePayload}; From d93ec1f36ac309cc70d49a5175e7617c537274b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:47:16 -0300 Subject: [PATCH 069/132] fix(p2p): keep range envelopes in batch order The chain actor keeps an envelope whose block has not imported only briefly and in small numbers, so range delivery works only if each batch's envelopes reach it before the next batch's blocks do. They did not: the next block request went out at once, and its blocks could overtake the previous batch's envelopes, which were then dropped by the actor's per-slot cap and TTL, leaving sync on serial by-root fetches. For a span that reaches gloas, range sync now holds the next request until the envelope answer has been checked and forwarded (all by one task, in slot order), or has failed. Pre-gloas pacing is unchanged. The release is a message to the p2p actor, so a failure, an error response or a refusal cannot leave the session waiting. The chain actor also stops asking for a parent's envelope by root the moment it holds a synced child, since the range answer already carries it and the fetch cost a post-state load and a signature check twice; the per-slot re-ask still recovers any the answer lacks. Smaller fixes: a response that arrives on the wrong protocol now retires its request instead of leaving the root in the dedup map; the envelope range ceiling is statically at least the block one; the docs describe the actor's real hold behavior. --- crates/blockchain/src/beacon_envelope.rs | 46 ++- crates/net/p2p/src/beacon/envelope_checks.rs | 41 ++- crates/net/p2p/src/lib.rs | 22 +- .../net/p2p/src/req_resp/envelope_client.rs | 290 +++++++++++++++--- crates/net/p2p/src/req_resp/handlers.rs | 70 ++++- 5 files changed, 405 insertions(+), 64 deletions(-) diff --git a/crates/blockchain/src/beacon_envelope.rs b/crates/blockchain/src/beacon_envelope.rs index d7568621c..1dfef2174 100644 --- a/crates/blockchain/src/beacon_envelope.rs +++ b/crates/blockchain/src/beacon_envelope.rs @@ -42,7 +42,7 @@ use std::collections::{HashMap, VecDeque}; -use ethlambda_network_api::FetchRequest; +use ethlambda_network_api::{BlockSource, FetchRequest}; use ethlambda_state_transition::beacon::fork::ForkName; use ethlambda_state_transition::beacon::fork_choice::{self, PayloadStatus, PayloadStatusEnum}; use ethlambda_state_transition::beacon::stf::ExecutionEngine; @@ -662,9 +662,16 @@ impl BlockChainServer { .entry(parent_root) .or_default() .insert(block_root, slot); + let arrived_by_sync = timings.source == Some(BlockSource::Sync); self.held_timings.insert(block_root, timings); self.publish_envelope_queues(); - self.request_missing_envelope(parent_root); + // A synced block comes from a range batch, whose envelopes are already + // being fetched alongside it. Asking again at once would fetch each + // envelope twice, and each costs a post-state load and a signature + // check; the once-a-slot re-ask covers any the range answer lacks. + if !arrived_by_sync { + self.request_missing_envelope(parent_root); + } } /// Ask p2p for `root`'s envelope by root. The request carries nothing but @@ -1387,6 +1394,41 @@ mod tests { assert!(server.pending_block_parents.is_empty()); } + /// A held child that came from range sync does not ask for its parent's + /// envelope at once, since the range answer carries it; one that came from + /// gossip does, as nothing else is fetching it. + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn only_a_synced_held_child_skips_the_immediate_envelope_request() { + for (source, asks) in [ + (Some(BlockSource::Sync), 0), + (Some(BlockSource::Gossip), 1), + (None, 1), + ] { + let case = Case::new("get_head", TIEBREAK); + let mut server = case.server(12); + let p2p = std::sync::Arc::new(crate::tests::RecordingP2P::default()); + server.p2p = Some(p2p.clone()); + let child = case.block(CHILD); + let parent = child.parent_root(); + + let timings = ImportTimings { + source, + ..ImportTimings::default() + }; + server.hold_block_for_parent_payload(child, timings); + + let fetches = p2p.fetches.lock().unwrap(); + assert_eq!(fetches.len(), asks, "source {source:?}"); + assert!(fetches.iter().all(|request| request.needs_envelope + && !request.needs_block + && request.block_root == parent)); + } + } + #[test] fn each_slot_re_asks_for_the_envelope_of_a_parent_with_held_children() { let (mut server, p2p) = diff --git a/crates/net/p2p/src/beacon/envelope_checks.rs b/crates/net/p2p/src/beacon/envelope_checks.rs index 01c367a99..00541d7a0 100644 --- a/crates/net/p2p/src/beacon/envelope_checks.rs +++ b/crates/net/p2p/src/beacon/envelope_checks.rs @@ -9,9 +9,12 @@ //! //! - `Accept`: forwarded. //! - `Queue`: forwarded too. The envelope's block (or its post-state) is not -//! here yet, and the actor holds such an envelope until the block imports, -//! which is exactly what a range answer needs, since its envelopes arrive -//! while the blocks they belong to are still queued in the actor's mailbox. +//! here yet, so the actor keeps it, but only briefly and in small numbers +//! (`ENVELOPE_AWAITING_BLOCK_TTL_SLOTS` by arrival slot, at most +//! `MAX_ENVELOPES_AWAITING_BLOCK_PER_SLOT` of them per slot), and the rest +//! are dropped. A range answer is therefore only safe to deliver while the +//! blocks it belongs to are imported in order, which is why range pacing +//! waits on it: see [`check_and_forward_then`]. //! - `Ignore` and `Reject`: dropped. //! //! Envelopes go to the actor in the order given. The caller sorts them by @@ -61,10 +64,36 @@ pub(crate) fn check_and_forward( server: &P2PServer, envelopes: Vec, ) { + check_and_forward_then(server, envelopes, None); +} + +/// A completion callback: runs once every envelope that passed has been handed +/// to the chain actor. +pub(crate) type Done = Box; + +/// [`check_and_forward`], then `done` once the last passing envelope has been +/// sent, or at once when there is nothing to check. +/// +/// This is how range sync paces itself on a gloas batch: the next batch's +/// blocks are requested only after `done`, so the actor never sees them ahead +/// of this batch's envelopes. All of the batch's envelopes are sent by one +/// task, in the order given, with nothing from another batch between them. +pub(crate) fn check_and_forward_then( + server: &P2PServer, + envelopes: Vec, + done: Option, +) { + let run_done = |done: Option| { + if let Some(done) = done { + done(); + } + }; if envelopes.is_empty() { + run_done(done); return; } let Some(blockchain) = server.blockchain.clone() else { + run_done(done); return; }; let store = server.store.clone(); @@ -75,6 +104,9 @@ pub(crate) fn check_and_forward( // The semaphore is never closed, so this only returns once a // permit is free. let Ok(permit) = permits.clone().acquire_owned().await else { + if let Some(done) = done { + done(); + } return; }; let store = store.clone(); @@ -107,6 +139,9 @@ pub(crate) fn check_and_forward( .new_execution_payload_envelope(Box::new(envelope), BlockArrival::now()) .inspect_err(|err| warn!(%err, "Failed to forward a fetched envelope")); } + if let Some(done) = done { + done(); + } }); } diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index c46157b90..ff11c0f4b 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -439,6 +439,14 @@ pub(crate) struct RangeSyncState { /// Latest advertised head slot for each peer. pub(crate) peer_set: HashMap, pub(crate) in_flight: bool, + /// Set while the envelopes of the batch just delivered are being fetched + /// and checked, which only gloas spans have. + /// + /// The chain actor holds an envelope whose block has not imported only + /// briefly and in small numbers, and imports a batch in order, so the next + /// batch's blocks must not reach it before this batch's envelopes have. + /// Holding the next request back on the answer is what guarantees that. + pub(crate) envelopes_pending: bool, /// When the next batch was first held back for custody, while it still is. /// Beacon-only; see [`RANGE_BATCH_CUSTODY_WAIT`]. pub(crate) custody_wait_since: Option, @@ -463,6 +471,7 @@ impl RangeSyncState { current_range, peer_set: HashMap::from([(peer, peer_head)]), in_flight: false, + envelopes_pending: false, custody_wait_since: None, } } @@ -507,7 +516,7 @@ impl RangeSyncState { } pub(crate) fn next_batch(&self) -> Option<(PeerId, Range)> { - if self.in_flight || self.current_range.is_empty() { + if self.in_flight || self.envelopes_pending || self.current_range.is_empty() { return None; } @@ -1368,6 +1377,8 @@ pub(crate) trait P2PProtocol: Send + Sync { #[allow(dead_code)] // invoked via send_after, not called directly fn retry_envelope_fetch(&self, block_root: H256) -> Result<(), ActorError>; #[allow(dead_code)] // invoked via send_after, not called directly + fn resume_range_after_envelopes(&self) -> Result<(), ActorError>; + #[allow(dead_code)] // invoked via send_after, not called directly fn retry_peer_redial(&self, peer_id: PeerId) -> Result<(), ActorError>; #[allow(dead_code)] // invoked via send_after, not called directly fn discover_peers(&self) -> Result<(), ActorError>; @@ -1404,6 +1415,15 @@ impl P2PServer { } } + #[send_handler] + async fn handle_resume_range_after_envelopes( + &mut self, + _msg: p2p_protocol::ResumeRangeAfterEnvelopes, + ctx: &Context, + ) { + req_resp::handlers::release_envelope_gate(self, ctx).await; + } + #[send_handler] async fn handle_retry_envelope_fetch( &mut self, diff --git a/crates/net/p2p/src/req_resp/envelope_client.rs b/crates/net/p2p/src/req_resp/envelope_client.rs index 315dc3c35..e1d275496 100644 --- a/crates/net/p2p/src/req_resp/envelope_client.rs +++ b/crates/net/p2p/src/req_resp/envelope_client.rs @@ -5,6 +5,7 @@ //! through `beacon::envelope_checks` before the chain actor sees it. use std::collections::HashSet; +use std::time::Duration; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::gloas::SignedExecutionPayloadEnvelope; @@ -15,13 +16,49 @@ use spawned_concurrency::tasks::{Context, send_after}; use tracing::{debug, error, trace}; use super::Request; -use super::handlers::{choose_fetch_peer, retire_root_attempt}; +use super::handlers::{choose_fetch_peer, request_next_beacon_range_batch, retire_root_attempt}; use crate::beacon::decode::fork_at_slot; use crate::beacon::envelope_checks; use crate::beacon::messages::ExecutionPayloadEnvelopesByRangeRequest; -use crate::beacon::protocols::MAX_REQUEST_PAYLOADS; +use crate::beacon::protocols::{MAX_REQUEST_BLOCKS_DENEB, MAX_REQUEST_PAYLOADS}; use crate::{P2PServer, PendingRequest, PendingRequestKind, ReqRespProtocol, p2p_protocol}; +// A range request is clamped to the smaller of the two ceilings, so a block +// batch is never wider than its envelope request can answer for. +const _: () = assert!(MAX_REQUEST_PAYLOADS >= MAX_REQUEST_BLOCKS_DENEB); + +/// What one failed attempt at an envelope lookup leaves to do. +#[derive(Debug, PartialEq, Eq)] +enum FailedAttempt { + /// Ask again after the backoff. + Retry(Duration), + /// The ladder is exhausted and the lookup is retired. + GaveUp, + /// Nothing is waiting on this root: a late or duplicate failure. + Stale, +} + +/// Charge `peer` with a failed attempt at `block_root`'s envelope, retiring the +/// lookup when the ladder runs out. +fn fail_envelope_attempt(server: &mut P2PServer, block_root: H256, peer: PeerId) -> FailedAttempt { + let Some(pending) = server.pending_envelope_requests.get_mut(&block_root) else { + return FailedAttempt::Stale; + }; + let attempts = pending.attempts; + match retire_root_attempt(pending, peer) { + Some(backoff) => { + debug!(%block_root, %peer, attempts, ?backoff, "Envelope fetch failed, scheduling retry"); + FailedAttempt::Retry(backoff) + } + None => { + error!(%block_root, %peer, attempts, + "Envelope fetch failed after max retries, giving up"); + server.pending_envelope_requests.remove(&block_root); + FailedAttempt::GaveUp + } + } +} + /// Retire one failed attempt at fetching `block_root`'s envelope. /// /// [`handle_fetch_failure`]'s counterpart, with the same invariant: every path @@ -35,25 +72,25 @@ pub(super) async fn handle_envelope_fetch_failure( peer: PeerId, ctx: &Context, ) { - let Some(pending) = server.pending_envelope_requests.get_mut(&block_root) else { - return; - }; + if let FailedAttempt::Retry(backoff) = fail_envelope_attempt(server, block_root, peer) { + send_after( + backoff, + ctx.clone(), + p2p_protocol::RetryEnvelopeFetch { block_root }, + ); + } +} - let attempts = pending.attempts; - let Some(backoff) = retire_root_attempt(pending, peer) else { - error!(%block_root, %peer, attempts, - "Envelope fetch failed after max retries, giving up"); - server.pending_envelope_requests.remove(&block_root); +/// Let range sync go on after a gloas batch's envelopes: whether they were +/// delivered, refused, or never came. Does nothing when no batch is waiting. +pub(crate) async fn release_envelope_gate(server: &mut P2PServer, ctx: &Context) { + let Some(state) = &mut server.range_sync_state else { return; }; - - debug!(%block_root, %peer, attempts, ?backoff, "Envelope fetch failed, scheduling retry"); - - send_after( - backoff, - ctx.clone(), - p2p_protocol::RetryEnvelopeFetch { block_root }, - ); + if !std::mem::take(&mut state.envelopes_pending) { + return; + } + request_next_beacon_range_batch(server, ctx).await; } /// Fetch the envelope of `block_root` from a random connected peer. @@ -143,6 +180,40 @@ pub(super) async fn handle_envelopes_by_root_response( envelopes: Vec, ctx: &Context, ) { + match settle_by_root_answer(server, peer, block_root, envelopes) { + ByRootAnswer::Forward(envelopes) => { + if server.blockchain.is_none() { + debug!(%peer, %block_root, "No blockchain handler available"); + return; + } + envelope_checks::check_and_forward(server, envelopes); + } + ByRootAnswer::Failed(FailedAttempt::Retry(backoff)) => { + send_after( + backoff, + ctx.clone(), + p2p_protocol::RetryEnvelopeFetch { block_root }, + ); + } + ByRootAnswer::Failed(FailedAttempt::GaveUp | FailedAttempt::Stale) => {} + } +} + +/// What a by-root answer comes to. +#[derive(Debug, PartialEq)] +enum ByRootAnswer { + /// The envelopes that answer the request; the lookup is retired. + Forward(Vec), + /// Nothing answered it: an attempt is spent. + Failed(FailedAttempt), +} + +fn settle_by_root_answer( + server: &mut P2PServer, + peer: PeerId, + block_root: H256, + envelopes: Vec, +) -> ByRootAnswer { let received = envelopes.len(); trace!(%peer, %block_root, received, "Received ExecutionPayloadEnvelopesByRoot response"); @@ -158,18 +229,11 @@ pub(super) async fn handle_envelopes_by_root_response( if envelopes.is_empty() { debug!(%peer, %block_root, "ExecutionPayloadEnvelopesByRoot response carried no matching envelope"); - handle_envelope_fetch_failure(server, block_root, peer, ctx).await; - return; + return ByRootAnswer::Failed(fail_envelope_attempt(server, block_root, peer)); } server.pending_envelope_requests.remove(&block_root); - - if server.blockchain.is_none() { - debug!(%peer, %block_root, "No blockchain handler available"); - return; - } - - envelope_checks::check_and_forward(server, envelopes); + ByRootAnswer::Forward(envelopes) } /// Whether a range sync batch over `span` can contain a gloas block, and so an @@ -187,12 +251,19 @@ fn range_reaches_gloas(config: &Config, span: &std::ops::RangeInclusive) -> /// finishes first: the blocks are already in its mailbox when this request /// goes out, and its answer is forwarded by a task spawned later still, so /// the actor sees every envelope after the blocks of the batch (its mailbox is -/// FIFO). It also never holds the batch's block delivery on this answer. +/// FIFO). It never holds the batch's block delivery on this answer. +/// +/// The chain actor keeps an envelope whose block has not imported only +/// briefly and in small numbers, so range delivery relies on the batches +/// reaching it in order. A batch's envelopes must therefore land before the +/// next batch's blocks do, and the next request is held back until the answer +/// has been checked and forwarded: this sets +/// `RangeSyncState::envelopes_pending`, and [`release_envelope_gate`] clears +/// it. An envelope the answer lacks is asked for by root once its block is a +/// parent missing one. /// -/// Best effort: nothing waits on the answer, and an envelope that does not -/// arrive is asked for by root once its block is a parent missing one. The -/// return value says whether a request went out. Does nothing for a span that -/// ends before gloas. +/// Returns whether a request went out, which is also whether range sync is now +/// waiting on it. Does nothing for a span that ends before gloas. pub(crate) async fn request_beacon_envelopes_by_range( server: &mut P2PServer, peer: PeerId, @@ -234,6 +305,9 @@ pub(crate) async fn request_beacon_envelopes_by_range( end_slot, }, ); + if let Some(state) = &mut server.range_sync_state { + state.envelopes_pending = true; + } true } @@ -259,22 +333,25 @@ fn retain_envelopes_in_range( /// Take delivery of an `ExecutionPayloadEnvelopesByRange` answer. /// -/// No failure handling, unlike the by-root path: nothing waits on this answer, -/// so a short or empty one is not an attempt spent. The envelopes it lacks are -/// asked for by root, since the chain actor re-asks every slot for each parent -/// still missing one. -pub(super) fn handle_envelopes_by_range_response( +/// No retry, unlike the by-root path: a short or empty answer is not an +/// attempt spent, and the envelopes it lacks are asked for by root, since the +/// chain actor re-asks every slot for each parent still missing one. Range +/// sync is released once the answer has been forwarded, or at once when there +/// is nothing to forward. +pub(super) async fn handle_envelopes_by_range_response( server: &mut P2PServer, peer: PeerId, start_slot: u64, end_slot: u64, envelopes: Vec, + ctx: &Context, ) { let received = envelopes.len(); trace!(%peer, start_slot, end_slot, received, "Received ExecutionPayloadEnvelopesByRange response"); if server.blockchain.is_none() { debug!(%peer, "No blockchain handler available"); + release_envelope_gate(server, ctx).await; return; } @@ -282,7 +359,15 @@ pub(super) fn handle_envelopes_by_range_response( if dropped > 0 { debug!(%peer, start_slot, end_slot, dropped, "Dropping out-of-range envelopes"); } - envelope_checks::check_and_forward(server, envelopes); + let resume = ctx.clone(); + let done: envelope_checks::Done = Box::new(move || { + send_after( + Duration::ZERO, + resume, + p2p_protocol::ResumeRangeAfterEnvelopes, + ); + }); + envelope_checks::check_and_forward_then(server, envelopes, Some(done)); } #[cfg(test)] @@ -473,8 +558,14 @@ mod tests { .connected_peers .insert(peer, ConnectionDirection::Outbound); + server.range_sync_state = Some(crate::RangeSyncState::new(0..100_000, peer, 99_999)); + assert!(!request_beacon_envelopes_by_range(&mut server, peer, 0, 64).await); assert!(server.outbound_requests.is_empty()); + // Pre-gloas pacing is unchanged: the next batch is not held back. + let state = server.range_sync_state.as_ref().unwrap(); + assert!(!state.envelopes_pending); + assert!(state.next_batch().is_some()); // The span is clamped to what the peer will answer for. assert!( @@ -491,6 +582,109 @@ mod tests { if *start_slot == first_gloas_slot && *end_slot == first_gloas_slot + MAX_REQUEST_PAYLOADS - 1 )); + // A gloas span holds the next batch back until its answer is delivered. + let state = server.range_sync_state.as_mut().unwrap(); + assert!(state.envelopes_pending); + assert!(state.next_batch().is_none()); + state.envelopes_pending = false; + assert!(state.next_batch().is_some()); + } + + /// The release runs the completion callback even with nothing to forward, + /// or no chain to forward to, so range sync cannot wait forever. + #[tokio::test] + async fn the_range_release_fires_when_there_is_nothing_to_forward() { + use crate::beacon::envelope_checks::tests::Recorder; + + let mut server = crate::test_support::unconnected_beacon_server(Config::mainnet(), 0).await; + let (done_tx, mut done_rx) = tokio::sync::mpsc::unbounded_channel(); + let callback = |tx: tokio::sync::mpsc::UnboundedSender<()>| -> envelope_checks::Done { + Box::new(move || { + let _ = tx.send(()); + }) + }; + + // No chain actor. + envelope_checks::check_and_forward_then( + &server, + vec![envelope_for(7, H256::repeat_byte(1))], + Some(callback(done_tx.clone())), + ); + assert_eq!(done_rx.recv().await, Some(())); + + // An empty answer. + let (sender, _received) = tokio::sync::mpsc::unbounded_channel(); + server.blockchain = Some(Arc::new(Recorder(sender))); + envelope_checks::check_and_forward_then(&server, Vec::new(), Some(callback(done_tx))); + assert_eq!(done_rx.recv().await, Some(())); + } + + fn pending_lookup(server: &mut P2PServer, root: H256, attempts: u32) { + server.pending_envelope_requests.insert( + root, + PendingRequest { + attempts, + failed_peers: HashSet::new(), + }, + ); + } + + /// An empty or non-matching by-root answer spends an attempt and schedules + /// a retry after the first rung of the backoff; a good one retires the + /// lookup and is forwarded. + #[tokio::test] + async fn a_by_root_answer_without_the_envelope_schedules_a_retry() { + let mut server = crate::test_support::unconnected_beacon_server(Config::mainnet(), 0).await; + let root = H256::repeat_byte(4); + let peer = PeerId::random(); + pending_lookup(&mut server, root, 1); + + let empty = settle_by_root_answer(&mut server, peer, root, Vec::new()); + assert_eq!( + empty, + ByRootAnswer::Failed(FailedAttempt::Retry(Duration::from_millis( + INITIAL_BACKOFF_MS + ))) + ); + assert_eq!(server.pending_envelope_requests[&root].attempts, 2); + assert!( + server.pending_envelope_requests[&root] + .failed_peers + .contains(&peer) + ); + + // An envelope for another root is as good as none. + let wrong = settle_by_root_answer( + &mut server, + peer, + root, + vec![envelope_for(9, H256::repeat_byte(5))], + ); + assert!(matches!( + wrong, + ByRootAnswer::Failed(FailedAttempt::Retry(_)) + )); + + let good = settle_by_root_answer(&mut server, peer, root, vec![envelope_for(9, root)]); + assert_eq!(good, ByRootAnswer::Forward(vec![envelope_for(9, root)])); + assert!(server.pending_envelope_requests.is_empty()); + } + + #[tokio::test] + async fn a_by_root_lookup_gives_up_when_the_ladder_is_spent() { + let mut server = crate::test_support::unconnected_beacon_server(Config::mainnet(), 0).await; + let root = H256::repeat_byte(4); + pending_lookup(&mut server, root, MAX_FETCH_RETRIES); + + let outcome = fail_envelope_attempt(&mut server, root, PeerId::random()); + + assert_eq!(outcome, FailedAttempt::GaveUp); + assert!(server.pending_envelope_requests.is_empty()); + // A late failure for a retired lookup is ignored. + assert_eq!( + fail_envelope_attempt(&mut server, root, PeerId::random()), + FailedAttempt::Stale + ); } /// The range answer reaches the chain actor after the blocks the batch @@ -515,17 +709,25 @@ mod tests { BlockArrival::now(), ); } - handle_envelopes_by_range_response( - &mut server, - PeerId::random(), - 20, - 21, - vec![envelope_for(21, root), envelope_for(20, root)], + // The handler's own steps, less the actor `Context` it also needs for + // the release message: filter to the span in slot order, then check + // and forward with a completion callback. + let (answer, _) = + retain_envelopes_in_range(vec![envelope_for(21, root), envelope_for(20, root)], 20, 21); + let (done_tx, mut done_rx) = tokio::sync::mpsc::unbounded_channel(); + envelope_checks::check_and_forward_then( + &server, + answer, + Some(Box::new(move || { + let _ = done_tx.send(()); + })), ); assert_eq!(received.recv().await, Some(Seen::Block(20))); assert_eq!(received.recv().await, Some(Seen::Block(21))); assert_eq!(received.recv().await, Some(Seen::Envelope(root, 20))); assert_eq!(received.recv().await, Some(Seen::Envelope(root, 21))); + // The release comes only after the last envelope was sent. + assert_eq!(done_rx.recv().await, Some(())); } } diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index 5a1abfaea..4aa6dd8ab 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -30,6 +30,7 @@ use ethlambda_types::checkpoint::Checkpoint; use ethlambda_types::primitives::HashTreeRoot as _; use ethlambda_types::{block::SignedBlock, primitives::H256}; +pub(crate) use super::envelope_client::release_envelope_gate; use super::envelope_client::{ handle_envelope_fetch_failure, handle_envelopes_by_range_response, handle_envelopes_by_root_response, request_beacon_envelopes_by_range, @@ -272,10 +273,10 @@ pub async fn handle_req_resp_message( // exactly the way #608 fixed, since no further // event will ever name this id again. Some( - PendingRequestKind::Root(_) + kind @ (PendingRequestKind::Root(_) | PendingRequestKind::Range { .. } | PendingRequestKind::EnvelopeRoot(_) - | PendingRequestKind::EnvelopeRange { .. }, + | PendingRequestKind::EnvelopeRange { .. }), ) => { error!( %peer, @@ -283,6 +284,7 @@ pub async fn handle_req_resp_message( count = sidecars.len(), "Data column sidecars response answered a non-column request id" ); + retire_unanswered(server, kind, peer, ctx).await; } None => { debug!( @@ -313,17 +315,18 @@ pub async fn handle_req_resp_message( end_slot, }) => { handle_envelopes_by_range_response( - server, peer, start_slot, end_slot, envelopes, - ); + server, peer, start_slot, end_slot, envelopes, ctx, + ) + .await; } // Unreachable by construction, as for the // column payload above: only the two envelope // kinds are sent on the envelope protocols. Some( - PendingRequestKind::Root(_) + kind @ (PendingRequestKind::Root(_) | PendingRequestKind::Range { .. } | PendingRequestKind::Columns(..) - | PendingRequestKind::ColumnRange { .. }, + | PendingRequestKind::ColumnRange { .. }), ) => { error!( %peer, @@ -331,6 +334,7 @@ pub async fn handle_req_resp_message( count = envelopes.len(), "Envelopes response answered a non-envelope request id" ); + retire_unanswered(server, kind, peer, ctx).await; } None => { debug!( @@ -380,15 +384,16 @@ pub async fn handle_req_resp_message( // data column protocol, which answers with // `DataColumnSidecars`, never with `Blocks`. Some( - PendingRequestKind::Columns(..) + kind @ (PendingRequestKind::Columns(..) | PendingRequestKind::ColumnRange { .. } | PendingRequestKind::EnvelopeRoot(_) - | PendingRequestKind::EnvelopeRange { .. }, + | PendingRequestKind::EnvelopeRange { .. }), ) => { error!( %peer, "Blocks response answered a non-block request id" ); + retire_unanswered(server, kind, peer, ctx).await; } None => { debug!(%peer, ?request_id, "Received blocks response for unknown request_id"); @@ -427,9 +432,12 @@ pub async fn handle_req_resp_message( // Same reasoning as the `Root` arm above. handle_envelope_fetch_failure(server, block_root, peer, ctx).await; } - // As for a column range: nothing waits on it, and - // the chain actor re-asks by root. - Some(PendingRequestKind::EnvelopeRange { .. }) => {} + // Nothing retries it, and the chain actor re-asks + // by root; but range pacing waits on the answer, + // so it must be released here. + Some(PendingRequestKind::EnvelopeRange { .. }) => { + release_envelope_gate(server, ctx).await; + } None => {} } } @@ -498,6 +506,7 @@ pub async fn handle_req_resp_message( end_slot, "ExecutionPayloadEnvelopesByRange request failed; envelopes fall back to the by-root path" ); + release_envelope_gate(server, ctx).await; } // The handshake is the only *tracked* request kind absent // here: every other outcome for a `Root`, `Range` or `Columns` @@ -1303,7 +1312,10 @@ async fn request_next_range_batch(server: &mut P2PServer) -> bool { /// nothing failed. The batch is re-checked when a peer's metadata arrives (see /// [`resume_range_batch_held_for_custody`]), on every call that would have sent /// it anyway, and at the deadline. -async fn request_next_beacon_range_batch(server: &mut P2PServer, ctx: &Context) -> bool { +pub(super) async fn request_next_beacon_range_batch( + server: &mut P2PServer, + ctx: &Context, +) -> bool { let Some((peer, batch)) = server .range_sync_state .as_ref() @@ -2611,8 +2623,10 @@ async fn handle_beacon_blocks_by_range_response( debug!(%peer, received, accepted, "Beacon blocks received"); // The batch's blocks are all in the actor's mailbox now; ask for the - // envelopes of the same span only after that (see the function's doc). - request_beacon_envelopes_by_range( + // envelopes of the same span only after that. On a gloas span the next + // batch is then held back until the answer has been delivered (see + // `RangeSyncState::envelopes_pending`). + let waiting_on_envelopes = request_beacon_envelopes_by_range( server, peer, start_slot, @@ -2633,9 +2647,37 @@ async fn handle_beacon_blocks_by_range_response( return; } } + if waiting_on_envelopes { + return; + } request_next_beacon_range_batch(server, ctx).await; } +/// Retire a request whose id was answered on a protocol that never carries its +/// kind. The codec cannot produce such an answer, but if one ever arrived the +/// exchange is over all the same: leaving the entry would keep its root in the +/// matching pending map, deduplicating every later ask for it, or leave range +/// sync waiting on a request that already ended. +async fn retire_unanswered( + server: &mut P2PServer, + kind: PendingRequestKind, + peer: PeerId, + ctx: &Context, +) { + match kind { + PendingRequestKind::Root(root) => handle_fetch_failure(server, root, peer, ctx).await, + PendingRequestKind::Range { .. } => fail_range_request(server, &peer), + PendingRequestKind::Columns(block_root, _) => { + handle_column_fetch_failure(server, block_root, peer, ctx).await; + } + PendingRequestKind::ColumnRange { .. } => {} + PendingRequestKind::EnvelopeRoot(block_root) => { + handle_envelope_fetch_failure(server, block_root, peer, ctx).await; + } + PendingRequestKind::EnvelopeRange { .. } => release_envelope_gate(server, ctx).await, + } +} + #[cfg(test)] pub(crate) mod tests { use super::*; From 0784bdbf9030ee0299a7bd0d8c46f6cca5579a5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:56:54 -0300 Subject: [PATCH 070/132] fix(beacon): reconcile the merged gloas branches The envelope fetch branch added a P2PToBlockChain test mock before the payload attestation method existed on the trait, so the merged tree did not compile its p2p tests. Two tests also pinned the old behavior where gloas was not followed: an undecodable gloas block is now a plain SSZ failure and is rejected, like at any other followed fork. --- crates/net/p2p/src/beacon/decode.rs | 7 ++----- crates/net/p2p/src/beacon/envelope_checks.rs | 7 +++++++ crates/net/p2p/src/gossipsub/handler.rs | 4 ++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/crates/net/p2p/src/beacon/decode.rs b/crates/net/p2p/src/beacon/decode.rs index 7a5c31d4f..b4fac1c01 100644 --- a/crates/net/p2p/src/beacon/decode.rs +++ b/crates/net/p2p/src/beacon/decode.rs @@ -465,17 +465,14 @@ mod tests { } #[test] - fn an_undecodable_block_is_unsupported_only_at_a_fork_the_node_does_not_follow() { + fn an_undecodable_block_is_an_ssz_failure_at_gloas_as_at_any_followed_fork() { let mut config = Config::mainnet(); config.gloas_fork_epoch = config.fulu_fork_epoch + 1; // Phase0-shaped bytes are not a valid block at either fork. let at_fulu = phase0_block(slot_of(config.fulu_fork_epoch)).to_ssz(); let at_gloas = phase0_block(slot_of(config.gloas_fork_epoch)).to_ssz(); assert_eq!(decode_block(&config, &at_fulu), Err(DecodeError::Ssz)); - assert_eq!( - decode_block(&config, &at_gloas), - Err(DecodeError::UnsupportedFork) - ); + assert_eq!(decode_block(&config, &at_gloas), Err(DecodeError::Ssz)); assert_eq!( decode_block(&config, &[0xff; 3]), Err(DecodeError::Truncated) diff --git a/crates/net/p2p/src/beacon/envelope_checks.rs b/crates/net/p2p/src/beacon/envelope_checks.rs index 00541d7a0..3f3a5c509 100644 --- a/crates/net/p2p/src/beacon/envelope_checks.rs +++ b/crates/net/p2p/src/beacon/envelope_checks.rs @@ -222,6 +222,13 @@ pub(crate) mod tests { )); Ok(()) } + fn new_payload_attestation_message( + &self, + _: ethlambda_types::beacon::containers::gloas::PayloadAttestationMessage, + _: BlockArrival, + ) -> Result<(), ActorError> { + Ok(()) + } } pub(crate) fn envelope(slot: u64, byte: u8) -> SignedExecutionPayloadEnvelope { diff --git a/crates/net/p2p/src/gossipsub/handler.rs b/crates/net/p2p/src/gossipsub/handler.rs index e09073ca1..4a18771e7 100644 --- a/crates/net/p2p/src/gossipsub/handler.rs +++ b/crates/net/p2p/src/gossipsub/handler.rs @@ -991,14 +991,14 @@ mod tests { } #[tokio::test] - async fn an_undecodable_block_at_a_gloas_slot_is_ignored() { + async fn an_undecodable_block_at_a_gloas_slot_is_rejected() { let mut config = Config::mainnet(); config.gloas_fork_epoch = config.fulu_fork_epoch + 1; let slot = config.gloas_fork_epoch * preset::SLOTS_PER_EPOCH; assert!(matches!( triage_block_under(config, &mismatched_block_bytes(slot)).await, - Dispatch::Report(Outcome::Ignore(IgnoreReason::UnsupportedFork)) + Dispatch::Report(Outcome::Reject(RejectReason::Decode)) )); } From f4ab3b5aa430c685a4164e96052d70d52b2482c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 04:09:40 -0300 Subject: [PATCH 071/132] feat(rpc): serve gloas execution payload envelopes A gloas block's payload lives in a separate envelope, so a node that follows gloas needs to serve it for other nodes to checkpoint-sync from and for operators to inspect. Add GET /eth/v1/beacon/execution_payload_envelopes/{block_id} (JSON and SSZ). It answers 404 for an unknown block and for any block the node holds no verified envelope for, since the store cannot tell a withheld payload from one not yet received, and the specification names no other status for a pre-gloas block. execution_optimistic now goes through one helper. An envelope is optimistic iff its payload verdict is not VALID. A block, header or state response for a gloas block is optimistic iff the latest FULL payload it builds on is not VALID, so a withheld payload of the block itself does not make the chain look unverified. Pre-gloas responses are unchanged. /eth/v1/config/spec reports gloas's preset and constant keys, read from the types crate. --- crates/net/rpc/src/beacon/blocks.rs | 4 +- crates/net/rpc/src/beacon/config.rs | 64 +++++++- crates/net/rpc/src/beacon/envelopes.rs | 196 ++++++++++++++++++++++++ crates/net/rpc/src/beacon/headers.rs | 2 +- crates/net/rpc/src/beacon/mod.rs | 2 + crates/net/rpc/src/beacon/states.rs | 6 +- crates/net/rpc/src/lib.rs | 115 +++++++++++++- crates/net/rpc/src/shared/mod.rs | 1 + crates/net/rpc/src/shared/optimistic.rs | 121 +++++++++++++++ docs/rpc.md | 65 ++++++-- 10 files changed, 553 insertions(+), 23 deletions(-) create mode 100644 crates/net/rpc/src/beacon/envelopes.rs create mode 100644 crates/net/rpc/src/shared/optimistic.rs diff --git a/crates/net/rpc/src/beacon/blocks.rs b/crates/net/rpc/src/beacon/blocks.rs index 790747914..3e9a90594 100644 --- a/crates/net/rpc/src/beacon/blocks.rs +++ b/crates/net/rpc/src/beacon/blocks.rs @@ -54,7 +54,7 @@ async fn get_block( Encoding::Ssz => ssz_response(block.to_ssz()), Encoding::Json => crate::json_response(Envelope { version: fork.as_str(), - execution_optimistic: store.is_beacon_optimistic(root), + execution_optimistic: crate::shared::optimistic::block_is_optimistic(&store, root), finalized: is_finalized(&store, block.slot()), data: block, }), @@ -70,7 +70,7 @@ async fn get_block_root(Path(block_id): Path, State(store): State }; crate::json_response(serde_json::json!({ - "execution_optimistic": store.is_beacon_optimistic(root), + "execution_optimistic": crate::shared::optimistic::block_is_optimistic(&store, root), "finalized": is_finalized(&store, block.slot()), "data": { "root": root }, })) diff --git a/crates/net/rpc/src/beacon/config.rs b/crates/net/rpc/src/beacon/config.rs index 2defba04f..0e2b1d2cf 100644 --- a/crates/net/rpc/src/beacon/config.rs +++ b/crates/net/rpc/src/beacon/config.rs @@ -18,13 +18,14 @@ //! //! The configuration keys include gloas's (`GLOAS_*` and the other gloas //! timing and churn keys), since `Config` holds them and this serializes the -//! whole of it. The preset and constant keys are the ones lighthouse reports, -//! less two kinds: gloas's own preset and constant keys (`DOMAIN_BEACON_BUILDER` -//! and `DOMAIN_PTC_ATTESTER` among them), which `beacon::preset` and -//! `beacon::constants` hold but this list does not report yet, and keys the -//! specification does not define at all (`GAS_LIMIT_ADJUSTMENT_FACTOR`, `RESP_TIMEOUT`, -//! `TTFB_TIMEOUT`). Constants lighthouse leaves out, such as -//! `JUSTIFICATION_BITS_LENGTH`, are left out here too. +//! whole of it. The preset and constant keys are the ones lighthouse reports +//! plus gloas's own, from `presets/*/gloas.yaml` and `beacon-chain.md`'s +//! constants (`PTC_SIZE`, `DOMAIN_BEACON_BUILDER`, ...), less keys the +//! specification does not define at all (`GAS_LIMIT_ADJUSTMENT_FACTOR`, +//! `RESP_TIMEOUT`, `TTFB_TIMEOUT`). Constants lighthouse leaves out, such as +//! `JUSTIFICATION_BITS_LENGTH`, are left out here too. Gloas's `MAX_*_SIZE` +//! preset keys bound gossip message sizes, which the node does not run on, so +//! they are left out as well. use axum::{Router, extract::State, response::Response, routing::get}; use ethlambda_storage::Store; @@ -141,6 +142,12 @@ fn extra_entries() -> impl Iterator { FIELD_ELEMENTS_PER_EXT_BLOB, CELLS_PER_EXT_BLOB, NUMBER_OF_COLUMNS, + // Gloas, in the order of the specification's `presets/*/gloas.yaml`. + PTC_SIZE, + MAX_PAYLOAD_ATTESTATIONS, + MAX_BUILDER_DEPOSIT_REQUESTS_PER_PAYLOAD, + MAX_BUILDER_EXIT_REQUESTS_PER_PAYLOAD, + MAX_BUILDERS_PER_WITHDRAWALS_SWEEP, ); let domains = entries!(hex_string; constants: @@ -156,6 +163,10 @@ fn extra_entries() -> impl Iterator { DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF, DOMAIN_CONTRIBUTION_AND_PROOF, DOMAIN_BLS_TO_EXECUTION_CHANGE, + DOMAIN_BEACON_BUILDER, + DOMAIN_PTC_ATTESTER, + DOMAIN_PROPOSER_PREFERENCES, + DOMAIN_BUILDER_DEPOSIT, ); // One-byte constants, so each goes out as a one-byte array would. @@ -163,6 +174,9 @@ fn extra_entries() -> impl Iterator { BLS_WITHDRAWAL_PREFIX, ETH1_ADDRESS_WITHDRAWAL_PREFIX, COMPOUNDING_WITHDRAWAL_PREFIX, + BUILDER_WITHDRAWAL_PREFIX, + BUILDER_DEPOSIT_REQUEST_TYPE, + BUILDER_EXIT_REQUEST_TYPE, ); // `VERSIONED_HASH_VERSION_KZG` is a `Bytes1` like the withdrawal prefixes, @@ -174,6 +188,11 @@ fn extra_entries() -> impl Iterator { VERSIONED_HASH_VERSION_KZG, UNSET_DEPOSIT_REQUESTS_START_INDEX, FULL_EXIT_REQUEST_AMOUNT, + BUILDER_INDEX_FLAG, + BUILDER_INDEX_SELF_BUILD, + BUILDER_PAYMENT_THRESHOLD_NUMERATOR, + BUILDER_PAYMENT_THRESHOLD_DENOMINATOR, + PAYLOAD_BUILDER_VERSION, ); preset @@ -264,6 +283,37 @@ mod tests { ); } + #[tokio::test] + async fn the_spec_carries_gloas_preset_and_constant_keys() { + let json = get_spec_json().await; + let data = &json["data"]; + + assert_eq!(data["PTC_SIZE"], preset::PTC_SIZE.to_string()); + assert_eq!( + data["MAX_PAYLOAD_ATTESTATIONS"], + preset::MAX_PAYLOAD_ATTESTATIONS.to_string() + ); + assert_eq!( + data["MAX_BUILDERS_PER_WITHDRAWALS_SWEEP"], + preset::MAX_BUILDERS_PER_WITHDRAWALS_SWEEP.to_string() + ); + assert_eq!(data["DOMAIN_BEACON_BUILDER"], "0x0b000000"); + assert_eq!(data["DOMAIN_PTC_ATTESTER"], "0x0c000000"); + assert_eq!(data["DOMAIN_PROPOSER_PREFERENCES"], "0x0d000000"); + assert_eq!(data["DOMAIN_BUILDER_DEPOSIT"], "0x0e000000"); + assert_eq!(data["BUILDER_WITHDRAWAL_PREFIX"], "0xb0"); + assert_eq!(data["BUILDER_DEPOSIT_REQUEST_TYPE"], "0x03"); + assert_eq!(data["BUILDER_EXIT_REQUEST_TYPE"], "0x04"); + assert_eq!(data["BUILDER_INDEX_SELF_BUILD"], "18446744073709551615"); + assert_eq!(data["BUILDER_INDEX_FLAG"], "1099511627776"); + assert_eq!( + data["BUILDER_PAYMENT_THRESHOLD_NUMERATOR"], + constants::BUILDER_PAYMENT_THRESHOLD_NUMERATOR.to_string() + ); + // A `Config` field, so it comes from the config half of the object. + assert!(data["MIN_BUILDER_WITHDRAWABILITY_DELAY"].is_string()); + } + #[test] fn no_key_is_reported_twice() { let config = Config::mainnet(); diff --git a/crates/net/rpc/src/beacon/envelopes.rs b/crates/net/rpc/src/beacon/envelopes.rs new file mode 100644 index 000000000..f525f804b --- /dev/null +++ b/crates/net/rpc/src/beacon/envelopes.rs @@ -0,0 +1,196 @@ +//! `/eth/v1/beacon/execution_payload_envelopes/{block_id}`. +//! +//! Gloas moves a block's execution payload out of the block into a separate +//! signed envelope. This serves the one the node verified, which is also what +//! makes the node usable as a checkpoint-sync source for the payload of its +//! finalized block. + +use axum::{ + Router, + extract::{Path, State}, + http::{HeaderMap, header}, + response::{IntoResponse, Response}, + routing::get, +}; +use ethlambda_storage::Store; +use ethlambda_types::beacon::fork::ForkName; +use libssz::SszEncode; + +use crate::{ + beacon::{ApiError, Envelope as ResponseEnvelope, blocks::is_finalized}, + shared::{ + block_id::{BlockId, IdError}, + content::{Encoding, ssz_response, with_consensus_version}, + optimistic::envelope_is_optimistic, + }, +}; + +pub(crate) fn routes() -> Router { + Router::new().route( + "/eth/v1/beacon/execution_payload_envelopes/{block_id}", + get(get_envelope), + ) +} + +/// The envelope for `block_id`. +/// +/// 404 covers an unknown block and a block with no stored envelope alike: the +/// store holds verified envelopes only, so a pre-gloas block, a withheld +/// payload and a payload not yet received are indistinguishable here, and the +/// specification names no other status for them. +async fn get_envelope( + Path(block_id): Path, + State(store): State, + headers: HeaderMap, +) -> Response { + let not_found = ApiError::NotFound("Execution payload envelope not found"); + let root = match BlockId::parse(&block_id).and_then(|id| id.resolve_beacon(&store)) { + Ok(root) => root, + Err(IdError::Malformed) => return ApiError::BadRequest("invalid block id").into_response(), + Err(IdError::NotFound) => return not_found.into_response(), + }; + let envelope = match store.get_execution_payload_envelope(&root) { + Ok(Some(envelope)) => envelope, + Ok(None) => return not_found.into_response(), + Err(_) => return ApiError::Internal("store read failed").into_response(), + }; + let slot = match store.get_signed_block(&root) { + Ok(Some(block)) => block.slot(), + Ok(None) => return not_found.into_response(), + Err(_) => return ApiError::Internal("store read failed").into_response(), + }; + + let accept = headers.get(header::ACCEPT).and_then(|v| v.to_str().ok()); + let response = match Encoding::from_accept(accept) { + Encoding::Ssz => ssz_response(envelope.to_ssz()), + Encoding::Json => crate::json_response(ResponseEnvelope { + version: ForkName::Gloas.as_str(), + execution_optimistic: envelope_is_optimistic(&store, root), + finalized: is_finalized(&store, slot), + data: envelope, + }), + }; + + with_consensus_version(response, ForkName::Gloas) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{beacon_fixture, gloas_envelope, gloas_fixture}; + use axum::{body::Body, http::Request, http::StatusCode}; + use ethlambda_types::{beacon::fork_choice::PayloadStatusEnum, primitives::H256}; + use http_body_util::BodyExt as _; + use tower::ServiceExt as _; + + async fn get(store: Store, uri: &str, accept: Option<&str>) -> axum::response::Response { + let mut request = Request::builder().uri(uri); + if let Some(accept) = accept { + request = request.header("accept", accept); + } + routes() + .with_state(store) + .oneshot(request.body(Body::empty()).unwrap()) + .await + .unwrap() + } + + fn url(root: H256) -> String { + format!("/eth/v1/beacon/execution_payload_envelopes/0x{root:x}") + } + + async fn json_of(response: axum::response::Response) -> serde_json::Value { + let body = response.into_body().collect().await.unwrap().to_bytes(); + serde_json::from_slice(&body).unwrap() + } + + #[tokio::test] + async fn a_stored_envelope_is_served_as_json_with_the_flags() { + let mut fixture = gloas_fixture(); + let (root, slot) = fixture.g1; + fixture + .store + .insert_verified_payload(slot, &gloas_envelope(root)); + + // No verdict recorded: NOT_VALIDATED, so optimistic. + let response = get(fixture.store.clone(), &url(root), None).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers().get("eth-consensus-version").unwrap(), + "gloas" + ); + let json = json_of(response).await; + assert_eq!(json["version"], "gloas"); + assert_eq!(json["execution_optimistic"], true); + assert_eq!(json["finalized"], false); + assert_eq!(json["data"]["message"]["builder_index"], "3"); + + fixture + .store + .insert_beacon_block_payload_status(root, slot, PayloadStatusEnum::Valid); + let json = json_of(get(fixture.store, &url(root), None).await).await; + assert_eq!(json["execution_optimistic"], false); + } + + #[tokio::test] + async fn a_stored_envelope_is_served_as_ssz_on_request() { + let mut fixture = gloas_fixture(); + let (root, slot) = fixture.g1; + let envelope = gloas_envelope(root); + fixture.store.insert_verified_payload(slot, &envelope); + + let response = get(fixture.store, &url(root), Some("application/octet-stream")).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response + .headers() + .get(axum::http::header::CONTENT_TYPE) + .unwrap(), + crate::SSZ_CONTENT_TYPE + ); + assert_eq!( + response.headers().get("eth-consensus-version").unwrap(), + "gloas" + ); + let body = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(body.as_ref(), envelope.to_ssz().as_slice()); + } + + #[tokio::test] + async fn a_gloas_block_without_a_stored_envelope_is_a_404() { + let fixture = gloas_fixture(); + let response = get(fixture.store, &url(fixture.g1.0), None).await; + assert_eq!(response.status(), StatusCode::NOT_FOUND); + let json = json_of(response).await; + assert_eq!(json["message"], "Execution payload envelope not found"); + } + + #[tokio::test] + async fn an_unknown_block_is_a_404_and_a_malformed_id_a_400() { + let fixture = beacon_fixture(64); + let unknown = url(H256::from([0xee; 32])); + assert_eq!( + get(fixture.store.clone(), &unknown, None).await.status(), + StatusCode::NOT_FOUND + ); + assert_eq!( + get( + fixture.store, + "/eth/v1/beacon/execution_payload_envelopes/nope", + None + ) + .await + .status(), + StatusCode::BAD_REQUEST + ); + } + + /// The specification's error list has no status for a pre-gloas block, and + /// such a block has no envelope, so it reads as not found. + #[tokio::test] + async fn a_pre_gloas_block_has_no_envelope() { + let fixture = gloas_fixture(); + let response = get(fixture.store, &url(fixture.head_root), None).await; + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } +} diff --git a/crates/net/rpc/src/beacon/headers.rs b/crates/net/rpc/src/beacon/headers.rs index 5345e4e74..de3a178f6 100644 --- a/crates/net/rpc/src/beacon/headers.rs +++ b/crates/net/rpc/src/beacon/headers.rs @@ -66,7 +66,7 @@ async fn get_header(Path(block_id): Path, State(store): State) -> // which is unreachable here: this router is only ever mounted on a beacon // store. crate::json_response(serde_json::json!({ - "execution_optimistic": store.is_beacon_optimistic(root), + "execution_optimistic": crate::shared::optimistic::block_is_optimistic(&store, root), "finalized": is_finalized(&store, block.slot()), "data": { "root": root, diff --git a/crates/net/rpc/src/beacon/mod.rs b/crates/net/rpc/src/beacon/mod.rs index 7498d5fad..c047aa299 100644 --- a/crates/net/rpc/src/beacon/mod.rs +++ b/crates/net/rpc/src/beacon/mod.rs @@ -18,6 +18,7 @@ use crate::shared::block_id::IdError; pub(crate) mod blocks; pub(crate) mod config; +pub(crate) mod envelopes; pub(crate) mod genesis; pub(crate) mod headers; pub(crate) mod node; @@ -123,6 +124,7 @@ impl IntoResponse for ApiError { pub(crate) fn routes(version: &'static str, peer_id: String) -> Router { Router::new() .merge(blocks::routes()) + .merge(envelopes::routes()) .merge(headers::routes()) .merge(states::routes()) .merge(genesis::routes()) diff --git a/crates/net/rpc/src/beacon/states.rs b/crates/net/rpc/src/beacon/states.rs index 9128132d9..2f83b1d1b 100644 --- a/crates/net/rpc/src/beacon/states.rs +++ b/crates/net/rpc/src/beacon/states.rs @@ -91,7 +91,7 @@ async fn get_state( // borrow. Encoding::Json => crate::json_response(Envelope { version: fork.as_str(), - execution_optimistic: store.is_beacon_optimistic(root), + execution_optimistic: crate::shared::optimistic::block_is_optimistic(&store, root), finalized: is_finalized(&store, state.slot()), data: state.as_ref(), }), @@ -114,7 +114,7 @@ async fn get_finality_checkpoints( // the state is the only place a *previous* justified checkpoint is kept // at all: the store keeps one justified row and one finalized row. crate::json_response(serde_json::json!({ - "execution_optimistic": store.is_beacon_optimistic(root), + "execution_optimistic": crate::shared::optimistic::block_is_optimistic(&store, root), "finalized": is_finalized(&store, state.slot()), "data": { "previous_justified": state.previous_justified_checkpoint(), @@ -324,7 +324,7 @@ fn validators_response(store: &Store, state_id: &str, request: ValidatorsRequest .collect(); crate::json_response(serde_json::json!({ - "execution_optimistic": store.is_beacon_optimistic(root), + "execution_optimistic": crate::shared::optimistic::block_is_optimistic(store, root), "finalized": is_finalized(store, state.slot()), "data": entries, })) diff --git a/crates/net/rpc/src/lib.rs b/crates/net/rpc/src/lib.rs index 95507c06d..439c21392 100644 --- a/crates/net/rpc/src/lib.rs +++ b/crates/net/rpc/src/lib.rs @@ -242,7 +242,9 @@ pub(crate) mod test_utils { use ethlambda_types::{ beacon::{ config::Config, - containers::{BeaconState, SignedBeaconBlock, phase0, shared::BeaconBlockHeader}, + containers::{ + BeaconState, SignedBeaconBlock, gloas, phase0, shared::BeaconBlockHeader, + }, preset, }, block::{Block, BlockBody, BlockHeader}, @@ -455,6 +457,117 @@ pub(crate) mod test_utils { } } + /// [`beacon_fixture`]'s chain with three gloas blocks on top of its head. + /// + /// ```text + /// anchor(phase0) - head(phase0) - g1 - g2 - g3 + /// ``` + /// + /// `g1` builds on a pre-gloas parent, so on its FULL branch. `g2`'s bid + /// names `g1`'s payload as its parent block hash (FULL). `g3`'s names + /// something else (EMPTY), so it builds on `g1`'s payload through `g2`'s + /// skipped one. No verdict or envelope is recorded for any of them. + pub(crate) struct GloasFixture { + pub(crate) store: Store, + pub(crate) head_root: H256, + pub(crate) g1: (H256, u64), + pub(crate) g2: (H256, u64), + pub(crate) g3: (H256, u64), + } + + /// A gloas block whose bid names `parent_block_hash` and `block_hash`. + fn gloas_beacon_block( + slot: u64, + parent_root: H256, + parent_block_hash: H256, + block_hash: H256, + ) -> SignedBeaconBlock { + let mut body = gloas::BeaconBlockBody::default(); + let bid = &mut body.signed_execution_payload_bid.message; + bid.parent_block_hash = parent_block_hash; + bid.block_hash = block_hash; + SignedBeaconBlock::Gloas(gloas::SignedBeaconBlock { + message: gloas::BeaconBlock { + slot, + proposer_index: 0, + parent_root, + state_root: H256::ZERO, + body, + }, + signature: Default::default(), + }) + } + + pub(crate) fn gloas_fixture() -> GloasFixture { + let BeaconFixture { + mut store, + head_root, + head_slot, + .. + } = beacon_fixture(64); + + let hash = |n: u8| H256::from([n; 32]); + let mut chain = Vec::new(); + let mut parent = (head_root, head_slot); + for (parent_hash, block_hash) in + [(hash(0), hash(1)), (hash(1), hash(2)), (hash(9), hash(3))] + { + let slot = parent.1 + 1; + let block = gloas_beacon_block(slot, parent.0, parent_hash, block_hash); + let root = block.message_hash_tree_root(); + store + .insert_signed_block(root, block) + .expect("insert gloas block"); + chain.push((root, slot)); + parent = (root, slot); + } + + GloasFixture { + store, + head_root, + g1: chain[0], + g2: chain[1], + g3: chain[2], + } + } + + /// A signed envelope for `root` with every other field at its zero value. + pub(crate) fn gloas_envelope(root: H256) -> gloas::SignedExecutionPayloadEnvelope { + let payload = gloas::ExecutionPayload { + parent_hash: Default::default(), + fee_recipient: Default::default(), + state_root: Default::default(), + receipts_root: Default::default(), + logs_bloom: vec![0u8; preset::BYTES_PER_LOGS_BLOOM] + .try_into() + .expect("built at exactly BYTES_PER_LOGS_BLOOM"), + prev_randao: Default::default(), + block_number: 7, + gas_limit: 0, + gas_used: 0, + timestamp: 0, + extra_data: Default::default(), + base_fee_per_gas: Default::default(), + block_hash: Default::default(), + transactions: Default::default(), + withdrawals: Default::default(), + blob_gas_used: 0, + excess_blob_gas: 0, + block_access_list: Default::default(), + slot_number: 0, + }; + gloas::SignedExecutionPayloadEnvelope { + message: gloas::ExecutionPayloadEnvelope { + payload, + execution_requests: Default::default(), + builder_index: 3, + beacon_block_root: root, + parent_beacon_block_root: H256::ZERO, + }, + signature: Default::default(), + } + } + /// A beacon store whose anchor, and so head, is `state`, under a block at /// the state's slot. Returns the store and that block's root. /// diff --git a/crates/net/rpc/src/shared/mod.rs b/crates/net/rpc/src/shared/mod.rs index b74db81db..a42313af4 100644 --- a/crates/net/rpc/src/shared/mod.rs +++ b/crates/net/rpc/src/shared/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod block_id; pub(crate) mod content; +pub(crate) mod optimistic; diff --git a/crates/net/rpc/src/shared/optimistic.rs b/crates/net/rpc/src/shared/optimistic.rs new file mode 100644 index 000000000..df1e3b5da --- /dev/null +++ b/crates/net/rpc/src/shared/optimistic.rs @@ -0,0 +1,121 @@ +//! The `execution_optimistic` flag a Beacon API response carries. +//! +//! Before gloas a block's payload ran inside the block, so the flag is whether +//! that block sits in the store's optimistic set. From gloas on the payload +//! travels in a separate envelope that may arrive late or never, so two +//! different questions share the flag: +//! +//! - an envelope response is optimistic iff that payload's verdict is not +//! `VALID` ([`envelope_is_optimistic`]); +//! - a block, state or header response is optimistic iff the latest FULL +//! payload the block builds on is not `VALID` ([`block_is_optimistic`]). +//! The block's own payload does not count: the block itself was imported +//! without it, and a withheld payload must not make the chain look +//! unverified. + +use ethlambda_state_transition::beacon::fork_choice::{self, PayloadStatus, PayloadStatusEnum}; +use ethlambda_storage::Store; +use ethlambda_types::primitives::H256; + +/// Whether the payload of the block at `root` is not yet vouched for by the +/// execution layer. +/// +/// An unrecorded gloas verdict reads as `NOT_VALIDATED`, so an envelope the +/// store holds but has no verdict for is optimistic. +pub(crate) fn envelope_is_optimistic(store: &Store, root: H256) -> bool { + fork_choice::block_payload_status(store, root) != PayloadStatusEnum::Valid +} + +/// Whether the block at `root` rests on a payload that is not `VALID`. +/// +/// Walks back from `root` through the blocks whose payload branch the child +/// skipped (`Empty`) until it reaches the FULL payload the chain actually +/// builds on, then reads that payload's verdict. A pre-gloas block is its own +/// payload, so it reads the optimistic set as before. A chain that runs out +/// (an anchor whose parent is below the retained window) has nothing left to +/// doubt, so it is not optimistic. +/// +/// A link that cannot be derived is reported optimistic: this flag gates +/// signing, so the answer for an unknown is the cautious one. +pub(crate) fn block_is_optimistic(store: &Store, root: H256) -> bool { + let mut current = root; + loop { + if fork_choice::ensure_payload_link(store, current).is_err() { + return true; + } + let Some(link) = store.payload_link(¤t) else { + // The block is not in the store, so there is no payload to doubt. + return false; + }; + if !link.is_gloas() { + return store.is_beacon_optimistic(current); + } + let Ok(Some(block)) = store.get_signed_block(¤t) else { + return false; + }; + let parent = block.parent_root(); + match link.parent_status() { + Some(PayloadStatus::Full) => { + return fork_choice::block_payload_status(store, parent) + != PayloadStatusEnum::Valid; + } + Some(PayloadStatus::Empty) => current = parent, + // A pending parent status never comes out of a block's own bid. + Some(PayloadStatus::Pending) | None => return false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{beacon_fixture, gloas_fixture}; + + #[test] + fn a_pre_gloas_block_reads_the_optimistic_set() { + let mut fixture = beacon_fixture(64); + let root = fixture.head_root; + assert!(!block_is_optimistic(&fixture.store, root)); + + fixture + .store + .insert_beacon_optimistic_root(root, fixture.head_slot); + assert!(block_is_optimistic(&fixture.store, root)); + } + + #[test] + fn a_gloas_block_is_optimistic_iff_the_payload_it_builds_on_is_not_valid() { + let mut fixture = gloas_fixture(); + let (g1, g1_slot) = fixture.g1; + let (g2, _) = fixture.g2; + let (g3, _) = fixture.g3; + + // g1 builds on its pre-gloas parent, which is not optimistic. + assert!(!block_is_optimistic(&fixture.store, g1)); + // g2 builds on g1's payload, which has no verdict yet. + assert!(block_is_optimistic(&fixture.store, g2)); + // g3 skipped g2's payload, so it too rests on g1's. + assert!(block_is_optimistic(&fixture.store, g3)); + + fixture + .store + .insert_beacon_block_payload_status(g1, g1_slot, PayloadStatusEnum::Valid); + assert!(!block_is_optimistic(&fixture.store, g2)); + assert!(!block_is_optimistic(&fixture.store, g3)); + + // g1's own payload being doubtful does not make g1 optimistic. + fixture + .store + .insert_beacon_block_payload_status(g1, g1_slot, PayloadStatusEnum::Syncing); + assert!(!block_is_optimistic(&fixture.store, g1)); + assert!(block_is_optimistic(&fixture.store, g2)); + } + + #[test] + fn a_gloas_block_on_an_optimistic_pre_gloas_parent_is_optimistic() { + let mut fixture = gloas_fixture(); + let head = fixture.head_root; + fixture.store.insert_beacon_optimistic_root(head, 65); + assert!(block_is_optimistic(&fixture.store, fixture.g1.0)); + } +} diff --git a/docs/rpc.md b/docs/rpc.md index ccb6fff70..212bd87e5 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -227,6 +227,7 @@ surface rather than sitting beside it; a `/lean/v0` path on a beacon node is a |--------|------|----------|-------------| | `GET` | `/eth/v2/beacon/blocks/{block_id}` | JSON or SSZ | `SignedBeaconBlock` at `block_id` | | `GET` | `/eth/v1/beacon/blocks/{block_id}/root` | JSON | That block's root | +| `GET` | `/eth/v1/beacon/execution_payload_envelopes/{block_id}` | JSON or SSZ | Gloas `SignedExecutionPayloadEnvelope` of that block (see below) | | `GET` | `/eth/v1/beacon/headers/{block_id}` | JSON | `SignedBeaconBlockHeader`, plus `canonical` | | `GET` | `/eth/v2/debug/beacon/states/{state_id}` | JSON or SSZ | `BeaconState` at `state_id` | | `GET` | `/eth/v1/beacon/states/{state_id}/finality_checkpoints` | JSON | That state's three checkpoints | @@ -353,6 +354,46 @@ Serving `/eth/v2/debug/beacon/states/finalized` as SSZ is what makes this client checkpoint-syncable from itself: it is the exact path [`checkpoint_sync.rs`](./checkpoint_sync.md) fetches from other clients. +### `GET /eth/v1/beacon/execution_payload_envelopes/{block_id}` + +Gloas moves a block's execution payload into a separate signed envelope. This +serves the one the node has verified, as `{version: "gloas", execution_optimistic, +finalized, data}` JSON or as SSZ on `Accept: application/octet-stream`, with +`Eth-Consensus-Version: gloas` either way. It is also what a checkpoint-syncing +node asks for the payload of its anchor (see +[`checkpoint_sync.md`](./checkpoint_sync.md)). + +| Status | When | +|--------|------| +| `400` | `block_id` is malformed | +| `404` | the block is unknown, or the node holds no envelope for it | + +The second `404` is deliberately one status for several situations: the store +keeps verified envelopes only, so a withheld payload, a payload not yet +received and a pre-gloas block (which has no envelope; the specification's +error list names no other status for it) are indistinguishable here. A caller +must read `404` as "unknown to this node", not as "the payload is empty". + +### `execution_optimistic` under gloas + +Before gloas a block's payload ran inside the block, so the flag is whether the +block is in the store's optimistic set. From gloas on the payload is a separate +object that may arrive late or never, and two rules share the flag: + +- an **envelope** response is optimistic iff that payload's verdict is not + `VALID` (an unrecorded verdict reads as `NOT_VALIDATED`); +- a **block, header, state or finality-checkpoints** response for a gloas block + is optimistic iff the latest FULL payload the block builds on is not `VALID`. + The walk starts at the block's parent: if the block's bid names the parent's + payload as its parent block hash it reads the parent's verdict, and otherwise + it steps back to the parent and repeats. A pre-gloas block on the way reads + the optimistic set. The block's own payload does not count, so a withheld + payload does not make the chain look unverified. + +Pre-gloas responses are unchanged. One helper implements both rules +(`shared/optimistic.rs`) and every response that carries the flag goes through +it. + ### `GET /eth/v1/config/spec` One flat object holding the network's configuration, the compiled preset, and @@ -360,15 +401,21 @@ the specification's constants, as the Beacon API asks. Validator clients depend on it: lighthouse's refuses a beacon node whose `PRESET_BASE` does not match its own, and treats an absent key as a mismatch. -The key set is lighthouse's, less gloas's preset and constant keys -(`beacon::preset` and `beacon::constants` hold them, such as -`DOMAIN_BEACON_BUILDER` and `DOMAIN_PTC_ATTESTER`, but this list does not -report them yet; gloas's schedule, timing and churn keys are reported, since -they are `Config` fields) and three keys the specification does not define -(`GAS_LIMIT_ADJUSTMENT_FACTOR`, `RESP_TIMEOUT`, `TTFB_TIMEOUT`). Domain types -and withdrawal prefixes are `0x`-prefixed hex; `VERSIONED_HASH_VERSION_KZG` is -a decimal, as lighthouse reports it. `GENESIS_TIME` is absent: -`/eth/v1/beacon/genesis` reports it. +The key set is lighthouse's plus gloas's preset and constant keys, read from +`beacon::preset` and `beacon::constants` rather than typed here: `PTC_SIZE`, +`MAX_PAYLOAD_ATTESTATIONS`, `MAX_BUILDER_DEPOSIT_REQUESTS_PER_PAYLOAD`, +`MAX_BUILDER_EXIT_REQUESTS_PER_PAYLOAD`, `MAX_BUILDERS_PER_WITHDRAWALS_SWEEP`, +the four gloas domains (`DOMAIN_BEACON_BUILDER`, `DOMAIN_PTC_ATTESTER`, +`DOMAIN_PROPOSER_PREFERENCES`, `DOMAIN_BUILDER_DEPOSIT`), the builder +withdrawal prefix and request types, and the `BUILDER_INDEX_*`, +`BUILDER_PAYMENT_THRESHOLD_*` and `PAYLOAD_BUILDER_VERSION` constants. Gloas's +schedule, timing and churn keys are `Config` fields, so they were already +reported. Three keys are left out because the specification does not define +them (`GAS_LIMIT_ADJUSTMENT_FACTOR`, `RESP_TIMEOUT`, `TTFB_TIMEOUT`), and so are +gloas's `MAX_*_SIZE` preset keys, which bound gossip message sizes the node does +not run on. Domain types and withdrawal prefixes are `0x`-prefixed hex; +`VERSIONED_HASH_VERSION_KZG` is a decimal, as lighthouse reports it. +`GENESIS_TIME` is absent: `/eth/v1/beacon/genesis` reports it. The configuration keys come from the `Config` the data directory was initialized with. Some of them (the custody and subnet counts, the From 92f39d1ece0895f3a73a89d55bb66de11a2df648 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 04:10:03 -0300 Subject: [PATCH 072/132] feat(beacon): fetch a gloas anchor's envelope during checkpoint sync A node anchored on a gloas block holds the block and state but not the payload its children may build on, and would otherwise wait for a FULL child to make the by-root request fetch it. After the chain actor starts, ask the checkpoint URLs for the anchor's envelope in the background and hand it to the actor through the same entry a gossiped envelope takes, so the bid, column and verification checks all apply. A 404 means the peer does not know the payload, not that it is empty; every failure is logged at info and ignored, since the by-root path recovers the envelope anyway. --- bin/ethlambda/src/checkpoint_sync.rs | 207 ++++++++++++++++++++++++++- bin/ethlambda/src/main.rs | 45 +++++- docs/checkpoint_sync.md | 18 +++ 3 files changed, 268 insertions(+), 2 deletions(-) diff --git a/bin/ethlambda/src/checkpoint_sync.rs b/bin/ethlambda/src/checkpoint_sync.rs index 12af5a821..619536d57 100644 --- a/bin/ethlambda/src/checkpoint_sync.rs +++ b/bin/ethlambda/src/checkpoint_sync.rs @@ -25,7 +25,7 @@ use std::time::Duration; use ethlambda_p2p::beacon::decode; use ethlambda_types::beacon::config::Config; -use ethlambda_types::beacon::containers::{BeaconState, SignedBeaconBlock}; +use ethlambda_types::beacon::containers::{BeaconState, SignedBeaconBlock, gloas}; use ethlambda_types::beacon::preset; use ethlambda_types::block::SignedBlock; use ethlambda_types::genesis::{GenesisMismatch, verify_state_genesis}; @@ -61,6 +61,12 @@ fn beacon_block_path(slot: u64) -> String { format!("/eth/v2/beacon/blocks/{slot}") } +/// Path of the execution-payload-envelope endpoint on a Beacon API server, for +/// the block at `root`. +fn beacon_envelope_path(root: H256) -> String { + format!("/eth/v1/beacon/execution_payload_envelopes/0x{root:x}") +} + /// Maximum attempts to refetch the anchor pair if the state and block roots don't match. const MAX_ANCHOR_FETCH_ATTEMPTS: u32 = 3; @@ -698,6 +704,77 @@ pub async fn fetch_beacon_anchor( Ok((state, block)) } +/// Why a peer did not hand over the anchor's envelope. +/// +/// `Unknown` is not "the payload is empty": a peer answers 404 both for a +/// payload it never saw and one that was withheld, and a checkpoint source +/// cannot tell the two apart for us either. +#[derive(Debug, thiserror::Error)] +enum EnvelopeFetchError { + #[error("the peer does not know the anchor's envelope")] + Unknown, + #[error("request failed: {0}")] + Request(#[from] reqwest::Error), + #[error("SSZ deserialization failed: {0:?}")] + Decode(DecodeError), + #[error("the envelope is for another block")] + WrongBlock, +} + +/// Fetch the execution payload envelope of the block at `anchor_root` from one +/// peer. +async fn fetch_envelope( + client: &Client, + base_url: &str, + anchor_root: H256, +) -> Result { + let url = format!("{base_url}{}", beacon_envelope_path(anchor_root)); + let response = client + .get(&url) + .header("Accept", "application/octet-stream") + .send() + .await?; + if response.status() == reqwest::StatusCode::NOT_FOUND { + return Err(EnvelopeFetchError::Unknown); + } + let bytes = response.error_for_status()?.bytes().await?; + let envelope = gloas::SignedExecutionPayloadEnvelope::from_ssz_bytes(&bytes) + .map_err(EnvelopeFetchError::Decode)?; + if envelope.message.beacon_block_root != anchor_root { + return Err(EnvelopeFetchError::WrongBlock); + } + Ok(envelope) +} + +/// Best-effort fetch of a gloas anchor's execution payload envelope, so the +/// node can serve and build on the anchor's payload without waiting for a FULL +/// child to make the req/resp by-root path fetch it. +/// +/// Tries each URL in order and returns the first envelope that names +/// `anchor_root`. Every failure is logged at `info!` and skipped, including a +/// 404: it means the peer does not know the payload, which is not evidence the +/// payload is empty, so the caller must not conclude that. Verification is the +/// chain actor's job; this only transports. +pub async fn fetch_anchor_envelope( + urls: &[String], + anchor_root: H256, +) -> Option { + let client = build_client() + .inspect_err(|err| info!(%err, "Skipping the anchor envelope fetch")) + .ok()?; + for url in urls { + let base_url = trim_trailing_slash(url); + match fetch_envelope(&client, base_url, anchor_root).await { + Ok(envelope) => { + info!(%url, "Fetched the anchor's execution payload envelope"); + return Some(envelope); + } + Err(err) => info!(%url, %err, "No anchor execution payload envelope from this peer"), + } + } + None +} + /// Try each checkpoint URL in order, then retry the whole round, exactly as /// the lean path does. The two loops are shared rather than duplicated: /// `try_urls_in_order` and the fixed backoff mean the same thing on both @@ -1287,3 +1364,131 @@ mod tests { )); } } + +/// The anchor-envelope fast path, against a one-shot HTTP server. +#[cfg(test)] +mod anchor_envelope_tests { + use super::*; + use libssz::SszEncode as _; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio::net::TcpListener; + + /// Serves `status` and `body` to every request until dropped, and returns + /// the base URL. The request line is captured on the channel. + async fn serve( + status: &'static str, + body: Vec, + ) -> (String, tokio::sync::mpsc::UnboundedReceiver) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + return; + }; + let mut buffer = vec![0u8; 4096]; + let read = socket.read(&mut buffer).await.unwrap_or(0); + let request = String::from_utf8_lossy(&buffer[..read]); + let line = request.lines().next().unwrap_or_default().to_owned(); + let _ = tx.send(line); + let head = format!( + "HTTP/1.1 {status}\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", + body.len() + ); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.write_all(&body).await; + let _ = socket.shutdown().await; + } + }); + (url, rx) + } + + fn envelope_for(root: H256) -> gloas::SignedExecutionPayloadEnvelope { + let payload = gloas::ExecutionPayload { + parent_hash: Default::default(), + fee_recipient: Default::default(), + state_root: Default::default(), + receipts_root: Default::default(), + logs_bloom: vec![0u8; preset::BYTES_PER_LOGS_BLOOM] + .try_into() + .expect("built at exactly BYTES_PER_LOGS_BLOOM"), + prev_randao: Default::default(), + block_number: 7, + gas_limit: 0, + gas_used: 0, + timestamp: 0, + extra_data: Default::default(), + base_fee_per_gas: Default::default(), + block_hash: Default::default(), + transactions: Default::default(), + withdrawals: Default::default(), + blob_gas_used: 0, + excess_blob_gas: 0, + block_access_list: Default::default(), + slot_number: 0, + }; + gloas::SignedExecutionPayloadEnvelope { + message: gloas::ExecutionPayloadEnvelope { + payload, + execution_requests: Default::default(), + builder_index: 3, + beacon_block_root: root, + parent_beacon_block_root: H256::ZERO, + }, + signature: Default::default(), + } + } + + #[tokio::test] + async fn the_envelope_is_fetched_from_the_peers_endpoint() { + let root = H256::from([7u8; 32]); + let envelope = envelope_for(root); + let (url, mut requests) = serve("200 OK", envelope.to_ssz()).await; + + let fetched = fetch_anchor_envelope(&[url], root).await; + + assert_eq!(fetched, Some(envelope)); + let line = requests.recv().await.unwrap(); + assert!( + line.starts_with(&format!( + "GET /eth/v1/beacon/execution_payload_envelopes/0x{root:x} " + )), + "unexpected request: {line}" + ); + } + + #[tokio::test] + async fn a_404_is_unknown_and_falls_through_to_the_next_peer() { + let root = H256::from([7u8; 32]); + let envelope = envelope_for(root); + let (unknown, _) = serve("404 Not Found", Vec::new()).await; + let (knows, _) = serve("200 OK", envelope.to_ssz()).await; + + assert_eq!( + fetch_anchor_envelope(std::slice::from_ref(&unknown), root).await, + None, + "a 404 is not an envelope" + ); + assert_eq!( + fetch_anchor_envelope(&[unknown, knows], root).await, + Some(envelope) + ); + } + + #[tokio::test] + async fn a_failing_or_wrong_envelope_is_ignored() { + let root = H256::from([7u8; 32]); + let (broken, _) = serve("500 Internal Server Error", Vec::new()).await; + let (garbage, _) = serve("200 OK", vec![1, 2, 3]).await; + let (other, _) = serve("200 OK", envelope_for(H256::from([8u8; 32])).to_ssz()).await; + + assert_eq!(fetch_anchor_envelope(&[broken], root).await, None); + assert_eq!(fetch_anchor_envelope(&[garbage], root).await, None); + assert_eq!( + fetch_anchor_envelope(&[other], root).await, + None, + "an envelope for another block must not be handed over" + ); + } +} diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index ec47f71ac..7b3d888d5 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -47,7 +47,8 @@ use ethlambda_crypto::signature::ValidatorSecretKey; use ethlambda_engine::types::ClientVersionV1; use ethlambda_engine::{EngineClient, JwtSecret}; use ethlambda_network_api::{ - InitBlockChain, InitP2P, ToBlockChainToP2PRef, ToP2PToBlockChainRef, ToRpcToP2PRef, + BlockArrival, InitBlockChain, InitP2P, ToBlockChainToP2PRef, ToP2PToBlockChainRef, + ToRpcToP2PRef, }; use ethlambda_p2p::{ LeanWireConfig, P2P, PeerId, SwarmConfig, WireConfig, attestation_subscription_subnets, @@ -204,6 +205,9 @@ enum ChainActor { /// `--execution-endpoint` was not given. engine: Option, safe_slots_to_import_optimistically: u64, + /// The checkpoint-sync URLs to ask for a gloas anchor's execution + /// payload envelope; empty when there is nothing to ask. + anchor_envelope_urls: Vec, }, } @@ -665,6 +669,7 @@ async fn run_node(options: Options) -> eyre::Result<()> { engine, safe_slots_to_import_optimistically: mainnet .safe_slots_to_import_optimistically, + anchor_envelope_urls: clean_checkpoint_urls.clone(), }, } } @@ -774,6 +779,7 @@ async fn run_node(options: Options) -> eyre::Result<()> { let _ = served.inspect_err(|err| error!(%err, "RPC server failed")); }); + let mut anchor_envelope_fetch: Option<(Vec, H256)> = None; let blockchain = match setup.chain { ChainActor::Lean(validator_keys, config) => { BlockChain::spawn(setup.store, validator_keys, config, events) @@ -782,8 +788,12 @@ async fn run_node(options: Options) -> eyre::Result<()> { custody_columns, engine, safe_slots_to_import_optimistically, + anchor_envelope_urls, } => { check_custody_set(&custody_columns)?; + anchor_envelope_fetch = missing_anchor_envelope_root(&setup.store) + .filter(|_| !anchor_envelope_urls.is_empty()) + .map(|root| (anchor_envelope_urls, root)); BlockChain::spawn_beacon( setup.store, sync_status, @@ -795,6 +805,22 @@ async fn run_node(options: Options) -> eyre::Result<()> { } }; + // Off the startup path: the fetch is best-effort and a slow peer must not + // delay the node. The envelope goes through the same chain-actor entry a + // gossiped one does, so the bid check, the column check and the + // verification all apply to it. + if let Some((urls, anchor_root)) = anchor_envelope_fetch { + let chain = blockchain.actor_ref().to_p2p_to_block_chain_ref(); + tokio::spawn(async move { + if let Some(envelope) = checkpoint_sync::fetch_anchor_envelope(&urls, anchor_root).await + { + let _ = chain + .new_execution_payload_envelope(Box::new(envelope), BlockArrival::now()) + .inspect_err(|err| warn!(%err, "Failed to hand over the anchor's envelope")); + } + }); + } + let p2p_ref = p2p.actor_ref(); let p2p_to_block_chain = p2p_ref.to_block_chain_to_p2p_ref(); @@ -824,6 +850,23 @@ async fn run_node(options: Options) -> eyre::Result<()> { Ok(()) } +/// The root of the anchor block when it is a gloas block whose execution +/// payload envelope the store does not hold. +/// +/// The anchor is the finalized block. A node that checkpoint-synced onto a +/// gloas block has the block and state but not the payload its children may +/// build on; asking the checkpoint source for it saves waiting for a FULL +/// child to make the by-root request fetch it. +fn missing_anchor_envelope_root(store: &Store) -> Option { + let root = store.latest_finalized().ok()?.root; + let block = store.get_signed_block(&root).ok()??; + if block.fork_name() != ForkName::Gloas { + return None; + } + let held = store.get_execution_payload_envelope(&root).ok()?.is_some(); + (!held).then_some(root) +} + /// Reject a beacon node with no custody set before it spawns. /// /// `data_availability_for` used to refuse this per block. It cannot any more: diff --git a/docs/checkpoint_sync.md b/docs/checkpoint_sync.md index 2316ebd7a..8706914dc 100644 --- a/docs/checkpoint_sync.md +++ b/docs/checkpoint_sync.md @@ -77,6 +77,24 @@ This is the recommended option for production deployments since it reduces trust 3. **Initialize**: the node stores the anchor block's header, its body (present unless the fetched block's own body happens to be empty, same as any other block), and the full state from the checkpoint. On `node`, persisting the block itself also means it can be served over `BlocksByRoot`; without that, peers requesting the anchor by root would get a synthetic block whose hash differs from `latest_finalized.root` and would score-penalize this node. `beacon`'s req/resp protocol has no `BlocksByRoot` handler yet, so that benefit doesn't apply there today; the block is stored anyway, since the pairing check above needs it. +#### Gloas anchor: the payload envelope (beacon only) + +A gloas block's execution payload is not in the block, so a node anchored on +one holds the block and state but not the payload its children may build on. +Once the chain actor is running, a node that anchored on a gloas block with no +stored envelope asks the checkpoint URLs for it, in order: +`GET /eth/v1/beacon/execution_payload_envelopes/{anchor_root}` as SSZ. A +success is handed to the chain actor as `new_execution_payload_envelope`, the +same entry a gossiped envelope takes, so the bid check, the column check and +the verification all apply. The fetch runs in the background and never delays +startup. + +It is a fast path and nothing depends on it. A `404` means the peer does not +know the payload, **not** that the payload is empty, so the node concludes +nothing from it; any failure (network, decoding, an envelope naming another +block) is logged at `info` and ignored. Without the envelope, the by-root +request that fires when a FULL child arrives recovers it. + ### Failure and success If any step fails (network error, decoding error, verification failure), the node logs the error and exits. There is no automatic retry; restart the node to try again. The database is not modified until verification succeeds, so a failed checkpoint sync leaves the data directory clean. From b5861c1ec3257c57bb90f926bc0d73107cb46988 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 04:18:10 -0300 Subject: [PATCH 073/132] fix(rpc): judge gloas optimism read-only and cautiously The execution_optimistic flag gates signing, so a store read that fails or a link that cannot be worked out must answer optimistic, not the opposite. The helper also called fork choice's ensure_payload_link, which writes to the store and can panic on a failed read; the RPC side now looks the link up or derives it without recording anything. Also: start the anchor-envelope fetch after both actors are wired so the chain actor has its p2p handle when an envelope waits for columns; gate it on gloas or any later fork instead of gloas alone; take the envelope response's slot from the payload instead of decoding the block; report electra's execution request types beside gloas's; list the tokio features the new tests use; and reword receive_envelope's doc, since callers may hand it envelopes nothing has checked. --- bin/ethlambda/Cargo.toml | 2 +- bin/ethlambda/src/main.rs | 141 ++++++++++++++++++++--- crates/blockchain/src/beacon_envelope.rs | 5 +- crates/net/rpc/src/beacon/config.rs | 12 +- crates/net/rpc/src/beacon/envelopes.rs | 11 +- crates/net/rpc/src/lib.rs | 6 +- crates/net/rpc/src/shared/optimistic.rs | 140 +++++++++++++++++++--- 7 files changed, 265 insertions(+), 52 deletions(-) diff --git a/bin/ethlambda/Cargo.toml b/bin/ethlambda/Cargo.toml index 1e239cae3..59b38642d 100644 --- a/bin/ethlambda/Cargo.toml +++ b/bin/ethlambda/Cargo.toml @@ -64,7 +64,7 @@ libc.workspace = true # `test-util` for `#[tokio::test(start_paused = true)]`: the checkpoint-sync # tests would otherwise wait out the real retry backoff. Dev-only, so the # feature never reaches the shipped binary. -tokio = { workspace = true, features = ["test-util"] } +tokio = { workspace = true, features = ["test-util", "net", "io-util", "macros", "rt"] } # Keygen writes a directory of key files and the benchmark corpus writes a # key set, so their tests need a throwaway directory. tempfile = "3" diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index 7b3d888d5..2dfa5206e 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -805,22 +805,6 @@ async fn run_node(options: Options) -> eyre::Result<()> { } }; - // Off the startup path: the fetch is best-effort and a slow peer must not - // delay the node. The envelope goes through the same chain-actor entry a - // gossiped one does, so the bid check, the column check and the - // verification all apply to it. - if let Some((urls, anchor_root)) = anchor_envelope_fetch { - let chain = blockchain.actor_ref().to_p2p_to_block_chain_ref(); - tokio::spawn(async move { - if let Some(envelope) = checkpoint_sync::fetch_anchor_envelope(&urls, anchor_root).await - { - let _ = chain - .new_execution_payload_envelope(Box::new(envelope), BlockArrival::now()) - .inspect_err(|err| warn!(%err, "Failed to hand over the anchor's envelope")); - } - }); - } - let p2p_ref = p2p.actor_ref(); let p2p_to_block_chain = p2p_ref.to_block_chain_to_p2p_ref(); @@ -840,6 +824,23 @@ async fn run_node(options: Options) -> eyre::Result<()> { }) .inspect_err(|err| error!(%err, "Failed to send InitBlockChain — actors not wired"))?; + // After both `Init*` sends, so the actor has its p2p handle when an + // envelope parks waiting for columns. Off the startup path: the fetch is best-effort and a slow peer must not + // delay the node. The envelope goes through the same chain-actor entry a + // gossiped one does, so the bid check, the column check and the + // verification all apply to it. + if let Some((urls, anchor_root)) = anchor_envelope_fetch { + let chain = blockchain.actor_ref().to_p2p_to_block_chain_ref(); + tokio::spawn(async move { + if let Some(envelope) = checkpoint_sync::fetch_anchor_envelope(&urls, anchor_root).await + { + let _ = chain + .new_execution_payload_envelope(Box::new(envelope), BlockArrival::now()) + .inspect_err(|err| warn!(%err, "Failed to hand over the anchor's envelope")); + } + }); + } + wait_for_shutdown(RunningNode { p2p, blockchain, @@ -860,7 +861,9 @@ async fn run_node(options: Options) -> eyre::Result<()> { fn missing_anchor_envelope_root(store: &Store) -> Option { let root = store.latest_finalized().ok()?.root; let block = store.get_signed_block(&root).ok()??; - if block.fork_name() != ForkName::Gloas { + // Gloas or any later fork (lean sorts last, and is not a beacon fork). + let fork = block.fork_name(); + if fork < ForkName::Gloas || fork == ForkName::Lean { return None; } let held = store.get_execution_payload_envelope(&root).ok()?.is_some(); @@ -3129,3 +3132,107 @@ validators: assert_eq!(fallback, mainnet_source.bootnodes()); } } + +#[cfg(test)] +mod anchor_envelope_root_tests { + use super::*; + use ethlambda_storage::backend::InMemoryBackend; + use ethlambda_types::beacon::config::Config; + use ethlambda_types::beacon::containers::gloas; + use ethlambda_types::checkpoint::Checkpoint; + + fn anchored_at(block: SignedBeaconBlock) -> (Store, H256, u64) { + let slot = block.slot(); + let root = block.message_hash_tree_root(); + let mut store = Store::init_beacon( + Arc::new(InMemoryBackend::default()), + 1_606_824_023, + Config::mainnet(), + root, + Checkpoint { root, slot }, + slot, + ); + store.insert_signed_block(root, block).expect("insert"); + (store, root, slot) + } + + fn gloas_block(slot: u64) -> SignedBeaconBlock { + SignedBeaconBlock::Gloas(gloas::SignedBeaconBlock { + message: gloas::BeaconBlock { + slot, + proposer_index: 0, + parent_root: H256::ZERO, + state_root: H256::ZERO, + body: Default::default(), + }, + signature: Default::default(), + }) + } + + #[test] + fn a_gloas_anchor_without_its_envelope_is_asked_about() { + let (store, root, _) = anchored_at(gloas_block(64)); + assert_eq!(missing_anchor_envelope_root(&store), Some(root)); + } + + #[test] + fn a_gloas_anchor_that_holds_its_envelope_is_not() { + let (mut store, root, slot) = anchored_at(gloas_block(64)); + let mut envelope = test_envelope(); + envelope.message.beacon_block_root = root; + store.insert_verified_payload(slot, &envelope); + assert_eq!(missing_anchor_envelope_root(&store), None); + } + + #[test] + fn a_pre_gloas_anchor_has_no_envelope_to_ask_for() { + let block = SignedBeaconBlock::Phase0(phase0::SignedBeaconBlock { + message: phase0::BeaconBlock { + slot: 64, + proposer_index: 0, + parent_root: H256::ZERO, + state_root: H256::ZERO, + body: phase0::BeaconBlockBody::default(), + }, + signature: Default::default(), + }); + let (store, _, _) = anchored_at(block); + assert_eq!(missing_anchor_envelope_root(&store), None); + } + + fn test_envelope() -> gloas::SignedExecutionPayloadEnvelope { + let payload = gloas::ExecutionPayload { + parent_hash: Default::default(), + fee_recipient: Default::default(), + state_root: Default::default(), + receipts_root: Default::default(), + logs_bloom: vec![0u8; ethlambda_types::beacon::preset::BYTES_PER_LOGS_BLOOM] + .try_into() + .expect("built at exactly BYTES_PER_LOGS_BLOOM"), + prev_randao: Default::default(), + block_number: 7, + gas_limit: 0, + gas_used: 0, + timestamp: 0, + extra_data: Default::default(), + base_fee_per_gas: Default::default(), + block_hash: Default::default(), + transactions: Default::default(), + withdrawals: Default::default(), + blob_gas_used: 0, + excess_blob_gas: 0, + block_access_list: Default::default(), + slot_number: 0, + }; + gloas::SignedExecutionPayloadEnvelope { + message: gloas::ExecutionPayloadEnvelope { + payload, + execution_requests: Default::default(), + builder_index: 3, + beacon_block_root: H256::ZERO, + parent_beacon_block_root: H256::ZERO, + }, + signature: Default::default(), + } + } +} diff --git a/crates/blockchain/src/beacon_envelope.rs b/crates/blockchain/src/beacon_envelope.rs index 1dfef2174..605b246e2 100644 --- a/crates/blockchain/src/beacon_envelope.rs +++ b/crates/blockchain/src/beacon_envelope.rs @@ -201,8 +201,9 @@ fn envelope_matches_bid( } impl BlockChainServer { - /// Take an envelope that passed gossip validation: verify it now, or hold - /// it until what it waits on arrives. The caller then settles the queues. + /// Take an envelope: verify it now, or hold it until what it waits on + /// arrives. Callers may hand over envelopes nothing has checked (a + /// checkpoint source's, for one); the actor verifies them itself. The caller then settles the queues. pub(crate) fn receive_envelope( &mut self, envelope: gloas::SignedExecutionPayloadEnvelope, diff --git a/crates/net/rpc/src/beacon/config.rs b/crates/net/rpc/src/beacon/config.rs index 0e2b1d2cf..10e2279f8 100644 --- a/crates/net/rpc/src/beacon/config.rs +++ b/crates/net/rpc/src/beacon/config.rs @@ -169,11 +169,15 @@ fn extra_entries() -> impl Iterator { DOMAIN_BUILDER_DEPOSIT, ); - // One-byte constants, so each goes out as a one-byte array would. - let withdrawal_prefixes = entries!(|prefix: u8| hex_string([prefix]); constants: + // One-byte constants (withdrawal prefixes and execution request types), so + // each goes out as a one-byte array would. + let one_byte_constants = entries!(|prefix: u8| hex_string([prefix]); constants: BLS_WITHDRAWAL_PREFIX, ETH1_ADDRESS_WITHDRAWAL_PREFIX, COMPOUNDING_WITHDRAWAL_PREFIX, + DEPOSIT_REQUEST_TYPE, + WITHDRAWAL_REQUEST_TYPE, + CONSOLIDATION_REQUEST_TYPE, BUILDER_WITHDRAWAL_PREFIX, BUILDER_DEPOSIT_REQUEST_TYPE, BUILDER_EXIT_REQUEST_TYPE, @@ -198,7 +202,7 @@ fn extra_entries() -> impl Iterator { preset .into_iter() .chain(domains) - .chain(withdrawal_prefixes) + .chain(one_byte_constants) .chain(other_constants) } @@ -301,6 +305,8 @@ mod tests { assert_eq!(data["DOMAIN_PTC_ATTESTER"], "0x0c000000"); assert_eq!(data["DOMAIN_PROPOSER_PREFERENCES"], "0x0d000000"); assert_eq!(data["DOMAIN_BUILDER_DEPOSIT"], "0x0e000000"); + assert_eq!(data["DEPOSIT_REQUEST_TYPE"], "0x00"); + assert_eq!(data["CONSOLIDATION_REQUEST_TYPE"], "0x02"); assert_eq!(data["BUILDER_WITHDRAWAL_PREFIX"], "0xb0"); assert_eq!(data["BUILDER_DEPOSIT_REQUEST_TYPE"], "0x03"); assert_eq!(data["BUILDER_EXIT_REQUEST_TYPE"], "0x04"); diff --git a/crates/net/rpc/src/beacon/envelopes.rs b/crates/net/rpc/src/beacon/envelopes.rs index f525f804b..5d077c881 100644 --- a/crates/net/rpc/src/beacon/envelopes.rs +++ b/crates/net/rpc/src/beacon/envelopes.rs @@ -54,11 +54,8 @@ async fn get_envelope( Ok(None) => return not_found.into_response(), Err(_) => return ApiError::Internal("store read failed").into_response(), }; - let slot = match store.get_signed_block(&root) { - Ok(Some(block)) => block.slot(), - Ok(None) => return not_found.into_response(), - Err(_) => return ApiError::Internal("store read failed").into_response(), - }; + // Verified against its block's bid, so the payload's slot is the block's. + let slot = envelope.message.payload.slot_number; let accept = headers.get(header::ACCEPT).and_then(|v| v.to_str().ok()); let response = match Encoding::from_accept(accept) { @@ -110,7 +107,7 @@ mod tests { let (root, slot) = fixture.g1; fixture .store - .insert_verified_payload(slot, &gloas_envelope(root)); + .insert_verified_payload(slot, &gloas_envelope(root, slot)); // No verdict recorded: NOT_VALIDATED, so optimistic. let response = get(fixture.store.clone(), &url(root), None).await; @@ -136,7 +133,7 @@ mod tests { async fn a_stored_envelope_is_served_as_ssz_on_request() { let mut fixture = gloas_fixture(); let (root, slot) = fixture.g1; - let envelope = gloas_envelope(root); + let envelope = gloas_envelope(root, slot); fixture.store.insert_verified_payload(slot, &envelope); let response = get(fixture.store, &url(root), Some("application/octet-stream")).await; diff --git a/crates/net/rpc/src/lib.rs b/crates/net/rpc/src/lib.rs index 439c21392..7958cbc72 100644 --- a/crates/net/rpc/src/lib.rs +++ b/crates/net/rpc/src/lib.rs @@ -476,7 +476,7 @@ pub(crate) mod test_utils { } /// A gloas block whose bid names `parent_block_hash` and `block_hash`. - fn gloas_beacon_block( + pub(crate) fn gloas_beacon_block( slot: u64, parent_root: H256, parent_block_hash: H256, @@ -532,7 +532,7 @@ pub(crate) mod test_utils { } /// A signed envelope for `root` with every other field at its zero value. - pub(crate) fn gloas_envelope(root: H256) -> gloas::SignedExecutionPayloadEnvelope { + pub(crate) fn gloas_envelope(root: H256, slot: u64) -> gloas::SignedExecutionPayloadEnvelope { let payload = gloas::ExecutionPayload { parent_hash: Default::default(), fee_recipient: Default::default(), @@ -554,7 +554,7 @@ pub(crate) mod test_utils { blob_gas_used: 0, excess_blob_gas: 0, block_access_list: Default::default(), - slot_number: 0, + slot_number: slot, }; gloas::SignedExecutionPayloadEnvelope { message: gloas::ExecutionPayloadEnvelope { diff --git a/crates/net/rpc/src/shared/optimistic.rs b/crates/net/rpc/src/shared/optimistic.rs index df1e3b5da..e7b127ad1 100644 --- a/crates/net/rpc/src/shared/optimistic.rs +++ b/crates/net/rpc/src/shared/optimistic.rs @@ -12,10 +12,85 @@ //! The block's own payload does not count: the block itself was imported //! without it, and a withheld payload must not make the chain look //! unverified. +//! +//! Everything here only reads the store, and an answer that cannot be worked +//! out is optimistic: this flag gates signing, so an unknown gets the cautious +//! reading. The payload links are therefore looked up or derived here instead +//! of through fork choice, whose `ensure_payload_link` records what it derives +//! and panics on a failed read. -use ethlambda_state_transition::beacon::fork_choice::{self, PayloadStatus, PayloadStatusEnum}; use ethlambda_storage::Store; -use ethlambda_types::primitives::H256; +use ethlambda_types::{ + beacon::{ + containers::SignedBeaconBlock, + fork_choice::{BlockPayloadLink, PayloadStatus, PayloadStatusEnum}, + }, + primitives::H256, +}; + +/// `root`'s payload link: the one fork choice recorded, or one derived from the +/// stored blocks without recording it. `None` when it cannot be worked out. +/// +/// The derivation is `derive_payload_link`'s: a gloas block's parent status is +/// FULL when its bid names the parent's block hash (or the parent is +/// pre-gloas), and unknown when the parent is not stored, which is the anchor. +fn payload_link(store: &Store, root: H256) -> Option { + if let Some(link) = store.payload_link(&root) { + return Some(link); + } + match store.get_signed_block(&root).ok()?? { + SignedBeaconBlock::Phase0(_) + | SignedBeaconBlock::Altair(_) + | SignedBeaconBlock::Bellatrix(_) + | SignedBeaconBlock::Capella(_) + | SignedBeaconBlock::Deneb(_) + | SignedBeaconBlock::Electra(_) + | SignedBeaconBlock::Fulu(_) => Some(BlockPayloadLink::PreGloas), + SignedBeaconBlock::Gloas(block) => { + let parent_status = match store.get_signed_block(&block.message.parent_root).ok()? { + None => None, + Some(SignedBeaconBlock::Gloas(parent)) => { + let wanted = block + .message + .body + .signed_execution_payload_bid + .message + .parent_block_hash; + let held = parent + .message + .body + .signed_execution_payload_bid + .message + .block_hash; + Some(if wanted == held { + PayloadStatus::Full + } else { + PayloadStatus::Empty + }) + } + Some(_) => Some(PayloadStatus::Full), + }; + Some(BlockPayloadLink::Gloas { parent_status }) + } + // A lean block is never in a beacon store. + SignedBeaconBlock::Lean(_) => None, + } +} + +/// Whether the payload of the block at `root` is not (known to be) `VALID`. +/// +/// A pre-gloas block's payload is its own, so it reads the optimistic set; a +/// gloas root reads its recorded verdict, where none recorded is +/// `NOT_VALIDATED`. Unknown links are not valid. +fn payload_is_not_valid(store: &Store, root: H256) -> bool { + match payload_link(store, root) { + Some(BlockPayloadLink::PreGloas) => store.is_beacon_optimistic(root), + Some(BlockPayloadLink::Gloas { .. }) => { + store.beacon_block_payload_status(root) != PayloadStatusEnum::Valid + } + None => true, + } +} /// Whether the payload of the block at `root` is not yet vouched for by the /// execution layer. @@ -23,7 +98,7 @@ use ethlambda_types::primitives::H256; /// An unrecorded gloas verdict reads as `NOT_VALIDATED`, so an envelope the /// store holds but has no verdict for is optimistic. pub(crate) fn envelope_is_optimistic(store: &Store, root: H256) -> bool { - fork_choice::block_payload_status(store, root) != PayloadStatusEnum::Valid + payload_is_not_valid(store, root) } /// Whether the block at `root` rests on a payload that is not `VALID`. @@ -33,32 +108,23 @@ pub(crate) fn envelope_is_optimistic(store: &Store, root: H256) -> bool { /// builds on, then reads that payload's verdict. A pre-gloas block is its own /// payload, so it reads the optimistic set as before. A chain that runs out /// (an anchor whose parent is below the retained window) has nothing left to -/// doubt, so it is not optimistic. -/// -/// A link that cannot be derived is reported optimistic: this flag gates -/// signing, so the answer for an unknown is the cautious one. +/// doubt, so it is not optimistic. A block or link that cannot be read is +/// optimistic. pub(crate) fn block_is_optimistic(store: &Store, root: H256) -> bool { let mut current = root; loop { - if fork_choice::ensure_payload_link(store, current).is_err() { + let Some(link) = payload_link(store, current) else { return true; - } - let Some(link) = store.payload_link(¤t) else { - // The block is not in the store, so there is no payload to doubt. - return false; }; if !link.is_gloas() { return store.is_beacon_optimistic(current); } - let Ok(Some(block)) = store.get_signed_block(¤t) else { - return false; + let parent = match store.get_signed_block(¤t) { + Ok(Some(block)) => block.parent_root(), + Ok(None) | Err(_) => return true, }; - let parent = block.parent_root(); match link.parent_status() { - Some(PayloadStatus::Full) => { - return fork_choice::block_payload_status(store, parent) - != PayloadStatusEnum::Valid; - } + Some(PayloadStatus::Full) => return payload_is_not_valid(store, parent), Some(PayloadStatus::Empty) => current = parent, // A pending parent status never comes out of a block's own bid. Some(PayloadStatus::Pending) | None => return false, @@ -118,4 +184,40 @@ mod tests { fixture.store.insert_beacon_optimistic_root(head, 65); assert!(block_is_optimistic(&fixture.store, fixture.g1.0)); } + + #[test] + fn a_block_the_store_cannot_produce_is_optimistic() { + let fixture = gloas_fixture(); + let unknown = H256::from([0xee; 32]); + assert!(block_is_optimistic(&fixture.store, unknown)); + assert!(envelope_is_optimistic(&fixture.store, unknown)); + } + + #[test] + fn a_gloas_anchor_with_an_unknown_parent_is_not_optimistic() { + let mut fixture = gloas_fixture(); + let block = crate::test_utils::gloas_beacon_block( + 500, + H256::from([0xab; 32]), + H256::ZERO, + H256::ZERO, + ); + let root = block.message_hash_tree_root(); + fixture + .store + .insert_signed_block(root, block) + .expect("insert"); + + assert!(!block_is_optimistic(&fixture.store, root)); + } + + #[test] + fn judging_a_block_records_nothing_in_the_store() { + let fixture = gloas_fixture(); + let (g2, _) = fixture.g2; + // Drop whatever import recorded, so the helper has to derive. + let before = fixture.store.payload_link(&g2); + block_is_optimistic(&fixture.store, g2); + assert_eq!(fixture.store.payload_link(&g2), before); + } } From 39bcb7c21222f91687f10f0f2e5adffc6aed0398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 04:31:26 -0300 Subject: [PATCH 074/132] fix(beacon): forward envelopes whose block state is not cached yet A gloas block held on its parent's unverified envelope asks p2p for that envelope by root. It arrives ~0.1 s later, but the envelope checks dropped it as state_unavailable: the parent was imported moments earlier, so its post-state is on disk but not in the cache p2p reads. The per-slot re-ask fetched it again ~12 s later, so sync advanced one block per slot and never closed the gap. Gossip envelopes were dropped (not forwarded) the same way. Return Queue(StateNotCached) instead. The chain actor fully verifies every envelope it receives, so forwarding without propagating is safe. --- .../src/beacon/gossip/envelope.rs | 68 +++++++++++++++++-- .../state_transition/src/beacon/gossip/mod.rs | 6 ++ crates/net/p2p/src/beacon/envelope_checks.rs | 1 + docs/metrics.md | 5 +- 4 files changed, 75 insertions(+), 5 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/gossip/envelope.rs b/crates/blockchain/state_transition/src/beacon/gossip/envelope.rs index 6557301cc..6fa257b64 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/envelope.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/envelope.rs @@ -13,7 +13,9 @@ //! - A block seen without a post-state is [`Outcome::Queue`] where the //! specification rejects it, the same deviation every other topic makes until //! a bad-block cache exists. A block never seen is `Queue` too (the -//! specification's "MAY be queued"). +//! specification's "MAY be queued"), and so is a block whose post-state +//! exists but is not cached, since the chain actor verifies the signature +//! itself. //! - The block-independent limits (withdrawal and request counts) run in //! [`cheap_checks`], so a message that no block could make valid is never //! queued. @@ -29,7 +31,8 @@ use lru::LruCache; use super::column::judgeable_block_gloas; use super::{ - IgnoreReason, Outcome, RejectReason, execution_requests_within_limits, finalized_start_slot, + IgnoreReason, Outcome, QueueReason, RejectReason, execution_requests_within_limits, + finalized_start_slot, }; use crate::beacon::containers::gloas::BuilderIndex; use crate::beacon::containers::{SignedBeaconBlock, gloas}; @@ -101,6 +104,17 @@ pub fn cheap_checks( /// The rules that need the envelope's block and its post-state, then the /// signature. Runs on a blocking thread. +/// +/// A block that is known and has a post-state, but whose state is not in the +/// cache this reads, is [`Outcome::Queue`] ([`QueueReason::StateNotCached`]), +/// not an ignore. The state is never rebuilt here, since a miss would stall +/// the verdict gossipsub waits on, and a block imported moments ago is +/// exactly when it is missing. The chain actor fully verifies every envelope +/// it receives (bid consistency, the builder's signature against the block's +/// post-state before parking it, then `verify_execution_payload_envelope`), +/// so forwarding it without propagating is safe, whereas dropping it stalls +/// sync: a fetched envelope for a block the actor is holding would be lost +/// until the next per-slot re-ask. pub fn stateful_checks(store: &Store, envelope: &gloas::SignedExecutionPayloadEnvelope) -> Outcome { let message = &envelope.message; let payload = &message.payload; @@ -146,7 +160,7 @@ pub fn stateful_checks(store: &Store, envelope: &gloas::SignedExecutionPayloadEn // post-state, never rebuilt: a miss would stall the verdict gossipsub // waits on. let Some(state) = store.cached_state(CacheKey::BlockState(message.beacon_block_root)) else { - return Outcome::Ignore(IgnoreReason::StateUnavailable); + return Outcome::Queue(QueueReason::StateNotCached); }; // An error (a builder the state has no entry for) is a signature that // cannot be valid. @@ -176,9 +190,10 @@ pub fn validate( #[cfg(test)] mod tests { + use std::sync::Arc; + use super::*; use crate::beacon::containers::electra; - use crate::beacon::gossip::QueueReason; use crate::beacon::gossip::test_support::{fulu_parent, seen_envelopes, store}; use crate::beacon::primitives::Slot; @@ -298,6 +313,51 @@ mod tests { ); } + /// A known gloas block with a post-state on disk, but with that state + /// evicted from the cache the checks read, must be forwarded (queued), not + /// dropped: the observed stall was a fetched envelope lost this way. + #[test] + fn an_envelope_whose_block_state_is_not_cached_is_queued_not_ignored() { + let envelope = envelope(5, 0); + let mut block = gloas::SignedBeaconBlock::default(); + block.message.slot = 5; + block + .message + .body + .signed_execution_payload_bid + .message + .execution_requests_root = envelope.message.execution_requests.hash_tree_root(); + let mut store = store(0); + store + .insert_pending_block(BLOCK_ROOT, SignedBeaconBlock::Gloas(block)) + .expect("insert the block"); + let state = fulu_parent(3); + store + .insert_state(BLOCK_ROOT, state.clone()) + .expect("insert the state"); + // Churn the cache past its capacity with other keys until the block's + // state is evicted; the pending-state buffer still answers `has_state`. + let filler = Arc::new(state); + for epoch in 0..1024 { + store.cache_state( + CacheKey::CheckpointState { + epoch, + root: Root::ZERO, + }, + filler.clone(), + ); + } + assert!( + store + .cached_state(CacheKey::BlockState(BLOCK_ROOT)) + .is_none() + ); + assert_eq!( + validate(&seen_envelopes(), &store, &envelope), + Outcome::Queue(QueueReason::StateNotCached) + ); + } + #[test] fn an_envelope_naming_a_non_gloas_block_is_rejected() { let mut store = store(0); diff --git a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs index 1a25b5756..945255d5f 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs @@ -84,6 +84,11 @@ pub enum QueueReason { BlockUnknown, /// A gloas sidecar names a block that is stored but has no post-state yet. BlockNotReady, + /// A gloas envelope names a block whose post-state is stored but not in + /// the cache the gossip checks read, as when the block was imported + /// moments ago. See `envelope::stateful_checks` for why this is not an + /// ignore. + StateNotCached, } impl QueueReason { @@ -94,6 +99,7 @@ impl QueueReason { Self::ShufflingUnavailable => "shuffling_unavailable", Self::BlockUnknown => "block_unknown", Self::BlockNotReady => "block_not_ready", + Self::StateNotCached => "state_not_cached", } } } diff --git a/crates/net/p2p/src/beacon/envelope_checks.rs b/crates/net/p2p/src/beacon/envelope_checks.rs index 3f3a5c509..7b3b7d71c 100644 --- a/crates/net/p2p/src/beacon/envelope_checks.rs +++ b/crates/net/p2p/src/beacon/envelope_checks.rs @@ -239,6 +239,7 @@ pub(crate) mod tests { fn only_accept_and_queue_reach_the_chain_actor() { assert!(forwards(&Outcome::Accept)); assert!(forwards(&Outcome::Queue(QueueReason::BlockUnknown))); + assert!(forwards(&Outcome::Queue(QueueReason::StateNotCached))); assert!(!forwards(&Outcome::Ignore(IgnoreReason::Finalized))); assert!(!forwards(&Outcome::Reject(RejectReason::BadSignature))); } diff --git a/docs/metrics.md b/docs/metrics.md index fcd7c9315..64644d7c1 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -367,7 +367,10 @@ label `data_column_sidecar` and every `beacon_attestation_{subnet_id}` sharing `payload_attestation_message` are labelled by their own name. `queue` means IGNORE to gossipsub while the chain actor still receives the object and parks it; of the gloas topics only `execution_payload` answers it, for an envelope -whose block is not known yet. The aggregate, attestation and +whose block is not known yet (`block_unknown`), has no post-state yet +(`block_not_ready`), or has one that is not in the cache the checks read +(`state_not_cached`, as when the block was imported moments ago; the actor +verifies the signature itself, so the envelope is forwarded rather than dropped). The aggregate, attestation and `payload_attestation_message` topics never answer `queue`, since the vote block's post-state is either cached or it is not (`IgnoreReason::UnknownBlock`/`StateUnavailable`), with nothing to hold the From d5fa1fedae31e8550c7b8d07310842b130869111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 04:12:37 -0300 Subject: [PATCH 075/132] feat(beacon): validate gloas payloads with newPayloadV5 and forkchoiceUpdated V4 A gloas payload arrives in an envelope, so the execution engine had no say over it: with an engine configured the status stayed absent. Ask newPayloadV5 before applying an envelope (after checking the builder signature, so an unsigned copy cannot earn an INVALID for the honest payload). VALID and SYNCING apply it and record Valid or Syncing; INVALID never applies it, records Invalid, drops children held for the payload and refuses later blocks that name it as their FULL parent. No verdict leaves it unapplied and refetches, as for a block. forkchoiceUpdated now names gloas hashes: the head node picks the payload (FULL: bid.block_hash, EMPTY: bid.parent_block_hash), finalized and safe use the checkpoint bids parent hash, and the call is V4 with the custody columns from a gloas head on. A VALID answer promotes optimistic payloads along the head chain; an INVALID one condemns from the first payload after latestValidHash, removing its FULL node and the children built on it while the block and its EMPTY branch stay. --- crates/blockchain/Cargo.toml | 2 + crates/blockchain/src/beacon_envelope.rs | 687 +++++++++++++++++++++-- crates/blockchain/src/beacon_payloads.rs | 354 ++++++++++++ crates/blockchain/src/lib.rs | 85 +-- crates/storage/src/store.rs | 51 ++ 5 files changed, 1110 insertions(+), 69 deletions(-) create mode 100644 crates/blockchain/src/beacon_payloads.rs diff --git a/crates/blockchain/Cargo.toml b/crates/blockchain/Cargo.toml index 3f9ade160..062e27ed7 100644 --- a/crates/blockchain/Cargo.toml +++ b/crates/blockchain/Cargo.toml @@ -38,6 +38,8 @@ hex.workspace = true hex = { workspace = true } libssz.workspace = true libssz-types.workspace = true +# The mock execution client the payload-validity tests talk to over a socket. +tokio = { workspace = true, features = ["io-util", "net", "sync"] } datatest-stable = "0.3.3" snap = "1.1.1" diff --git a/crates/blockchain/src/beacon_envelope.rs b/crates/blockchain/src/beacon_envelope.rs index 605b246e2..2d442b950 100644 --- a/crates/blockchain/src/beacon_envelope.rs +++ b/crates/blockchain/src/beacon_envelope.rs @@ -42,9 +42,12 @@ use std::collections::{HashMap, VecDeque}; +use ethlambda_engine::EngineError; use ethlambda_network_api::{BlockSource, FetchRequest}; use ethlambda_state_transition::beacon::fork::ForkName; -use ethlambda_state_transition::beacon::fork_choice::{self, PayloadStatus, PayloadStatusEnum}; +use ethlambda_state_transition::beacon::fork_choice::{ + self, PayloadStatus, PayloadStatusEnum, PayloadValidity, +}; use ethlambda_state_transition::beacon::stf::ExecutionEngine; use ethlambda_state_transition::beacon::stf::gloas as stf_gloas; use ethlambda_types::ShortRoot; @@ -52,6 +55,7 @@ use ethlambda_types::beacon::containers::{DataColumnSidecar, SignedBeaconBlock, use ethlambda_types::primitives::{H256, HashTreeRoot as _}; use tracing::{debug, error, info, trace, warn}; +use crate::beacon_engine; use crate::import_timing::ImportTimings; use crate::{BlockChainServer, custody_columns_present, metrics}; @@ -203,21 +207,34 @@ fn envelope_matches_bid( impl BlockChainServer { /// Take an envelope: verify it now, or hold it until what it waits on /// arrives. Callers may hand over envelopes nothing has checked (a - /// checkpoint source's, for one); the actor verifies them itself. The caller then settles the queues. - pub(crate) fn receive_envelope( + /// checkpoint source's, for one); the actor verifies them itself. The + /// caller then settles the queues. + pub(crate) async fn receive_envelope( &mut self, envelope: gloas::SignedExecutionPayloadEnvelope, event_ms: u64, ) { let config = self.store.config(); let arrival_slot = fork_choice::get_current_slot(&self.store, &config); - self.apply_or_hold_envelope(HeldEnvelope::new(envelope, arrival_slot, event_ms)); + self.apply_or_hold_envelope(HeldEnvelope::new(envelope, arrival_slot, event_ms)) + .await; } /// Verify `held` against its block, or put it back in the queue for what it /// is waiting on. `true` when the envelope was verified and recorded. - fn apply_or_hold_envelope(&mut self, held: HeldEnvelope) -> bool { + async fn apply_or_hold_envelope(&mut self, held: HeldEnvelope) -> bool { let root = held.envelope.message.beacon_block_root; + if self + .store + .beacon_block_payload_status(root) + .is_invalidated() + { + trace!( + block_root = %ShortRoot(&root.0), + "Dropping an envelope whose payload the execution client found invalid" + ); + return false; + } if self.store.has_verified_payload(&root) { trace!( block_root = %ShortRoot(&root.0), @@ -322,12 +339,78 @@ impl BlockChainServer { } }; + // The execution client's verdict comes before the envelope is applied, + // as `verify_execution_payload_envelope` orders it: its part is the + // last check, and an `INVALID` payload never becomes a FULL node. + // Without a client the follower trusts the payload, as it already does + // for every pre-gloas block it imports (`NotRequired`). + let status = match self.engine.clone() { + None => PayloadStatusEnum::Valid, + Some(client) => { + // The client is asked only about an envelope the builder + // signed: an unsigned copy carrying the honest block hash but a + // different body would otherwise earn an `INVALID` verdict + // that lands on the honest payload. + if !self.envelope_signature_holds(root, &held.envelope) { + warn!( + %slot, + block_root = %ShortRoot(&root.0), + "Dropping an envelope with an invalid signature" + ); + self.refetch_envelope_if_children_wait(root); + return false; + } + match beacon_engine::ask_envelope(&client, &block, &held.envelope).await { + Ok(PayloadValidity::Validated | PayloadValidity::NotRequired) => { + PayloadStatusEnum::Valid + } + Ok(PayloadValidity::Optimistic) => PayloadStatusEnum::Syncing, + Ok(PayloadValidity::Invalidated { latest_valid_hash }) => { + warn!( + %slot, + block_root = %ShortRoot(&root.0), + "Execution client found an envelope's payload invalid; not applying it" + ); + self.condemn_payload_chain(root, latest_valid_hash); + // Children held for this payload are gone, and the + // head may have named a branch the verdict removed. + self.envelopes.verified.push_back(root); + return false; + } + Err(EngineError::EnvelopeMismatch(reason)) => { + warn!( + %slot, + block_root = %ShortRoot(&root.0), + reason, + "Dropping an envelope that does not match its block" + ); + self.refetch_envelope_if_children_wait(root); + return false; + } + // No answer after the whole retry ladder. As for a block + // (`optimistic-sync.md`), the payload is not applied; the + // envelope is fetched again instead of held, so a client + // that comes back is asked afresh. + Err(err) => { + warn!( + %slot, + block_root = %ShortRoot(&root.0), + %err, + "No verdict from the execution client; not applying the envelope" + ); + metrics::inc_engine_no_verdict(); + self.request_missing_envelope(root); + return false; + } + } + } + }; + self.advance_beacon_clock_to(held.event_ms); let config = self.store.config(); - // Without an execution client the follower trusts the payload, as it - // already does for every pre-gloas block it imports (the engine's - // verdict is `NotRequired` there), so the engine's part of - // `verify_execution_payload_envelope` is answered "valid". + // The engine's part of `verify_execution_payload_envelope` is answered + // above; `VALID` and `SYNCING` both let the payload in (the CL has + // verified it; the EL has not finished). let verdict = fork_choice::on_execution_payload_envelope( &mut self.store, &held.envelope, @@ -340,18 +423,17 @@ impl BlockChainServer { info!( %slot, block_root = %ShortRoot(&root.0), + ?status, "Execution payload envelope verified" ); - // With a client configured the verdict is the client's to give - // (a later change asks it); recording VALID without asking - // would claim a validation that never happened. - if self.engine.is_none() { - self.store.insert_beacon_block_payload_status( - root, - slot, - PayloadStatusEnum::Valid, - ); - } + self.store + .insert_beacon_block_payload_status(root, slot, status); + // Lets `latestValidHash` find this FULL node. + self.store.insert_beacon_el_block_hash( + root, + slot, + held.envelope.message.payload.block_hash, + ); self.envelopes.verified.push_back(root); true } @@ -504,6 +586,10 @@ impl BlockChainServer { if !self.envelopes.awaiting_block.contains_key(&parent_root) && !self.envelopes.awaiting_columns.contains_key(&parent_root) && !self.store.has_verified_payload(&parent_root) + && !self + .store + .beacon_block_payload_status(parent_root) + .is_invalidated() { self.request_missing_envelope(parent_root); } @@ -586,7 +672,7 @@ impl BlockChainServer { // payload, so a second bid-consistent candidate differs only // in a signature, which a parked one has had checked. for held in self.take_ready_envelopes(root) { - if self.apply_or_hold_envelope(held) { + if self.apply_or_hold_envelope(held).await { break; } } @@ -629,6 +715,23 @@ impl BlockChainServer { let block_root = block.message_hash_tree_root(); let parent_root = block.parent_root(); + // The parent's payload was found invalid, so this block builds on a + // FULL node that will never exist, and neither will its descendants. + if self + .store + .beacon_block_payload_status(parent_root) + .is_invalidated() + { + warn!( + %slot, + block_root = %ShortRoot(&block_root.0), + parent_root = %ShortRoot(&parent_root.0), + "Dropping a block: it builds on a payload the execution client found invalid" + ); + self.discard_pending_subtree(block_root); + return; + } + let held_for_parent = self .envelopes .blocks_awaiting_parent_payload @@ -677,7 +780,7 @@ impl BlockChainServer { /// Ask p2p for `root`'s envelope by root. The request carries nothing but /// the flag, since an envelope names its block and nothing else. - fn request_missing_envelope(&self, root: H256) { + pub(crate) fn request_missing_envelope(&self, root: H256) { if let Some(ref p2p) = self.p2p { let _ = p2p .fetch_block(FetchRequest { @@ -929,7 +1032,9 @@ mod tests { let case = Case::new("get_head", TIEBREAK); let mut server = case.server(12); - server.receive_envelope(case.envelope(PARENT_ENVELOPE), 0); + server + .receive_envelope(case.envelope(PARENT_ENVELOPE), 0) + .await; server.settle_envelopes().await; assert!(server.envelopes.awaiting_block.contains_key(&root(PARENT))); assert!(!server.store.has_verified_payload(&root(PARENT))); @@ -988,7 +1093,9 @@ mod tests { assert!(server.held_timings.contains_key(&root(FULL_CHILD))); assert_eq!(head(&server).0, root(FULL_PARENT)); - server.receive_envelope(valid_envelope_for_full_parent(), 0); + server + .receive_envelope(valid_envelope_for_full_parent(), 0) + .await; server.settle_envelopes().await; assert!(server.store.has_state(&root(FULL_CHILD)).unwrap()); @@ -1020,7 +1127,9 @@ mod tests { .await; assert!(server.envelopes.is_block_held(&root(child))); - server.receive_envelope(case.envelope(bad_envelope), 0); + server + .receive_envelope(case.envelope(bad_envelope), 0) + .await; server.settle_envelopes().await; assert!(!server.store.has_verified_payload(&root(parent))); @@ -1041,10 +1150,14 @@ mod tests { server .on_block(case.block(PARENT), ImportTimings::default()) .await; - server.receive_envelope(case.envelope(PARENT_ENVELOPE), 0); + server + .receive_envelope(case.envelope(PARENT_ENVELOPE), 0) + .await; assert!(server.store.has_verified_payload(&root(PARENT))); - server.receive_envelope(case.envelope(PARENT_ENVELOPE), 0); + server + .receive_envelope(case.envelope(PARENT_ENVELOPE), 0) + .await; assert!(server.envelopes.awaiting_block.is_empty()); assert!(server.envelopes.awaiting_columns.is_empty()); @@ -1156,10 +1269,10 @@ mod tests { }; for byte in 1..=MAX_ENVELOPES_AWAITING_BLOCK_PER_SLOT as u8 + 2 { - server.receive_envelope(with_root(byte), 0); + server.receive_envelope(with_root(byte), 0).await; } // A second delivery of a held root changes nothing. - server.receive_envelope(with_root(1), 0); + server.receive_envelope(with_root(1), 0).await; assert_eq!( server.envelopes.awaiting_block.len(), @@ -1271,12 +1384,12 @@ mod tests { // The forgeries arrive first and fill what the root may hold, bar one. for gas_limit in 1..MAX_ENVELOPES_PER_UNMATCHED_ROOT as u64 { - server.receive_envelope(forged(gas_limit), 0); + server.receive_envelope(forged(gas_limit), 0).await; } - server.receive_envelope(honest.clone(), 0); + server.receive_envelope(honest.clone(), 0).await; // A re-delivery of the honest one and a surplus forgery add nothing. - server.receive_envelope(honest.clone(), 0); - server.receive_envelope(forged(99), 0); + server.receive_envelope(honest.clone(), 0).await; + server.receive_envelope(forged(99), 0).await; assert_eq!( server.envelopes.awaiting_block[&root(PARENT)].len(), MAX_ENVELOPES_PER_UNMATCHED_ROOT @@ -1575,4 +1688,514 @@ mod tests { .contains(&Some(true)) ); } + + // --------------------------------------------------------------------- + // The execution client's part: newPayloadV5 on envelope import and + // forkchoiceUpdated for a gloas head. + // --------------------------------------------------------------------- + + use ethlambda_engine::{EngineClient, JwtSecret}; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio::net::TcpListener; + + /// What the mock execution client answers and has been asked. + #[derive(Default)] + struct MockState { + /// The `payloadStatus` object `engine_newPayloadV5` answers with. + new_payload: String, + /// The `payloadStatus` object `engine_forkchoiceUpdated*` answers with. + forkchoice: String, + /// Request bodies, in arrival order. + requests: Vec, + } + + fn status_json(status: &str, latest_valid_hash: Option) -> String { + let hash = latest_valid_hash.map_or("null".to_string(), |hash| format!("\"{hash:?}\"")); + format!(r#"{{"status":"{status}","latestValidHash":{hash},"validationError":null}}"#) + } + + /// A hand-rolled JSON-RPC server answering by method name, like + /// `ethlambda-engine`'s wire smoke tests but for many requests. + async fn mock_engine( + new_payload: String, + forkchoice: String, + ) -> (EngineClient, Arc>) { + let state = Arc::new(std::sync::Mutex::new(MockState { + new_payload, + forkchoice, + requests: Vec::new(), + })); + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("a loopback port"); + let addr = listener.local_addr().expect("bound address"); + let shared = Arc::clone(&state); + tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + return; + }; + let shared = Arc::clone(&shared); + tokio::spawn(async move { + let mut pending: Vec = Vec::new(); + loop { + // One request: headers up to the blank line, then + // `Content-Length` bytes of body. + let (head_end, body_len) = loop { + if let Some(end) = + pending.windows(4).position(|window| window == b"\r\n\r\n") + { + let head = String::from_utf8_lossy(&pending[..end]).to_lowercase(); + let len = head + .lines() + .find_map(|line| line.strip_prefix("content-length:")) + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0); + break (end + 4, len); + } + let mut chunk = [0u8; 16 * 1024]; + match socket.read(&mut chunk).await { + Ok(0) | Err(_) => return, + Ok(read) => pending.extend_from_slice(&chunk[..read]), + } + }; + while pending.len() < head_end + body_len { + let mut chunk = [0u8; 16 * 1024]; + match socket.read(&mut chunk).await { + Ok(0) | Err(_) => return, + Ok(read) => pending.extend_from_slice(&chunk[..read]), + } + } + let body = String::from_utf8_lossy(&pending[head_end..head_end + body_len]) + .to_string(); + pending.drain(..head_end + body_len); + + let result = { + let mut state = shared.lock().unwrap(); + state.requests.push(body.clone()); + if body.contains("engine_newPayloadV5") { + state.new_payload.clone() + } else { + state.forkchoice.clone() + } + }; + let result = if body.contains("engine_newPayloadV5") { + result + } else { + format!(r#"{{"payloadStatus":{result},"payloadId":null}}"#) + }; + let payload = format!(r#"{{"jsonrpc":"2.0","id":1,"result":{result}}}"#); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{payload}", + payload.len() + ); + if socket.write_all(response.as_bytes()).await.is_err() { + return; + } + } + }); + } + }); + let client = EngineClient::new(format!("http://{addr}"), JwtSecret::new([0x0f; 32])) + .expect("a client"); + (client, state) + } + + fn methods(state: &Arc>) -> Vec { + state + .lock() + .unwrap() + .requests + .iter() + .map(|body| { + let rest = body.split("\"method\":\"").nth(1).expect("a method"); + rest.split('"').next().expect("closing quote").to_string() + }) + .collect() + } + + /// The `on_execution_payload_envelope` chain: a parent (FULL once its + /// envelope is applied) and a child that builds on the FULL parent, held + /// until the envelope verifies. + async fn server_with_held_child(client: EngineClient) -> BlockChainServer { + let case = Case::new("on_execution_payload_envelope", FULL_CHILD_CASE); + let mut server = case.server(12); + server.engine = Some(client); + server + .on_block(case.block(FULL_PARENT), ImportTimings::default()) + .await; + fork_choice::on_tick(&mut server.store, 24, &fixture_config()); + server + .on_block(case.block(FULL_CHILD), ImportTimings::default()) + .await; + assert!(server.envelopes.is_block_held(&root(FULL_CHILD))); + server + } + + fn payload_hash(envelope: &gloas::SignedExecutionPayloadEnvelope) -> H256 { + envelope.message.payload.block_hash + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn a_valid_verdict_applies_the_envelope_and_records_valid() { + let (client, mock) = + mock_engine(status_json("VALID", None), status_json("SYNCING", None)).await; + let mut server = server_with_held_child(client).await; + let envelope = valid_envelope_for_full_parent(); + + server.receive_envelope(envelope.clone(), 0).await; + server.settle_envelopes().await; + + assert!(server.store.has_verified_payload(&root(FULL_PARENT))); + assert_eq!( + server.store.beacon_block_payload_status(root(FULL_PARENT)), + PayloadStatusEnum::Valid + ); + assert_eq!( + server.store.beacon_el_block_hash(root(FULL_PARENT)), + Some(payload_hash(&envelope)) + ); + assert!(server.store.has_state(&root(FULL_CHILD)).unwrap()); + let methods = methods(&mock); + assert_eq!( + methods + .iter() + .filter(|m| *m == "engine_newPayloadV5") + .count(), + 1 + ); + // The head is gloas, so forkchoiceUpdated is V4. + assert!(methods.iter().any(|m| m == "engine_forkchoiceUpdatedV4")); + assert!(!methods.iter().any(|m| m == "engine_forkchoiceUpdatedV3")); + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn a_syncing_verdict_applies_the_envelope_as_not_validated() { + let (client, _mock) = + mock_engine(status_json("SYNCING", None), status_json("SYNCING", None)).await; + let mut server = server_with_held_child(client).await; + + server + .receive_envelope(valid_envelope_for_full_parent(), 0) + .await; + server.settle_envelopes().await; + + assert!(server.store.has_verified_payload(&root(FULL_PARENT))); + let status = server.store.beacon_block_payload_status(root(FULL_PARENT)); + assert_eq!(status, PayloadStatusEnum::Syncing); + assert!(status.is_not_validated()); + assert!(server.store.has_state(&root(FULL_CHILD)).unwrap()); + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn an_invalid_verdict_leaves_no_full_node_and_drops_the_full_child() { + let (client, mock) = + mock_engine(status_json("INVALID", None), status_json("SYNCING", None)).await; + let mut server = server_with_held_child(client).await; + + server + .receive_envelope(valid_envelope_for_full_parent(), 0) + .await; + server.settle_envelopes().await; + + assert!(!server.store.has_verified_payload(&root(FULL_PARENT))); + assert_eq!( + server.store.beacon_block_payload_status(root(FULL_PARENT)), + PayloadStatusEnum::Invalid + ); + // The child built on the FULL node, so it is gone; the parent block and + // its EMPTY branch stay. + assert!(!server.envelopes.is_block_held(&root(FULL_CHILD))); + assert!(!server.store.has_state(&root(FULL_CHILD)).unwrap()); + assert!(server.store.block_index().contains_key(&root(FULL_PARENT))); + assert_eq!(head(&server), (root(FULL_PARENT), PayloadStatus::Empty)); + + // A second delivery is not asked about again, and a later block that + // builds on the FULL node is dropped instead of held. + server + .receive_envelope(valid_envelope_for_full_parent(), 0) + .await; + let case = Case::new("on_execution_payload_envelope", FULL_CHILD_CASE); + server + .on_block(case.block(FULL_CHILD), ImportTimings::default()) + .await; + assert!(!server.envelopes.is_block_held(&root(FULL_CHILD))); + let asked = methods(&mock) + .iter() + .filter(|m| *m == "engine_newPayloadV5") + .count(); + assert_eq!(asked, 1); + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn no_verdict_does_not_apply_the_envelope() { + // Port 1 has nothing listening, so every attempt is refused at once. + let client = EngineClient::new("http://127.0.0.1:1".to_string(), JwtSecret::new([0u8; 32])) + .expect("client builds"); + let mut server = server_with_held_child(client).await; + + server + .receive_envelope(valid_envelope_for_full_parent(), 0) + .await; + + assert!(!server.store.has_verified_payload(&root(FULL_PARENT))); + assert!(server.envelopes.is_block_held(&root(FULL_CHILD))); + } + + /// The fixture payloads all hash to zero, which `forkchoiceUpdated` rightly + /// treats as "nothing to say"; give `root`'s bid and cached payload a real + /// hash so the call can be observed. + fn give_payload_a_hash(server: &mut BlockChainServer, root: H256, hash: H256) { + let mut block = server + .store + .get_signed_block(&root) + .unwrap() + .expect("block stored"); + let slot = block.slot(); + match &mut block { + SignedBeaconBlock::Gloas(inner) => { + inner + .message + .body + .signed_execution_payload_bid + .message + .block_hash = hash; + } + _ => panic!("a gloas block"), + } + server + .store + .insert_pending_block(root, block) + .expect("overwrite"); + server.store.insert_beacon_el_block_hash(root, slot, hash); + } + + /// A server whose only block is `FULL_PARENT`, its envelope applied with + /// `new_payload`'s verdict, then given a nonzero payload hash and made the + /// FULL head. + async fn full_head_server( + new_payload: &str, + forkchoice: &str, + ) -> (BlockChainServer, Arc>, H256) { + let (client, mock) = mock_engine( + status_json(new_payload, None), + status_json(forkchoice, None), + ) + .await; + let case = Case::new("on_execution_payload_envelope", FULL_CHILD_CASE); + let mut server = case.server(12); + server.engine = Some(client); + server + .on_block(case.block(FULL_PARENT), ImportTimings::default()) + .await; + server + .receive_envelope(valid_envelope_for_full_parent(), 0) + .await; + server.settle_envelopes().await; + assert!(server.store.has_verified_payload(&root(FULL_PARENT))); + + let hash = H256::repeat_byte(0x42); + give_payload_a_hash(&mut server, root(FULL_PARENT), hash); + fork_choice::get_head(&mut server.store, &fixture_config()).expect("head"); + server + .store + .set_head_payload_status(root(FULL_PARENT), PayloadStatus::Full); + (server, mock, hash) + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn the_forkchoice_hashes_follow_the_head_node_and_the_checkpoints_bids() { + let (server, _mock, hash) = full_head_server("VALID", "SYNCING").await; + let parent = server + .store + .get_signed_block(&root(FULL_PARENT)) + .unwrap() + .unwrap(); + let bid = crate::beacon_payloads::bid_of(&parent).expect("gloas block"); + let anchor_root = server.store.beacon_finalized_checkpoint().root; + let anchor = server + .store + .get_signed_block(&anchor_root) + .unwrap() + .unwrap(); + let anchor_parent_hash = crate::beacon_payloads::bid_of(&anchor) + .expect("gloas anchor") + .parent_block_hash; + + // A FULL head names the revealed payload; finalized and safe are the + // checkpoint blocks' bids' parent hashes. + let plan = crate::beacon_payloads::forkchoice_plan(&server.store, root(FULL_PARENT)) + .expect("a plan"); + assert!(plan.gloas); + assert_eq!(plan.state.head_block_hash, hash); + assert_eq!(plan.state.finalized_block_hash, anchor_parent_hash); + assert_eq!(plan.state.safe_block_hash, anchor_parent_hash); + + // An EMPTY head names the payload before it. + server + .store + .set_head_payload_status(root(FULL_PARENT), PayloadStatus::Empty); + let plan = crate::beacon_payloads::forkchoice_plan(&server.store, root(FULL_PARENT)) + .expect("a plan"); + assert_eq!(plan.state.head_block_hash, bid.parent_block_hash); + + // No status for the head yet: nothing to say. + server + .store + .set_head_payload_status(H256::repeat_byte(1), PayloadStatus::Full); + assert!( + crate::beacon_payloads::forkchoice_plan(&server.store, root(FULL_PARENT)).is_none() + ); + } + + #[test] + fn a_pre_gloas_head_keeps_its_own_hash_and_uses_forkchoice_v3() { + let mut store = beacon_store(GENESIS_TIME, 0); + let block = crate::tests::fulu_block(H256::ZERO, 1, 0); + let block_root = block.message_hash_tree_root(); + store + .insert_pending_block(block_root, block) + .expect("insert"); + store.insert_beacon_el_block_hash(block_root, 1, H256::repeat_byte(7)); + + let plan = crate::beacon_payloads::forkchoice_plan(&store, block_root).expect("a plan"); + + assert!(!plan.gloas); + assert_eq!(plan.state.head_block_hash, H256::repeat_byte(7)); + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn a_gloas_head_sends_forkchoice_v4_with_the_custody_columns() { + let (mut server, mock, hash) = full_head_server("VALID", "SYNCING").await; + server.custody_columns = vec![0, 9]; + + server.notify_forkchoice_updated().await; + + let last = mock + .lock() + .unwrap() + .requests + .last() + .cloned() + .expect("a request"); + assert!(last.contains("engine_forkchoiceUpdatedV4")); + assert!(last.contains(&format!("{hash:?}"))); + // Columns 0 and 9 as a little-endian bitvector. + assert!(last.contains("0x01020000000000000000000000000000")); + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn a_valid_forkchoice_response_promotes_optimistic_payloads() { + let (mut server, mock, _hash) = full_head_server("SYNCING", "SYNCING").await; + assert_eq!( + server.store.beacon_block_payload_status(root(FULL_PARENT)), + PayloadStatusEnum::Syncing + ); + + // Still syncing: nothing changes. + server.notify_forkchoice_updated().await; + assert_eq!( + server.store.beacon_block_payload_status(root(FULL_PARENT)), + PayloadStatusEnum::Syncing + ); + + mock.lock().unwrap().forkchoice = status_json("VALID", None); + server.notify_forkchoice_updated().await; + + assert_eq!( + server.store.beacon_block_payload_status(root(FULL_PARENT)), + PayloadStatusEnum::Valid + ); + assert!(!server.store.has_unvalidated_block_payloads()); + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn an_invalid_forkchoice_response_unverifies_the_payload_and_keeps_the_block() { + let (mut server, _mock, _hash) = full_head_server("SYNCING", "INVALID").await; + + server.notify_forkchoice_updated().await; + + assert!(!server.store.has_verified_payload(&root(FULL_PARENT))); + assert_eq!( + server.store.beacon_block_payload_status(root(FULL_PARENT)), + PayloadStatusEnum::Invalid + ); + assert!(server.store.block_index().contains_key(&root(FULL_PARENT))); + assert_eq!(head(&server), (root(FULL_PARENT), PayloadStatus::Empty)); + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn an_invalid_forkchoice_response_removes_a_child_built_on_the_full_payload() { + let (mut server, _mock, hash) = full_head_server("SYNCING", "INVALID").await; + // A child whose bid names the parent's payload as its own parent: the + // fixture child, retargeted at the hash this test gave the payload. + let case = Case::new("on_execution_payload_envelope", FULL_CHILD_CASE); + let mut child = case.block(FULL_CHILD); + let child_root = child.message_hash_tree_root(); + match &mut child { + SignedBeaconBlock::Gloas(inner) => { + inner + .message + .body + .signed_execution_payload_bid + .message + .parent_block_hash = hash; + } + _ => panic!("a gloas block"), + } + server + .store + .insert_pending_block(child_root, child) + .expect("insert"); + let child_slot = server + .store + .get_signed_block(&child_root) + .unwrap() + .unwrap() + .slot(); + server + .store + .insert_live_chain_entry(child_slot, child_root, root(FULL_PARENT)); + assert!(server.store.block_index().contains_key(&child_root)); + + server.notify_forkchoice_updated().await; + + assert!(!server.store.block_index().contains_key(&child_root)); + } } diff --git a/crates/blockchain/src/beacon_payloads.rs b/crates/blockchain/src/beacon_payloads.rs new file mode 100644 index 000000000..4ac488b65 --- /dev/null +++ b/crates/blockchain/src/beacon_payloads.rs @@ -0,0 +1,354 @@ +//! Execution-layer bookkeeping for gloas payloads. +//! +//! Before gloas a block carries its payload, so the execution client's verdict +//! on it is the block's and the optimistic-sync machinery in +//! `ethlambda_state_transition::beacon::fork_choice` works on blocks. In gloas +//! the payload arrives later, in an envelope, and fork choice weighs a FULL and +//! an EMPTY node per block. A payload the execution client rejects must take +//! only its FULL node (and whatever builds on it) with it; the block and its +//! EMPTY branch stay viable. This module is that second granularity: +//! +//! ```text +//! envelope ──► newPayloadV5 ──► VALID apply, status Valid +//! ├─► SYNCING apply, status Syncing (NOT_VALIDATED) +//! └─► INVALID do not apply, status Invalid, +//! drop children built on the FULL node +//! +//! forkchoiceUpdated(V4) ──► VALID Syncing payloads on the head's ancestry -> Valid +//! └─► INVALID first invalid payload after latestValidHash: +//! unverify it, drop its FULL children +//! ``` +//! +//! # Which hash names what +//! +//! A gloas block's EL hash depends on the node: its FULL node is the payload +//! the envelope revealed (the bid's `block_hash`); its EMPTY node is the last +//! payload before it (the bid's `parent_block_hash`). `SignedBeaconBlock:: +//! execution_block_hash` is `None` for gloas on purpose, so the helpers here +//! read the bid. + +use std::collections::HashMap; + +use ethlambda_engine::ForkchoiceStateV1; +use ethlambda_state_transition::beacon::fork_choice::{ + self, PayloadStatus, PayloadStatusEnum, PayloadValidity, +}; +use ethlambda_storage::Store; +use ethlambda_types::ShortRoot; +use ethlambda_types::beacon::containers::{SignedBeaconBlock, gloas}; +use ethlambda_types::beacon::primitives::ExecutionBlockHash; +use ethlambda_types::beacon::signing::compute_start_slot_at_epoch; +use ethlambda_types::primitives::H256; +use tracing::{error, warn}; + +use crate::BlockChainServer; + +/// The bid a gloas block carries, or `None` for any other fork. +pub(crate) fn bid_of(block: &SignedBeaconBlock) -> Option<&gloas::ExecutionPayloadBid> { + match block { + SignedBeaconBlock::Gloas(inner) => { + Some(&inner.message.body.signed_execution_payload_bid.message) + } + SignedBeaconBlock::Phase0(_) + | SignedBeaconBlock::Altair(_) + | SignedBeaconBlock::Bellatrix(_) + | SignedBeaconBlock::Capella(_) + | SignedBeaconBlock::Deneb(_) + | SignedBeaconBlock::Electra(_) + | SignedBeaconBlock::Fulu(_) + | SignedBeaconBlock::Lean(_) => None, + } +} + +/// The execution hash of the payload `root` carries on the chain, or `None` if +/// it carries none: a pre-gloas block's own payload (cached at import), or a +/// gloas block's verified payload (cached when the envelope is applied, and +/// read back off the bid for a payload verified before a restart emptied the +/// cache). +pub(crate) fn payload_hash_of(store: &Store, root: H256) -> Option { + if let Some(hash) = store.beacon_el_block_hash(root) { + return Some(hash); + } + if !store.has_verified_payload(&root) { + return None; + } + let block = store.get_signed_block(&root).ok().flatten()?; + bid_of(&block).map(|bid| bid.block_hash) +} + +/// The execution hash a checkpoint root stands for in `forkchoiceUpdated`. +/// +/// A gloas block is `bid.parent_block_hash` (gloas `fork-choice.md`, "Modified +/// `notify_forkchoice_updated`": the payload a checkpoint block builds on is +/// the one a node can know to be final, since the block's own payload may +/// never be revealed). A pre-gloas block keeps its own payload's hash. +fn checkpoint_hash(store: &Store, root: H256) -> ExecutionBlockHash { + if let Some(block) = store.get_signed_block(&root).ok().flatten() + && let Some(bid) = bid_of(&block) + { + return bid.parent_block_hash; + } + store.beacon_el_block_hash(root).unwrap_or(H256::ZERO) +} + +/// What `forkchoiceUpdated` should carry for the current head, or `None` when +/// there is nothing to say yet. +pub(crate) struct ForkchoicePlan { + pub(crate) state: ForkchoiceStateV1, + /// Whether the head block is gloas or later, which selects + /// `engine_forkchoiceUpdatedV4` (with custody columns) over V3. + pub(crate) gloas: bool, +} + +/// Builds the `forkchoiceUpdated` payload for `head_root`. +/// +/// Head: a gloas head's FULL node is the payload its envelope revealed +/// (`bid.block_hash`), its EMPTY node the payload before it +/// (`bid.parent_block_hash`). `None` while the head's payload status has not +/// been computed (a gloas head is ambiguous without it), and while the head +/// hash is zero, as before. +/// +/// Finalized: the finalized block's `bid.parent_block_hash` (spec). Safe: the +/// spec's `get_safe_execution_block_hash` is fast-confirmation based, which +/// this follower does not run, so the justified checkpoint block's +/// `bid.parent_block_hash` stands in (see `docs/spec_deviations.md`). A +/// pre-gloas checkpoint block, including one under a gloas head, uses its own +/// payload hash. +pub(crate) fn forkchoice_plan(store: &Store, head_root: H256) -> Option { + let head_block = store.get_signed_block(&head_root).ok().flatten(); + let head_bid = head_block.as_ref().and_then(bid_of); + let head_block_hash = match head_bid { + Some(bid) => match store.head_payload_status()? { + PayloadStatus::Full => bid.block_hash, + PayloadStatus::Empty | PayloadStatus::Pending => bid.parent_block_hash, + }, + None => store.beacon_el_block_hash(head_root).unwrap_or(H256::ZERO), + }; + if head_block_hash.is_zero() { + return None; + } + let state = ForkchoiceStateV1 { + head_block_hash, + safe_block_hash: checkpoint_hash(store, store.beacon_justified_checkpoint().root), + finalized_block_hash: checkpoint_hash(store, store.beacon_finalized_checkpoint().root), + }; + Some(ForkchoicePlan { + state, + gloas: head_bid.is_some(), + }) +} + +/// The nearest ancestor-or-self of `from` whose payload hash is `hash`: the +/// block a `forkchoiceUpdated` verdict about `hash` is about. +pub(crate) fn payload_holder( + store: &Store, + index: &HashMap, + from: H256, + hash: ExecutionBlockHash, +) -> Option { + let mut cursor = from; + loop { + if payload_hash_of(store, cursor) == Some(hash) { + return Some(cursor); + } + cursor = index.get(&cursor)?.1; + } +} + +/// The block whose payload is the first invalid one on `holder`'s chain, per +/// `optimistic-sync.md`'s `latestValidHash` table, reading payloads rather +/// than blocks. +/// +/// [`fork_choice::resolve_invalid_block`] walks parents while each carries a +/// cached payload hash, which stops at the first gloas block whose payload was +/// never revealed. Here a root with no payload is skipped, since an EMPTY node +/// inherits its parent's payload and a payload built on a FULL ancestor +/// descends from it in the execution layer whatever sits between. +/// +/// | `latest_valid_hash` | result | +/// |---|---| +/// | a payload hash found on this chain | the next payload-carrying block after it | +/// | all zeroes | the earliest payload-carrying ancestor | +/// | `None`, or a hash not on this chain | `holder` itself | +pub(crate) fn resolve_invalid_payload( + store: &Store, + index: &HashMap, + holder: H256, + latest_valid_hash: Option, +) -> H256 { + let Some(latest_valid_hash) = latest_valid_hash else { + return holder; + }; + let mut condemned = holder; + let mut cursor = holder; + while let Some(&(_slot, parent)) = index.get(&cursor) { + cursor = parent; + match payload_hash_of(store, cursor) { + Some(hash) if hash == latest_valid_hash => return condemned, + Some(_) => condemned = cursor, + None => {} + } + } + if latest_valid_hash.is_zero() { + condemned + } else { + // Ran off the top of what this store indexes without finding it: + // `optimistic-sync.md` says to treat it as `null`. + holder + } +} + +/// Moves `root` and every optimistic ancestor to validated: pre-gloas blocks +/// leave `optimistic_roots`, gloas payloads recorded `NOT_VALIDATED` become +/// `VALID`. `optimistic-sync.md`: a block's ancestors transition with it. +/// +/// A single walk of the whole ancestry rather than +/// [`fork_choice::mark_validated`]'s stop at the first non-optimistic parent, +/// since a gloas block with a settled payload can sit between two optimistic +/// ones. Skipped entirely, with no `block_index` scan, when nothing is +/// optimistic. +pub(crate) fn mark_payloads_validated(store: &mut Store, root: H256) { + fork_choice::mark_validated(store, root); + if !store.has_unvalidated_block_payloads() && !store.has_beacon_optimistic_roots() { + return; + } + let index = store.block_index(); + let mut cursor = root; + while let Some(&(slot, parent)) = index.get(&cursor) { + store.remove_beacon_optimistic_root(cursor); + if store.has_verified_payload(&cursor) + && store.beacon_block_payload_status(cursor).is_not_validated() + { + store.insert_beacon_block_payload_status(cursor, slot, PayloadStatusEnum::Valid); + } + cursor = parent; + } +} + +impl BlockChainServer { + /// [`Self::apply_forkchoice_verdict`] for a gloas head, whose `head_hash` + /// names a payload. + /// + /// `VALID` promotes every optimistic payload on the holder's ancestry; + /// `INVALID` condemns from the first payload after `latestValidHash`. + /// `SYNCING` and `ACCEPTED` change nothing. + pub(crate) fn apply_gloas_forkchoice_verdict( + &mut self, + head_root: H256, + head_hash: ExecutionBlockHash, + status: ðlambda_engine::PayloadStatusV1, + ) { + let verdict = crate::beacon_engine::verdict(status); + let (PayloadValidity::Validated | PayloadValidity::Invalidated { .. }) = verdict else { + return; + }; + let index = self.store.block_index(); + let Some(holder) = payload_holder(&self.store, &index, head_root, head_hash) else { + warn!( + head_root = %ShortRoot(&head_root.0), + "No block on the head's chain carries the payload forkchoiceUpdated named" + ); + return; + }; + match verdict { + PayloadValidity::Validated => mark_payloads_validated(&mut self.store, holder), + PayloadValidity::Invalidated { latest_valid_hash } => { + if self.condemn_payload_chain(holder, latest_valid_hash) { + // Not `recompute_beacon_head`: this runs inside + // `forkchoiceUpdated`'s own response handling. + self.update_head_from_fork_choice(); + } + } + PayloadValidity::Optimistic | PayloadValidity::NotRequired => {} + } + } + + /// Applies an `INVALID` verdict about the payload of `holder` (an imported + /// block, or a gloas block whose envelope was just refused). + /// + /// Resolves how far the verdict reaches ([`resolve_invalid_payload`]) and + /// condemns that block's payload: a pre-gloas block is removed with its + /// subtree, as `forkchoiceUpdated` always did; a gloas payload loses its + /// FULL node and every child built on it, and the block itself stays with + /// its EMPTY branch. Returns whether anything left fork choice. + pub(crate) fn condemn_payload_chain( + &mut self, + holder: H256, + latest_valid_hash: Option, + ) -> bool { + let index = self.store.block_index(); + let condemned = resolve_invalid_payload(&self.store, &index, holder, latest_valid_hash); + let mut removed = self.condemn_gloas_payload(condemned, &index); + // The holder's own payload is invalid whichever ancestor the verdict + // reached; when it is a gloas payload it must not stay FULL. + if condemned != holder { + let index = self.store.block_index(); + removed |= self.condemn_gloas_payload(holder, &index); + } + removed + } + + /// Records `root`'s payload as `INVALID`, removes its FULL node, and drops + /// every child built on it, held or imported. + /// + /// A pre-gloas `root` takes the block-level path instead: its payload is + /// the block's, so the whole subtree goes. Refuses a block at or below + /// finality, like [`fork_choice::invalidate_subtree`]. + fn condemn_gloas_payload(&mut self, root: H256, index: &HashMap) -> bool { + let Some(block) = self.store.get_signed_block(&root).ok().flatten() else { + return false; + }; + let Some(bid) = bid_of(&block) else { + return fork_choice::invalidate_subtree(&mut self.store, root) > 0; + }; + let hash = bid.block_hash; + let slot = block.slot(); + let finalized = self.store.beacon_finalized_checkpoint(); + if root == finalized.root || slot <= compute_start_slot_at_epoch(finalized.epoch) { + error!( + condemned = %ShortRoot(&root.0), + "The execution client condemned a finalized payload; refusing to invalidate. \ + The execution and consensus layers disagree about finalized history and \ + this node needs operator attention" + ); + return false; + } + + warn!( + %slot, + block_root = %ShortRoot(&root.0), + payload_hash = %ShortRoot(&hash.0), + "Execution client found a payload invalid; dropping its FULL node" + ); + self.store + .insert_beacon_block_payload_status(root, slot, PayloadStatusEnum::Invalid); + let mut removed = self.store.remove_verified_payload(&root); + + // Children held for this payload build on it by construction. + if let Some(held) = self.envelopes.blocks_awaiting_parent_payload.remove(&root) { + for (child_root, _slot) in held { + self.discard_pending_subtree(child_root); + } + self.publish_envelope_queues(); + } + + // Imported children whose bid names this payload as their parent's. + let full_children: Vec = index + .iter() + .filter(|(_, (_, parent))| *parent == root) + .map(|(child, _)| *child) + .filter(|child| { + self.store + .get_signed_block(child) + .ok() + .flatten() + .and_then(|block| bid_of(&block).map(|bid| bid.parent_block_hash == hash)) + .unwrap_or(false) + }) + .collect(); + for child in full_children { + removed |= fork_choice::invalidate_subtree(&mut self.store, child) > 0; + } + removed + } +} diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 8a7b9a4c9..34f848d8e 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -1,4 +1,4 @@ -use ethlambda_engine::{EngineClient, ForkchoiceStateV1, PayloadStatusV1 as EnginePayloadStatus}; +use ethlambda_engine::{CustodyColumns, EngineClient, PayloadStatusV1 as EnginePayloadStatus}; use ethlambda_network_api::{ AggregateArrival, BlockArrival, BlockChainToP2PRef, BlockSource, DeferredFrom, FetchRequest, InitP2P, @@ -58,6 +58,7 @@ mod beacon_aggregates; mod beacon_columns; pub mod beacon_engine; mod beacon_envelope; +mod beacon_payloads; pub mod block_builder; pub(crate) mod coverage; pub mod events; @@ -2418,7 +2419,7 @@ impl BlockChainServer { /// A failure here means fork choice could not find a head (for instance /// every known block is unjustifiable), which is a condition to log and /// wait out, not a reason to crash a follower. - fn update_head_from_fork_choice(&mut self) -> HeadTimings { + pub(crate) fn update_head_from_fork_choice(&mut self) -> HeadTimings { let _timing = metrics::time_beacon_head_compute(); let mut timings = HeadTimings { head_start: Some(Instant::now()), @@ -2482,35 +2483,14 @@ impl BlockChainServer { let client = self.engine.clone()?; let (_head_slot, head_root) = self.store.beacon_head()?; - let justified_root = self.store.beacon_justified_checkpoint().root; - let finalized_root = self.store.beacon_finalized_checkpoint().root; - - // `H256::ZERO` explicitly rather than `unwrap_or_default()`: the zero - // hash is a meaningful value here, not an absence. EIP-3675 requires - // `finalized_block_hash` to be zero before a post-transition block is - // finalized, and the specification's own - // `get_safe_execution_block_hash` returns zero when no payload is - // justified yet. - let state = ForkchoiceStateV1 { - head_block_hash: self - .store - .beacon_el_block_hash(head_root) - .unwrap_or(H256::ZERO), - safe_block_hash: self - .store - .beacon_el_block_hash(justified_root) - .unwrap_or(H256::ZERO), - finalized_block_hash: self - .store - .beacon_el_block_hash(finalized_root) - .unwrap_or(H256::ZERO), - }; - - // Nothing to say yet: a follower whose head has no cached payload hash - // is still on its checkpoint anchor. - if state.head_block_hash.is_zero() { - return None; - } + // A gloas head is ambiguous without its payload status (FULL names the + // revealed payload, EMPTY the one before), and a head whose status has + // not been computed yet is skipped. The plan also carries the hashes + // for a pre-gloas head, which keeps its own cached payload hash; + // `None` there means a follower still on its checkpoint anchor. + let plan = beacon_payloads::forkchoice_plan(&self.store, head_root)?; + let state = plan.state; + let gloas = plan.gloas; // `head_root` stays valid across the await: this is a single-threaded // actor, so no other message is handled until this one returns. An @@ -2521,8 +2501,18 @@ impl BlockChainServer { // `apply_forkchoice_verdict` recomputes the head itself rather than // leaving it to the next tick. let start = Instant::now(); - match client.forkchoice_updated(&state).await { - Ok(status) => self.apply_forkchoice_verdict(head_root, &status), + // V4 from gloas on: it carries the custody columns the execution + // client must keep. The follower sends `null` attributes either way. + let response = if gloas { + let custody = CustodyColumns::from_indices(self.custody_columns.iter().copied()); + client.forkchoice_updated_v4(&state, custody).await + } else { + client.forkchoice_updated(&state).await + }; + match response { + Ok(status) => { + self.apply_forkchoice_verdict(head_root, state.head_block_hash, gloas, &status) + } Err(err) => warn!(%err, "forkchoiceUpdated failed"), } Some((start, Instant::now())) @@ -2533,7 +2523,23 @@ impl BlockChainServer { /// `VALID` clears the head and every optimistic ancestor; `INVALID` cuts the /// condemned branch out of fork choice. `SYNCING` and `ACCEPTED` say the /// execution client is still working and change nothing. - fn apply_forkchoice_verdict(&mut self, head_root: H256, status: &EnginePayloadStatus) { + /// + /// For a gloas head the verdict is about `head_hash`, which names a payload + /// rather than a block: the head's own (FULL) or an ancestor's (EMPTY). It + /// is resolved to the block carrying that payload first, and an `INVALID` + /// verdict then takes that payload's FULL node and what builds on it, not + /// the block (see `beacon_payloads`). + fn apply_forkchoice_verdict( + &mut self, + head_root: H256, + head_hash: H256, + gloas: bool, + status: &EnginePayloadStatus, + ) { + if gloas { + self.apply_gloas_forkchoice_verdict(head_root, head_hash, status); + return; + } match beacon_engine::verdict(status) { fork_choice::PayloadValidity::Validated => { fork_choice::mark_validated(&mut self.store, head_root); @@ -3087,7 +3093,7 @@ impl BlockChainServer { /// /// Used when a block is rejected (e.g., at/below finalized slot) to clean up /// children that would otherwise remain stuck in the pending maps indefinitely. - fn discard_pending_subtree(&mut self, block_root: H256) { + pub(crate) fn discard_pending_subtree(&mut self, block_root: H256) { // A block held for its custody columns already had a known parent // when it was held (see `hold_block_for_columns`), so it carries no // entry of its own in `pending_blocks` and would not be reached by @@ -3823,7 +3829,8 @@ impl Handler for BlockChainServer { }; let arrival_ms = unix_now_ms(); let picked_up = Instant::now(); - self.receive_envelope(*msg.envelope, msg.arrival.wire_ms(picked_up, arrival_ms)); + self.receive_envelope(*msg.envelope, msg.arrival.wire_ms(picked_up, arrival_ms)) + .await; self.settle_envelopes().await; } } @@ -4797,7 +4804,11 @@ mod tests { /// caller of this helper is exercising, so nothing here needs to be real: /// only structurally present, so `message_hash_tree_root` and `to_ssz` /// succeed. - fn fulu_block(parent_root: H256, slot: u64, commitment_count: usize) -> SignedBeaconBlock { + pub(crate) fn fulu_block( + parent_root: H256, + slot: u64, + commitment_count: usize, + ) -> SignedBeaconBlock { let payload = deneb::ExecutionPayload { parent_hash: Default::default(), fee_recipient: Default::default(), diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index c0df581e4..8190d678a 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -3422,6 +3422,25 @@ impl Store { self.beacon.lock().unwrap().verified_payloads.insert(root); } + /// Gloas: forgets that `root`'s payload is verified and deletes its stored + /// envelope, for a payload the execution layer later found invalid. + /// + /// The FULL node of the block stops existing for fork choice (it is + /// derived from [`Self::has_verified_payload`]) and the envelope is no + /// longer served to peers. Returns whether `root` was verified. + pub fn remove_verified_payload(&mut self, root: &H256) -> bool { + let was_verified = self.beacon.lock().unwrap().verified_payloads.remove(root); + if let Some(slot) = beacon_block_slot(self.backend.as_ref(), root) { + let keys = vec![encode_slot_root_key(slot, root)]; + let mut batch = self.backend.begin_write().expect("write batch"); + batch + .delete_batch(Table::ExecutionPayloadEnvelopes, keys) + .expect("delete execution payload envelope"); + batch.commit().expect("commit"); + } + was_verified + } + /// Gloas: the verified envelope stored for `root`, if any. pub fn get_execution_payload_envelope( &self, @@ -3819,6 +3838,20 @@ impl Store { .map_or(PayloadStatusEnum::Syncing, |(_slot, status)| *status) } + /// Whether any recorded gloas payload verdict is still `NOT_VALIDATED`. + /// + /// The cheap half of resolving optimistic payloads, like + /// [`Self::has_beacon_optimistic_roots`]: with a healthy execution client + /// it is always `false`, so callers can skip a `block_index` scan. + pub fn has_unvalidated_block_payloads(&self) -> bool { + self.beacon + .lock() + .unwrap() + .block_payload_statuses + .values() + .any(|(_slot, status)| status.is_not_validated()) + } + /// Records the execution client's verdict on `root`'s payload, against the /// block's slot. A later verdict replaces an earlier one, since an /// optimistic payload is resolved one way or the other. @@ -7650,6 +7683,24 @@ mod tests { ); } + #[test] + fn an_unverified_payload_is_forgotten_across_reopen() { + let root = H256::from([1u8; 32]); + let backend = beacon_backend_finalized_at(root); + let mut store = reopen(backend.clone()); + store + .insert_signed_block(root, gloas_test_block(SLOTS_PER_EPOCH, H256::ZERO)) + .expect("insert block"); + store.insert_verified_payload(SLOTS_PER_EPOCH, &test_envelope(root)); + + assert!(store.remove_verified_payload(&root)); + assert!(!store.remove_verified_payload(&root)); + + assert!(!store.has_verified_payload(&root)); + assert_eq!(store.get_execution_payload_envelope(&root).unwrap(), None); + assert!(!reopen(backend).has_verified_payload(&root)); + } + #[test] fn a_verified_payload_survives_reopen() { let root = H256::from([1u8; 32]); From 029e8fb8f934e6b71198f7866a32c94b26111364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 04:12:50 -0300 Subject: [PATCH 076/132] docs(beacon): record how the execution client judges gloas payloads The engine verdict is asked before the envelope is applied and has three outcomes where the spec has two, and the safe hash falls back to the justified block bid because fast confirmation is out of scope. Both are deliberate departures worth finding in one place. --- docs/spec_deviations.md | 51 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 6a85d7372..874b76b1d 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -274,3 +274,54 @@ node, and each call walks every latest message and every ancestor step. `gloas_get_weight` remain, as the references the walk is compared with, at every `head` check of every fork-choice fixture and on randomized trees; `compute_head` is the pre-gloas reference. + +## The execution client judges a gloas payload before the envelope is applied + +`verify_execution_payload_envelope` asks the execution engine last, as part of +one boolean function. The follower asks it first, with `engine_newPayloadV5`, +and applies the envelope with the engine's part answered "valid", because the +answer has three outcomes and the specification's `ExecutionEngine` has two. + +- **`VALID`:** applied; the payload is recorded `VALID`. +- **`SYNCING` / `ACCEPTED`:** applied, since the consensus checks passed and the + payload is `NOT_VALIDATED` (`optimistic-sync.md`); recorded `SYNCING`, which + the attestation gossip rules already read as not yet validated. A later + `forkchoiceUpdated` `VALID` for the head, or for a descendant payload, moves + it and every optimistic payload on that chain to `VALID`. Unlike a pre-gloas + block there is no `is_optimistic_candidate_block` gate: a payload is applied + to a block the consensus layer already imported, so there is no + merge-transition poisoning to guard against. +- **`INVALID`:** not applied, so the FULL node of the block never exists; the + payload is recorded `INVALID`, children held for it are dropped and later + blocks that name it as their FULL parent are dropped instead of held. The + block itself and its EMPTY branch stay. `latestValidHash` can reach further + back, in which case the first payload after it is condemned the same way + (`beacon_payloads::resolve_invalid_payload`, which skips blocks that carry no + payload, unlike the pre-gloas `resolve_invalid_block`). A `forkchoiceUpdated` + `INVALID` for a payload already applied removes it again (its FULL node, its + stored envelope, and every child built on it). +- **No answer after the retry ladder:** as for a block, the payload is not + applied; the envelope is fetched again rather than held. + +The engine is asked only about an envelope whose builder signature holds: an +unsigned copy with the honest block hash and a different body would otherwise +earn an `INVALID` that lands on the honest payload. + +## `forkchoiceUpdated` for a gloas head + +- The head hash follows the head node: `bid.block_hash` for a FULL head, + `bid.parent_block_hash` for an EMPTY one. A head whose payload status has not + been computed yet is skipped. +- `finalized_block_hash` is the finalized block's `bid.parent_block_hash`, as + the specification has it. +- `safe_block_hash` is the justified block's `bid.parent_block_hash`. The + specification's `get_safe_execution_block_hash` is built on fast + confirmation, which the follower does not run, so this is the fallback. It + can only be older than the confirmed block, which keeps it safe. +- A pre-gloas checkpoint block, including one beneath a gloas head, keeps its + own payload hash. +- The call is V4 with the node's custody columns when the head block is gloas, + and V3 before; `engine_newPayloadV5` is for envelopes and V4 for pre-gloas + blocks. +- A hash of zero is "nothing to say" on either fork, so a payload whose hash is + zero (the fixtures' placeholder) sends no `forkchoiceUpdated`. From 281aceaac08279230f58fffc062b33a32599a1a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 04:29:24 -0300 Subject: [PATCH 077/132] fix(beacon): follow the execution chain, and check consensus before asking the engine A verdict about a gloas payload walked every verified ancestor as if it were an execution ancestor. A block that built on an ancestor EMPTY node has that ancestor payload skipped, so an INVALID with a latestValidHash could delete a valid envelope (and the chain on it), and a VALID could mark it validated. One walk now follows the bid parent hash (and a pre-gloas block its parent), for both, bounded at finality and without a block_index scan. The envelope is checked by the consensus rules before the engine is asked, so the engine only sees envelopes this node would accept. INVALID_BLOCK_HASH now refuses only that envelope: its contents do not match the hash, which says nothing about the payload the bid names. An engine error keeps the consensus-valid envelope and retries once a slot instead of re-fetching it. Also: payloads restored from disk get a verdict so a VALID can promote them; removing a payload drops its cached hash; the proposer path uses the same checkpoint hash rule as forkchoiceUpdated. --- crates/blockchain/Cargo.toml | 2 +- crates/blockchain/src/beacon_engine.rs | 19 +- crates/blockchain/src/beacon_envelope.rs | 592 +++++++++++++++--- crates/blockchain/src/beacon_payloads.rs | 170 +++-- crates/blockchain/src/lib.rs | 10 +- .../src/beacon/fork_choice.rs | 43 +- crates/net/rpc/src/beacon/proposal.rs | 13 +- crates/storage/src/store.rs | 21 +- docs/spec_deviations.md | 28 +- 9 files changed, 729 insertions(+), 169 deletions(-) diff --git a/crates/blockchain/Cargo.toml b/crates/blockchain/Cargo.toml index 062e27ed7..0b7c66e6f 100644 --- a/crates/blockchain/Cargo.toml +++ b/crates/blockchain/Cargo.toml @@ -39,7 +39,7 @@ hex = { workspace = true } libssz.workspace = true libssz-types.workspace = true # The mock execution client the payload-validity tests talk to over a socket. -tokio = { workspace = true, features = ["io-util", "net", "sync"] } +tokio = { workspace = true, features = ["io-util", "macros", "net", "rt"] } datatest-stable = "0.3.3" snap = "1.1.1" diff --git a/crates/blockchain/src/beacon_engine.rs b/crates/blockchain/src/beacon_engine.rs index 377c71b56..38f34c39e 100644 --- a/crates/blockchain/src/beacon_engine.rs +++ b/crates/blockchain/src/beacon_engine.rs @@ -215,16 +215,29 @@ pub async fn ask_envelope( block: &SignedBeaconBlock, envelope: &containers::gloas::SignedExecutionPayloadEnvelope, ) -> Result { + ask_envelope_status(client, block, envelope) + .await + .map(|status| verdict(&status)) +} + +/// [`ask_envelope`] with the engine's raw answer, for a caller that must tell +/// `INVALID_BLOCK_HASH` (the envelope's contents do not hash to its claimed +/// block hash, so only that envelope is bad) from `INVALID` (the payload the +/// hash names is bad). +pub async fn ask_envelope_status( + client: &EngineClient, + block: &SignedBeaconBlock, + envelope: &containers::gloas::SignedExecutionPayloadEnvelope, +) -> Result { let request = gloas_new_payload_request(block, envelope)?; - let status = client + client .new_payload_v5( request.execution_payload, &request.versioned_hashes, request.parent_beacon_block_root, &request.execution_requests, ) - .await?; - Ok(verdict(&status)) + .await } #[cfg(test)] diff --git a/crates/blockchain/src/beacon_envelope.rs b/crates/blockchain/src/beacon_envelope.rs index 2d442b950..544a21502 100644 --- a/crates/blockchain/src/beacon_envelope.rs +++ b/crates/blockchain/src/beacon_envelope.rs @@ -43,6 +43,7 @@ use std::collections::{HashMap, VecDeque}; use ethlambda_engine::EngineError; +use ethlambda_engine::types::PayloadStatusValue; use ethlambda_network_api::{BlockSource, FetchRequest}; use ethlambda_state_transition::beacon::fork::ForkName; use ethlambda_state_transition::beacon::fork_choice::{ @@ -92,6 +93,13 @@ pub(crate) const ENVELOPE_AWAITING_BLOCK_TTL_SLOTS: u64 = 2; /// chain skipped past it, plus equivocations. pub(crate) const MAX_BLOCKS_HELD_PER_PARENT_PAYLOAD: usize = 4; +/// How many envelope roots the execution client refused for their own +/// contents are remembered. +/// +/// Enough to cover the copies one slot can bring; an older one asked about +/// again costs one engine call and is refused again. +pub(crate) const MAX_REJECTED_ENVELOPES: usize = 64; + /// An envelope the actor is holding until it can be verified. pub(crate) struct HeldEnvelope { envelope: gloas::SignedExecutionPayloadEnvelope, @@ -133,6 +141,17 @@ pub(crate) struct EnvelopeQueues { /// Envelopes whose bid has commitments and whose sampled columns are not /// all stored, by the block's root. pub(crate) awaiting_columns: HashMap, + /// Consensus-valid envelopes the execution client gave no answer for, by + /// the block's root, asked again once a slot. + pub(crate) awaiting_engine: HashMap, + /// Roots of envelopes the execution client refused for their own contents + /// (`INVALID_BLOCK_HASH`), newest last and bounded by + /// [`MAX_REJECTED_ENVELOPES`], so a copy that arrives again is not asked + /// about again. + rejected: VecDeque, + /// Set when an engine verdict changed what fork choice sees without a + /// payload being verified, so the head is recomputed. + pub(crate) head_dirty: bool, /// Blocks whose parent is FULL with a payload not yet verified, by the /// parent's root, each with its own slot. The block's bytes are in /// `Table::BlockHeaders`/`BlockBodies` as for any other held block. @@ -152,9 +171,23 @@ impl EnvelopeQueues { .any(|children| children.contains_key(block_root)) } + /// Whether the execution client already refused the envelope with this + /// root. + fn is_rejected(&self, envelope_root: &H256) -> bool { + self.rejected.contains(envelope_root) + } + + fn reject(&mut self, envelope_root: H256) { + if self.rejected.len() >= MAX_REJECTED_ENVELOPES { + self.rejected.pop_front(); + } + self.rejected.push_back(envelope_root); + } + fn is_empty(&self) -> bool { self.awaiting_block.is_empty() && self.awaiting_columns.is_empty() + && self.awaiting_engine.is_empty() && self.blocks_awaiting_parent_payload.is_empty() } @@ -242,6 +275,14 @@ impl BlockChainServer { ); return false; } + if self + .envelopes + .awaiting_engine + .get(&root) + .is_some_and(|other| other.envelope_root == held.envelope_root) + { + return false; + } let has_state = self.store.has_state(&root).expect("DB read should succeed"); let block = if has_state { @@ -267,25 +308,13 @@ impl BlockChainServer { return false; } - let bid = match &block { - SignedBeaconBlock::Gloas(inner) => { - &inner.message.body.signed_execution_payload_bid.message - } - SignedBeaconBlock::Phase0(_) - | SignedBeaconBlock::Altair(_) - | SignedBeaconBlock::Bellatrix(_) - | SignedBeaconBlock::Capella(_) - | SignedBeaconBlock::Deneb(_) - | SignedBeaconBlock::Electra(_) - | SignedBeaconBlock::Fulu(_) - | SignedBeaconBlock::Lean(_) => { - warn!( - %slot, - block_root = %ShortRoot(&root.0), - "Dropping an envelope: its block is not a gloas block" - ); - return false; - } + let Some(bid) = crate::beacon_payloads::bid_of(&block) else { + warn!( + %slot, + block_root = %ShortRoot(&root.0), + "Dropping an envelope: its block is not a gloas block" + ); + return false; }; let commitment_count = bid.blob_kzg_commitments.len(); @@ -339,44 +368,80 @@ impl BlockChainServer { } }; - // The execution client's verdict comes before the envelope is applied, - // as `verify_execution_payload_envelope` orders it: its part is the - // last check, and an `INVALID` payload never becomes a FULL node. + // The consensus checks come first, as `verify_execution_payload_envelope` + // orders them, so the execution client is only ever asked about an + // envelope this node would accept: signed by the builder, matching + // the bid and the state, and with its data available. + self.advance_beacon_clock_to(held.event_ms); + let config = self.store.config(); + if let Err(err) = fork_choice::check_execution_payload_envelope( + &self.store, + &held.envelope, + &config, + &sidecars, + &ExecutionEngine::valid(), + ) { + // p2p judged the envelope before it got here, so no peer is + // penalized from this side. + warn!( + %slot, + block_root = %ShortRoot(&root.0), + ?err, + "Dropping an execution payload envelope that failed verification" + ); + self.refetch_envelope_if_children_wait(root); + return false; + } + + // Then the execution client, whose answer has three outcomes where the + // specification's engine has two (see `docs/spec_deviations.md`). // Without a client the follower trusts the payload, as it already does // for every pre-gloas block it imports (`NotRequired`). + if self.envelopes.is_rejected(&held.envelope_root) { + trace!( + block_root = %ShortRoot(&root.0), + "Dropping an envelope the execution client already rejected" + ); + return false; + } let status = match self.engine.clone() { None => PayloadStatusEnum::Valid, Some(client) => { - // The client is asked only about an envelope the builder - // signed: an unsigned copy carrying the honest block hash but a - // different body would otherwise earn an `INVALID` verdict - // that lands on the honest payload. - if !self.envelope_signature_holds(root, &held.envelope) { - warn!( - %slot, - block_root = %ShortRoot(&root.0), - "Dropping an envelope with an invalid signature" - ); - self.refetch_envelope_if_children_wait(root); - return false; - } - match beacon_engine::ask_envelope(&client, &block, &held.envelope).await { - Ok(PayloadValidity::Validated | PayloadValidity::NotRequired) => { - PayloadStatusEnum::Valid - } - Ok(PayloadValidity::Optimistic) => PayloadStatusEnum::Syncing, - Ok(PayloadValidity::Invalidated { latest_valid_hash }) => { + match beacon_engine::ask_envelope_status(&client, &block, &held.envelope).await { + // The payload does not hash to the block hash it claims: + // this envelope is bad, the payload the bid names may not + // be. Only this envelope is refused, so the builder's real + // one can still arrive. + Ok(raw) if raw.status == PayloadStatusValue::InvalidBlockHash => { warn!( %slot, block_root = %ShortRoot(&root.0), - "Execution client found an envelope's payload invalid; not applying it" + "Execution client found an envelope's contents do not match its \ + block hash; rejecting this envelope" ); - self.condemn_payload_chain(root, latest_valid_hash); - // Children held for this payload are gone, and the - // head may have named a branch the verdict removed. - self.envelopes.verified.push_back(root); + self.envelopes.reject(held.envelope_root); + self.refetch_envelope_if_children_wait(root); return false; } + Ok(raw) => match beacon_engine::verdict(&raw) { + PayloadValidity::Validated | PayloadValidity::NotRequired => { + PayloadStatusEnum::Valid + } + PayloadValidity::Optimistic => PayloadStatusEnum::Syncing, + PayloadValidity::Invalidated { latest_valid_hash } => { + warn!( + %slot, + block_root = %ShortRoot(&root.0), + "Execution client found an envelope's payload invalid; \ + not applying it" + ); + self.condemn_payload_chain(root, latest_valid_hash); + // Children held for this payload are gone, and the + // head may have named a branch the verdict removed. + self.envelopes.head_dirty = true; + return false; + } + }, Err(EngineError::EnvelopeMismatch(reason)) => { warn!( %slot, @@ -387,69 +452,58 @@ impl BlockChainServer { self.refetch_envelope_if_children_wait(root); return false; } - // No answer after the whole retry ladder. As for a block - // (`optimistic-sync.md`), the payload is not applied; the - // envelope is fetched again instead of held, so a client - // that comes back is asked afresh. + // No answer after the whole retry ladder. The envelope is + // consensus-valid, so it is kept, and the per-slot redrive + // asks again; fetching it would only repeat the ladder + // for every copy that arrives. Err(err) => { warn!( %slot, block_root = %ShortRoot(&root.0), %err, - "No verdict from the execution client; not applying the envelope" + "No verdict from the execution client; holding the envelope" ); metrics::inc_engine_no_verdict(); - self.request_missing_envelope(root); + self.hold_envelope_awaiting_engine(root, slot, held); return false; } } } }; - self.advance_beacon_clock_to(held.event_ms); - let config = self.store.config(); - // The engine's part of `verify_execution_payload_envelope` is answered - // above; `VALID` and `SYNCING` both let the payload in (the CL has - // verified it; the EL has not finished). - let verdict = fork_choice::on_execution_payload_envelope( - &mut self.store, - &held.envelope, - &config, - &sidecars, - &ExecutionEngine::valid(), + // `VALID` and `SYNCING` both let the payload in: the consensus layer + // has verified it, the execution layer may not have finished. + fork_choice::accept_execution_payload_envelope(&mut self.store, &held.envelope); + info!( + %slot, + block_root = %ShortRoot(&root.0), + ?status, + "Execution payload envelope verified" ); - match verdict { - Ok(()) => { - info!( - %slot, - block_root = %ShortRoot(&root.0), - ?status, - "Execution payload envelope verified" - ); - self.store - .insert_beacon_block_payload_status(root, slot, status); - // Lets `latestValidHash` find this FULL node. - self.store.insert_beacon_el_block_hash( - root, - slot, - held.envelope.message.payload.block_hash, - ); - self.envelopes.verified.push_back(root); - true - } - Err(err) => { - // p2p judged the envelope before it got here, so no peer is - // penalized from this side. - warn!( - %slot, - block_root = %ShortRoot(&root.0), - ?err, - "Dropping an execution payload envelope that failed verification" - ); - self.refetch_envelope_if_children_wait(root); - false - } - } + self.store + .insert_beacon_block_payload_status(root, slot, status); + // Lets `latestValidHash` find this FULL node. + self.store.insert_beacon_el_block_hash( + root, + slot, + held.envelope.message.payload.block_hash, + ); + self.envelopes.verified.push_back(root); + true + } + + /// Keep a consensus-valid envelope until the execution client answers. + fn hold_envelope_awaiting_engine(&mut self, root: H256, slot: u64, mut held: HeldEnvelope) { + held.slot = slot; + self.envelopes.awaiting_engine.entry(root).or_insert(held); + self.publish_envelope_queues(); + } + + /// Once a slot: ask the execution client again about every envelope it + /// has not answered for. + pub(crate) fn redrive_envelopes_awaiting_engine(&mut self) { + let roots: Vec = self.envelopes.awaiting_engine.keys().copied().collect(); + self.envelopes.work.extend(roots); } /// Every sampled column stored for `root`, decoded as gloas sidecars, or @@ -585,6 +639,7 @@ impl BlockChainServer { for &parent_root in self.envelopes.blocks_awaiting_parent_payload.keys() { if !self.envelopes.awaiting_block.contains_key(&parent_root) && !self.envelopes.awaiting_columns.contains_key(&parent_root) + && !self.envelopes.awaiting_engine.contains_key(&parent_root) && !self.store.has_verified_payload(&parent_root) && !self .store @@ -635,6 +690,9 @@ impl BlockChainServer { .remove(&root) .unwrap_or_default(); } + if let Some(held) = self.envelopes.awaiting_engine.remove(&root) { + return vec![held]; + } let Some(slot) = self .envelopes .awaiting_columns @@ -660,10 +718,13 @@ impl BlockChainServer { pub(crate) async fn settle_envelopes(&mut self) { // Nothing to do on most ticks, and none at all on lean: leave the // gauges alone rather than rewriting them every interval. - if self.envelopes.work.is_empty() && self.envelopes.verified.is_empty() { + if self.envelopes.work.is_empty() + && self.envelopes.verified.is_empty() + && !self.envelopes.head_dirty + { return; } - let mut changed = false; + let mut changed = std::mem::take(&mut self.envelopes.head_dirty); loop { if let Some(root) = self.envelopes.work.pop_front() { // The first that verifies wins; the rest are forgeries or @@ -682,6 +743,7 @@ impl BlockChainServer { } else { break; } + changed |= std::mem::take(&mut self.envelopes.head_dirty); } self.publish_envelope_queues(); if changed { @@ -889,6 +951,9 @@ impl BlockChainServer { self.envelopes .awaiting_columns .retain(|root, held| held.slot >= finalized_slot || *root == finalized.root); + self.envelopes + .awaiting_engine + .retain(|root, held| held.slot >= finalized_slot || *root == finalized.root); let stale: Vec = self .envelopes @@ -1784,7 +1849,11 @@ mod tests { } else { format!(r#"{{"payloadStatus":{result},"payloadId":null}}"#) }; - let payload = format!(r#"{{"jsonrpc":"2.0","id":1,"result":{result}}}"#); + let payload = if result == "ERROR" { + r#"{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"mock failure"}}"#.to_string() + } else { + format!(r#"{{"jsonrpc":"2.0","id":1,"result":{result}}}"#) + }; let response = format!( "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{payload}", payload.len() @@ -2198,4 +2267,331 @@ mod tests { assert!(!server.store.block_index().contains_key(&child_root)); } + + fn count(mock: &Arc>, method: &str) -> usize { + methods(mock).iter().filter(|m| *m == method).count() + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn the_engine_is_not_asked_about_an_envelope_that_fails_consensus_checks() { + let (client, mock) = + mock_engine(status_json("VALID", None), status_json("SYNCING", None)).await; + let case = Case::new("on_execution_payload_envelope", FULL_CHILD_CASE); + let bad_envelope = "73c5948d041ccb171b9331c0eac4a3bb8cdcf01994916fee4c7493e63e890559"; + let mut server = case.server(12); + server.engine = Some(client); + server + .on_block(case.block(FULL_PARENT), ImportTimings::default()) + .await; + + server + .receive_envelope(case.envelope(bad_envelope), 0) + .await; + + assert_eq!(count(&mock, "engine_newPayloadV5"), 0); + assert!(!server.store.has_verified_payload(&root(FULL_PARENT))); + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn an_invalid_block_hash_rejects_only_that_envelope() { + let (client, mock) = mock_engine( + status_json("INVALID_BLOCK_HASH", None), + status_json("SYNCING", None), + ) + .await; + let mut server = server_with_held_child(client).await; + + server + .receive_envelope(valid_envelope_for_full_parent(), 0) + .await; + server.settle_envelopes().await; + + // The root's payload is not condemned: another envelope may still + // reveal it, and the child keeps waiting for one. + assert!(!server.store.has_verified_payload(&root(FULL_PARENT))); + assert!( + !server + .store + .beacon_block_payload_status(root(FULL_PARENT)) + .is_invalidated() + ); + assert!(server.envelopes.is_block_held(&root(FULL_CHILD))); + + // The same envelope is not asked about again. + server + .receive_envelope(valid_envelope_for_full_parent(), 0) + .await; + assert_eq!(count(&mock, "engine_newPayloadV5"), 1); + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn an_engine_error_holds_the_envelope_and_asks_again_once_a_slot() { + let (client, mock) = mock_engine("ERROR".to_string(), status_json("SYNCING", None)).await; + let mut server = server_with_held_child(client).await; + let p2p = Arc::new(crate::tests::RecordingP2P::default()); + server.p2p = Some(p2p.clone()); + let fetches_before = p2p.fetches.lock().unwrap().len(); + + server + .receive_envelope(valid_envelope_for_full_parent(), 0) + .await; + assert!( + server + .envelopes + .awaiting_engine + .contains_key(&root(FULL_PARENT)) + ); + assert_eq!(count(&mock, "engine_newPayloadV5"), 1); + + // The same copy again, and the slot's other redrives, add no asks. + server + .receive_envelope(valid_envelope_for_full_parent(), 0) + .await; + server.redrive_missing_envelopes(); + assert_eq!(count(&mock, "engine_newPayloadV5"), 1); + + // One ask per slot while it keeps failing, and no fetch at all. + for asks in 2..=3 { + server.redrive_envelopes_awaiting_engine(); + server.settle_envelopes().await; + assert_eq!(count(&mock, "engine_newPayloadV5"), asks); + } + assert_eq!(p2p.fetches.lock().unwrap().len(), fetches_before); + assert!(!server.store.has_verified_payload(&root(FULL_PARENT))); + + // The client recovers: the next redrive applies it. + mock.lock().unwrap().new_payload = status_json("VALID", None); + server.redrive_envelopes_awaiting_engine(); + server.settle_envelopes().await; + assert!(server.store.has_verified_payload(&root(FULL_PARENT))); + assert!(server.envelopes.awaiting_engine.is_empty()); + assert!(server.store.has_state(&root(FULL_CHILD)).unwrap()); + } + + /// A chain `q <- p <- r` of gloas blocks above the anchor, each with a + /// verified payload and the cached hash, where `r` built on `p`'s EMPTY + /// node: its bid's parent hash is `q`'s payload, not `p`'s. + /// + /// Returns `(server, [q, p, r], [hq, hp, hr])`. Blocks are the fixture + /// parent with its slot, parent and bid hashes rewritten, stored under + /// made-up roots, which is all the walks read. + fn skipped_payload_chain() -> (BlockChainServer, [H256; 3], [H256; 3]) { + let case = Case::new("on_execution_payload_envelope", FULL_CHILD_CASE); + let mut server = case.server(12); + let anchor = server.store.latest_finalized().unwrap(); + let roots = [ + H256::repeat_byte(0xa1), + H256::repeat_byte(0xa2), + H256::repeat_byte(0xa3), + ]; + let hashes = [ + H256::repeat_byte(0xb1), + H256::repeat_byte(0xb2), + H256::repeat_byte(0xb3), + ]; + // (parent root, bid.parent_block_hash) per block. + let links = [ + (anchor.root, H256::repeat_byte(0xb0)), + (roots[0], hashes[0]), + (roots[1], hashes[0]), + ]; + for i in 0..3 { + let slot = anchor.slot + 1 + i as u64; + let mut block = case.block(FULL_PARENT); + match &mut block { + SignedBeaconBlock::Gloas(inner) => { + inner.message.slot = slot; + inner.message.parent_root = links[i].0; + let bid = &mut inner.message.body.signed_execution_payload_bid.message; + bid.block_hash = hashes[i]; + bid.parent_block_hash = links[i].1; + } + _ => panic!("a gloas block"), + } + server + .store + .insert_pending_block(roots[i], block) + .expect("insert"); + server + .store + .insert_live_chain_entry(slot, roots[i], links[i].0); + let mut envelope = valid_envelope_for_full_parent(); + envelope.message.beacon_block_root = roots[i]; + server.store.insert_verified_payload(slot, &envelope); + server + .store + .insert_beacon_el_block_hash(roots[i], slot, hashes[i]); + server.store.insert_beacon_block_payload_status( + roots[i], + slot, + PayloadStatusEnum::Syncing, + ); + } + (server, roots, hashes) + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn a_latest_valid_hash_does_not_reach_a_payload_the_block_skipped() { + let (mut server, [q, p, r], [hq, _hp, _hr]) = skipped_payload_chain(); + + // r's EL parent is q: p was skipped (r built on p's EMPTY node). + assert_eq!( + crate::beacon_payloads::resolve_invalid_payload(&server.store, r, Some(hq)), + r + ); + assert_eq!( + crate::beacon_payloads::resolve_invalid_payload(&server.store, r, None), + r + ); + // A hash that is not on the chain reads as null. + assert_eq!( + crate::beacon_payloads::resolve_invalid_payload( + &server.store, + r, + Some(H256::repeat_byte(0xee)) + ), + r + ); + // All zeroes: the earliest block on the chain, which is q, not p. + assert_eq!( + crate::beacon_payloads::resolve_invalid_payload(&server.store, r, Some(H256::ZERO)), + q + ); + + server.condemn_payload_chain(r, Some(hq)); + + assert!(!server.store.has_verified_payload(&r)); + assert!( + server.store.has_verified_payload(&p), + "p's payload survives" + ); + assert!(server.store.has_verified_payload(&q)); + assert_eq!( + server.store.beacon_block_payload_status(r), + PayloadStatusEnum::Invalid + ); + assert_eq!( + server.store.beacon_block_payload_status(p), + PayloadStatusEnum::Syncing + ); + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn validation_does_not_promote_a_payload_the_block_skipped() { + let (mut server, [q, p, r], _hashes) = skipped_payload_chain(); + + crate::beacon_payloads::mark_payloads_validated(&mut server.store, r); + + assert_eq!( + server.store.beacon_block_payload_status(r), + PayloadStatusEnum::Valid + ); + assert_eq!( + server.store.beacon_block_payload_status(q), + PayloadStatusEnum::Valid + ); + assert_eq!( + server.store.beacon_block_payload_status(p), + PayloadStatusEnum::Syncing, + "r never built on p's payload" + ); + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn an_invalid_payload_takes_its_full_child_and_spares_the_empty_one() { + let (mut server, [q, p, r], [_hq, hp, _hr]) = skipped_payload_chain(); + // Two children of p: `full` names p's payload as its parent's, `empty` + // names the payload before it. Condemning p's payload takes only `full`. + let slot = server.store.get_signed_block(&p).unwrap().unwrap().slot() + 1; + let full = H256::repeat_byte(0xc1); + let empty = H256::repeat_byte(0xc2); + for (root, parent_hash) in [(full, hp), (empty, H256::repeat_byte(0xb1))] { + let mut block = server.store.get_signed_block(&r).unwrap().unwrap(); + match &mut block { + SignedBeaconBlock::Gloas(inner) => { + inner.message.slot = slot; + inner.message.parent_root = p; + inner + .message + .body + .signed_execution_payload_bid + .message + .parent_block_hash = parent_hash; + } + _ => panic!("a gloas block"), + } + server + .store + .insert_pending_block(root, block) + .expect("insert"); + server.store.insert_live_chain_entry(slot, root, p); + } + + server.condemn_payload_chain(p, None); + + let index = server.store.block_index(); + assert!(!server.store.has_verified_payload(&p)); + assert!(index.contains_key(&p), "the block stays"); + assert!(!index.contains_key(&full), "the FULL child goes"); + assert!(index.contains_key(&empty), "the EMPTY child stays"); + assert!(server.store.has_verified_payload(&q)); + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn a_restored_payload_is_seeded_with_a_verdict_a_valid_answer_can_promote() { + let (mut server, [q, p, r], _hashes) = skipped_payload_chain(); + // A restart empties the status map; the verified set is on disk. + for root in [q, p, r] { + server + .store + .insert_beacon_block_payload_status(root, 0, PayloadStatusEnum::Valid); + } + let mut restarted = server; + + crate::beacon_payloads::seed_resumed_payload_statuses(&mut restarted.store, true); + assert_eq!( + restarted.store.beacon_block_payload_status(r), + PayloadStatusEnum::Syncing + ); + assert!(restarted.store.has_unvalidated_block_payloads()); + crate::beacon_payloads::mark_payloads_validated(&mut restarted.store, r); + assert_eq!( + restarted.store.beacon_block_payload_status(r), + PayloadStatusEnum::Valid + ); + + crate::beacon_payloads::seed_resumed_payload_statuses(&mut restarted.store, false); + assert_eq!( + restarted.store.beacon_block_payload_status(p), + PayloadStatusEnum::Valid + ); + } } diff --git a/crates/blockchain/src/beacon_payloads.rs b/crates/blockchain/src/beacon_payloads.rs index 4ac488b65..a935fb240 100644 --- a/crates/blockchain/src/beacon_payloads.rs +++ b/crates/blockchain/src/beacon_payloads.rs @@ -39,7 +39,7 @@ use ethlambda_types::beacon::containers::{SignedBeaconBlock, gloas}; use ethlambda_types::beacon::primitives::ExecutionBlockHash; use ethlambda_types::beacon::signing::compute_start_slot_at_epoch; use ethlambda_types::primitives::H256; -use tracing::{error, warn}; +use tracing::{debug, error, warn}; use crate::BlockChainServer; @@ -82,7 +82,7 @@ pub(crate) fn payload_hash_of(store: &Store, root: H256) -> Option ExecutionBlockHash { +pub fn checkpoint_hash(store: &Store, root: H256) -> ExecutionBlockHash { if let Some(block) = store.get_signed_block(&root).ok().flatten() && let Some(bid) = bid_of(&block) { @@ -138,90 +138,161 @@ pub(crate) fn forkchoice_plan(store: &Store, head_root: H256) -> Option (u64, H256) { + let finalized = store.beacon_finalized_checkpoint(); + (compute_start_slot_at_epoch(finalized.epoch), finalized.root) +} + +/// The block whose payload `root`'s payload builds on in the execution layer, +/// or `None` when it is below what this store knows (an anchor's parent) or +/// null. +/// +/// A gloas block names it in its bid: the ancestor whose payload hash is +/// `bid.parent_block_hash`, skipping every block between that built on an +/// EMPTY node (their payloads are not in this chain). A pre-gloas block +/// builds on its parent's payload. The one rule both +/// [`resolve_invalid_payload`] and [`mark_payloads_validated`] walk by, since +/// treating every verified ancestor as an EL ancestor would condemn or +/// validate payloads the block never built on. +fn el_parent(store: &Store, root: H256) -> Option { + let block = store.get_signed_block(&root).ok().flatten()?; + let (_slot, parent) = store.block_entry(&root)?; + let Some(bid) = bid_of(&block) else { + return payload_hash_of(store, parent).map(|_| parent); + }; + if bid.parent_block_hash.is_zero() { + return None; + } + let (floor_slot, floor_root) = finality_floor(store); + let mut cursor = parent; + loop { + if payload_hash_of(store, cursor) == Some(bid.parent_block_hash) { + return Some(cursor); + } + // Nothing below finality is searched: the walk would only scan the + // whole window for an answer no verdict can act on. + let (slot, next) = store.block_entry(&cursor)?; + if cursor == floor_root || slot <= floor_slot { + return None; + } + cursor = next; + } +} + +/// `start` and then each EL parent in turn, ending at the first block at or +/// below finality (included) or the first with no EL parent. +fn el_chain(store: &Store, start: H256) -> Vec { + let (floor_slot, floor_root) = finality_floor(store); + let mut chain = vec![start]; + let mut cursor = start; + while let Some(parent) = el_parent(store, cursor) { + chain.push(parent); + let at_floor = store + .block_entry(&parent) + .is_some_and(|(slot, _)| slot <= floor_slot); + if parent == floor_root || at_floor { + break; + } + cursor = parent; + } + chain +} + /// The nearest ancestor-or-self of `from` whose payload hash is `hash`: the /// block a `forkchoiceUpdated` verdict about `hash` is about. -pub(crate) fn payload_holder( - store: &Store, - index: &HashMap, - from: H256, - hash: ExecutionBlockHash, -) -> Option { +pub(crate) fn payload_holder(store: &Store, from: H256, hash: ExecutionBlockHash) -> Option { + let (floor_slot, floor_root) = finality_floor(store); let mut cursor = from; loop { if payload_hash_of(store, cursor) == Some(hash) { return Some(cursor); } - cursor = index.get(&cursor)?.1; + let (slot, next) = store.block_entry(&cursor)?; + if cursor == floor_root || slot <= floor_slot { + return None; + } + cursor = next; } } -/// The block whose payload is the first invalid one on `holder`'s chain, per -/// `optimistic-sync.md`'s `latestValidHash` table, reading payloads rather +/// The block whose payload is the first invalid one on `holder`'s EL chain, +/// per `optimistic-sync.md`'s `latestValidHash` table, reading payloads rather /// than blocks. /// /// [`fork_choice::resolve_invalid_block`] walks parents while each carries a /// cached payload hash, which stops at the first gloas block whose payload was -/// never revealed. Here a root with no payload is skipped, since an EMPTY node -/// inherits its parent's payload and a payload built on a FULL ancestor -/// descends from it in the execution layer whatever sits between. +/// never revealed. Here the walk follows [`el_parent`], so a payload applied +/// to an ancestor that `holder` did not build on (it built on that ancestor's +/// EMPTY node) is never reached, and an `INVALID` verdict about `holder` +/// cannot delete it. /// /// | `latest_valid_hash` | result | /// |---|---| -/// | a payload hash found on this chain | the next payload-carrying block after it | -/// | all zeroes | the earliest payload-carrying ancestor | +/// | a payload hash found on this chain | the next block after it on the chain | +/// | all zeroes | the earliest block on the chain | /// | `None`, or a hash not on this chain | `holder` itself | pub(crate) fn resolve_invalid_payload( store: &Store, - index: &HashMap, holder: H256, latest_valid_hash: Option, ) -> H256 { let Some(latest_valid_hash) = latest_valid_hash else { return holder; }; - let mut condemned = holder; - let mut cursor = holder; - while let Some(&(_slot, parent)) = index.get(&cursor) { - cursor = parent; - match payload_hash_of(store, cursor) { - Some(hash) if hash == latest_valid_hash => return condemned, - Some(_) => condemned = cursor, - None => {} + let chain = el_chain(store, holder); + for pair in chain.windows(2) { + if payload_hash_of(store, pair[1]) == Some(latest_valid_hash) { + return pair[0]; } } if latest_valid_hash.is_zero() { - condemned + *chain.last().expect("the chain holds its start") } else { - // Ran off the top of what this store indexes without finding it: - // `optimistic-sync.md` says to treat it as `null`. + // Not on this chain: `optimistic-sync.md` says to treat it as `null`. holder } } -/// Moves `root` and every optimistic ancestor to validated: pre-gloas blocks -/// leave `optimistic_roots`, gloas payloads recorded `NOT_VALIDATED` become -/// `VALID`. `optimistic-sync.md`: a block's ancestors transition with it. +/// Moves `root` and every optimistic block on its EL chain to validated: +/// pre-gloas blocks leave `optimistic_roots`, gloas payloads recorded +/// `NOT_VALIDATED` become `VALID`. `optimistic-sync.md`: a block's ancestors +/// transition with it. Only EL ancestors do: a payload applied to a block the +/// chain skipped past was not part of what the execution client validated. /// -/// A single walk of the whole ancestry rather than -/// [`fork_choice::mark_validated`]'s stop at the first non-optimistic parent, -/// since a gloas block with a settled payload can sit between two optimistic -/// ones. Skipped entirely, with no `block_index` scan, when nothing is -/// optimistic. +/// Skipped entirely when nothing is optimistic, and bounded at finality +/// otherwise. pub(crate) fn mark_payloads_validated(store: &mut Store, root: H256) { - fork_choice::mark_validated(store, root); if !store.has_unvalidated_block_payloads() && !store.has_beacon_optimistic_roots() { return; } - let index = store.block_index(); - let mut cursor = root; - while let Some(&(slot, parent)) = index.get(&cursor) { - store.remove_beacon_optimistic_root(cursor); - if store.has_verified_payload(&cursor) - && store.beacon_block_payload_status(cursor).is_not_validated() + for block in el_chain(store, root) { + store.remove_beacon_optimistic_root(block); + if store.has_verified_payload(&block) + && store.beacon_block_payload_status(block).is_not_validated() + && let Some((slot, _parent)) = store.block_entry(&block) { - store.insert_beacon_block_payload_status(cursor, slot, PayloadStatusEnum::Valid); + store.insert_beacon_block_payload_status(block, slot, PayloadStatusEnum::Valid); + } + } +} + +/// Gives a payload restored from disk a verdict: the status map is scratch, so +/// without this a verified payload from before a restart has no entry, reads +/// as `NOT_VALIDATED`, and is never promoted. With an engine configured they +/// start `SYNCING`, so the first `forkchoiceUpdated` `VALID` for the head +/// validates them; without one the follower trusts them, as it trusts every +/// envelope it applies. +pub(crate) fn seed_resumed_payload_statuses(store: &mut Store, engine_configured: bool) { + let status = if engine_configured { + PayloadStatusEnum::Syncing + } else { + PayloadStatusEnum::Valid + }; + for root in store.verified_payload_roots() { + if let Some((slot, _parent)) = store.block_entry(&root) { + store.insert_beacon_block_payload_status(root, slot, status); } - cursor = parent; } } @@ -242,9 +313,10 @@ impl BlockChainServer { let (PayloadValidity::Validated | PayloadValidity::Invalidated { .. }) = verdict else { return; }; - let index = self.store.block_index(); - let Some(holder) = payload_holder(&self.store, &index, head_root, head_hash) else { - warn!( + let Some(holder) = payload_holder(&self.store, head_root, head_hash) else { + // Expected for a head whose parent payload is below the index (an + // anchor): the specification treats it as known-null. + debug!( head_root = %ShortRoot(&head_root.0), "No block on the head's chain carries the payload forkchoiceUpdated named" ); @@ -276,8 +348,8 @@ impl BlockChainServer { holder: H256, latest_valid_hash: Option, ) -> bool { + let condemned = resolve_invalid_payload(&self.store, holder, latest_valid_hash); let index = self.store.block_index(); - let condemned = resolve_invalid_payload(&self.store, &index, holder, latest_valid_hash); let mut removed = self.condemn_gloas_payload(condemned, &index); // The holder's own payload is invalid whichever ancestor the verdict // reached; when it is a gloas payload it must not stay FULL. diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 34f848d8e..597ba52b8 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -59,6 +59,7 @@ mod beacon_columns; pub mod beacon_engine; mod beacon_envelope; mod beacon_payloads; +pub use beacon_payloads::checkpoint_hash; pub mod block_builder; pub(crate) mod coverage; pub mod events; @@ -379,7 +380,7 @@ impl BlockChain { /// `safe_slots_to_import_optimistically`, which nothing on that arm reads. #[allow(clippy::too_many_arguments)] fn start_actor( - store: Store, + mut store: Store, sync_status: SyncStatusTracker, sync_status_controller: SyncStatusController, events: EventBus, @@ -390,6 +391,12 @@ impl BlockChain { ) -> BlockChain { let genesis_time = store.config().genesis_time; + // Payloads verified before a restart have no recorded verdict, since + // the map is scratch; see `seed_resumed_payload_statuses`. + if matches!(duties, ChainDuties::Beacon) { + beacon_payloads::seed_resumed_payload_statuses(&mut store, engine.is_some()); + } + // `sidecars_awaiting_parent` starts empty below, and it is the only // index into `Table::PendingDataColumns`. Anything a previous run // parked there is unreachable from here on, so it goes now rather than @@ -1048,6 +1055,7 @@ impl BlockChainServer { self.evict_envelope_queues_at_or_below_finality(); self.redrive_held_blocks().await; self.redrive_envelopes_awaiting_columns(); + self.redrive_envelopes_awaiting_engine(); self.redrive_missing_envelopes(); self.settle_envelopes().await; diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index d23523a14..bf0df95e0 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -4923,6 +4923,28 @@ pub fn on_execution_payload_envelope( config: &Config, sidecars: &[gloas::DataColumnSidecar], engine: &stf::ExecutionEngine, +) -> Result<()> { + check_execution_payload_envelope(store, signed_envelope, config, sidecars, engine)?; + accept_execution_payload_envelope(store, signed_envelope); + Ok(()) +} + +/// Every check of [`on_execution_payload_envelope`], without recording the +/// payload. +/// +/// Split out so a caller whose execution engine answers over the network can +/// run the pure consensus checks first, ask the engine only about an envelope +/// that passed them, and record the payload with +/// [`accept_execution_payload_envelope`] once the answer allows it. The +/// `engine` argument still answers the engine's part of +/// `verify_execution_payload_envelope`; such a caller passes +/// [`stf::ExecutionEngine::valid`] and consults the real engine itself. +pub fn check_execution_payload_envelope( + store: &Store, + signed_envelope: &gloas::SignedExecutionPayloadEnvelope, + config: &Config, + sidecars: &[gloas::DataColumnSidecar], + engine: &stf::ExecutionEngine, ) -> Result<()> { let envelope = &signed_envelope.message; let block_root = envelope.beacon_block_root; @@ -4952,13 +4974,26 @@ pub fn on_execution_payload_envelope( // Verify the execution payload envelope. stf::gloas::verify_execution_payload_envelope(&state, signed_envelope, config, engine)?; - // Add execution payload envelope to the store. Persisted, so a restarted - // follower keeps the full branch of this block. - store.insert_verified_payload(block.slot(), signed_envelope); - Ok(()) } +/// The recording half of [`on_execution_payload_envelope`]: adds a checked +/// envelope to the store. Persisted, so a restarted follower keeps the full +/// branch of this block. +/// +/// The caller must have run [`check_execution_payload_envelope`] on the same +/// envelope; a block the store does not hold is the only case this refuses. +pub fn accept_execution_payload_envelope( + store: &mut Store, + signed_envelope: &gloas::SignedExecutionPayloadEnvelope, +) { + let block_root = signed_envelope.message.beacon_block_root; + let Some((slot, _parent)) = store.block_entry(&block_root) else { + return; + }; + store.insert_verified_payload(slot, signed_envelope); +} + /// Validates `attestation` and, if valid, records it as each attester's /// latest message. /// diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs index fb0162b37..6de77cf45 100644 --- a/crates/net/rpc/src/beacon/proposal.rs +++ b/crates/net/rpc/src/beacon/proposal.rs @@ -369,11 +369,18 @@ async fn build_payload( ); ExecutionAddress::ZERO }); - let el_hash = |root: H256| store.beacon_el_block_hash(root).unwrap_or(H256::ZERO); + // The same rule `forkchoiceUpdated` uses for a checkpoint block (a gloas + // one is its bid's parent hash), so the two calls never disagree. let forkchoice = ForkchoiceStateV1 { head_block_hash: inputs.parent_hash, - safe_block_hash: el_hash(store.beacon_justified_checkpoint().root), - finalized_block_hash: el_hash(store.beacon_finalized_checkpoint().root), + safe_block_hash: ethlambda_blockchain::checkpoint_hash( + store, + store.beacon_justified_checkpoint().root, + ), + finalized_block_hash: ethlambda_blockchain::checkpoint_hash( + store, + store.beacon_finalized_checkpoint().root, + ), }; let attributes = PayloadAttributesV3 { timestamp: inputs.timestamp, diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 8190d678a..37499f6c7 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -3429,7 +3429,13 @@ impl Store { /// derived from [`Self::has_verified_payload`]) and the envelope is no /// longer served to peers. Returns whether `root` was verified. pub fn remove_verified_payload(&mut self, root: &H256) -> bool { - let was_verified = self.beacon.lock().unwrap().verified_payloads.remove(root); + let was_verified = { + let mut scratch = self.beacon.lock().unwrap(); + // The cached hash goes too: nothing may resolve a payload this + // store no longer holds as a FULL node. + scratch.el_block_hashes.remove(root); + scratch.verified_payloads.remove(root) + }; if let Some(slot) = beacon_block_slot(self.backend.as_ref(), root) { let keys = vec![encode_slot_root_key(slot, root)]; let mut batch = self.backend.begin_write().expect("write batch"); @@ -3441,6 +3447,17 @@ impl Store { was_verified } + /// Gloas: the roots whose payload is verified, in no particular order. + pub fn verified_payload_roots(&self) -> Vec { + self.beacon + .lock() + .unwrap() + .verified_payloads + .iter() + .copied() + .collect() + } + /// Gloas: the verified envelope stored for `root`, if any. pub fn get_execution_payload_envelope( &self, @@ -7692,8 +7709,10 @@ mod tests { .insert_signed_block(root, gloas_test_block(SLOTS_PER_EPOCH, H256::ZERO)) .expect("insert block"); store.insert_verified_payload(SLOTS_PER_EPOCH, &test_envelope(root)); + store.insert_beacon_el_block_hash(root, SLOTS_PER_EPOCH, H256::from([9u8; 32])); assert!(store.remove_verified_payload(&root)); + assert_eq!(store.beacon_el_block_hash(root), None); assert!(!store.remove_verified_payload(&root)); assert!(!store.has_verified_payload(&root)); diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 874b76b1d..3ed32007a 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -278,9 +278,12 @@ node, and each call walks every latest message and every ancestor step. ## The execution client judges a gloas payload before the envelope is applied `verify_execution_payload_envelope` asks the execution engine last, as part of -one boolean function. The follower asks it first, with `engine_newPayloadV5`, -and applies the envelope with the engine's part answered "valid", because the -answer has three outcomes and the specification's `ExecutionEngine` has two. +one boolean function. The follower runs the pure consensus checks first +(`check_execution_payload_envelope`: signature, bid, state, data availability), +then asks the engine with `engine_newPayloadV5` only about an envelope that +passed them, and records it with `accept_execution_payload_envelope`. The +engine's part is answered "valid" inside the checks because its answer has +three outcomes and the specification's `ExecutionEngine` has two. - **`VALID`:** applied; the payload is recorded `VALID`. - **`SYNCING` / `ACCEPTED`:** applied, since the consensus checks passed and the @@ -300,12 +303,19 @@ answer has three outcomes and the specification's `ExecutionEngine` has two. payload, unlike the pre-gloas `resolve_invalid_block`). A `forkchoiceUpdated` `INVALID` for a payload already applied removes it again (its FULL node, its stored envelope, and every child built on it). -- **No answer after the retry ladder:** as for a block, the payload is not - applied; the envelope is fetched again rather than held. - -The engine is asked only about an envelope whose builder signature holds: an -unsigned copy with the honest block hash and a different body would otherwise -earn an `INVALID` that lands on the honest payload. +- **`INVALID_BLOCK_HASH`:** the envelope's contents do not hash to the block + hash it claims, so only that envelope (by its own root) is refused; the + root's payload is not condemned, since the builder's real envelope may still + arrive. +- **No answer after the retry ladder:** the payload is not applied, as for a + block, but unlike a block the envelope is consensus-valid and already held, + so it is kept and the per-slot redrive asks again. It is not fetched again: + that would repeat the ladder for every copy that arrives. + +`latestValidHash` and `VALID` follow the execution-layer chain, not every +verified ancestor: a gloas block's EL parent is the ancestor whose payload hash +is its `bid.parent_block_hash`, skipping blocks it built past on their EMPTY +node, and a pre-gloas block's is its parent. Both walks stop at finality. ## `forkchoiceUpdated` for a gloas head From 4a8b2445b6f98227c2ee478476dbea53a5b2d13f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 04:34:27 -0300 Subject: [PATCH 078/132] fix(beacon): scope hash-mismatch verdicts to the envelope and probe a down engine once ethrex has no INVALID_BLOCK_HASH status: it answers a block hash mismatch with INVALID and a null latestValidHash, and a payload that executed and failed with INVALID naming its last valid ancestor. A null one is now read as a mismatch (only that envelope is refused, so the honest copy can still arrive) and a non-null one as the payload being invalid. With the engine down each held envelope would spend a whole retry ladder per slot, and the held set grows with the finality lag. The redrive now asks about the oldest one, and releases the rest only once an ask is answered. A refused copy is also looked up before the consensus checks, so a redelivery costs nothing. --- crates/blockchain/src/beacon_envelope.rs | 134 ++++++++++++++++++++--- docs/spec_deviations.md | 18 ++- 2 files changed, 134 insertions(+), 18 deletions(-) diff --git a/crates/blockchain/src/beacon_envelope.rs b/crates/blockchain/src/beacon_envelope.rs index 544a21502..ed0fa8d19 100644 --- a/crates/blockchain/src/beacon_envelope.rs +++ b/crates/blockchain/src/beacon_envelope.rs @@ -275,6 +275,15 @@ impl BlockChainServer { ); return false; } + // Before any work: a copy the execution client already refused costs + // nothing. + if self.envelopes.is_rejected(&held.envelope_root) { + trace!( + block_root = %ShortRoot(&root.0), + "Dropping an envelope the execution client already rejected" + ); + return false; + } if self .envelopes .awaiting_engine @@ -397,22 +406,31 @@ impl BlockChainServer { // specification's engine has two (see `docs/spec_deviations.md`). // Without a client the follower trusts the payload, as it already does // for every pre-gloas block it imports (`NotRequired`). - if self.envelopes.is_rejected(&held.envelope_root) { - trace!( - block_root = %ShortRoot(&root.0), - "Dropping an envelope the execution client already rejected" - ); - return false; - } let status = match self.engine.clone() { None => PayloadStatusEnum::Valid, Some(client) => { - match beacon_engine::ask_envelope_status(&client, &block, &held.envelope).await { + let asked = + beacon_engine::ask_envelope_status(&client, &block, &held.envelope).await; + // The client answered, so the envelopes waiting on it can be + // asked about in this same pass. + if asked.is_ok() { + self.release_envelopes_awaiting_engine(); + } + match asked { // The payload does not hash to the block hash it claims: // this envelope is bad, the payload the bid names may not // be. Only this envelope is refused, so the builder's real // one can still arrive. - Ok(raw) if raw.status == PayloadStatusValue::InvalidBlockHash => { + // Some execution clients have no `INVALID_BLOCK_HASH` and + // answer a hash mismatch with `INVALID` and a null + // `latestValidHash` (ethrex does), whereas a payload that + // executed and failed names its last valid ancestor. So a + // null one is read the same way. + Ok(raw) + if raw.status == PayloadStatusValue::InvalidBlockHash + || (raw.status == PayloadStatusValue::Invalid + && raw.latest_valid_hash.is_none()) => + { warn!( %slot, block_root = %ShortRoot(&root.0), @@ -499,9 +517,25 @@ impl BlockChainServer { self.publish_envelope_queues(); } - /// Once a slot: ask the execution client again about every envelope it - /// has not answered for. + /// Once a slot: ask the execution client again about the oldest envelope + /// it has not answered for. + /// + /// One, not all: with the client down each ask spends the whole retry + /// ladder, and the held envelopes grow with the finality lag. The rest are + /// released by [`Self::release_envelopes_awaiting_engine`] once an ask gets + /// any answer. pub(crate) fn redrive_envelopes_awaiting_engine(&mut self) { + let oldest = self + .envelopes + .awaiting_engine + .iter() + .min_by_key(|(root, held)| (held.slot, held.event_ms, **root)) + .map(|(root, _)| *root); + self.envelopes.work.extend(oldest); + } + + /// Queue every envelope held for the execution client. + fn release_envelopes_awaiting_engine(&mut self) { let roots: Vec = self.envelopes.awaiting_engine.keys().copied().collect(); self.envelopes.work.extend(roots); } @@ -1970,8 +2004,12 @@ mod tests { ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" )] async fn an_invalid_verdict_leaves_no_full_node_and_drops_the_full_child() { - let (client, mock) = - mock_engine(status_json("INVALID", None), status_json("SYNCING", None)).await; + // A non-null `latestValidHash`: the payload executed and failed. + let (client, mock) = mock_engine( + status_json("INVALID", Some(H256::repeat_byte(0x11))), + status_json("SYNCING", None), + ) + .await; let mut server = server_with_held_child(client).await; server @@ -2594,4 +2632,74 @@ mod tests { PayloadStatusEnum::Valid ); } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn an_invalid_verdict_with_a_null_latest_valid_hash_rejects_only_that_envelope() { + // How ethrex reports a payload whose contents do not hash to its block + // hash: INVALID, `latestValidHash` null. + let (client, mock) = + mock_engine(status_json("INVALID", None), status_json("SYNCING", None)).await; + let mut server = server_with_held_child(client).await; + + server + .receive_envelope(valid_envelope_for_full_parent(), 0) + .await; + server.settle_envelopes().await; + + assert!(!server.store.has_verified_payload(&root(FULL_PARENT))); + assert!( + !server + .store + .beacon_block_payload_status(root(FULL_PARENT)) + .is_invalidated() + ); + assert!(server.envelopes.is_block_held(&root(FULL_CHILD))); + server + .receive_envelope(valid_envelope_for_full_parent(), 0) + .await; + assert_eq!(count(&mock, "engine_newPayloadV5"), 1); + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn a_down_engine_is_probed_once_per_redrive_however_many_envelopes_wait() { + let (client, mock) = mock_engine("ERROR".to_string(), status_json("SYNCING", None)).await; + let mut server = server_with_held_child(client).await; + server + .receive_envelope(valid_envelope_for_full_parent(), 0) + .await; + // More envelopes waiting, for blocks later than the real one. + for byte in 1..=4u8 { + let mut envelope = valid_envelope_for_full_parent(); + envelope.message.beacon_block_root = H256::repeat_byte(byte); + server.envelopes.awaiting_engine.insert( + H256::repeat_byte(byte), + HeldEnvelope::new(envelope, 1_000 + u64::from(byte), 0), + ); + } + assert_eq!(server.envelopes.awaiting_engine.len(), 5); + assert_eq!(count(&mock, "engine_newPayloadV5"), 1); + + for asks in 2..=3 { + server.redrive_envelopes_awaiting_engine(); + server.settle_envelopes().await; + assert_eq!(count(&mock, "engine_newPayloadV5"), asks); + assert_eq!(server.envelopes.awaiting_engine.len(), 5); + } + + // The engine answers: the oldest is applied and the rest are released + // in the same redrive. + mock.lock().unwrap().new_payload = status_json("VALID", None); + server.redrive_envelopes_awaiting_engine(); + server.settle_envelopes().await; + assert!(server.store.has_verified_payload(&root(FULL_PARENT))); + assert!(server.envelopes.awaiting_engine.is_empty()); + } } diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 3ed32007a..91607f83d 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -303,13 +303,21 @@ three outcomes and the specification's `ExecutionEngine` has two. payload, unlike the pre-gloas `resolve_invalid_block`). A `forkchoiceUpdated` `INVALID` for a payload already applied removes it again (its FULL node, its stored envelope, and every child built on it). -- **`INVALID_BLOCK_HASH`:** the envelope's contents do not hash to the block - hash it claims, so only that envelope (by its own root) is refused; the - root's payload is not condemned, since the builder's real envelope may still - arrive. +- **A hash mismatch (`INVALID_BLOCK_HASH`, or `INVALID` with a null + `latestValidHash`):** the envelope's contents do not hash to the block hash it + claims, so only that envelope (by its own root) is refused; the root's + payload is not condemned, since the builder's real envelope may still arrive. + ethrex has no `INVALID_BLOCK_HASH` status: it answers a block hash mismatch + with `INVALID` and a null `latestValidHash`, and an executed-and-failed + payload with `INVALID` and its last valid ancestor. So a null one is read as + a mismatch and a non-null one as the payload being invalid. A payload that + failed with no valid ancestor to name is read as a mismatch too, which only + costs a refused envelope. - **No answer after the retry ladder:** the payload is not applied, as for a block, but unlike a block the envelope is consensus-valid and already held, - so it is kept and the per-slot redrive asks again. It is not fetched again: + so it is kept and the per-slot redrive asks again, about the oldest held + envelope only and the rest once that ask is answered, so a down client costs + one retry ladder per slot however many wait. It is not fetched again: that would repeat the ladder for every copy that arrives. `latestValidHash` and `VALID` follow the execution-layer chain, not every From 1ab5abe6f2d01831afc9f4f084d89794a4fb52d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 04:48:50 -0300 Subject: [PATCH 079/132] docs(beacon): describe the gloas follower as it runs The follower now follows gloas, but CLAUDE.md, README.md and several docs still said it refuses gloas blocks and anchors, that the envelope protocols are served but never asked, and that the beacon subcommand imports nothing. Describe what is true: the envelope pipeline and its queues, the gloas gossip rules and their deliberate departures, runtime fork-digest rollover and its ENR gap, the envelope req/resp client, the new storage tables and their resume behavior, and how the execution client judges a gloas payload. --- CLAUDE.md | 93 +++++++++++++++++++++++++++++++++-------- README.md | 8 ++-- docs/beacon_engine.md | 63 ++++++++++++++++++++++++---- docs/beacon_stf.md | 30 +++++++++++++ docs/beacon_wire.md | 52 ++++++++++++++++++----- docs/checkpoint_sync.md | 2 +- docs/cli.md | 53 ++++++++++------------- docs/data_storage.md | 9 ++++ docs/discovery.md | 7 +++- docs/spec_deviations.md | 5 ++- 10 files changed, 248 insertions(+), 74 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 5ef12fd3a..874395e78 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,7 +45,7 @@ crates/ └─ test-fixtures/ # Spec-fixture loading (prod dep of rpc's Hive test driver) net/ ├─ api/ # Actor protocol traits wiring BlockChain ↔ P2P - ├─ p2p/ # libp2p: gossipsub + req-resp (Status, BlocksByRoot, BlocksByRange) + ├─ p2p/ # libp2p: gossipsub + req-resp (Status, BlocksByRoot, BlocksByRange, gloas envelopes) │ ├─ src/gossipsub/ # Topic encoding, message handling │ ├─ src/req_resp/ # Request/response codec and handlers │ └─ src/metrics.rs # Peer connection/disconnection tracking @@ -334,12 +334,27 @@ actual_slot = finalized_slot + 1 + relative_index - Beacon wire: `validate_messages()` is on, so every beacon message waits for a verdict (~4.2s before gossipsub's cache evicts it). Rules in `state_transition::beacon::gossip` (cheap half inline, stateful half on a bounded `spawn_blocking` task); plumbing in `p2p/src/beacon/verdict.rs`. Lean gossip still auto-forwards - Data columns: every check runs in p2p. A column gossip did not accept (`Queue`/`Overloaded`), every fetched column, and parked columns replayed after their parent imports go through `column::chain_checks` in `p2p/src/beacon/column_checks.rs`. The chain actor stores what it gets unchecked; only debug builds re-run `chain_checks` there - Beacon subscribes seven global topics plus two node-id-derived subnet families: custody - columns and backbone attestation subnets. `beacon_aggregate_and_proof` and + columns and backbone attestation subnets. Gloas digests add two more topics, + `execution_payload` and `payload_attestation_message` (`BeaconTopics::for_fork`; earlier + digests never carry them). Gloas has its own rules for `beacon_block`, + `data_column_sidecar` (fork enum `DataColumnSidecar`, fork from the topic's digest), + aggregates (`SignedAggregateAndProof::Gloas`, aggregation-bits length bounded before + expansion) and attestations (`verify_attestation_payload_status`). Deliberate + departures: with no bad-block cache a block seen without a post-state is queued or + ignored where the spec rejects; an envelope whose block state is not cached is queued and + forwarded to the actor (which verifies every envelope) rather than dropped; payload votes + are applied without a second signature check. `beacon_aggregate_and_proof` and `beacon_attestation_{subnet_id}` validate in p2p like blocks/columns, with their own permit pool. The actor applies only accepted aggregates, attesting indices already gossip-verified, held one slot per `validate_on_attestation`'s `current_slot >= data.slot + 1`; subnet attestations are verified and relayed but never applied. See [`docs/beacon_wire.md`](docs/beacon_wire.md) + - Fork-digest rollover happens at runtime (`ForkSchedule`, `beacon::transition`): the next + digest's topics are joined `SUBSCRIBE_LEAD_EPOCHS` before its boundary and the previous + one's left `UNSUBSCRIBE_LAG_EPOCHS` after, blob-parameter (BPO) boundaries included. + **Known gap**: ethrex's `DiscoveryServer` cannot replace the served ENR at runtime, so its + `eth2` entry keeps the startup digest until a restart; peers discovering the node after a + boundary reject the record - **Req/Resp**: Status, BlocksByRoot, BlocksByRange (snappy frame compression + varint length) - Beacon adds `beacon_blocks_by_{range,root}/2` alongside its Status/Ping/MetaData/Goodbye set. Both serve from the checkpoint-anchored store, and `build_status` advertises it @@ -349,6 +364,11 @@ actual_slot = finalized_slot + 1 + relative_index the chain actor, since the store's head trails a delivered batch by the whole actor mailbox) and the store's head (which gossip imports move and the watermark never sees). Fetched blocks reach the actor as `BlockSource::Sync`. + - Gloas adds `execution_payload_envelopes_by_{range,root}/1`. Server: canonical FULL + envelopes only, and the head's envelope only when the head node is FULL. Client: by root + on `needs_envelope` (same dedup and retry ladder as block fetches), by range after a + gloas span's blocks were handed to the actor, with range sync holding the next batch + until the envelope answer is forwarded (the actor keeps unmatched envelopes only briefly) Their chunks carry four `` (the block's own epoch's fork digest), which is why `BeaconWire` and the codec carry `genesis_validators_root`. See [`docs/beacon_wire.md`](docs/beacon_wire.md) @@ -451,20 +471,23 @@ against them. `beacon::tests::the_built_in_network_derives_what_it_always_did` pins the parsed mainnet config to `Config::mainnet()`, so the file and the Rust constant cannot drift apart. Sepolia's config schedules gloas: its `GLOAS_*` keys are claimed (`ForkName::Gloas` exists, and `Config` carries the -fork's schedule, timing and churn fields), and its state transition and fork -choice exist, but the live follower refuses gloas blocks -(`process_or_pend_block`) and a gloas anchor (`refuse_unfollowable_fork` in -`main.rs`), since nothing delivers payload envelopes or payload attestations to -it, so a Sepolia follower stops tracking the chain at `GLOAS_FORK_EPOCH` -regardless. `network::warn_if_unfollowed_fork_scheduled` (shared by -`BuiltInNetwork::resolve` and a loaded `NetworkDir`) warns about this -explicitly at startup, once per scheduled fork for which `ForkName::is_followed` -answers no (the one place that says which forks the node follows), separately -from the ignored-keys warning, which now only ever names heze's keys and +fork's schedule, timing and churn fields), and the live follower follows it +(`ForkName::Gloas.is_followed()`): the chain actor imports gloas blocks and +their payload envelopes, the p2p layer delivers both and the payload votes, and +the follower crosses `GLOAS_FORK_EPOCH` without a restart (fork-digest +rollover, see Networking). `refuse_unfollowable_fork` in `main.rs` and +`network::warn_if_unfollowed_fork_scheduled` (shared by +`BuiltInNetwork::resolve` and a loaded `NetworkDir`) are now inert guards for +the next fork the node cannot follow: the first refuses such an anchor with +`UnsupportedFork`, the second warns once per scheduled fork for which +`ForkName::is_followed` answers no (the one place that says which forks the +node follows). The ignored-keys warning only ever names heze's keys and `GAS_LIMIT_SCHEDULE`/`INCLUSION_LIST_DUE_BPS`. Platåberget (the Glamsterdam testnet) is already past its `GLOAS_FORK_EPOCH`, -so its checkpoint anchor is a gloas state that `refuse_unfollowable_fork` -refuses; its entry is there for when the live follower takes gloas blocks. Its +so its checkpoint anchor is a gloas state: it is accepted, starts with no +payload known (head EMPTY), and its first FULL child is held until the anchor's +envelope arrives (best-effort from the checkpoint URL's +`execution_payload_envelopes` endpoint, else the by-root request). Its file says `CONFIG_NAME: 'testnet'` (a placeholder for Prysm), so `BuiltInNetwork::resolve` sets every built-in's `CONFIG_NAME` to its `--network` name. @@ -662,6 +685,11 @@ transitions are in `ethlambda-types`, per the section above. Nothing above tests inside the module, are `#[cfg_attr(not(feature = ...), ignore)]` so they report as ignored rather than silently absent. Turning the feature on is what `make test-beacon` does, and it runs `--lib` too so those 15 are not missed. + `ethlambda-blockchain` has its own `beacon-spec-tests` feature gating the + fixture-driven chain-actor tests (the gloas envelope cases in its `--lib` + target, which read `consensus-spec-tests`); `make test-beacon-mainnet` runs + them, and without the feature they report as ignored, so a plain `cargo test` + needs no download. - Every fixture case is its own test, named `////`, so a failure names the case and not the suite around it. The spec binary therefore supplies its own harness (`harness = false`), since a case is only @@ -725,9 +753,29 @@ transitions are in `ethlambda-types`, per the section above. Nothing above suites. - **Gloas** lives in `stf/gloas.rs`, `stf/epoch/gloas.rs`, `helpers/gloas.rs`, the gloas section of `fork_choice.rs`, and `containers/gloas.rs` in - `ethlambda-types`. **The live follower does not follow gloas**: it refuses - gloas blocks and a gloas anchor until envelopes and payload attestations are - delivered to it. `get_head_node` is one bottom-up walk for every fork, with + `ethlambda-types`. **The live follower follows gloas.** The chain actor's + envelope path is `beacon_envelope.rs`: an envelope waits for its block's + post-state, then for every sampled column when the bid has commitments + (data availability moves from the block to the envelope), then goes to + `on_execution_payload_envelope`; a block whose parent is FULL with an + unverified payload is held until that envelope verifies. All queues are + capped, aged out and swept at finality, a missing parent envelope is re-asked + once a slot (`FetchRequest.needs_envelope`), and `beacon_columns.rs` bounds + the parking of gloas column sidecars (they carry no signature). Fork-choice + events are timed at arrival, not at the slot tick, and the head's payload + status is kept with its root (`Store::head_payload_status`, recomputed after + a restart). The node stays a follower: the validator-client endpoints still + refuse a gloas epoch (`refuse_validator_duties_from_gloas`). + With `--execution-endpoint`, an envelope that passes the consensus checks is + judged by `engine_newPayloadV5` (INVALID with a null `latestValidHash` is a + hash mismatch that refuses only that envelope; INVALID with one condemns the + payload via the execution-chain walk; an engine error keeps the envelope and + retries once a slot, probing with the oldest), and a gloas head sends + `engine_forkchoiceUpdatedV4` with the custody columns (head hash by the head + node's FULL or EMPTY status, finalized and safe from the checkpoint blocks' + `bid.parent_block_hash`, safe being a fallback for fast confirmation). See + [`docs/beacon_engine.md`](docs/beacon_engine.md). + `get_head_node` is one bottom-up walk for every fork, with the current slot's fork selecting the payload rules and the boost gate, so a follower on any network that schedules gloas (Sepolia does) pays work proportional to its votes plus blocks every slot and decodes no block in the @@ -822,11 +870,20 @@ snapshot (`States`) + diff (`StateDiffs`) pairs; `BlockRoots` and `LiveChain` index by slot for range serving and fork choice. Attestations and gossip signatures are not persisted; they live in in-memory `Store` buffers consumed during the tick pipeline. See [`docs/data_storage.md`](docs/data_storage.md) -for the full reference: what each of the ten tables holds and how it's +for the full reference: what each of the twelve tables holds and how it's keyed, the snapshot/diff reconstruction algorithm, the block-import write sequence, pruning rules, what never changes at runtime, and startup/restore behavior. +- `ExecutionPayloadEnvelopes` (verified gloas envelopes, keyed by slot and + root) and `BlockTimeliness` (the gloas part of `block_timeliness`) are + reloaded on resume for the unfinalized window; the PTC vote vectors are + reseeded empty and each payload gets a verdict seeded for the execution + client to promote. Adding them did not bump `DB_VERSION`. +- A restart of a beacon data directory currently hits + `SpecAssert("block_root in store.unrealized_justifications")` in head + computation until lambdaclass/ethlambda#628 (persists unrealized + justifications) lands; that is not gloas-specific. - `BlockProof` is the only pruned block table (below the finalized boundary); `get_signed_block` returns `None` for a pruned finalized block. - A `StateDiff` omits `config` and `validators`, trusting they never mutate; diff --git a/README.md b/README.md index e0fb368bd..7b139b5d6 100644 --- a/README.md +++ b/README.md @@ -132,9 +132,11 @@ provider. See [`ethlambda beacon`](#ethlambda-beacon--the-ethereum-beacon-chain) below for the remaining flags. -`--network plataberget` (the Glamsterdam testnet) is built in as well, but -Platåberget has run gloas since its `GLOAS_FORK_EPOCH` and the follower does -not take gloas blocks or a gloas anchor yet, so the node cannot follow it yet. +`--network plataberget` (the Glamsterdam testnet) is built in as well. It has +run gloas since its `GLOAS_FORK_EPOCH`, which the follower follows: a gloas +checkpoint anchor is accepted, and payload envelopes are fetched and imported. +It stays a follower, so the validator-client endpoints still refuse a gloas +epoch. ### Lean consensus devnet diff --git a/docs/beacon_engine.md b/docs/beacon_engine.md index 27665762e..f26847faf 100644 --- a/docs/beacon_engine.md +++ b/docs/beacon_engine.md @@ -14,20 +14,22 @@ The wire is ## Which methods, and why so few -Five: +Six: | Method | Introduced | Why | |---|---|---| | `engine_newPayloadV4` | prague | Validate one block's payload | | `engine_newPayloadV5` | amsterdam | Validate the payload a gloas envelope reveals | | `engine_forkchoiceUpdatedV3` | cancun | Say where the head, safe and finalized blocks are | +| `engine_forkchoiceUpdatedV4` | amsterdam | The same, from a gloas head on; also carries the node's custody columns | | `engine_exchangeCapabilities` | common | Startup handshake | | `engine_getClientVersionV1` | identification | Log what the execution client is | Osaka introduces **no new `newPayload`**. Its own document adds only `engine_getPayloadV5` and `engine_getBlobsV2`/`V3`; `engine_newPayloadV5` belongs to Amsterdam, where gloas's envelope import uses it. So the Osaka-current call for a payload is prague's V4, and the -Osaka-current fork choice notification is cancun's V3. +Osaka-current fork choice notification is cancun's V3. Amsterdam's pair (newPayload V5, forkchoiceUpdated V4) is +used from gloas on; the startup handshake warns when the execution client does not advertise either, but only on a network whose schedule includes gloas. Three method families a full client would have are deliberately absent: @@ -60,6 +62,10 @@ process_block (beacon arm) └─ record the block's execution hash and its optimistic status ``` +This diagram is the pre-gloas path. A gloas block carries no payload, only a +builder's bid, so nothing is asked about it at import; the question moves to the +envelope that reveals the payload (see [Gloas](#gloas) below). + The order is the specification's own: `is_data_available` runs before `state_transition`, so a block about to be held for its custody columns is never one this node asks an execution client about. @@ -120,11 +126,11 @@ finality: an execution client doing a long state sync answers `NOT_VALIDATED` to every block, so without that bound the set would take one root per import for the life of the process. -Nothing outside `mark_validated`'s own ancestor walk reads the set yet. The -readers it is waiting for are the ones that need to answer "is my head -optimistic?": the Beacon API's `execution_optimistic` response field, and a sync -status that separates a head this node has vouched for from one it has merely -imported. +Besides `mark_validated`'s own ancestor walk, the Beacon API's +`execution_optimistic` response field reads the set (see +[rpc.md](./rpc.md#execution_optimistic-under-gloas)). A sync status that +separates a head this node has vouched for from one it has merely imported does +not exist yet. ### `is_optimistic_candidate_block` @@ -195,6 +201,40 @@ itself. An execution client condemning finalized history means it and this node disagree about what is final, which is an operator emergency rather than something to resolve by emptying fork choice. +## Gloas + +A gloas block commits to a builder's bid and the payload arrives later in a +`SignedExecutionPayloadEnvelope`, so the execution client judges the envelope, +not the block. The order differs from the specification's, which asks the engine +last inside one boolean function: the follower runs the pure consensus checks +first (signature, bid, state, data availability), and only an envelope that +passed them is sent to `engine_newPayloadV5`. That way an unsigned copy with the +honest block hash and a different body cannot earn an `INVALID` that lands on the +honest payload, and the engine never spends time on an envelope this node would +refuse anyway. + +| Answer | What happens to the envelope | +|---|---| +| `VALID` | Applied; the payload is recorded `VALID` | +| `SYNCING`, `ACCEPTED` | Applied and recorded `SYNCING` (not yet validated); a later `forkchoiceUpdated` `VALID` promotes it | +| `INVALID` with a `latestValidHash` | The payload is invalid: not applied, so the block's FULL node never exists; the block and its EMPTY branch stay | +| `INVALID` with a null `latestValidHash`, or `INVALID_BLOCK_HASH` | The envelope's contents do not hash to the block hash it claims: only that envelope is refused, since the builder's real one may still arrive | +| no answer after the retry ladder | Kept (it is consensus-valid) and asked again once a slot | + +The null-`latestValidHash` row exists because ethrex has no `INVALID_BLOCK_HASH` +status: it answers a mismatch with a null hash and an executed-and-failed payload +with its last valid ancestor. `latestValidHash` and `VALID` follow the execution +chain (each block's `bid.parent_block_hash`), not every verified ancestor, since +a block built past an ancestor's EMPTY node skips that ancestor's payload. The +details, and why each departs from the specification, are in +[spec_deviations.md](./spec_deviations.md#the-execution-client-judges-a-gloas-payload-before-the-envelope-is-applied). + +With the engine down, the per-slot retry asks about the oldest held envelope +only and releases the rest once that ask is answered, so an unreachable client +costs one retry ladder per slot however many envelopes wait. This is the gloas +counterpart of the limitation below, and unlike a block the envelope is not +dropped. + ## `forkchoiceUpdated` Sent once after every head recompute, which is once per import cascade and once @@ -219,6 +259,15 @@ epoch boundary leaves the finalized block below the bound. Dropping its hash would make every later call carry `finalized_block_hash = 0x00..0` and stop the execution client advancing its own finalized block for the life of the process. +For a gloas head the hashes come from bids rather than from a cached payload +hash: the head node's `bid.block_hash` when it is FULL and its +`bid.parent_block_hash` when it is EMPTY, the finalized block's +`bid.parent_block_hash`, and the justified block's `bid.parent_block_hash` as +the safe hash. The last is a fallback, since the specification builds the safe +hash on fast confirmation, which the follower does not run. The call is V4 with +the node's custody columns. A head whose payload status has not been computed +yet (the first moments after a restart) sends nothing. + The response carries a `PayloadStatusV1` of its own, and that is the channel by which a block imported on `SYNCING` later becomes `VALID` or is found to be `INVALID`. An invalidation reaching the follower this way can remove the head diff --git a/docs/beacon_stf.md b/docs/beacon_stf.md index 0b5bcbf18..2f41c2db2 100644 --- a/docs/beacon_stf.md +++ b/docs/beacon_stf.md @@ -326,6 +326,36 @@ computation after a restart), which the envelope by-range server reads to withhold the head's envelope while the head is the EMPTY node. The node is still only a follower: the validator-client endpoints refuse a gloas epoch. +### The envelope pipeline in the chain actor + +Order of arrival is not order of validity: an envelope can beat its block, a +block can name a parent payload that has not arrived, and the columns an +envelope's commitments need can lag both. `beacon_envelope.rs` holds each case +in a queue with its own bound, and `beacon_columns.rs` bounds the parked +sidecars of the same fork. + +| Queue | Holds | Released when | +|---|---|---| +| `awaiting_block` | an envelope whose block has no post-state | the block imports; evicted after `ENVELOPE_AWAITING_BLOCK_TTL_SLOTS`, capped per slot and per root | +| `awaiting_columns` | a consensus-checked envelope whose bid has commitments and whose sampled columns are not all stored | the last column is stored | +| `awaiting_engine` | a consensus-valid envelope the execution client gave no answer for | the per-slot redrive gets an answer (see [beacon_engine.md](./beacon_engine.md#gloas)) | +| `blocks_awaiting_parent_payload` | a block whose parent is FULL with a payload not yet verified, capped per parent | that parent's envelope verifies | + +Data availability moves with the payload: a gloas block's commitments are in its +bid, so the columns gate the envelope that reveals the payload, not the block's +own import. A held block's parent envelope is requested through +`FetchRequest.needs_envelope`, and the tick re-asks once a slot for every parent +still missing one, which is what recovers a fetch that gave up. Everything is +swept at finality, except the finalized block's own envelope. + +Fork-choice events are timed at their arrival: the store clock is advanced to +the moment an envelope, block or payload vote reached the node before its +handler runs, because the tick fires once per slot and the timeliness deadlines +would otherwise be judged at the slot's start. Payload votes already verified +in p2p are applied without a second signature check +(`apply_verified_payload_attestation`). The PTC vote vectors are not persisted +and are reseeded empty on resume; see [data_storage.md](./data_storage.md). + ### The deferred payload A gloas block has no execution payload. Its body commits to a builder's bid diff --git a/docs/beacon_wire.md b/docs/beacon_wire.md index 2f4a62f85..83addec59 100644 --- a/docs/beacon_wire.md +++ b/docs/beacon_wire.md @@ -97,13 +97,13 @@ since it is valid only within its own slot. | Topic | Decoded as | | --- | --- | | `beacon_block` | `SignedBeaconBlock`, fork chosen by the block's slot | -| `beacon_aggregate_and_proof` | `SignedAggregateAndProof`, phase0 or electra | +| `beacon_aggregate_and_proof` | `SignedAggregateAndProof`, phase0, electra or gloas | | `attester_slashing` | `AttesterSlashing`, phase0 or electra | | `voluntary_exit` | `SignedVoluntaryExit` | | `proposer_slashing` | `ProposerSlashing` | | `bls_to_execution_change` | `SignedBLSToExecutionChange` | | `sync_committee_contribution_and_proof` | `SignedContributionAndProof` | -| `beacon_attestation_{subnet_id}` | `Attestation`, phase0 or electra | +| `beacon_attestation_{subnet_id}` | `Attestation`, phase0 or electra's `SingleAttestation` (gloas keeps the latter) | | `execution_payload` | `SignedExecutionPayloadEnvelope`, gloas digests only | | `payload_attestation_message` | `PayloadAttestationMessage`, gloas digests only | @@ -142,14 +142,17 @@ plus `SUBNETS_PER_NODE`, still far short of a full subscription to every attestation, sync-committee and data-column subnet, and narrower still than `NUMBER_OF_CUSTODY_GROUPS` columns of custody, which is what a supernode would carry alone. A sidecar decodes as -`fulu::DataColumnSidecar`; the checks it passes before this node keeps or +the fork-agnostic `DataColumnSidecar` enum, its fork taken from the topic's +digest (the bytes carry no tag): fulu's names its block by a signed header and +inclusion proof, gloas's by a slot and block root, with the commitments it is +checked against living in that block's bid. The checks it passes before this node keeps or forwards it are described just below. How a kept sidecar is later served back out over req/resp is under [Data column sidecars](#data-column-sidecars). Every beacon message is held by gossipsub until it has a verdict (`validate_messages()` is on for this wire only). Blocks, data column sidecars, aggregates and subnet attestations are all validated by fulu's -gossip rules (`ethlambda_state_transition::beacon::gossip`, one module per +gossip rules, or gloas's modified ones from the fork on (`ethlambda_state_transition::beacon::gossip`, one module per topic family): the checks that need no state run inline in the p2p actor, the rest on a bounded `spawn_blocking` task whose verdict comes back to the actor (`crate::beacon::verdict`). Two permit pools bound how many of these run at @@ -183,7 +186,22 @@ the block's envelope has been seen and its payload validated (`verify_attestation_payload_status`). Gloas is a followed fork, so a block at a gloas slot that fails to decode is REJECTed like one at any other followed fork; `unsupported_fork` is kept for a fork the node does not follow -(`ForkName::is_followed`), none today. +(`ForkName::is_followed`), none today. A gloas aggregate's aggregation bits are +an unbounded progressive bitlist, so one longer than electra's type bound is +REJECTed first, reading its length without expanding it, since expanding is +what the seen-set check does before any other. + +Three deliberate departures from the gloas gossip rules, each for want of state +this node does not keep. Where the specification rejects a message whose block +failed validation, this node (having no bad-block cache) cannot tell that from +a block it has not finished importing, so it queues or ignores instead; the +matching `reject_block_failed_validation` vectors are in `SKIPPED`. An +`execution_payload` whose block state is not in the cache p2p reads (a parent +imported moments ago, say) is queued and forwarded to the chain actor, which +verifies every envelope itself, without being propagated. And a +`payload_attestation_message` is applied to fork choice after p2p verified its +signature, with the actor re-checking only the store-state conditions (known +block, current slot, committee seat). The remaining five global topics are decoded, logged at `debug`, and IGNOREd, since nothing consumes them; an undecodable payload on any topic is REJECTed. @@ -284,8 +302,8 @@ choice. | `beacon_blocks_by_root/2` | both | | `data_column_sidecars_by_root/1` | both | | `data_column_sidecars_by_range/1` | both | -| `execution_payload_envelopes_by_range/1` | both; served only, nothing asks yet | -| `execution_payload_envelopes_by_root/1` | both; served only, nothing asks yet | +| `execution_payload_envelopes_by_range/1` | both | +| `execution_payload_envelopes_by_root/1` | both | The two data column sidecar protocols are registered because this node custodies the columns its node id selects and can answer for them out of @@ -395,8 +413,8 @@ network's. ### Execution payload envelopes -Both gloas envelope protocols are registered `Full` so the server side can -answer; no request is issued yet. A chunk carries one +Both gloas envelope protocols are registered `Full`: the server answers, and +the client (`req_resp/envelope_client.rs`) asks. A chunk carries one `SignedExecutionPayloadEnvelope` under the fork digest of its block's epoch. The envelope has no slot of its own, so the epoch comes from `payload.slot_number`, which `verify_execution_payload_envelope` pins to the @@ -406,7 +424,7 @@ not the digest its slot implies, ends the stream. | Protocol | Served from | | --- | --- | | `execution_payload_envelopes_by_root/1` | a point lookup per root in `Table::ExecutionPayloadEnvelopes`; unknown roots are left out, answers follow the order asked | -| `execution_payload_envelopes_by_range/1` | the canonical blocks of the window (`BlockRoots`), each envelope kept only when the next canonical block builds on its payload; see [spec_deviations.md](./spec_deviations.md) for the head | +| `execution_payload_envelopes_by_range/1` | the canonical blocks of the window (`BlockRoots`), each envelope kept only when the next canonical block builds on its payload; the head's own envelope only when the head node is FULL (`Store::head_payload_status`, which pairs the status with the head root it describes, so a stale status reads as unknown and the envelope is withheld) | Both are bounded by `MAX_REQUEST_PAYLOADS`: a by-root list over it fails to decode, and a by-range `count` over it is truncated, which the specification's @@ -414,6 +432,20 @@ decode, and a by-range `count` over it is truncated, which the specification's A by-range request starting below `Store::anchor_slot` gets `RESOURCE_UNAVAILABLE`, and a window before gloas is simply empty. +The client asks in two situations. By root: the chain actor's +`FetchRequest.needs_envelope` (a block whose child waits on its payload) starts +a lookup with the block fetch's dedup, peer choice and retry ladder; a give-up +is recovered by the actor's per-slot re-ask. By range: after a +`beacon_blocks_by_range/2` answer for a span that reaches gloas has been handed +to the actor, the same peer is asked for that span's envelopes, and range sync +holds the next block request until that answer is checked and forwarded (or has +failed). The pause exists because the actor keeps an envelope whose block has +not imported only briefly and in small numbers, so a batch's envelopes must +arrive before the next batch's blocks do. Everything fetched goes through the +same rules as an `execution_payload` gossip message +(`beacon/envelope_checks.rs`) before the actor sees it; a `Queue` verdict is +forwarded too. + ### Data column sidecars Both protocols are registered `Full`, since with the columns on disk this node diff --git a/docs/checkpoint_sync.md b/docs/checkpoint_sync.md index 8706914dc..9331e6e9c 100644 --- a/docs/checkpoint_sync.md +++ b/docs/checkpoint_sync.md @@ -99,7 +99,7 @@ request that fires when a FULL child arrives recovers it. If any step fails (network error, decoding error, verification failure), the node logs the error and exits. There is no automatic retry; restart the node to try again. The database is not modified until verification succeeds, so a failed checkpoint sync leaves the data directory clean. -After successful initialization, the node starts normally: `node` connects to the P2P network and begins participating from the checkpoint slot; `beacon` joins the resolved network's gossip and logs what it decodes, without advancing its state past the anchor (see `docs/cli.md`, "What `ethlambda beacon` does today"). +After successful initialization, the node starts normally: `node` connects to the P2P network and begins participating from the checkpoint slot; `beacon` joins the resolved network's gossip and follows the chain from the anchor, importing blocks (a gloas anchor starts EMPTY, see [the envelope](#gloas-anchor-the-payload-envelope-beacon-only)). ## Restarts and Existing State diff --git a/docs/cli.md b/docs/cli.md index 4661706ad..a013c07a8 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -166,11 +166,11 @@ numbers are accepted quoted or bare, and unrecognised keys (on a current config, the heze schedule this build does not claim) are dropped with one warning line naming each. Its `PRESET_BASE` is checked against the compiled preset; a mismatch is a hard startup error naming the cargo feature -that would fix it. Sepolia's config schedules gloas, whose keys this build -does claim but whose blocks the chain actor refuses, since nothing delivers -payload envelopes or payload attestations to it yet: a separate, explicit -warning at startup names that boundary, since the chain follower stops -tracking the chain there regardless of what parsed cleanly. The keys this +that would fix it. Sepolia's config schedules gloas, which this build follows, so +it parses and runs like any other fork. A scheduled fork the build claims but +does not follow gets a separate, explicit warning at startup, since the chain +follower would stop tracking the chain there regardless of what parsed +cleanly; none exists today. The keys this build runs on compile-time constants for (the custody and subnet counts, the `MAX_REQUEST_*` limits, `MAX_PAYLOAD_SIZE`, the snappy message domains and `MAXIMUM_GOSSIP_CLOCK_DISPARITY`) must equal those @@ -192,30 +192,25 @@ consistency (see [`checkpoint_sync.md`](checkpoint_sync.md)). The full anchor precedence is: a resumable data directory, then `--checkpoint-sync-url`, then, for a loaded network only, that directory's own `genesis.ssz`, then abort. The URL is therefore **required** on a fresh data directory only for a built-in -network: this follower imports nothing past its anchor, so anchoring a built-in -network at genesis would leave it parked at slot 0 while claiming to follow a -live chain. A loaded network's own genesis +network: no built-in network carries a genesis state, and anchoring one at genesis +would leave the follower replaying a chain that has been live for years. A loaded network's own genesis state is a legitimate anchor instead, since a freshly started devnet has no checkpoint provider at slot 0 and this is the only way to join one. A directory already anchored from a previous run resumes without the flag either way, the same way `node`'s does. -`--api-port` is bound here too, off `beacon`'s own anchored store now rather -than an empty one: one HTTP call site (`start_rpc_server`) serves both chains. -The lean-shaped `/lean/v0/...` routes read metadata keys and state variants a -beacon directory never carries, though, so calling one of them, e.g. `GET -/lean/v0/states/finalized`, panics that request rather than answering for a -chain that isn't running. This is deliberate for now: giving the beacon -follower its own HTTP surface is a change of its own. Treat `/metrics` on -`--metrics-port` as the only meaningful HTTP surface of a `beacon` run today. +`--api-port` is bound here too, off `beacon`'s own anchored store: one HTTP call +site serves both chains, but it picks the router by the store's chain, so a +`beacon` run serves the Beacon API under `/eth/v1` and `/eth/v2` and answers a +`/lean/v0/...` path with a 404. See [`rpc.md`](./rpc.md). ## What `ethlambda beacon` does today -It follows the resolved network's gossip and nothing above it. It now anchors a real, -RocksDB-backed store at a checkpoint-synced (or resumed) finalized state, but -does nothing more with it: no state transition, no fork choice, and no block -import past that anchor. The node joins the network, decodes what arrives, and -logs it. See [`beacon_wire.md`](./beacon_wire.md) for what goes on the wire and +It follows the resolved network as a non-validating follower. It anchors a +RocksDB-backed store at a checkpoint-synced (or resumed) finalized state, joins +the network's gossip and req/resp, and the chain actor imports blocks from there +(phase0 through gloas), runs fork choice, and, with `--execution-endpoint`, +asks an execution client about payloads. It never proposes or attests. See [`beacon_wire.md`](./beacon_wire.md) for what goes on the wire and how to check a run against the live network. ### One entry point, two chains @@ -239,16 +234,16 @@ not chain-specific, branches once, and shares the shutdown: | **the one `match`**: produce a `ChainSetup` | genesis config, validator keys, checkpoint sync or resume onto the shared backend, subnets | `beacon::wire_params` (genesis metadata, epoch, fork digest), then checkpoint sync or resume onto the same backend | | build the swarm, spawn P2P, start discv5 | shared; discv5 only with `--discovery.enable` | shared | | start the HTTP server | shared | shared | -| spawn the chain actor and wire it to P2P | yes | no: it imports nothing | +| spawn the chain actor and wire it to P2P | yes | yes: `BlockChain::spawn_beacon` | | ctrl-c, stop and join the actors | shared | shared | `ChainSetup` is what the `match` produces: the wire configuration, the ENR entries that describe it, the store the req/resp handlers answer from, the -node-name roster, and an `Option` holding the validator keys and -`BlockChainConfig`. The operator-supplied half of the discv5 configuration (node +node-name roster, and a `ChainActor` saying what to spawn: lean's validator keys +and `BlockChainConfig`, or beacon's marker. The operator-supplied half of the discv5 configuration (node key, ports, bootnodes, peer target) is the same on either chain, so it is filled -in once below the match. That `Option` being `None` is what ends the mainnet path: -`run_node` returns straight into the shared shutdown after starting the wire. +in once below the match; everything after it, including the chain actor spawn +and the shutdown, is shared. `beacon::wire_params` reads `genesis_time` and `genesis_validators_root` off the resolved network (a built-in network's two constants, or a loaded @@ -259,11 +254,7 @@ digest. It builds nothing: one `build_swarm` serves both chains, dispatching on the `WireConfig` variant for the topics, the req/resp protocol set, the gossipsub `seen_ttl`, the identify version and the connection limits. -The chain actor is the one thing mainnet has none of, so `RunningNode` -carries it as an `Option` and the shared shutdown skips it. - -Not implemented here: block import and fork choice past the anchor. A decoded -block is logged and dropped. +Both chains run a chain actor, so `RunningNode` carries a plain `BlockChain`. ## `validator` flags diff --git a/docs/data_storage.md b/docs/data_storage.md index 51079b266..73080bdc6 100644 --- a/docs/data_storage.md +++ b/docs/data_storage.md @@ -461,6 +461,15 @@ vectors empty for each gloas block in that window. The votes themselves are not persisted: the next block's payload attestations refill them. Neither table is pruned; rows outside the window are simply not read. +The execution client's verdict on each reloaded payload is in-memory scratch +too, so the chain actor seeds one at startup: `SYNCING` when an execution +client is configured, so its first `forkchoiceUpdated` `VALID` for the head +promotes them, and `VALID` without one, as the follower trusts every envelope it +applies then. The head node's payload status (`Store::head_payload_status`) is +likewise not stored; it is unknown until the first head computation after a +restart, and the envelope by-range server withholds the head's envelope until it +is known. + ### PendingDataColumns Same key and same encoding as `DataColumns`, holding sidecars that have **not diff --git a/docs/discovery.md b/docs/discovery.md index f8285524f..3040ea64a 100644 --- a/docs/discovery.md +++ b/docs/discovery.md @@ -93,8 +93,11 @@ A discovered peer is admitted only if: - it advertises a `quic` port, a `tcp` port, or both. The digest compared against is the caller's, not a constant: lean passes its -hardcoded dummy and `beacon` passes the digest it derived at startup, so one -admission policy serves both chains. +hardcoded dummy and `beacon` passes the digest it derived at startup and moves +it at each fork-digest boundary, so one admission policy serves both chains. The +record this node serves is not refreshed at a boundary (ethrex's +`DiscoveryServer` cannot replace it at runtime), so its `eth2` entry keeps the +startup digest until a restart; see [beacon_wire.md](./beacon_wire.md#the-fork-digest). A differing `next_fork_version` or `next_fork_epoch` is *not* grounds for rejection: the spec permits connecting to a peer that is incompatible with an diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 91607f83d..fd194d45f 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -192,8 +192,9 @@ that the tests compare like with like. it, and its vote walks the block table and contributes nothing. - **Missing timeliness** (`should_apply_proposer_boost`). A candidate with no `block_timeliness` entry reads as not timely by either deadline, where the - specification indexes the entry directly. The entries are in memory only, so a - restart empties them. The reading can only miss withholding a boost from an + specification indexes the entry directly. A gloas block's entry is persisted + (`Table::BlockTimeliness`) and reloaded on resume; a pre-gloas block's is in + memory only, so a restart empties it. The reading can only miss withholding a boost from an early equivocation, never withhold one wrongly. - **Equivalence:** each is the specification's answer wherever the specification has one; they differ only where it would have raised. From 1ab6dee04d3384e8278d1e6fea1bed264a84cf36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 05:07:00 -0300 Subject: [PATCH 080/132] fix(beacon): ask for envelopes of held blocks at once and keep those of stored blocks A catching-up gloas follower imported one block per slot. Two causes: - A synced held block skipped its immediate parent-envelope request on the premise that a range batch brings it. BlockSource::Sync also covers by-root answers and cascade releases, and a failed range answer falls back to the same ask, so only the once-a-slot redrive ever released them. The ask is now unconditional; p2p dedups in-flight roots. - A batch's envelopes arrive when all but its first block are stored but unimported, and were capped per arrival slot as if their roots were unauthenticated. An envelope whose block is stored is now checked against that block's bid and held in awaiting_import, bounded by the stored blocks and evicted with them and at finality. The per-slot cap and TTL remain for roots with no stored block. Also adds ForkName::has_payload_envelopes, replacing three fork checks that mishandled a fork after gloas or lean, and rewrites the clock_is_past_followed_forks doc for the runtime digest rollover. Docs describe both envelope queues. --- CLAUDE.md | 6 +- crates/blockchain/src/beacon_envelope.rs | 325 ++++++++++++++++-- crates/blockchain/src/lib.rs | 11 +- .../src/beacon/gossip/column.rs | 4 +- crates/common/types/src/beacon/fork.rs | 29 ++ .../net/p2p/src/req_resp/envelope_client.rs | 4 +- docs/beacon_stf.md | 20 +- 7 files changed, 357 insertions(+), 42 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 874395e78..06357a87d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -760,7 +760,11 @@ transitions are in `ethlambda-types`, per the section above. Nothing above `on_execution_payload_envelope`; a block whose parent is FULL with an unverified payload is held until that envelope verifies. All queues are capped, aged out and swept at finality, a missing parent envelope is re-asked - once a slot (`FetchRequest.needs_envelope`), and `beacon_columns.rs` bounds + once a slot (`FetchRequest.needs_envelope`, asked at once when the block is + held, whatever its source, since `BlockSource::Sync` also covers by-root + answers), an envelope for a block that is stored but not imported is checked + against that block's bid and held outside the per-arrival-slot cap + (`awaiting_import`, which range catch-up needs), and `beacon_columns.rs` bounds the parking of gloas column sidecars (they carry no signature). Fork-choice events are timed at arrival, not at the slot tick, and the head's payload status is kept with its root (`Store::head_payload_status`, recomputed after diff --git a/crates/blockchain/src/beacon_envelope.rs b/crates/blockchain/src/beacon_envelope.rs index ed0fa8d19..f4b0c3221 100644 --- a/crates/blockchain/src/beacon_envelope.rs +++ b/crates/blockchain/src/beacon_envelope.rs @@ -6,8 +6,9 @@ //! and each has a bounded queue here: //! //! ```text -//! envelope ─► block known (with post-state)? ─no─► awaiting_block[root] -//! │yes +//! envelope ─► block imported (has post-state)? ─no─► block stored? ─yes (bid-checked)─► awaiting_import[root] +//! │yes │no +//! │ └─► awaiting_block[root] //! ▼ //! bid has commitments and a sampled column missing? ─yes─► awaiting_columns[root] //! │no @@ -33,19 +34,24 @@ //! //! # What bounds this //! -//! An envelope for a block this node has not imported cannot be judged, so +//! An envelope for a block this node has never seen cannot be judged, so //! nobody has vouched for it: at most [`MAX_ENVELOPES_AWAITING_BLOCK_PER_SLOT`] -//! are kept for the slot they arrived in, and one per root. Blocks held for a -//! parent's payload are capped per parent by -//! [`MAX_BLOCKS_HELD_PER_PARENT_PAYLOAD`]. All three queues are swept at -//! finality by [`BlockChainServer::evict_envelope_queues_at_or_below_finality`]. +//! are kept for the slot they arrived in, and a few per root. An envelope for a +//! block that is stored but not imported (pending on its parent, or held for +//! columns or a parent's payload) is checked against that block's bid instead, +//! and held in `awaiting_import` outside that cap, bounded by the stored blocks +//! it names. Range catch-up depends on it: a batch's envelopes arrive when all +//! but its first block are in that state. Blocks held for a parent's payload +//! are capped per parent by [`MAX_BLOCKS_HELD_PER_PARENT_PAYLOAD`]. Every queue +//! is swept at finality by +//! [`BlockChainServer::evict_envelope_queues_at_or_below_finality`], and +//! `awaiting_import` also when its block is discarded. use std::collections::{HashMap, VecDeque}; use ethlambda_engine::EngineError; use ethlambda_engine::types::PayloadStatusValue; -use ethlambda_network_api::{BlockSource, FetchRequest}; -use ethlambda_state_transition::beacon::fork::ForkName; +use ethlambda_network_api::FetchRequest; use ethlambda_state_transition::beacon::fork_choice::{ self, PayloadStatus, PayloadStatusEnum, PayloadValidity, }; @@ -138,6 +144,15 @@ pub(crate) struct EnvelopeQueues { /// Envelopes whose block has no post-state yet, by the block's root, up to /// [`MAX_ENVELOPES_PER_UNMATCHED_ROOT`] distinct ones each. pub(crate) awaiting_block: HashMap>, + /// Envelopes for blocks that are stored but not imported (pending on a + /// missing parent, held for columns or for their parent's payload), by the + /// block's root. Each was checked against the stored block's bid, which + /// commits to the whole payload, so they sit outside the per-arrival-slot + /// cap and are bounded by the stored blocks instead. Up to + /// [`MAX_ENVELOPES_PER_UNMATCHED_ROOT`] each, since the signature is not + /// checked until the block imports and a forged copy must not shut the + /// honest one out. + pub(crate) awaiting_import: HashMap>, /// Envelopes whose bid has commitments and whose sampled columns are not /// all stored, by the block's root. pub(crate) awaiting_columns: HashMap, @@ -186,6 +201,7 @@ impl EnvelopeQueues { fn is_empty(&self) -> bool { self.awaiting_block.is_empty() + && self.awaiting_import.is_empty() && self.awaiting_columns.is_empty() && self.awaiting_engine.is_empty() && self.blocks_awaiting_parent_payload.is_empty() @@ -300,7 +316,9 @@ impl BlockChainServer { None }; let Some(block) = block else { - self.hold_envelope_awaiting_block(root, held); + if let Some(held) = self.hold_envelope_for_stored_block(root, held) { + self.hold_envelope_awaiting_block(root, held); + } return false; }; @@ -562,6 +580,69 @@ impl BlockChainServer { .collect() } + /// Hold `held` for a block that is in the store but has no post-state yet, + /// after checking it against that block's bid. + /// + /// Range catch-up delivers a batch's envelopes right behind its blocks, + /// when all but the first block are still pending or held for a parent's + /// payload. Those envelopes are not strangers: the stored block names what + /// payload it committed to, so one that matches is held outside the + /// per-arrival-slot cap, which exists for roots nobody has vouched for. + /// `Err(held)` hands the envelope back when the block is not stored at all, + /// for the unverified path. + fn hold_envelope_for_stored_block( + &mut self, + root: H256, + mut held: HeldEnvelope, + ) -> Option { + if !self.store.has_block(&root) { + return Some(held); + } + let Ok(Some(block)) = self.store.get_signed_block(&root) else { + return Some(held); + }; + let slot = block.slot(); + let Some(bid) = crate::beacon_payloads::bid_of(&block) else { + warn!( + %slot, + block_root = %ShortRoot(&root.0), + "Dropping an envelope: its block is not a gloas block" + ); + return None; + }; + if !envelope_matches_bid(&held.envelope.message, bid, slot) { + warn!( + %slot, + block_root = %ShortRoot(&root.0), + "Dropping an envelope that does not match its stored block's bid" + ); + return None; + } + let finalized = self + .store + .latest_finalized() + .expect("finalized checkpoint exists"); + if slot < finalized.slot && root != finalized.root { + return None; + } + held.slot = slot; + let held_for_root = self.envelopes.awaiting_import.entry(root).or_default(); + if held_for_root.len() < MAX_ENVELOPES_PER_UNMATCHED_ROOT + && !held_for_root + .iter() + .any(|other| other.envelope_root == held.envelope_root) + { + debug!( + %slot, + block_root = %ShortRoot(&root.0), + "Holding an envelope until its stored block is imported" + ); + held_for_root.push(held); + } + self.publish_envelope_queues(); + None + } + fn hold_envelope_awaiting_block(&mut self, root: H256, held: HeldEnvelope) { let queues = &mut self.envelopes; if let Some(held_for_root) = queues.awaiting_block.get_mut(&root) { @@ -654,7 +735,9 @@ impl BlockChainServer { /// Queue `root`'s envelope for a look in [`Self::settle_envelopes`], if one /// is waiting for the block to import. pub(crate) fn note_block_imported_for_envelope(&mut self, root: H256) { - if self.envelopes.awaiting_block.contains_key(&root) { + if self.envelopes.awaiting_block.contains_key(&root) + || self.envelopes.awaiting_import.contains_key(&root) + { self.envelopes.work.push_back(root); } } @@ -715,14 +798,22 @@ impl BlockChainServer { /// The held envelopes for `root` if what they waited on is now there. fn take_ready_envelopes(&mut self, root: H256) -> Vec { - if self.envelopes.awaiting_block.contains_key(&root) + if (self.envelopes.awaiting_block.contains_key(&root) + || self.envelopes.awaiting_import.contains_key(&root)) && self.store.has_state(&root).expect("DB read should succeed") { - return self + let mut ready = self .envelopes - .awaiting_block + .awaiting_import .remove(&root) .unwrap_or_default(); + ready.extend( + self.envelopes + .awaiting_block + .remove(&root) + .unwrap_or_default(), + ); + return ready; } if let Some(held) = self.envelopes.awaiting_engine.remove(&root) { return vec![held]; @@ -789,7 +880,7 @@ impl BlockChainServer { /// parent's payload is not yet verified: `on_block` would reject it with /// `is_payload_verified(store, block.parent_root)`. pub(crate) fn parent_payload_unverified(&self, block: &SignedBeaconBlock) -> bool { - if block.fork_name() != ForkName::Gloas { + if !block.fork_name().has_payload_envelopes() { return false; } matches!( @@ -862,16 +953,17 @@ impl BlockChainServer { .entry(parent_root) .or_default() .insert(block_root, slot); - let arrived_by_sync = timings.source == Some(BlockSource::Sync); self.held_timings.insert(block_root, timings); self.publish_envelope_queues(); - // A synced block comes from a range batch, whose envelopes are already - // being fetched alongside it. Asking again at once would fetch each - // envelope twice, and each costs a post-state load and a signature - // check; the once-a-slot re-ask covers any the range answer lacks. - if !arrived_by_sync { - self.request_missing_envelope(parent_root); - } + // Asked at once whatever the block's source. A synced block is not + // covered by a range batch's envelopes: `BlockSource::Sync` also marks + // by-root answers and pending children a cascade released, and a range + // answer that fails or comes back empty falls back to this same ask. + // Waiting for the once-a-slot redrive instead is what held a catching-up + // follower to one block per slot. p2p dedups in-flight roots, so a + // range batch whose envelopes are on their way costs at most one extra + // request per block. + self.request_missing_envelope(parent_root); } /// Ask p2p for `root`'s envelope by root. The request carries nothing but @@ -951,6 +1043,14 @@ impl BlockChainServer { self.publish_envelope_queues(); } + /// Drop the envelopes held for `block_root`'s import, which is not coming. + /// Called from the funnel every block eviction goes through. + pub(crate) fn forget_envelopes_awaiting_import(&mut self, block_root: H256) { + if self.envelopes.awaiting_import.remove(&block_root).is_some() { + self.publish_envelope_queues(); + } + } + /// Drop every queued envelope and held block that finality has superseded. /// /// The counterpart of [`Self::evict_held_blocks_at_or_below_finality`], run @@ -982,6 +1082,10 @@ impl BlockChainServer { }); !held.is_empty() }); + self.envelopes.awaiting_import.retain(|root, held| { + held.retain(|other| other.slot >= finalized_slot || *root == finalized.root); + !held.is_empty() + }); self.envelopes .awaiting_columns .retain(|root, held| held.slot >= finalized_slot || *root == finalized.root); @@ -1004,7 +1108,13 @@ impl BlockChainServer { } pub(crate) fn publish_envelope_queues(&self) { - let unmatched: usize = self.envelopes.awaiting_block.values().map(Vec::len).sum(); + let unmatched: usize = self + .envelopes + .awaiting_block + .values() + .chain(self.envelopes.awaiting_import.values()) + .map(Vec::len) + .sum(); metrics::set_envelopes_awaiting_block(unmatched as u64); metrics::set_envelopes_awaiting_columns(self.envelopes.awaiting_columns.len() as u64); metrics::set_blocks_awaiting_parent_payload(self.envelopes.held_blocks() as u64); @@ -1023,6 +1133,8 @@ mod tests { use super::*; use crate::tests::{GENESIS_TIME, beacon_server, beacon_store}; + use ethlambda_network_api::BlockSource; + use ethlambda_state_transition::beacon::fork::ForkName; /// The mainnet preset with every fork active from genesis, as the fork /// choice fixtures that carry no `config.yaml` of their own are run. @@ -1607,17 +1719,17 @@ mod tests { assert!(server.pending_block_parents.is_empty()); } - /// A held child that came from range sync does not ask for its parent's - /// envelope at once, since the range answer carries it; one that came from - /// gossip does, as nothing else is fetching it. + /// A held child asks for its parent's envelope at once whatever its + /// source: `BlockSource::Sync` also marks by-root answers and cascade + /// releases, which no range batch is bringing an envelope for. #[tokio::test] #[cfg_attr( not(feature = "beacon-spec-tests"), ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" )] - async fn only_a_synced_held_child_skips_the_immediate_envelope_request() { + async fn a_held_child_asks_for_the_envelope_at_once_whatever_its_source() { for (source, asks) in [ - (Some(BlockSource::Sync), 0), + (Some(BlockSource::Sync), 1), (Some(BlockSource::Gossip), 1), (None, 1), ] { @@ -1642,6 +1754,161 @@ mod tests { } } + /// A stored, unimported gloas block at `slot_offset` past the fixture + /// child's slot, whose bid names the valid envelope's payload, and that + /// envelope rewritten to belong to it. The stand-in for one block of a + /// range batch. + fn stored_block_with_matching_envelope( + server: &mut BlockChainServer, + slot_offset: u64, + ) -> (H256, gloas::SignedExecutionPayloadEnvelope) { + let case = Case::new("on_execution_payload_envelope", FULL_CHILD_CASE); + let mut envelope = valid_envelope_for_full_parent(); + let mut block = case.block(FULL_CHILD); + let SignedBeaconBlock::Gloas(inner) = &mut block else { + unreachable!("the fixture block is a gloas block"); + }; + inner.message.slot += slot_offset; + let slot = inner.message.slot; + let bid = &mut inner.message.body.signed_execution_payload_bid.message; + bid.builder_index = envelope.message.builder_index; + bid.block_hash = envelope.message.payload.block_hash; + bid.prev_randao = envelope.message.payload.prev_randao; + bid.gas_limit = envelope.message.payload.gas_limit; + bid.execution_requests_root = envelope.message.execution_requests.hash_tree_root(); + bid.blob_kzg_commitments = Default::default(); + envelope.message.payload.slot_number = slot; + let block_root = block.message_hash_tree_root(); + envelope.message.beacon_block_root = block_root; + server + .store + .insert_pending_block(block_root, block) + .expect("pending block stores"); + (block_root, envelope) + } + + /// Range catch-up: a batch's later blocks are stored but unimported when + /// their envelopes arrive, all in one arrival slot. The per-arrival-slot + /// cap is for roots nobody has vouched for, so none of these may count + /// against it. + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn envelopes_for_stored_blocks_are_kept_outside_the_per_slot_cap() { + let case = Case::new("on_execution_payload_envelope", FULL_CHILD_CASE); + let mut server = case.server(12); + let count = MAX_ENVELOPES_AWAITING_BLOCK_PER_SLOT as u64 * 3; + let mut roots = Vec::new(); + let mut envelopes = Vec::new(); + for offset in 0..count { + let (block_root, envelope) = stored_block_with_matching_envelope(&mut server, offset); + roots.push(block_root); + envelopes.push(envelope); + } + + for envelope in envelopes { + server.receive_envelope(envelope, 0).await; + server.settle_envelopes().await; + } + + for block_root in &roots { + assert!( + server.envelopes.awaiting_import.contains_key(block_root), + "every stored block keeps its envelope" + ); + } + assert!(server.envelopes.awaiting_block.is_empty()); + + // The cap still binds roots nothing is stored for. + let stranger = valid_envelope_for_full_parent(); + for byte in 1..=MAX_ENVELOPES_AWAITING_BLOCK_PER_SLOT as u8 + 2 { + let mut envelope = stranger.clone(); + envelope.message.beacon_block_root = H256::repeat_byte(byte); + server.receive_envelope(envelope, 0).await; + } + assert_eq!( + server.envelopes.awaiting_block.len(), + MAX_ENVELOPES_AWAITING_BLOCK_PER_SLOT + ); + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn an_envelope_that_contradicts_the_stored_bid_is_dropped() { + let case = Case::new("on_execution_payload_envelope", FULL_CHILD_CASE); + let mut server = case.server(12); + let (block_root, mut envelope) = stored_block_with_matching_envelope(&mut server, 0); + envelope.message.payload.gas_limit += 1; + + server.receive_envelope(envelope, 0).await; + + assert!(server.envelopes.awaiting_import.is_empty()); + assert!(!server.envelopes.awaiting_block.contains_key(&block_root)); + } + + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn an_envelope_held_for_a_stored_block_goes_with_the_block() { + let case = Case::new("on_execution_payload_envelope", FULL_CHILD_CASE); + let mut server = case.server(12); + let (block_root, envelope) = stored_block_with_matching_envelope(&mut server, 0); + server.receive_envelope(envelope, 0).await; + assert!(server.envelopes.awaiting_import.contains_key(&block_root)); + + server.discard_pending_subtree(block_root); + + assert!(server.envelopes.awaiting_import.is_empty()); + } + + /// The whole catch-up chain imports on deliveries alone: the child arrives + /// ahead of its parent, then the parent, then the parent's envelope, and + /// the clock is not ticked in between. + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn catch_up_imports_the_chain_without_a_tick() { + let case = Case::new("on_execution_payload_envelope", FULL_CHILD_CASE); + let mut server = case.server(24); + let p2p = std::sync::Arc::new(crate::tests::RecordingP2P::default()); + server.p2p = Some(p2p.clone()); + let time_before = server.store.ms_since_genesis(); + + let timings = || ImportTimings { + source: Some(BlockSource::Sync), + ..ImportTimings::default() + }; + server.on_block(case.block(FULL_CHILD), timings()).await; + server.on_block(case.block(FULL_PARENT), timings()).await; + assert!(server.envelopes.is_block_held(&root(FULL_CHILD))); + // Asked for at once, though the child came from sync. + assert!( + p2p.fetches + .lock() + .unwrap() + .iter() + .any(|request| request.needs_envelope && request.block_root == root(FULL_PARENT)) + ); + + server + .receive_envelope(valid_envelope_for_full_parent(), 0) + .await; + server.settle_envelopes().await; + + assert!(server.store.has_state(&root(FULL_CHILD)).unwrap()); + assert!(server.envelopes.blocks_awaiting_parent_payload.is_empty()); + assert_eq!(server.store.ms_since_genesis(), time_before); + } + #[test] fn each_slot_re_asks_for_the_envelope_of_a_parent_with_held_children() { let (mut server, p2p) = diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 597ba52b8..969420aed 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -3116,6 +3116,8 @@ impl BlockChainServer { } self.forget_block_held_for_parent_payload(block_root); + // An envelope held for this block has nothing left to wait for. + self.forget_envelopes_awaiting_import(block_root); // Both eviction paths reach this function, so removing here is what // keeps `held_timings` from outliving the blocks it describes. @@ -3510,10 +3512,11 @@ impl BlockChainServer { /// network as stalled `NETWORK_STALL_THRESHOLD` slots after the fork and /// report `Synced` for good on a node that follows nothing. Nothing else /// tells the node the network is ahead: gossip blocks past the fork are - /// ignored in p2p, and a node that runs across the fork keeps the fork - /// digest it computed at startup, so peers' Status messages are dropped - /// and no range session starts. The clock is the one signal that is always - /// there, and treating the network as fresh lets head lag decide. + /// ignored in p2p. The fork digest rolls over at runtime, so a node that + /// runs across a fork it can follow switches topics and Status digests on + /// its own; for a fork it cannot follow that switch brings nothing in. The + /// clock is the one signal that is always there, and treating the network + /// as fresh lets head lag decide. fn clock_is_past_followed_forks(config: &Config, current_slot: u64, head_slot: u64) -> bool { let fork_at = |slot: u64| config.fork_at_epoch(compute_epoch_at_slot(slot)); !fork_at(current_slot).is_followed() && fork_at(head_slot).is_followed() diff --git a/crates/blockchain/state_transition/src/beacon/gossip/column.rs b/crates/blockchain/state_transition/src/beacon/gossip/column.rs index 27d6c442f..9c40961ed 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/column.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/column.rs @@ -24,7 +24,6 @@ use crate::beacon::containers::DataColumnSidecar as AnyDataColumnSidecar; use crate::beacon::containers::SignedBeaconBlock; use crate::beacon::containers::fulu::DataColumnSidecar; use crate::beacon::containers::gloas; -use crate::beacon::fork::ForkName; use crate::beacon::fork_choice::{self, Store}; use crate::beacon::helpers::accessors::get_beacon_proposer_index; use crate::beacon::helpers::misc::{compute_domain, compute_epoch_at_slot, compute_signing_root}; @@ -498,7 +497,7 @@ pub fn is_parkable_gloas( sidecar: &gloas::DataColumnSidecar, ) -> bool { let epoch = compute_epoch_at_slot(sidecar.slot); - config.fork_at_epoch(epoch) == ForkName::Gloas + config.fork_at_epoch(epoch).has_payload_envelopes() && (sidecar.index as usize) < preset::NUMBER_OF_COLUMNS && !sidecar.column.is_empty() && sidecar.column.len() == sidecar.kzg_proofs.len() @@ -588,6 +587,7 @@ mod tests { use super::*; use crate::beacon::constants::MAXIMUM_GOSSIP_CLOCK_DISPARITY as DISPARITY; use crate::beacon::containers::{SignedBeaconBlock, electra, fulu, shared}; + use crate::beacon::fork::ForkName; use crate::beacon::gossip::test_support::{fulu_parent, seen_columns, slot_start_ms, store}; use crate::beacon::gossip::{IgnoreReason, Outcome, QueueReason, RejectReason}; use crate::beacon::helpers::misc::compute_start_slot_at_epoch; diff --git a/crates/common/types/src/beacon/fork.rs b/crates/common/types/src/beacon/fork.rs index 8f0c4c918..c3ccd42a2 100644 --- a/crates/common/types/src/beacon/fork.rs +++ b/crates/common/types/src/beacon/fork.rs @@ -138,6 +138,27 @@ impl ForkName { } } + /// Whether blocks of this fork reveal their execution payload in a separate + /// envelope (gloas, EIP-7732, and any fork after it). + /// + /// An explicit match rather than `self >= ForkName::Gloas`: the derived + /// order puts [`ForkName::Lean`] after every beacon fork, so that + /// comparison is true for lean, which has no envelopes. A fork added after + /// gloas must be listed here, so the compiler forces the decision. + pub const fn has_payload_envelopes(self) -> bool { + match self { + ForkName::Gloas => true, + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Fulu + | ForkName::Lean => false, + } + } + /// The one-byte tag this fork is stored under in a `States` value. /// /// Spelled out rather than `self as u8`. The variant order is already @@ -222,6 +243,14 @@ mod tests { assert!(ForkName::Fulu > ForkName::Phase0); } + #[test] + fn only_gloas_and_later_beacon_forks_have_payload_envelopes() { + for fork in ForkName::ALL { + assert_eq!(fork.has_payload_envelopes(), fork >= ForkName::Gloas); + } + assert!(!ForkName::Lean.has_payload_envelopes()); + } + #[test] fn parse_round_trips_every_fork() { for fork in ForkName::ALL { diff --git a/crates/net/p2p/src/req_resp/envelope_client.rs b/crates/net/p2p/src/req_resp/envelope_client.rs index e1d275496..a75b3d097 100644 --- a/crates/net/p2p/src/req_resp/envelope_client.rs +++ b/crates/net/p2p/src/req_resp/envelope_client.rs @@ -9,7 +9,6 @@ use std::time::Duration; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::gloas::SignedExecutionPayloadEnvelope; -use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::primitives::H256; use libp2p::PeerId; use spawned_concurrency::tasks::{Context, send_after}; @@ -240,7 +239,7 @@ fn settle_by_root_answer( /// envelope worth asking for. Earlier forks have no envelopes, and the server /// answers such a request empty at best. fn range_reaches_gloas(config: &Config, span: &std::ops::RangeInclusive) -> bool { - fork_at_slot(config, *span.end()) >= ForkName::Gloas + fork_at_slot(config, *span.end()).has_payload_envelopes() } /// Ask `peer` for the envelopes in `[start_slot, start_slot + count)`, the span @@ -378,6 +377,7 @@ mod tests { use ethlambda_network_api::{BlockArrival, BlockSource}; use ethlambda_types::beacon::containers::SignedBeaconBlock; + use ethlambda_types::beacon::fork::ForkName; use super::*; use crate::req_resp::handlers::tests::unconnected_server; diff --git a/docs/beacon_stf.md b/docs/beacon_stf.md index 2f41c2db2..a6ee43e5a 100644 --- a/docs/beacon_stf.md +++ b/docs/beacon_stf.md @@ -336,7 +336,8 @@ sidecars of the same fork. | Queue | Holds | Released when | |---|---|---| -| `awaiting_block` | an envelope whose block has no post-state | the block imports; evicted after `ENVELOPE_AWAITING_BLOCK_TTL_SLOTS`, capped per slot and per root | +| `awaiting_import` | an envelope whose block is stored but has no post-state (pending on a parent, held for columns or for its parent's payload), checked against the stored block's bid | the block imports; evicted with the block when it is discarded and at finality; bounded by the stored blocks, not by arrival slot | +| `awaiting_block` | an envelope whose block is not stored at all | the block imports; evicted after `ENVELOPE_AWAITING_BLOCK_TTL_SLOTS`, capped per slot and per root | | `awaiting_columns` | a consensus-checked envelope whose bid has commitments and whose sampled columns are not all stored | the last column is stored | | `awaiting_engine` | a consensus-valid envelope the execution client gave no answer for | the per-slot redrive gets an answer (see [beacon_engine.md](./beacon_engine.md#gloas)) | | `blocks_awaiting_parent_payload` | a block whose parent is FULL with a payload not yet verified, capped per parent | that parent's envelope verifies | @@ -344,9 +345,20 @@ sidecars of the same fork. Data availability moves with the payload: a gloas block's commitments are in its bid, so the columns gate the envelope that reveals the payload, not the block's own import. A held block's parent envelope is requested through -`FetchRequest.needs_envelope`, and the tick re-asks once a slot for every parent -still missing one, which is what recovers a fetch that gave up. Everything is -swept at finality, except the finalized block's own envelope. +`FetchRequest.needs_envelope` the moment the block is held, whatever its source: +`BlockSource::Sync` also marks by-root answers and pending children a cascade +released, so no range batch can be assumed to bring the envelope, and p2p dedups +in-flight roots. The tick re-asks once a slot for every parent still missing +one, which is only what recovers a fetch that gave up; catch-up does not wait +for it. + +The two envelope-before-block queues differ in who has vouched for the root. +A range batch's envelopes arrive when all but its first block are stored and +unimported, so a stored block's own bid is the authority: an envelope matching +it (`envelope_matches_bid`) goes to `awaiting_import` outside the per-slot cap, +and one contradicting it is dropped. Only a root with no stored block, which +nobody has authenticated, is held under the per-slot cap and the TTL. +Everything is swept at finality, except the finalized block's own envelope. Fork-choice events are timed at their arrival: the store clock is advanced to the moment an envelope, block or payload vote reached the node before its From f1b0e33c806e9d3882a5e8674b1f8f509e67f61a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 1 Oct 2026 05:11:13 -0300 Subject: [PATCH 081/132] fix(beacon): bound envelopes held for stored blocks awaiting_import was bounded per root and by finality only, but range sync's fetch lead over import is unbounded across sessions, so a stalled actor (missing envelope, engine down) with sync running grew it with the lag while nothing finalized to sweep it. It now has a global cap of a few sync batches; past it the envelopes nearest the import frontier (lowest slots) stay and the highest are dropped, to be asked for by root once their child is held. The per-root duplicate and cap check now runs before the stored block is decoded, so repeat copies cost nothing. A held block no longer asks for its parent's envelope when any queue already holds it, which saved one by-root request and signature check per block of a cascade. --- crates/blockchain/src/beacon_envelope.rs | 164 +++++++++++++++++++---- 1 file changed, 141 insertions(+), 23 deletions(-) diff --git a/crates/blockchain/src/beacon_envelope.rs b/crates/blockchain/src/beacon_envelope.rs index f4b0c3221..ca2046014 100644 --- a/crates/blockchain/src/beacon_envelope.rs +++ b/crates/blockchain/src/beacon_envelope.rs @@ -83,6 +83,26 @@ pub(crate) const MAX_ENVELOPES_AWAITING_BLOCK_PER_SLOT: usize = 4; /// when the block imports each is tried until one verifies. pub(crate) const MAX_ENVELOPES_PER_UNMATCHED_ROOT: usize = 3; +/// How many envelopes one by-range envelope answer can carry, mirroring p2p's +/// `MAX_REQUEST_PAYLOADS` (this crate does not depend on p2p). +const SYNC_BATCH_ENVELOPES: usize = 128; + +/// How many sync batches of envelopes `awaiting_import` may hold. +const SYNC_BATCHES_AWAITING_IMPORT: usize = 4; + +/// The most envelopes held for stored, unimported blocks, across all roots. +/// +/// The per-root cap bounds one block, and finality bounds the stored blocks, +/// but p2p's fetch lead over import is unbounded across sessions: with the +/// actor stalled (a missing envelope, the engine down) range sync keeps +/// delivering while nothing finalizes to sweep the queue. A few batches is the +/// working set catch-up needs, since the envelopes of one batch arrive before +/// the next is released. Past it the envelopes nearest the import frontier +/// (lowest slots) are kept and the highest dropped: those are the furthest +/// from import, and are asked for again by root once their child is held. +pub(crate) const MAX_ENVELOPES_AWAITING_IMPORT: usize = + SYNC_BATCH_ENVELOPES * SYNC_BATCHES_AWAITING_IMPORT; + /// How many slots an envelope for a block this node has not imported is kept. /// /// A builder reveals its payload after the block it bid in, so the block @@ -199,6 +219,30 @@ impl EnvelopeQueues { self.rejected.push_back(envelope_root); } + /// Drop envelopes from `awaiting_import`, highest slot first, until it + /// holds at most [`MAX_ENVELOPES_AWAITING_IMPORT`]. + fn trim_awaiting_import(&mut self) { + let mut total: usize = self.awaiting_import.values().map(Vec::len).sum(); + while total > MAX_ENVELOPES_AWAITING_IMPORT { + let Some(furthest) = self + .awaiting_import + .iter() + .filter_map(|(root, held)| held.iter().map(|h| h.slot).max().map(|s| (s, *root))) + .max() + .map(|(_, root)| root) + else { + return; + }; + if let Some(held) = self.awaiting_import.get_mut(&furthest) { + held.pop(); + total -= 1; + if held.is_empty() { + self.awaiting_import.remove(&furthest); + } + } + } + } + fn is_empty(&self) -> bool { self.awaiting_block.is_empty() && self.awaiting_import.is_empty() @@ -595,6 +639,16 @@ impl BlockChainServer { root: H256, mut held: HeldEnvelope, ) -> Option { + // Repeat copies cost nothing: the per-root bound and the duplicate + // check come before the block is decoded. + if let Some(held_for_root) = self.envelopes.awaiting_import.get(&root) + && (held_for_root.len() >= MAX_ENVELOPES_PER_UNMATCHED_ROOT + || held_for_root + .iter() + .any(|other| other.envelope_root == held.envelope_root)) + { + return None; + } if !self.store.has_block(&root) { return Some(held); } @@ -626,19 +680,17 @@ impl BlockChainServer { return None; } held.slot = slot; - let held_for_root = self.envelopes.awaiting_import.entry(root).or_default(); - if held_for_root.len() < MAX_ENVELOPES_PER_UNMATCHED_ROOT - && !held_for_root - .iter() - .any(|other| other.envelope_root == held.envelope_root) - { - debug!( - %slot, - block_root = %ShortRoot(&root.0), - "Holding an envelope until its stored block is imported" - ); - held_for_root.push(held); - } + debug!( + %slot, + block_root = %ShortRoot(&root.0), + "Holding an envelope until its stored block is imported" + ); + self.envelopes + .awaiting_import + .entry(root) + .or_default() + .push(held); + self.envelopes.trim_awaiting_import(); self.publish_envelope_queues(); None } @@ -754,20 +806,26 @@ impl BlockChainServer { /// does not stall the child for good. pub(crate) fn redrive_missing_envelopes(&self) { for &parent_root in self.envelopes.blocks_awaiting_parent_payload.keys() { - if !self.envelopes.awaiting_block.contains_key(&parent_root) - && !self.envelopes.awaiting_columns.contains_key(&parent_root) - && !self.envelopes.awaiting_engine.contains_key(&parent_root) - && !self.store.has_verified_payload(&parent_root) - && !self - .store - .beacon_block_payload_status(parent_root) - .is_invalidated() - { + if self.parent_envelope_is_unaccounted_for(&parent_root) { self.request_missing_envelope(parent_root); } } } + /// Whether `parent_root`'s envelope is neither held in any queue nor + /// verified (nor its payload found invalid), so asking for it can help. + fn parent_envelope_is_unaccounted_for(&self, parent_root: &H256) -> bool { + !self.envelopes.awaiting_block.contains_key(parent_root) + && !self.envelopes.awaiting_import.contains_key(parent_root) + && !self.envelopes.awaiting_columns.contains_key(parent_root) + && !self.envelopes.awaiting_engine.contains_key(parent_root) + && !self.store.has_verified_payload(parent_root) + && !self + .store + .beacon_block_payload_status(*parent_root) + .is_invalidated() + } + /// Once a slot: look at every envelope waiting for columns, and ask peers /// for whatever is still missing. pub(crate) fn redrive_envelopes_awaiting_columns(&mut self) { @@ -963,7 +1021,12 @@ impl BlockChainServer { // follower to one block per slot. p2p dedups in-flight roots, so a // range batch whose envelopes are on their way costs at most one extra // request per block. - self.request_missing_envelope(parent_root); + // Unless a queue already holds it: a cascade of held children whose + // parents' envelopes arrived with their range batch would otherwise + // cost one by-root request and signature check per block. + if self.parent_envelope_is_unaccounted_for(&parent_root) { + self.request_missing_envelope(parent_root); + } } /// Ask p2p for `root`'s envelope by root. The request carries nothing but @@ -1909,6 +1972,61 @@ mod tests { assert_eq!(server.store.ms_since_genesis(), time_before); } + #[tokio::test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + async fn a_held_envelope_for_the_parent_suppresses_the_immediate_ask() { + let case = Case::new("get_head", TIEBREAK); + let mut server = case.server(12); + let p2p = std::sync::Arc::new(crate::tests::RecordingP2P::default()); + server.p2p = Some(p2p.clone()); + let child = case.block(CHILD); + let envelope = case.envelope(PARENT_ENVELOPE); + server + .envelopes + .awaiting_import + .insert(child.parent_root(), vec![HeldEnvelope::new(envelope, 0, 0)]); + + server.hold_block_for_parent_payload(child, ImportTimings::default()); + + assert!(p2p.fetches.lock().unwrap().is_empty()); + } + + #[test] + #[cfg_attr( + not(feature = "beacon-spec-tests"), + ignore = "needs the fork choice fixtures; run `make consensus-spec-tests`" + )] + fn awaiting_import_keeps_the_lowest_slots_when_over_its_cap() { + let mut queues = EnvelopeQueues::default(); + let envelope = Case::new("get_head", TIEBREAK).envelope(PARENT_ENVELOPE); + let total = MAX_ENVELOPES_AWAITING_IMPORT as u64 + 5; + for slot in 0..total { + let mut root_bytes = [0u8; 32]; + root_bytes[..8].copy_from_slice(&slot.to_be_bytes()); + queues.awaiting_import.insert( + H256(root_bytes), + vec![HeldEnvelope::new(envelope.clone(), slot, 0)], + ); + } + + queues.trim_awaiting_import(); + + let mut kept: Vec = queues + .awaiting_import + .values() + .flatten() + .map(|held| held.slot) + .collect(); + kept.sort(); + assert_eq!( + kept, + (0..MAX_ENVELOPES_AWAITING_IMPORT as u64).collect::>() + ); + } + #[test] fn each_slot_re_asks_for_the_envelope_of_a_parent_with_held_children() { let (mut server, p2p) = From d1ef2fe516ff6419ca6738263cab20c50513a863 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 2 Oct 2026 13:31:06 -0300 Subject: [PATCH 082/132] fix(beacon): let the finalized block's own gloas columns through A follower checkpoint-synced onto a gloas anchor whose bid carries blobs never imported past it. The anchor's envelope waits for its sampled columns, and every column fetched for it was dropped by chain_checks_gloas as finalized: the anchor is the finalized block, and the rule ignored any sidecar at or below the finalized slot. With the envelope never verified, the first FULL child was held, and every later block waited on it. An anchor with no blobs needs no columns, which is why this went unnoticed. Gloas has no finalized bound on data column sidecars (its envelope rule even admits the finalized slot). The bound stays for every other block, so a parked row still cannot name an old slot, but the finalized root is exempt. Matched by root rather than by slot, since an anchor state advanced past empty slots finalizes a block from before its epoch's start slot. --- .../src/beacon/gossip/column.rs | 50 +++++++++++++++++-- docs/checkpoint_sync.md | 8 +++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/gossip/column.rs b/crates/blockchain/state_transition/src/beacon/gossip/column.rs index 9c40961ed..a607ec58e 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/column.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/column.rs @@ -26,7 +26,9 @@ use crate::beacon::containers::fulu::DataColumnSidecar; use crate::beacon::containers::gloas; use crate::beacon::fork_choice::{self, Store}; use crate::beacon::helpers::accessors::get_beacon_proposer_index; -use crate::beacon::helpers::misc::{compute_domain, compute_epoch_at_slot, compute_signing_root}; +use crate::beacon::helpers::misc::{ + compute_domain, compute_epoch_at_slot, compute_signing_root, compute_start_slot_at_epoch, +}; use crate::beacon::lean_boundary::lean_block_unreachable; use crate::beacon::precheck; use crate::beacon::preset; @@ -207,8 +209,20 @@ pub fn chain_checks_gloas( if is_future_slot(&config, sidecar.slot, now_ms) { return ChainVerdict::Drop(Outcome::Ignore(IgnoreReason::FutureSlot)); } - // [IGNORE] From a slot greater than the latest finalized slot. - if sidecar.slot <= finalized_start_slot(store) { + // [IGNORE] From a slot greater than the latest finalized slot, unless it + // is a column of the finalized block itself. Not a gloas rule: gloas has + // no finalized bound on sidecars, and this one stays only to bound what a + // parked row can name. The finalized block is exempt because its envelope + // can still be waiting on these columns: a checkpoint-synced anchor is the + // finalized block, and no child building on its payload imports until its + // sampled columns arrive. Matched by root rather than by slot, since an + // anchor state advanced past empty slots finalizes a block from before its + // epoch's start slot. A column naming that root is never parked: the block + // and its post-state are both stored, so it is judged at once. + let finalized = store.beacon_finalized_checkpoint(); + if sidecar.slot <= compute_start_slot_at_epoch(finalized.epoch) + && sidecar.beacon_block_root != finalized.root + { return ChainVerdict::Drop(Outcome::Ignore(IgnoreReason::Finalized)); } // [IGNORE] Already stored, as in [`chain_checks`]. @@ -1234,6 +1248,36 @@ mod tests { ); } + /// A checkpoint-synced anchor is the finalized block, and its envelope + /// waits on its columns, so the finalized rule lets that block's own + /// columns through: at the finalized slot, and below it, where an anchor + /// state advanced past empty slots puts its block. Any other block's + /// columns there are still dropped. + #[test] + fn chain_checks_let_the_finalized_blocks_own_gloas_sidecars_through() { + let store = store(32); + let now = slot_start_ms(&store, 40); + let finalized_root = store.beacon_finalized_checkpoint().root; + for slot in [32, 30] { + let of_finalized = gloas::DataColumnSidecar { + beacon_block_root: finalized_root, + ..gloas_sidecar(slot, 0) + }; + // This store holds no block, so reaching the block lookup is as + // far as it goes. + assert_eq!( + chain_checks_gloas(&store, &of_finalized, now), + ChainVerdict::AwaitParent, + "the finalized block's column at slot {slot}" + ); + assert_eq!( + chain_checks_gloas(&store, &gloas_sidecar(slot, 0), now), + ChainVerdict::Drop(Outcome::Ignore(IgnoreReason::Finalized)), + "another block's column at slot {slot}" + ); + } + } + #[test] fn a_gloas_sidecar_naming_a_non_gloas_block_is_rejected() { let mut store = store(0); diff --git a/docs/checkpoint_sync.md b/docs/checkpoint_sync.md index 9331e6e9c..53569364e 100644 --- a/docs/checkpoint_sync.md +++ b/docs/checkpoint_sync.md @@ -95,6 +95,14 @@ nothing from it; any failure (network, decoding, an envelope naming another block) is logged at `info` and ignored. Without the envelope, the by-root request that fires when a FULL child arrives recovers it. +An anchor whose bid carries blobs also needs its sampled columns before its +envelope verifies, and those are fetched by root like any other block's. They +sit at the finalized slot, or below it for an anchor state advanced past empty +slots, which is where the chain checks drop every other block's columns. So +the finalized block's own columns are exempt from that rule +(`column::chain_checks_gloas`); without the exemption such an anchor never gets +its payload, and the node never imports past it. + ### Failure and success If any step fails (network error, decoding error, verification failure), the node logs the error and exits. There is no automatic retry; restart the node to try again. The database is not modified until verification succeeds, so a failed checkpoint sync leaves the data directory clean. From ee15eeb695a7042f30c42a12d6413879311748e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:31:15 -0300 Subject: [PATCH 083/132] chore(beacon): lay out the interfaces gloas validator duties will share The gloas validator duties touch the network actor, the Beacon API and block production at once. Fixing the seams first lets each part be built against them independently: two RpcToP2P methods for the gossip a validator client's envelope and payload votes need, a payload attestation pool that gossip and the API fill and block production reads, empty route modules for the new endpoints, JSON decoding for the gloas containers the API accepts, and the electra/gloas attestation conversions a pool holding electra's shape needs to serve gloas. --- .../state_transition/src/beacon/mod.rs | 1 + .../src/beacon/payload_attestation_pool.rs | 84 +++++++++++++ .../types/src/beacon/containers/gloas.rs | 111 ++++++++++++++++-- crates/net/api/src/lib.rs | 19 +++ crates/net/p2p/src/gossipsub/handler.rs | 34 ++++++ crates/net/p2p/src/gossipsub/mod.rs | 1 + crates/net/p2p/src/lib.rs | 13 ++ crates/net/rpc/src/beacon/gloas_proposal.rs | 13 ++ crates/net/rpc/src/beacon/mod.rs | 4 + crates/net/rpc/src/beacon/ptc.rs | 13 ++ crates/net/rpc/src/lib.rs | 28 +++++ 11 files changed, 314 insertions(+), 7 deletions(-) create mode 100644 crates/blockchain/state_transition/src/beacon/payload_attestation_pool.rs create mode 100644 crates/net/rpc/src/beacon/gloas_proposal.rs create mode 100644 crates/net/rpc/src/beacon/ptc.rs diff --git a/crates/blockchain/state_transition/src/beacon/mod.rs b/crates/blockchain/state_transition/src/beacon/mod.rs index fc9e41458..bae1ac018 100644 --- a/crates/blockchain/state_transition/src/beacon/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/mod.rs @@ -79,6 +79,7 @@ pub mod gossip; pub mod hash; pub mod helpers; pub mod kzg; +pub mod payload_attestation_pool; pub mod precheck; pub mod stf; pub mod upgrade; diff --git a/crates/blockchain/state_transition/src/beacon/payload_attestation_pool.rs b/crates/blockchain/state_transition/src/beacon/payload_attestation_pool.rs new file mode 100644 index 000000000..0105bb5e3 --- /dev/null +++ b/crates/blockchain/state_transition/src/beacon/payload_attestation_pool.rs @@ -0,0 +1,84 @@ +//! Payload timeliness committee votes, held until a proposer packs them. +//! +//! Gloas `validator.md` ("Payload attestations"): the proposer of slot `N` +//! listens to `payload_attestation_message`, keeps the votes on its parent +//! block (slot `N - 1`), and aggregates every vote sharing one +//! `PayloadAttestationData` into a single `PayloadAttestation` for its body. +//! +//! Only messages that already passed validation go in (gossip's +//! `payload_attestation_message` verdict, or the Beacon API's pool endpoint +//! after the same checks), since that is what makes combining their +//! signatures safe: one bad share fails the whole aggregate, and with it the +//! block that carries it. +//! +//! The method signatures below are the shared contract between the tasks +//! that fill the pool and block production, which only calls +//! [`messages_for`]. The storage behind them is a placeholder. +//! +//! [`messages_for`]: PayloadAttestationPool::messages_for + +use std::sync::{Arc, Mutex}; + +use ethlambda_types::beacon::{ + containers::gloas::PayloadAttestationMessage, + primitives::{Root, Slot}, +}; + +/// The pool, shared between whatever fills it (the Beacon API's pool +/// endpoint, the `payload_attestation_message` gossip verdict) and what reads +/// it (block production, `GET /eth/v1/beacon/pool/payload_attestations`). +/// There must be exactly one per node. +pub type SharedPayloadAttestationPool = Arc>; + +#[derive(Debug, Default)] +pub struct PayloadAttestationPool { + /// Placeholder storage; the PTC task may restructure it freely. + messages: Vec, +} + +impl PayloadAttestationPool { + /// Record a validated message. Returns `false` (and keeps the first) when + /// the pool already holds a message from the same validator for the same + /// `data`. Drops every message for a slot more than one before the + /// message's own. + pub fn insert(&mut self, message: PayloadAttestationMessage) -> bool { + self.prune_before(message.data.slot.saturating_sub(1)); + let held = self.messages.iter().any(|held| { + held.validator_index == message.validator_index && held.data == message.data + }); + if !held { + self.messages.push(message); + } + !held + } + + /// Every held message voting on `beacon_block_root` at `slot`, in any + /// order, one per (validator, data). + pub fn messages_for( + &self, + slot: Slot, + beacon_block_root: Root, + ) -> Vec { + self.messages + .iter() + .filter(|held| { + held.data.slot == slot && held.data.beacon_block_root == beacon_block_root + }) + .cloned() + .collect() + } + + /// Every held message, or only those for `slot` when given. + pub fn all(&self, slot: Option) -> Vec { + self.messages + .iter() + .filter(|held| slot.is_none_or(|slot| held.data.slot == slot)) + .cloned() + .collect() + } + + /// Drop every message for a slot before `slot`. + pub fn prune_before(&mut self, slot: Slot) { + self.messages.retain(|held| held.data.slot >= slot); + } +} diff --git a/crates/common/types/src/beacon/containers/gloas.rs b/crates/common/types/src/beacon/containers/gloas.rs index 8c6731e03..51d236b7a 100644 --- a/crates/common/types/src/beacon/containers/gloas.rs +++ b/crates/common/types/src/beacon/containers/gloas.rs @@ -259,7 +259,17 @@ pub struct BuilderExitRequest { /// What the payload timeliness committee attests to: whether the slot's /// payload was revealed on time and whether its blob data is available. #[derive( - Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Copy, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] pub struct PayloadAttestationData { pub beacon_block_root: Root, @@ -281,7 +291,17 @@ pub struct PayloadAttestation { /// One payload timeliness committee member's unaggregated vote, gossiped /// before an aggregator folds it into a [`PayloadAttestation`]. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] pub struct PayloadAttestationMessage { #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub validator_index: ValidatorIndex, @@ -464,17 +484,74 @@ pub struct NewPayloadRequest { /// Electra's shape (EIP-7549's wide `committee_bits`/`aggregation_bits`), /// with `aggregation_bits` now the unbounded [`AggregationBits`] rather than /// electra's bounded `SszBitlist`. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] #[ssz(progressive_container)] pub struct Attestation { - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub aggregation_bits: AggregationBits, pub data: AttestationData, pub signature: BlsSignature, - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub committee_bits: CommitteeBits, } +impl From<&super::electra::Attestation> for Attestation { + /// The same vote in gloas's container: every field carries over unchanged, + /// only `aggregation_bits` moves from electra's bounded bitlist to the + /// progressive one. A pool that holds electra-shaped aggregates (the + /// shape gossip and the Beacon API agree on until the fork) serves gloas + /// readers through this. + fn from(attestation: &super::electra::Attestation) -> Self { + let source = &attestation.aggregation_bits; + let mut aggregation_bits = AggregationBits::with_length(source.len()); + for index in (0..source.len()).filter(|&index| source.get(index) == Some(true)) { + aggregation_bits + .set(index, true) + .expect("the index is below the length the bitlist was built with"); + } + Self { + aggregation_bits, + data: attestation.data, + signature: attestation.signature, + committee_bits: attestation.committee_bits.clone(), + } + } +} + +impl TryFrom<&Attestation> for super::electra::Attestation { + type Error = libssz_types::TypeError; + + /// The inverse of the conversion above, for a gloas aggregate entering a + /// pool that stores electra's shape. Fails only when the bits exceed + /// electra's `MAX_VALIDATORS_PER_SLOT` bound, which no aggregate a real + /// committee assignment produces can reach. + fn try_from(attestation: &Attestation) -> Result { + let source = &attestation.aggregation_bits; + let mut aggregation_bits = super::electra::AggregationBits::with_length(source.len())?; + for index in (0..source.len()).filter(|&index| source.get(index) == Some(true)) { + aggregation_bits + .set(index, true) + .expect("the index is below the length the bitlist was built with"); + } + Ok(Self { + aggregation_bits, + data: attestation.data, + signature: attestation.signature, + committee_bits: attestation.committee_bits.clone(), + }) + } +} + /// An attestation with its attesters named rather than bit-encoded, the same /// role electra's plays, with `attesting_indices` now the unbounded /// [`AttestingIndices`]. @@ -499,7 +576,17 @@ pub struct AttesterSlashing { /// An aggregate together with proof that its aggregator was selected to /// produce it. Unchanged in shape from electra: `aggregate` simply carries /// gloas's own, wider [`Attestation`] now. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] pub struct AggregateAndProof { #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub aggregator_index: ValidatorIndex, @@ -507,7 +594,17 @@ pub struct AggregateAndProof { pub selection_proof: BlsSignature, } -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] pub struct SignedAggregateAndProof { pub message: AggregateAndProof, pub signature: BlsSignature, diff --git a/crates/net/api/src/lib.rs b/crates/net/api/src/lib.rs index 705e98f8e..7c3c3305d 100644 --- a/crates/net/api/src/lib.rs +++ b/crates/net/api/src/lib.rs @@ -354,6 +354,25 @@ pub trait RpcToP2P: Send + Sync { /// delivers a node its own messages, so without the second half this node /// would not follow its own proposal. Checked by the caller as above. fn publish_beacon_block(&self, block: SignedBeaconBlock) -> Result<(), ActorError>; + /// Gossip a gloas execution payload envelope a validator client signed on + /// `execution_payload`, and its blobs' data column sidecars on their + /// subnets, then hand both to the chain actor: as for a block, gossip never + /// delivers a node its own messages. Checked by the caller as above. + /// + /// Boxed for the reason [`P2PToBlockChain::new_execution_payload_envelope`] + /// is: the envelope carries a whole execution payload. + fn publish_execution_payload_envelope( + &self, + envelope: Box, + sidecars: Vec, + ) -> Result<(), ActorError>; + /// Gossip a payload timeliness committee member's vote on + /// `payload_attestation_message` and hand it to the chain actor. Checked by + /// the caller as above. + fn publish_payload_attestation_message( + &self, + message: PayloadAttestationMessage, + ) -> Result<(), ActorError>; } // --- Init messages --- diff --git a/crates/net/p2p/src/gossipsub/handler.rs b/crates/net/p2p/src/gossipsub/handler.rs index 4a18771e7..bc9b100b0 100644 --- a/crates/net/p2p/src/gossipsub/handler.rs +++ b/crates/net/p2p/src/gossipsub/handler.rs @@ -788,6 +788,40 @@ pub async fn publish_beacon_block(server: &mut P2PServer, block: SignedBeaconBlo } } +/// Gossip a gloas execution payload envelope a validator client signed, +/// handed over by the Beacon API, on `execution_payload`, its data column +/// sidecars on their subnets, and pass all of them to the chain actor. +/// +/// STUB: owned by the block-production task (see the contract), which +/// replaces this body. +pub async fn publish_execution_payload_envelope( + _server: &mut P2PServer, + envelope: ethlambda_types::beacon::containers::gloas::SignedExecutionPayloadEnvelope, + sidecars: Vec, +) { + error!( + block_root = %ShortRoot(&envelope.message.beacon_block_root.0), + sidecars = sidecars.len(), + "Publishing execution payload envelopes is not implemented; dropping it" + ); +} + +/// Gossip a payload attestation message a validator client signed, handed +/// over by the Beacon API, on `payload_attestation_message`, and pass it to +/// the chain actor. +/// +/// STUB: owned by the PTC task (see the contract), which replaces this body. +pub async fn publish_payload_attestation_message( + _server: &mut P2PServer, + message: ethlambda_types::beacon::containers::gloas::PayloadAttestationMessage, +) { + error!( + slot = message.data.slot, + validator = message.validator_index, + "Publishing payload attestation messages is not implemented; dropping it" + ); +} + /// The beacon wall-clock slot, from the wire's genesis and slot duration. fn beacon_wall_slot(wire: &BeaconWire) -> u64 { let genesis_ms = wire.genesis_time.saturating_mul(1000); diff --git a/crates/net/p2p/src/gossipsub/mod.rs b/crates/net/p2p/src/gossipsub/mod.rs index 608584a2d..1701c5f3a 100644 --- a/crates/net/p2p/src/gossipsub/mod.rs +++ b/crates/net/p2p/src/gossipsub/mod.rs @@ -7,5 +7,6 @@ pub use handler::{ handle_gossip_message, join_aggregator_subnets, leave_expired_aggregator_subnets, prune_attestation_pool, publish_aggregated_attestation, publish_attestation, publish_beacon_aggregate, publish_beacon_attestation, publish_beacon_block, publish_block, + publish_execution_payload_envelope, publish_payload_attestation_message, }; pub use messages::{aggregation_topic, attestation_subnet_topic, block_topic, topic_kind}; diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index ff11c0f4b..053638ecd 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -45,6 +45,7 @@ use ethlambda_network_api::{ }, rpc_to_p2p::{ PublishBeaconAggregate, PublishBeaconAttestation, PublishBeaconBlock, + PublishExecutionPayloadEnvelope, PublishPayloadAttestationMessage, SubscribeAttestationSubnets, }, }; @@ -1596,6 +1597,18 @@ impl Handler for P2PServer { } } +impl Handler for P2PServer { + async fn handle(&mut self, msg: PublishExecutionPayloadEnvelope, _ctx: &Context) { + gossipsub::publish_execution_payload_envelope(self, *msg.envelope, msg.sidecars).await; + } +} + +impl Handler for P2PServer { + async fn handle(&mut self, msg: PublishPayloadAttestationMessage, _ctx: &Context) { + gossipsub::publish_payload_attestation_message(self, msg.message).await; + } +} + impl Handler for P2PServer { async fn handle(&mut self, msg: SubscribeAttestationSubnets, _ctx: &Context) { gossipsub::join_aggregator_subnets(self, msg.subnets); diff --git a/crates/net/rpc/src/beacon/gloas_proposal.rs b/crates/net/rpc/src/beacon/gloas_proposal.rs new file mode 100644 index 000000000..846086c33 --- /dev/null +++ b/crates/net/rpc/src/beacon/gloas_proposal.rs @@ -0,0 +1,13 @@ +//! Gloas block production and payload envelope publication. +//! +//! STUB: owned by the block-production task (see the contract), which fills +//! in `POST /eth/v4/validator/blocks/{slot}`, +//! `GET /eth/v1/validator/execution_payload_envelopes/{slot}/{beacon_block_root}` +//! and `POST /eth/v1/beacon/execution_payload_envelopes`. + +use axum::Router; +use ethlambda_storage::Store; + +pub(crate) fn routes() -> Router { + Router::new() +} diff --git a/crates/net/rpc/src/beacon/mod.rs b/crates/net/rpc/src/beacon/mod.rs index c047aa299..a28b3beaf 100644 --- a/crates/net/rpc/src/beacon/mod.rs +++ b/crates/net/rpc/src/beacon/mod.rs @@ -20,10 +20,12 @@ pub(crate) mod blocks; pub(crate) mod config; pub(crate) mod envelopes; pub(crate) mod genesis; +pub(crate) mod gloas_proposal; pub(crate) mod headers; pub(crate) mod node; pub(crate) mod pool; pub(crate) mod proposal; +pub(crate) mod ptc; pub(crate) mod states; pub(crate) mod validator; #[cfg(test)] @@ -133,6 +135,8 @@ pub(crate) fn routes(version: &'static str, peer_id: String) -> Router { .merge(validator::routes()) .merge(pool::routes()) .merge(proposal::routes()) + .merge(gloas_proposal::routes()) + .merge(ptc::routes()) } #[cfg(test)] diff --git a/crates/net/rpc/src/beacon/ptc.rs b/crates/net/rpc/src/beacon/ptc.rs new file mode 100644 index 000000000..181616235 --- /dev/null +++ b/crates/net/rpc/src/beacon/ptc.rs @@ -0,0 +1,13 @@ +//! The payload timeliness committee's validator-facing endpoints. +//! +//! STUB: owned by the PTC task (see the contract), which fills in +//! `POST /eth/v1/validator/duties/ptc/{epoch}`, +//! `GET /eth/v1/validator/payload_attestation_data` and +//! `GET`/`POST /eth/v1/beacon/pool/payload_attestations`. + +use axum::Router; +use ethlambda_storage::Store; + +pub(crate) fn routes() -> Router { + Router::new() +} diff --git a/crates/net/rpc/src/lib.rs b/crates/net/rpc/src/lib.rs index 7958cbc72..4c312d8f0 100644 --- a/crates/net/rpc/src/lib.rs +++ b/crates/net/rpc/src/lib.rs @@ -417,6 +417,15 @@ pub(crate) mod test_utils { pub(crate) subscriptions: std::sync::Mutex>, pub(crate) blocks: std::sync::Mutex>, + pub(crate) envelopes: std::sync::Mutex< + Vec<( + ethlambda_types::beacon::containers::gloas::SignedExecutionPayloadEnvelope, + Vec, + )>, + >, + pub(crate) payload_attestations: std::sync::Mutex< + Vec, + >, } impl ethlambda_network_api::RpcToP2P for RecordingNetwork { @@ -455,6 +464,25 @@ pub(crate) mod test_utils { self.blocks.lock().unwrap().push(block); Ok(()) } + + fn publish_execution_payload_envelope( + &self, + envelope: Box< + ethlambda_types::beacon::containers::gloas::SignedExecutionPayloadEnvelope, + >, + sidecars: Vec, + ) -> Result<(), spawned_concurrency::error::ActorError> { + self.envelopes.lock().unwrap().push((*envelope, sidecars)); + Ok(()) + } + + fn publish_payload_attestation_message( + &self, + message: ethlambda_types::beacon::containers::gloas::PayloadAttestationMessage, + ) -> Result<(), spawned_concurrency::error::ActorError> { + self.payload_attestations.lock().unwrap().push(message); + Ok(()) + } } /// [`beacon_fixture`]'s chain with three gloas blocks on top of its head. From bc02069b81714f5c698a7ea8192b0c9b49fad706 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:40:54 -0300 Subject: [PATCH 084/132] feat(validator): pick duty deadlines by the fork of the slot Gloas moves the attester and aggregator offsets earlier and adds a payload timeliness committee deadline. The clock now keeps the whole Config and picks the basis points by the fork of the slot's epoch, so a client running across the boundary is on the right offsets on both sides of it. The spec response parser reads the four new keys and keeps the specification defaults when a node omits them. --- crates/validator/src/beacon_node/dto.rs | 42 ++++++++ crates/validator/src/lib.rs | 7 +- crates/validator/src/slot_clock.rs | 121 ++++++++++++++++++++++-- 3 files changed, 155 insertions(+), 15 deletions(-) diff --git a/crates/validator/src/beacon_node/dto.rs b/crates/validator/src/beacon_node/dto.rs index df5708f45..3ec6ba076 100644 --- a/crates/validator/src/beacon_node/dto.rs +++ b/crates/validator/src/beacon_node/dto.rs @@ -476,6 +476,27 @@ pub fn config_from_spec_response(value: &serde_json::Value) -> Result { if let Some(bps) = spec_u64(value, "AGGREGATE_DUE_BPS")? { config.aggregate_due_bps = bps; } + // The gloas offsets and the payload deadlines. Absent keeps the + // specification's defaults already in `Config` (2500, 5000, 5000, 7500). + for (key, field) in [ + ( + "ATTESTATION_DUE_BPS_GLOAS", + &mut config.attestation_due_bps_gloas, + ), + ( + "AGGREGATE_DUE_BPS_GLOAS", + &mut config.aggregate_due_bps_gloas, + ), + ("PAYLOAD_DUE_BPS", &mut config.payload_due_bps), + ( + "PAYLOAD_ATTESTATION_DUE_BPS", + &mut config.payload_attestation_due_bps, + ), + ] { + if let Some(bps) = spec_u64(value, key)? { + *field = bps; + } + } if let Some(version) = spec_version(value, "GENESIS_FORK_VERSION")? { config = config.with_fork_version(ForkName::Phase0, version); } @@ -910,6 +931,27 @@ mod tests { assert_eq!(config.aggregate_due_bps, 5_000); } + #[test] + fn the_gloas_offsets_are_read_and_default_to_the_specification() { + let response = serde_json::json!({ + "ATTESTATION_DUE_BPS_GLOAS": "2000", + "AGGREGATE_DUE_BPS_GLOAS": "4000", + "PAYLOAD_DUE_BPS": "5500", + "PAYLOAD_ATTESTATION_DUE_BPS": "8000", + }); + let config = config_from_spec_response(&response).expect("builds"); + assert_eq!(config.attestation_due_bps_gloas, 2_000); + assert_eq!(config.aggregate_due_bps_gloas, 4_000); + assert_eq!(config.payload_due_bps, 5_500); + assert_eq!(config.payload_attestation_due_bps, 8_000); + + let config = config_from_spec_response(&serde_json::json!({})).expect("builds"); + assert_eq!(config.attestation_due_bps_gloas, 2_500); + assert_eq!(config.aggregate_due_bps_gloas, 5_000); + assert_eq!(config.payload_due_bps, 5_000); + assert_eq!(config.payload_attestation_due_bps, 7_500); + } + /// A node that has not moved to the basis-point form keeps the defaults, /// which are the right answer for every network that predates it. #[test] diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index 9eca256e0..a54a39fe6 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -167,12 +167,7 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { "Validator client starting" ); - let clock = SlotClock::new( - genesis.genesis_time, - spec.slot_duration_ms, - spec.attestation_due_bps, - spec.aggregate_due_bps, - ); + let clock = SlotClock::from_config(genesis.genesis_time, &spec); let context = Arc::new(SigningContext { config: spec, genesis_validators_root: genesis.genesis_validators_root, diff --git a/crates/validator/src/slot_clock.rs b/crates/validator/src/slot_clock.rs index dc5bf6adf..e402b80b6 100644 --- a/crates/validator/src/slot_clock.rs +++ b/crates/validator/src/slot_clock.rs @@ -7,6 +7,9 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use ethlambda_types::beacon::config::Config; +use ethlambda_types::beacon::constants::FAR_FUTURE_EPOCH; +use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::preset::SLOTS_PER_EPOCH; use ethlambda_types::beacon::primitives::{Epoch, Slot}; @@ -29,12 +32,19 @@ const BASIS_POINTS: u64 = 10_000; /// both offsets to 2500 and 5000 basis points, which no fixed fraction /// expresses. Holding the basis points means this clock follows the network it /// was told about rather than the one it was compiled for. -#[derive(Debug, Clone, Copy)] +/// +/// # The offsets depend on the fork of the slot +/// +/// Gloas moves the attester and aggregator offsets earlier (2500 and 5000 +/// basis points) and adds a payload timeliness committee deadline. So the clock +/// holds the whole [`Config`] and answers by the fork of the slot's own epoch +/// (`Config::fork_at_epoch`), which keeps a client running across the boundary +/// on the right offsets on both sides of it. +#[derive(Debug, Clone)] pub struct SlotClock { genesis_time: u64, slot_duration_ms: u64, - attestation_due_bps: u64, - aggregate_due_bps: u64, + config: Config, } impl SlotClock { @@ -51,15 +61,33 @@ impl SlotClock { attestation_due_bps: u64, aggregate_due_bps: u64, ) -> Self { - assert!(slot_duration_ms > 0, "slot_duration_ms must be nonzero"); + let mut config = Config::mainnet().with_fork_epoch(ForkName::Gloas, FAR_FUTURE_EPOCH); + config.slot_duration_ms = slot_duration_ms; + config.attestation_due_bps = attestation_due_bps; + config.aggregate_due_bps = aggregate_due_bps; + Self::from_config(genesis_time, &config) + } + + /// A clock for the network `config` describes, which is how the offsets + /// of every fork reach it. See the type's docs for why it keeps the whole + /// configuration rather than one pair of offsets. + pub fn from_config(genesis_time: u64, config: &Config) -> Self { + assert!( + config.slot_duration_ms > 0, + "slot_duration_ms must be nonzero" + ); Self { genesis_time, - slot_duration_ms, - attestation_due_bps, - aggregate_due_bps, + slot_duration_ms: config.slot_duration_ms, + config: config.clone(), } } + /// Whether `slot` is a gloas slot. + pub fn is_gloas(&self, slot: Slot) -> bool { + self.config.fork_at_epoch(self.epoch_of(slot)) >= ForkName::Gloas + } + /// The slot containing `now`, or `None` before genesis. /// /// Divides in milliseconds, not seconds, so a chain whose slot is not a @@ -94,7 +122,20 @@ impl SlotClock { /// When the attester duty for `slot` should run. pub fn attestation_time(&self, slot: Slot) -> SystemTime { - self.offset_into(slot, self.attestation_due_bps) + let bps = if self.is_gloas(slot) { + self.config.attestation_due_bps_gloas + } else { + self.config.attestation_due_bps + }; + self.offset_into(slot, bps) + } + + /// When the payload timeliness committee's duty for `slot` should run. + /// + /// Only meaningful for a gloas slot; the offset is the configured one + /// whatever the slot, and callers gate on [`Self::is_gloas`]. + pub fn payload_attestation_time(&self, slot: Slot) -> SystemTime { + self.offset_into(slot, self.config.payload_attestation_due_bps) } /// When the aggregation duty for `slot` should run. @@ -103,7 +144,12 @@ impl SlotClock { /// necessarily: an aggregator folds together votes its beacon node has /// collected, and before the attesters have voted there is nothing to fold. pub fn aggregation_time(&self, slot: Slot) -> SystemTime { - self.offset_into(slot, self.aggregate_due_bps) + let bps = if self.is_gloas(slot) { + self.config.aggregate_due_bps_gloas + } else { + self.config.aggregate_due_bps + }; + self.offset_into(slot, bps) } /// `bps` basis points of the way into `slot`. @@ -170,6 +216,17 @@ impl SlotClock { .unwrap_or(Duration::ZERO) } + /// How long from `now` until the payload timeliness committee's duty for + /// `slot`, or zero once that instant has passed. + /// + /// The PTC counterpart to [`Self::until_aggregation`]: the loop sleeps on + /// it between aggregation and the committee vote. + pub fn until_payload_attestation(&self, slot: Slot, now: SystemTime) -> Duration { + self.payload_attestation_time(slot) + .duration_since(now) + .unwrap_or(Duration::ZERO) + } + /// The next slot to serve, given the last one served, and how long until /// it begins. /// @@ -500,6 +557,52 @@ mod tests { assert_eq!(now + delay, clock.start_of(0)); } + /// Slots before the gloas epoch keep the pre-gloas offsets and slots from + /// it on take the gloas ones, on both sides of the boundary. + #[test] + fn the_offsets_change_at_the_gloas_boundary() { + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 2); + let clock = SlotClock::from_config(GENESIS, &config); + let last_fulu = 2 * SLOTS_PER_EPOCH - 1; + let first_gloas = 2 * SLOTS_PER_EPOCH; + let ms = + |t: SystemTime, slot: u64| t.duration_since(clock.start_of(slot)).expect("after start"); + + assert!(!clock.is_gloas(last_fulu)); + assert!(clock.is_gloas(first_gloas)); + assert_eq!( + ms(clock.attestation_time(last_fulu), last_fulu), + Duration::from_millis(3_999) + ); + assert_eq!( + ms(clock.aggregation_time(last_fulu), last_fulu), + Duration::from_millis(8_000) + ); + assert_eq!( + ms(clock.attestation_time(first_gloas), first_gloas), + Duration::from_millis(3_000) + ); + assert_eq!( + ms(clock.aggregation_time(first_gloas), first_gloas), + Duration::from_millis(6_000) + ); + } + + #[test] + fn the_payload_attestation_deadline_is_three_quarters_in() { + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 0); + let clock = SlotClock::from_config(GENESIS, &config); + let slot = 5; + assert_eq!( + clock.until_payload_attestation(slot, clock.start_of(slot)), + Duration::from_millis(9_000) + ); + assert_eq!( + clock.until_payload_attestation(slot, clock.start_of(slot) + Duration::from_secs(10)), + Duration::ZERO + ); + } + #[test] fn the_wait_until_the_attester_duty_is_the_rest_of_the_offset() { let clock = clock(); From ba8a33667af2db141033a697d374113d8271c2ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:52:14 -0300 Subject: [PATCH 085/132] feat(validator): propose gloas blocks and reveal the self-built envelope From gloas the block commits to a bid and the payload travels in an envelope, so a proposal is now two publications. The client asks produceBlockV4 with include_payload so the envelope arrives with the block, decodes the body by Eth-Execution-Payload-Included, signs and publishes the bare block, then signs the envelope under the builder domain with the proposer's own key and publishes it inside the same budget. When the node returned no payload the envelope is fetched by slot and block root and published bare. The envelope is checked to be for this block and this self-build before it is signed, since the signature covers it as it stands. A bid naming a builder is left to that builder. A failed envelope does not fail the proposal, whose block is already out, and has its own failure counter because it is the failure that withholds the payload. The proposal guard is untouched: envelopes are not slashable. The same change widens BeaconNodeApi (and its http, mock and fallback implementations) with the gloas aggregate, payload timeliness committee and envelope calls, so that the services built on top of them in the following commits do not each reopen the trait. --- crates/validator/src/aggregation.rs | 58 ++- crates/validator/src/attestation.rs | 11 +- .../src/beacon_node/block_contents.rs | 385 +++++++++++++++- crates/validator/src/beacon_node/dto.rs | 12 + crates/validator/src/beacon_node/fallback.rs | 74 ++- crates/validator/src/beacon_node/http.rs | 434 +++++++++++++++--- crates/validator/src/beacon_node/mock.rs | 300 +++++++++++- crates/validator/src/beacon_node/mod.rs | 146 +++++- crates/validator/src/metrics.rs | 56 +++ crates/validator/src/proposal.rs | 369 ++++++++++++++- crates/validator/src/signing.rs | 61 ++- 11 files changed, 1776 insertions(+), 130 deletions(-) diff --git a/crates/validator/src/aggregation.rs b/crates/validator/src/aggregation.rs index 493ae187e..eca92222d 100644 --- a/crates/validator/src/aggregation.rs +++ b/crates/validator/src/aggregation.rs @@ -35,7 +35,8 @@ use std::collections::BTreeMap; use std::sync::Arc; -use ethlambda_types::beacon::containers::electra::{AggregateAndProof, SignedAggregateAndProof}; +use ethlambda_types::beacon::containers::electra; +use ethlambda_types::beacon::containers::gloas; use ethlambda_types::beacon::containers::shared::AttestationData; use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::primitives::{HashTreeRoot as _, Slot}; @@ -43,12 +44,19 @@ use tokio::sync::RwLock; use tracing::{info, warn}; use crate::aggregation_selection::selection_for; -use crate::beacon_node::BeaconNodeApi; use crate::beacon_node::dto::{AttesterDutyDto, parse_pubkey}; +use crate::beacon_node::{AggregateKind, BeaconNodeApi, SignedAggregates}; use crate::error::{Error, Result}; use crate::keys::ValidatorStore; use crate::signing::SigningContext; +/// An `AggregateAndProof` of either fork, held between building it and +/// signing it. +enum Wrapper { + Electra(electra::AggregateAndProof), + Gloas(gloas::AggregateAndProof), +} + pub struct AggregationService { beacon_node: Arc, context: Arc, @@ -155,7 +163,8 @@ impl AggregationService { let signed = { let store = store.read().await; - let mut signed = Vec::with_capacity(selected.len()); + let mut electra_signed = Vec::new(); + let mut gloas_signed = Vec::new(); for (duty, selection_proof) in &selected { let Some(aggregate) = aggregates.get(&duty.committee_index) else { continue; @@ -168,15 +177,31 @@ impl AggregationService { } }; - let message = AggregateAndProof { - aggregator_index: duty.validator_index, - aggregate: aggregate.attestation.clone(), - selection_proof: *selection_proof, + // The wrapper is the fork's own container, and its root is taken + // from it: gloas's `Attestation` hashes differently from + // electra's, so a root computed over the wrong one would sign a + // message no node can verify. + let (root, wrapper) = match &aggregate.attestation { + AggregateKind::Electra(attestation) => { + let message = electra::AggregateAndProof { + aggregator_index: duty.validator_index, + aggregate: attestation.clone(), + selection_proof: *selection_proof, + }; + (message.hash_tree_root(), Wrapper::Electra(message)) + } + AggregateKind::Gloas(attestation) => { + let message = gloas::AggregateAndProof { + aggregator_index: duty.validator_index, + aggregate: attestation.clone(), + selection_proof: *selection_proof, + }; + (message.hash_tree_root(), Wrapper::Gloas(message)) + } }; // Signed over the whole wrapper, not over the aggregate: that // is what binds this validator's index and its selection proof // to the votes it is republishing. - let root = message.hash_tree_root(); let signature = match self .context .sign_aggregate_and_proof(&store, &pubkey, root, slot) @@ -187,9 +212,22 @@ impl AggregationService { continue; } }; - signed.push(SignedAggregateAndProof { message, signature }); + match wrapper { + Wrapper::Electra(message) => { + electra_signed.push(electra::SignedAggregateAndProof { message, signature }) + } + Wrapper::Gloas(message) => { + gloas_signed.push(gloas::SignedAggregateAndProof { message, signature }) + } + } + } + // Every aggregate in a batch is one fork (see `batch_fork`), so only + // one of the two lists is ever filled. + if fork >= ForkName::Gloas { + SignedAggregates::Gloas(gloas_signed) + } else { + SignedAggregates::Electra(electra_signed) } - signed }; if signed.is_empty() { diff --git a/crates/validator/src/attestation.rs b/crates/validator/src/attestation.rs index c274f2bfb..a9080561d 100644 --- a/crates/validator/src/attestation.rs +++ b/crates/validator/src/attestation.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use ethlambda_types::beacon::containers::shared::AttestationData; use ethlambda_types::beacon::primitives::Slot; +use ethlambda_types::beacon::signing::compute_epoch_at_slot; use tokio::sync::RwLock; use tracing::{error, info, warn}; @@ -105,7 +106,15 @@ impl AttestationService { }); } - let data = self.beacon_node.attestation_data(slot).await?; + // The fork of the slot, by this client's own schedule, tells the node + // which form of the query to answer: from gloas the `committee_index` + // is omitted and the answer's `index` is the payload signal. It is + // signed as received, never reset to zero. + let slot_fork = self + .context + .config + .fork_at_epoch(compute_epoch_at_slot(slot)); + let data = self.beacon_node.attestation_data(slot, slot_fork).await?; // Checked again here, having already been checked by the // implementation this call went through (see the contract on diff --git a/crates/validator/src/beacon_node/block_contents.rs b/crates/validator/src/beacon_node/block_contents.rs index 7dcb40197..c4d2806ca 100644 --- a/crates/validator/src/beacon_node/block_contents.rs +++ b/crates/validator/src/beacon_node/block_contents.rs @@ -42,7 +42,18 @@ //! | Fork | Produced | Published | //! |---|---|---| //! | Deneb, Electra, Fulu | `BlockContents` | `SignedBlockContents` | -//! | Gloas and later | bare block; blobs move to the payload envelope | bare signed block | +//! | Gloas and later | `GloasBlockContents` (block and self-built envelope) or a bare block | bare signed block, then the envelope separately | +//! +//! # Gloas +//! +//! The block no longer carries the payload (EIP-7732): it commits to a bid, and +//! the payload travels in an `ExecutionPayloadEnvelope` the builder reveals +//! afterwards. With `include_payload=true` the node hands back the block and the +//! envelope for a self-built payload together as [`GloasBlockContents`], and +//! says so in `Eth-Execution-Payload-Included`; without it the body is a bare +//! block and the envelope is fetched separately. Publication is split the same +//! way: the bare signed block first, then the signed envelope, with the blobs +//! and cell proofs moving into the envelope's contents. //! //! Deneb shares that envelope but not the block inside it: its //! `BeaconBlockBody` has twelve fields where electra's has thirteen, so the two @@ -76,6 +87,7 @@ use ethlambda_types::beacon::containers::deneb::Blob; use ethlambda_types::beacon::containers::electra::{BeaconBlock, SignedBeaconBlock}; +use ethlambda_types::beacon::containers::gloas; use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::preset; use ethlambda_types::beacon::primitives::{BlsSignature, KzgProof}; @@ -142,6 +154,64 @@ pub struct FuluSignedBlockContents { pub blobs: Blobs, } +/// `produceBlockV4`'s body for gloas when the node includes the payload: the +/// block, the envelope revealing the self-built payload it commits to, and the +/// blob material that goes out with the envelope. Not a consensus container; see +/// the module doc. +#[derive(Debug, Clone, PartialEq, SszEncode, SszDecode)] +pub struct GloasBlockContents { + pub block: gloas::BeaconBlock, + pub execution_payload_envelope: gloas::ExecutionPayloadEnvelope, + pub kzg_proofs: CellKzgProofs, + pub blobs: Blobs, +} + +/// `publishExecutionPayloadEnvelope`'s body for gloas when the client supplies +/// the blobs (`Eth-Blob-Data-Included: true`). +#[derive(Debug, Clone, PartialEq, SszEncode, SszDecode)] +pub struct SignedExecutionPayloadEnvelopeContents { + pub signed_execution_payload_envelope: gloas::SignedExecutionPayloadEnvelope, + pub kzg_proofs: CellKzgProofs, + pub blobs: Blobs, +} + +/// The envelope and blob material that came with a gloas block, kept apart from +/// the block because it is published separately and after it. +#[derive(Debug, Clone, PartialEq)] +pub struct GloasPayload { + pub envelope: gloas::ExecutionPayloadEnvelope, + pub kzg_proofs: CellKzgProofs, + pub blobs: Blobs, +} + +impl GloasPayload { + /// Attach `signature` and encode the body for `Eth-Blob-Data-Included: true`. + pub fn into_signed_contents_ssz(self, signature: BlsSignature) -> Vec { + SignedExecutionPayloadEnvelopeContents { + signed_execution_payload_envelope: gloas::SignedExecutionPayloadEnvelope { + message: self.envelope, + signature, + }, + kzg_proofs: self.kzg_proofs, + blobs: self.blobs, + } + .to_ssz() + } +} + +/// Encode a bare signed envelope, the body for `Eth-Blob-Data-Included: false`, +/// where the node attaches the blobs it cached at production time. +pub fn signed_envelope_ssz( + envelope: gloas::ExecutionPayloadEnvelope, + signature: BlsSignature, +) -> Vec { + gloas::SignedExecutionPayloadEnvelope { + message: envelope, + signature, + } + .to_ssz() +} + /// A block a beacon node produced, with whatever travelled alongside it. /// /// The proofs and blobs are kept in the shape they arrived in rather than @@ -180,6 +250,12 @@ pub enum Contents { kzg_proofs: CellKzgProofs, blobs: Blobs, }, + /// Gloas: a bare block, and the self-built payload's envelope when the + /// node included it. + Gloas { + block: gloas::BeaconBlock, + payload: Option>, + }, } impl ProducedBlock { @@ -230,16 +306,18 @@ impl ProducedBlock { }, }) } - // ePBS (EIP-7732) replaces the embedded payload this client - // decodes here with a builder's bid, a shape neither - // `Contents` variant models yet, so this is refused the same - // way a pre-electra fork is rather than mis-decoded. + // Gloas needs one more fact than the fork: whether the node + // included the payload, which only the response header says. See + // `Self::from_gloas_ssz`. + ForkName::Gloas => Err(Error::InconsistentResponse( + "a gloas block cannot be decoded without Eth-Execution-Payload-Included" + .to_string(), + )), ForkName::Phase0 | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb - | ForkName::Gloas | ForkName::Lean => Err(Error::InconsistentResponse(format!( "beacon node produced a block for fork {}, which this client does not propose \ under; electra is the earliest supported", @@ -248,21 +326,121 @@ impl ProducedBlock { } } - /// The block itself, whichever wrapper it came in. - pub fn block(&self) -> &BeaconBlock { + /// Decode a gloas `produceBlockV4` body. `payload_included` is the node's + /// `Eth-Execution-Payload-Included`, which selects between + /// [`GloasBlockContents`] and a bare block: SSZ carries no tag, so a wrong + /// guess would decode garbage or fail in a way that names neither. + pub fn from_gloas_ssz(payload_included: bool, bytes: &[u8]) -> crate::error::Result { + let decode = |what: &str, err: libssz::DecodeError| { + Error::Decode(format!( + "produced gloas block: {what} did not decode: {err:?}" + )) + }; + let (block, payload) = if payload_included { + let contents = GloasBlockContents::from_ssz_bytes(bytes) + .map_err(|err| decode("block contents", err))?; + ( + contents.block, + Some(Box::new(GloasPayload { + envelope: contents.execution_payload_envelope, + kzg_proofs: contents.kzg_proofs, + blobs: contents.blobs, + })), + ) + } else { + let block = gloas::BeaconBlock::from_ssz_bytes(bytes) + .map_err(|err| decode("bare block", err))?; + (block, None) + }; + Ok(Self { + fork: ForkName::Gloas, + contents: Contents::Gloas { block, payload }, + }) + } + + /// The block's slot, whichever fork's block it is. + pub fn slot(&self) -> ethlambda_types::beacon::primitives::Slot { + match &self.contents { + Contents::WithBlobProofs { block, .. } | Contents::WithCellProofs { block, .. } => { + block.slot + } + Contents::Gloas { block, .. } => block.slot, + } + } + + /// The proposer the block names. + pub fn proposer_index(&self) -> ethlambda_types::beacon::primitives::ValidatorIndex { + match &self.contents { + Contents::WithBlobProofs { block, .. } | Contents::WithCellProofs { block, .. } => { + block.proposer_index + } + Contents::Gloas { block, .. } => block.proposer_index, + } + } + + /// The block's `hash_tree_root`, taken from the decoded container of + /// whichever fork it is, which is what gets signed. + pub fn block_root(&self) -> ethlambda_types::beacon::primitives::Root { + use ethlambda_types::beacon::primitives::HashTreeRoot as _; + match &self.contents { + Contents::WithBlobProofs { block, .. } | Contents::WithCellProofs { block, .. } => { + block.hash_tree_root() + } + Contents::Gloas { block, .. } => block.hash_tree_root(), + } + } + + /// Where the block pays its execution-layer rewards: the payload's fee + /// recipient before gloas, the bid's from it on. + pub fn fee_recipient(&self) -> ethlambda_types::beacon::primitives::ExecutionAddress { match &self.contents { Contents::WithBlobProofs { block, .. } | Contents::WithCellProofs { block, .. } => { + block.body.execution_payload.fee_recipient + } + Contents::Gloas { block, .. } => { block + .body + .signed_execution_payload_bid + .message + .fee_recipient } } } + /// The builder the gloas bid names, `None` before gloas. The self-build + /// sentinel means this client's own proposer key signs the envelope. + pub fn builder_index(&self) -> Option { + match &self.contents { + Contents::Gloas { block, .. } => Some( + block + .body + .signed_execution_payload_bid + .message + .builder_index, + ), + _ => None, + } + } + + /// Take the envelope and blob material out of a gloas block, leaving the + /// block to be signed and published alone. `None` for every earlier fork + /// and for a gloas block the node returned without its payload. + pub fn take_gloas_payload(&mut self) -> Option { + match &mut self.contents { + Contents::Gloas { payload, .. } => payload.take().map(|payload| *payload), + _ => None, + } + } + /// How many blobs came with the block. For logging. pub fn blob_count(&self) -> usize { match &self.contents { Contents::WithBlobProofs { blobs, .. } | Contents::WithCellProofs { blobs, .. } => { blobs.len() } + Contents::Gloas { payload, .. } => { + payload.as_ref().map_or(0, |payload| payload.blobs.len()) + } } } @@ -299,6 +477,14 @@ impl ProducedBlock { blobs, } .to_ssz(), + // Bare: the envelope and blobs are published on their own, after + // this body, and are expected to have been taken out already by + // `take_gloas_payload`. + Contents::Gloas { block, .. } => gloas::SignedBeaconBlock { + message: block, + signature, + } + .to_ssz(), } } } @@ -372,6 +558,66 @@ pub fn empty_block_for( } } +/// The smallest gloas block for `slot` and `proposer_index`, whose bid names +/// `builder_index`. +#[cfg(test)] +pub fn empty_gloas_block_for( + slot: ethlambda_types::beacon::primitives::Slot, + proposer_index: ethlambda_types::beacon::primitives::ValidatorIndex, + builder_index: gloas::BuilderIndex, +) -> gloas::BeaconBlock { + let mut block = gloas::BeaconBlock { + slot, + proposer_index, + ..Default::default() + }; + block + .body + .signed_execution_payload_bid + .message + .builder_index = builder_index; + block +} + +/// A gloas envelope for the block whose root is `beacon_block_root`, with an +/// otherwise zeroed payload. +#[cfg(test)] +pub fn empty_gloas_envelope_for( + beacon_block_root: ethlambda_types::beacon::primitives::Root, + builder_index: gloas::BuilderIndex, +) -> gloas::ExecutionPayloadEnvelope { + let payload = gloas::ExecutionPayload { + parent_hash: Default::default(), + fee_recipient: Default::default(), + state_root: Default::default(), + receipts_root: Default::default(), + logs_bloom: vec![0u8; preset::BYTES_PER_LOGS_BLOOM] + .try_into() + .expect("a logs bloom is BYTES_PER_LOGS_BLOOM long by construction"), + prev_randao: Default::default(), + block_number: 0, + gas_limit: 0, + gas_used: 0, + timestamp: 0, + extra_data: Default::default(), + base_fee_per_gas: Default::default(), + block_hash: Default::default(), + transactions: Default::default(), + withdrawals: Default::default(), + blob_gas_used: 0, + excess_blob_gas: 0, + block_access_list: Default::default(), + slot_number: 0, + }; + gloas::ExecutionPayloadEnvelope { + payload, + execution_requests: Default::default(), + builder_index, + beacon_block_root, + parent_beacon_block_root: Default::default(), + } +} + /// A zeroed blob. `Blob` deliberately has no `Default`, because zeroing 128 KiB /// by accident is exactly the mistake that derive would invite. #[cfg(test)] @@ -410,8 +656,8 @@ mod tests { let bytes = contents_bytes(2, 2); let produced = ProducedBlock::from_ssz(ForkName::Electra, &bytes).expect("decodes"); - assert_eq!(produced.block().slot, 1234); - assert_eq!(produced.block().proposer_index, 7); + assert_eq!(produced.slot(), 1234); + assert_eq!(produced.proposer_index(), 7); assert_eq!(produced.blob_count(), 2); assert_eq!(produced.fork, ForkName::Electra); assert!(matches!(produced.contents, Contents::WithBlobProofs { .. })); @@ -427,7 +673,7 @@ mod tests { let bytes = contents_bytes(1, 1); let produced = ProducedBlock::from_ssz(ForkName::Electra, &bytes).expect("decodes"); - assert_eq!(produced.block().hash_tree_root(), expected); + assert_eq!(produced.block_root(), expected); let signed = produced.into_signed_ssz(BlsSignature([9; 96])); let decoded = SignedBlockContents::from_ssz_bytes(&signed).expect("decodes"); @@ -536,4 +782,121 @@ mod tests { .expect_err("must fail"); assert!(matches!(err, Error::Decode(_)), "got {err:?}"); } + + fn gloas_contents_bytes( + builder_index: u64, + proofs: usize, + blobs: usize, + ) -> (Vec, ethlambda_types::beacon::primitives::Root) { + let block = empty_gloas_block_for(77, 5, builder_index); + let root = block.hash_tree_root(); + let bytes = GloasBlockContents { + block, + execution_payload_envelope: empty_gloas_envelope_for(root, builder_index), + kzg_proofs: vec![KzgProof([3; 48]); proofs] + .try_into() + .expect("in bounds"), + blobs: vec![blob(); blobs].try_into().expect("in bounds"), + } + .to_ssz(); + (bytes, root) + } + + /// With the payload included the body is the contents container, and the + /// envelope, proofs and blobs come out in one piece for the second + /// publication. + #[test] + fn a_gloas_block_with_its_payload_decodes_and_hands_the_payload_over() { + let (bytes, root) = gloas_contents_bytes(u64::MAX, 3, 2); + let mut produced = ProducedBlock::from_gloas_ssz(true, &bytes).expect("decodes"); + + assert_eq!(produced.fork, ForkName::Gloas); + assert_eq!(produced.slot(), 77); + assert_eq!(produced.proposer_index(), 5); + assert_eq!(produced.block_root(), root); + assert_eq!(produced.blob_count(), 2); + assert_eq!(produced.builder_index(), Some(u64::MAX)); + + let payload = produced + .take_gloas_payload() + .expect("the payload came along"); + assert_eq!(payload.envelope.beacon_block_root, root); + assert_eq!(payload.kzg_proofs.len(), 3); + assert_eq!(payload.blobs.len(), 2); + assert!(produced.take_gloas_payload().is_none(), "taken once"); + + // The block published afterwards is bare, and its root is untouched. + let signed = produced.into_signed_ssz(BlsSignature([4; 96])); + let decoded = + gloas::SignedBeaconBlock::from_ssz_bytes(&signed).expect("a bare gloas block"); + assert_eq!(decoded.message.hash_tree_root(), root); + assert_eq!(decoded.signature, BlsSignature([4; 96])); + } + + #[test] + fn a_bare_gloas_block_has_no_payload_to_hand_over() { + let block = empty_gloas_block_for(78, 6, 1); + let mut produced = ProducedBlock::from_gloas_ssz(false, &block.to_ssz()).expect("decodes"); + assert_eq!(produced.slot(), 78); + assert_eq!(produced.blob_count(), 0); + assert!(produced.take_gloas_payload().is_none()); + } + + /// The header selects the container, so a wrong value is a decode error and + /// not a block that silently differs. + #[test] + fn the_included_header_selects_the_container() { + let (bytes, _) = gloas_contents_bytes(u64::MAX, 0, 0); + let err = + ProducedBlock::from_gloas_ssz(false, &bytes).expect_err("contents are not a block"); + assert!(matches!(err, Error::Decode(_)), "got {err:?}"); + } + + #[test] + fn a_gloas_body_cannot_be_decoded_without_the_included_header() { + let (bytes, _) = gloas_contents_bytes(u64::MAX, 0, 0); + let err = ProducedBlock::from_ssz(ForkName::Gloas, &bytes).expect_err("must refuse"); + assert!(matches!(err, Error::InconsistentResponse(_)), "got {err:?}"); + } + + /// From gloas the fee recipient lives in the bid, not in a payload the + /// block no longer carries. + #[test] + fn a_gloas_blocks_fee_recipient_is_the_bids() { + let mut block = empty_gloas_block_for(1, 1, u64::MAX); + block + .body + .signed_execution_payload_bid + .message + .fee_recipient = ethlambda_types::beacon::primitives::H160([0xcd; 20]); + let produced = ProducedBlock::from_gloas_ssz(false, &block.to_ssz()).expect("decodes"); + assert_eq!( + produced.fee_recipient(), + ethlambda_types::beacon::primitives::H160([0xcd; 20]) + ); + } + + /// An envelope contents body is the signed envelope, then proofs, then + /// blobs, and decodes back to what was put in. + #[test] + fn the_envelope_contents_round_trip() { + let (bytes, root) = gloas_contents_bytes(u64::MAX, 2, 1); + let mut produced = ProducedBlock::from_gloas_ssz(true, &bytes).expect("decodes"); + let payload = produced.take_gloas_payload().expect("payload"); + let signed = payload.into_signed_contents_ssz(BlsSignature([8; 96])); + let decoded = SignedExecutionPayloadEnvelopeContents::from_ssz_bytes(&signed).expect("ok"); + assert_eq!( + decoded + .signed_execution_payload_envelope + .message + .beacon_block_root, + root + ); + assert_eq!( + decoded.signed_execution_payload_envelope.signature, + BlsSignature([8; 96]) + ); + assert_eq!(decoded.kzg_proofs.len(), 2); + assert_eq!(decoded.blobs.len(), 1); + } } diff --git a/crates/validator/src/beacon_node/dto.rs b/crates/validator/src/beacon_node/dto.rs index 3ec6ba076..22dd42ddb 100644 --- a/crates/validator/src/beacon_node/dto.rs +++ b/crates/validator/src/beacon_node/dto.rs @@ -205,6 +205,18 @@ pub struct ProposerDutyDto { pub slot: Slot, } +/// One entry of `POST /eth/v1/validator/duties/ptc/{epoch}`: the same three +/// fields a proposer duty has, since a committee member too is named by key, +/// index and the one slot it votes in. +#[derive(Debug, Clone, Deserialize)] +pub struct PtcDutyDto { + pub pubkey: String, + #[serde(with = "quoted_u64")] + pub validator_index: ValidatorIndex, + #[serde(with = "quoted_u64")] + pub slot: Slot, +} + #[derive(Debug, Clone, Deserialize)] pub struct ValidatorEntryDto { #[serde(with = "quoted_u64")] diff --git a/crates/validator/src/beacon_node/fallback.rs b/crates/validator/src/beacon_node/fallback.rs index c1d3eb352..c5eb5c8d9 100644 --- a/crates/validator/src/beacon_node/fallback.rs +++ b/crates/validator/src/beacon_node/fallback.rs @@ -9,7 +9,7 @@ use std::pin::Pin; use async_trait::async_trait; use ethlambda_types::beacon::config::Config; -use ethlambda_types::beacon::containers::electra::SignedAggregateAndProof; +use ethlambda_types::beacon::containers::gloas; use ethlambda_types::beacon::containers::shared::AttestationData; use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::primitives::{BlsPubkey, Epoch, Root, Slot, ValidatorIndex}; @@ -21,7 +21,7 @@ use crate::beacon_node::dto::{ }; use crate::beacon_node::{ AggregateAttestation, AttesterDuties, BeaconNodeApi, BlockRequest, Genesis, ProposerDuties, - Published, ValidatorEntry, + PtcDuties, Published, SignedAggregates, ValidatorEntry, }; use crate::error::{Error, Result}; @@ -223,8 +223,8 @@ impl BeaconNodeApi for FallbackBeaconNode { .await } - async fn attestation_data(&self, slot: Slot) -> Result { - self.try_each("attestation_data", |node| node.attestation_data(slot)) + async fn attestation_data(&self, slot: Slot, fork: ForkName) -> Result { + self.try_each("attestation_data", |node| node.attestation_data(slot, fork)) .await } @@ -251,6 +251,61 @@ impl BeaconNodeApi for FallbackBeaconNode { .await } + /// Fails over like any query: the envelope the node cached is only on the + /// node that produced the block, and one that did not answers 404. + async fn execution_payload_envelope( + &self, + slot: Slot, + block_root: Root, + ) -> Result { + self.try_each("execution_payload_envelope", |node| { + node.execution_payload_envelope(slot, block_root) + }) + .await + } + + /// First node that accepts it wins, for the reason [`Self::publish_block`] + /// is: the accepting node gossips it. + async fn publish_execution_payload_envelope( + &self, + body: &[u8], + blob_data_included: bool, + ) -> Result<()> { + self.try_each("publish_execution_payload_envelope", |node| { + node.publish_execution_payload_envelope(body, blob_data_included) + }) + .await + } + + async fn ptc_duties(&self, epoch: Epoch, indices: &[ValidatorIndex]) -> Result { + self.try_each("ptc_duties", |node| node.ptc_duties(epoch, indices)) + .await + } + + /// A 204 is an answer, so the first node that knows no block ends the + /// walk, even though another might have one. That is the right trade: PTC + /// votes are due at three quarters of the slot, and a node that has seen + /// nothing by then is one this client should not be voting through. + async fn payload_attestation_data( + &self, + slot: Slot, + ) -> Result> { + self.try_each("payload_attestation_data", |node| { + node.payload_attestation_data(slot) + }) + .await + } + + async fn submit_payload_attestations( + &self, + messages: &[gloas::PayloadAttestationMessage], + ) -> Result { + self.try_each("submit_payload_attestations", |node| { + node.submit_payload_attestations(messages) + }) + .await + } + async fn submit_attestations( &self, attestations: &[SingleAttestationDto], @@ -280,7 +335,7 @@ impl BeaconNodeApi for FallbackBeaconNode { async fn publish_aggregates( &self, fork: ForkName, - aggregates: &[SignedAggregateAndProof], + aggregates: &SignedAggregates, ) -> Result<()> { self.try_each("publish_aggregates", |node| { node.publish_aggregates(fork, aggregates) @@ -354,7 +409,7 @@ mod tests { let fallback = FallbackBeaconNode::new(vec![stale, fresh]); let data = fallback - .attestation_data(96) + .attestation_data(96, ForkName::Electra) .await .expect("the second node answers correctly"); @@ -374,7 +429,7 @@ mod tests { ]); let err = fallback - .attestation_data(96) + .attestation_data(96, ForkName::Electra) .await .expect_err("no node answered about the requested slot"); @@ -477,6 +532,7 @@ mod tests { use ethlambda_types::beacon::primitives::{BlsSignature, Bytes32}; BlockRequest { slot, + fork: ForkName::Electra, proposer_index, randao_reveal: BlsSignature([0; 96]), graffiti: Bytes32::default(), @@ -497,7 +553,7 @@ mod tests { .produce_block(&request(96, 7)) .await .expect("the second node answers correctly"); - assert_eq!(block.block().slot, 96); + assert_eq!(block.slot(), 96); } /// A node on a different fork computes a different proposer. Signing its @@ -513,7 +569,7 @@ mod tests { .produce_block(&request(96, 7)) .await .expect("the second node names the expected proposer"); - assert_eq!(block.block().proposer_index, 7); + assert_eq!(block.proposer_index(), 7); } #[tokio::test] diff --git a/crates/validator/src/beacon_node/http.rs b/crates/validator/src/beacon_node/http.rs index dee38b01d..71a845fbc 100644 --- a/crates/validator/src/beacon_node/http.rs +++ b/crates/validator/src/beacon_node/http.rs @@ -24,7 +24,8 @@ use std::time::Duration; use async_trait::async_trait; use ethlambda_types::beacon::config::Config; -use ethlambda_types::beacon::containers::electra::{Attestation, SignedAggregateAndProof}; +use ethlambda_types::beacon::containers::electra::Attestation; +use ethlambda_types::beacon::containers::gloas; use ethlambda_types::beacon::containers::shared::AttestationData; use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::primitives::{BlsPubkey, Epoch, Root, Slot, ValidatorIndex}; @@ -32,16 +33,19 @@ use reqwest::{Client, StatusCode}; use serde::Serialize; use tracing::{debug, warn}; +use libssz::SszDecode as _; + use crate::beacon_node::block_contents::ProducedBlock; use crate::beacon_node::dto::{ AttestationDataDto, AttestationDto, AttesterDutyDto, CommitteeSubscriptionDto, DataResponse, DutiesResponse, GenesisDto, IndexedErrorResponse, ProposerDutyDto, ProposerPreparationDto, - SignedAggregateAndProofOutDto, SingleAttestationDto, SyncingDto, ValidatorEntryDto, + PtcDutyDto, SignedAggregateAndProofOutDto, SingleAttestationDto, SyncingDto, ValidatorEntryDto, VersionedResponse, config_from_spec_response, encode_hex, parse_pubkey, parse_root, }; use crate::beacon_node::{ - AggregateAttestation, AttesterDuties, BeaconNodeApi, BlockRequest, Genesis, ProposerDuties, - Published, ValidatorEntry, validate_attestation_data, validate_produced_block, + AggregateAttestation, AggregateKind, AttesterDuties, BeaconNodeApi, BlockRequest, Genesis, + ProposerDuties, PtcDuties, Published, SignedAggregates, ValidatorEntry, + validate_attestation_data, validate_produced_block, }; use crate::error::{BeaconNodeFailure, Error, Result}; @@ -54,6 +58,23 @@ use crate::error::{BeaconNodeFailure, Error, Result}; /// a stalled request could then outlive the slot it was serving. const REQUEST_TIMEOUT: Duration = Duration::from_secs(8); +/// What an SSZ answer carries besides its bytes: the fork, which only a header +/// can say, and the two booleans the block endpoints use to say what the bytes +/// are. +struct SszResponse { + fork: ForkName, + /// `Eth-Execution-Payload-Blinded`, sent by `produceBlockV3` only. + blinded: Option, + /// `Eth-Execution-Payload-Included`, sent by `produceBlockV4` only. + payload_included: Option, + body: Vec, +} + +/// The body `produceBlockV4` requires. Self-build only, so no minimum bid, no +/// boost and no builders: the node decodes it, as the specification demands, +/// and builds locally. +const BUILDER_CONFIG: &str = r#"{"min_bid":"0","builder_boost_factor":"0","builders":[]}"#; + /// A [`BeaconNodeApi`] backed by one beacon node's standard REST Beacon API. pub struct HttpBeaconNode { base_url: String, @@ -170,24 +191,43 @@ impl HttpBeaconNode { /// root that produces. The comparison is case-insensitive: the schema's /// enum is lowercase but nothing in the specification says a client must /// match it that way. - /// - /// Returns the blinded flag as an `Option`, `None` when the header is - /// absent, because only one endpoint sends it and only that endpoint's - /// caller knows whether its absence is an error. - async fn get_ssz(&self, path: &str) -> Result<(ForkName, Option, Vec)> { + async fn get_ssz(&self, path: &str) -> Result { let url = format!("{}{path}", self.base_url); debug!(%url, "Beacon API GET (ssz)"); - let response = self + let request = self .client .get(&url) + .header(reqwest::header::ACCEPT, "application/octet-stream"); + Self::read_ssz(url, request).await + } + + /// POST a JSON body and read the answer as SSZ, the way [`Self::get_ssz`] + /// does. For `produceBlockV4`, which is a POST because it carries the + /// builder configuration, and answers in SSZ like its predecessor. + async fn post_json_for_ssz( + &self, + path: &str, + json_body: &'static str, + consensus_version: &str, + ) -> Result { + let url = format!("{}{path}", self.base_url); + debug!(%url, "Beacon API POST (json in, ssz out)"); + let request = self + .client + .post(&url) .header(reqwest::header::ACCEPT, "application/octet-stream") - .send() - .await - .map_err(|err| Error::BeaconNode { - url: url.clone(), - failure: BeaconNodeFailure::classify(&err), - detail: err.to_string(), - })?; + .header(reqwest::header::CONTENT_TYPE, "application/json") + .header("Eth-Consensus-Version", consensus_version) + .body(json_body); + Self::read_ssz(url, request).await + } + + async fn read_ssz(url: String, request: reqwest::RequestBuilder) -> Result { + let response = request.send().await.map_err(|err| Error::BeaconNode { + url: url.clone(), + failure: BeaconNodeFailure::classify(&err), + detail: err.to_string(), + })?; let status = response.status(); if status == StatusCode::SERVICE_UNAVAILABLE { @@ -233,26 +273,111 @@ impl HttpBeaconNode { )) })?; - // Reported, not judged. Only `produceBlockV3` sends this header, so - // requiring it here would break every other caller of this helper; - // whether its absence matters is the caller's question. - let blinded = match header("eth-execution-payload-blinded") { - Some(value) if value.eq_ignore_ascii_case("true") => Some(true), - Some(value) if value.eq_ignore_ascii_case("false") => Some(false), - Some(value) => { - return Err(Error::InconsistentResponse(format!( - "Eth-Execution-Payload-Blinded is {value}, expected true or false" - ))); - } - None => None, - }; + // Reported, not judged. Each is sent by one endpoint only, so requiring + // it here would break every other caller of this helper; whether its + // absence matters is the caller's question. + let blinded = Self::bool_header(header("eth-execution-payload-blinded"), "Blinded")?; + let payload_included = + Self::bool_header(header("eth-execution-payload-included"), "Included")?; let body = response.bytes().await.map_err(|err| Error::BeaconNode { url, failure: BeaconNodeFailure::classify(&err), detail: err.to_string(), })?; - Ok((fork, blinded, body.to_vec())) + Ok(SszResponse { + fork, + blinded, + payload_included, + body: body.to_vec(), + }) + } + + /// A `true`/`false` header, `None` when absent, an error when it is + /// anything else. + fn bool_header(value: Option, name: &str) -> Result> { + match value { + Some(value) if value.eq_ignore_ascii_case("true") => Ok(Some(true)), + Some(value) if value.eq_ignore_ascii_case("false") => Ok(Some(false)), + Some(value) => Err(Error::InconsistentResponse(format!( + "Eth-Execution-Payload-{name} is {value}, expected true or false" + ))), + None => Ok(None), + } + } + + /// A GET whose success may be a 204, which is an answer ("nothing to + /// report") and not a failure. + async fn get_optional(&self, path: &str) -> Result> { + let url = format!("{}{path}", self.base_url); + debug!(%url, "Beacon API GET"); + let response = self + .client + .get(&url) + .send() + .await + .map_err(|err| Error::BeaconNode { + url: url.clone(), + failure: BeaconNodeFailure::classify(&err), + detail: err.to_string(), + })?; + if response.status() == StatusCode::NO_CONTENT { + return Ok(None); + } + Self::decode(response).await.map(Some) + } + + /// `produceBlockV4`: a POST carrying the builder configuration, asking for + /// the self-built payload to be included. + /// + /// `include_payload=true` saves the proposal a round trip, which matters + /// because the whole proposal, envelope included, has to fit before the + /// attesters' deadline. The node says whether it complied in + /// `Eth-Execution-Payload-Included`, and a missing header is an error for + /// the reason the blinded flag's absence is on the v3 path: it selects which + /// container the bytes are. + async fn produce_gloas_block(&self, request: &BlockRequest) -> Result { + let path = format!( + "/eth/v4/validator/blocks/{}?randao_reveal={}&graffiti={}&include_payload=true", + request.slot, + encode_hex(&request.randao_reveal.0), + encode_hex(&request.graffiti.0), + ); + let response = self + .post_json_for_ssz(&path, BUILDER_CONFIG, request.fork.as_str()) + .await?; + if response.fork != ForkName::Gloas { + return Err(Error::InconsistentResponse(format!( + "asked for a gloas block for slot {}, node answered with a {} one", + request.slot, + response.fork.as_str() + ))); + } + let included = response.payload_included.ok_or_else(|| { + Error::InconsistentResponse( + "response carries no Eth-Execution-Payload-Included, so whether the body is \ + block contents or a bare block is unknown" + .to_string(), + ) + })?; + let block = ProducedBlock::from_gloas_ssz(included, &response.body)?; + // Enforced here rather than at the call site so failover works: see the + // contract on `BeaconNodeApi::produce_block`. + validate_produced_block(request, &block)?; + Ok(block) + } + + /// The path for one slot's attestation data. + /// + /// Gloas omits `committee_index`: the parameter is optional there, and the + /// answer's `index` is the payload signal rather than a committee, so + /// asking about committee 0 would only suggest the old meaning. + fn attestation_data_path(slot: Slot, fork: ForkName) -> String { + if fork >= ForkName::Gloas { + format!("/eth/v1/validator/attestation_data?slot={slot}") + } else { + format!("/eth/v1/validator/attestation_data?slot={slot}&committee_index=0") + } } /// The query string for one aggregate. @@ -438,12 +563,9 @@ impl BeaconNodeApi for HttpBeaconNode { }) } - async fn attestation_data(&self, slot: Slot) -> Result { - let response: DataResponse = self - .get(&format!( - "/eth/v1/validator/attestation_data?slot={slot}&committee_index=0" - )) - .await?; + async fn attestation_data(&self, slot: Slot, fork: ForkName) -> Result { + let response: DataResponse = + self.get(&Self::attestation_data_path(slot, fork)).await?; let data = AttestationData::try_from(&response.data)?; // Enforced here rather than at the call site so that failover works: // see the contract on `BeaconNodeApi::attestation_data`. An `Err` here @@ -473,13 +595,21 @@ impl BeaconNodeApi for HttpBeaconNode { /// burn the slot's proposal guard entry for a block that can never be /// sent. async fn produce_block(&self, request: &BlockRequest) -> Result { + if request.fork >= ForkName::Gloas { + return self.produce_gloas_block(request).await; + } let path = format!( "/eth/v3/validator/blocks/{}?randao_reveal={}&graffiti={}&builder_boost_factor=0", request.slot, encode_hex(&request.randao_reveal.0), encode_hex(&request.graffiti.0), ); - let (fork, blinded, body) = self.get_ssz(&path).await?; + let SszResponse { + fork, + blinded, + body, + .. + } = self.get_ssz(&path).await?; // Required on this endpoint, so its absence is an error rather than a // default. The specification marks it required precisely because it // selects which container the bytes are, and a blinded body is a @@ -540,6 +670,136 @@ impl BeaconNodeApi for HttpBeaconNode { }) } + async fn execution_payload_envelope( + &self, + slot: Slot, + block_root: Root, + ) -> Result { + let path = format!( + "/eth/v1/validator/execution_payload_envelopes/{slot}/{}", + encode_hex(&block_root.0) + ); + let response = self.get_ssz(&path).await?; + if response.fork != ForkName::Gloas { + return Err(Error::InconsistentResponse(format!( + "node answered an envelope request with a {} body", + response.fork.as_str() + ))); + } + let envelope = gloas::ExecutionPayloadEnvelope::from_ssz_bytes(&response.body) + .map_err(|err| Error::Decode(format!("execution payload envelope: {err:?}")))?; + // The envelope is signed as it stands, so one for another block must + // never reach the signing path. Enforced here so failover can act on it. + if envelope.beacon_block_root != block_root { + return Err(Error::InconsistentResponse(format!( + "asked for the envelope of block {}, node answered with one for block {}", + encode_hex(&block_root.0), + encode_hex(&envelope.beacon_block_root.0) + ))); + } + Ok(envelope) + } + + async fn publish_execution_payload_envelope( + &self, + body: &[u8], + blob_data_included: bool, + ) -> Result<()> { + let url = format!( + "{}/eth/v1/beacon/execution_payload_envelopes", + self.base_url + ); + let response = self + .client + .post(&url) + .header("Eth-Consensus-Version", ForkName::Gloas.as_str()) + .header("Eth-Blob-Data-Included", blob_data_included.to_string()) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .body(body.to_vec()) + .send() + .await + .map_err(|err| Error::BeaconNode { + url: url.clone(), + failure: BeaconNodeFailure::classify(&err), + detail: err.to_string(), + })?; + let status = response.status(); + if status.is_success() { + return Ok(()); + } + let body = response.text().await.unwrap_or_default(); + Err(Error::BeaconNodeStatus { + status: status.as_u16(), + body, + }) + } + + async fn ptc_duties(&self, epoch: Epoch, indices: &[ValidatorIndex]) -> Result { + let body: Vec = indices.iter().map(|index| index.to_string()).collect(); + let response: DutiesResponse> = self + .post( + &format!("/eth/v1/validator/duties/ptc/{epoch}"), + &body, + None, + ) + .await?; + Ok(PtcDuties { + dependent_root: parse_root(&response.dependent_root)?, + duties: response.data, + }) + } + + async fn payload_attestation_data( + &self, + slot: Slot, + ) -> Result> { + let response: Option> = self + .get_optional(&format!( + "/eth/v1/validator/payload_attestation_data?slot={slot}" + )) + .await?; + let Some(response) = response else { + return Ok(None); + }; + // Enforced here for the reason the other fetches' checks are: this is + // signable material, and a node answering about another slot must be + // failed over from rather than signed for. + if response.data.slot != slot { + return Err(Error::InconsistentResponse(format!( + "requested payload attestation data for slot {slot}, node answered for slot {}", + response.data.slot + ))); + } + Ok(Some(response.data)) + } + + /// The same partial-success reading as attestations: see + /// [`Self::handle_pool_submission`]. + async fn submit_payload_attestations( + &self, + messages: &[gloas::PayloadAttestationMessage], + ) -> Result { + let url = format!("{}/eth/v1/beacon/pool/payload_attestations", self.base_url); + let response = self + .client + .post(&url) + .header("Eth-Consensus-Version", ForkName::Gloas.as_str()) + .json(messages) + .send() + .await + .map_err(|err| Error::BeaconNode { + url: url.clone(), + failure: BeaconNodeFailure::classify(&err), + detail: err.to_string(), + })?; + let status = response.status(); + if status.is_success() { + return Ok(messages.len()); + } + let body = response.text().await.unwrap_or_default(); + Self::handle_pool_submission(status, body, messages.len()) + } + /// Distinct from `post_no_content`: this endpoint's 400s are not always /// total failures. See [`Self::handle_pool_submission`]. async fn submit_attestations( @@ -583,7 +843,10 @@ impl BeaconNodeApi for HttpBeaconNode { // JSON, not SSZ, and not by preference: Lighthouse answers this endpoint // in JSON whatever `Accept` says. See `AttestationDto` for why that is // safe here when it would not be for a block. - let response: VersionedResponse = self.get(&path).await?; + // + // The body is read as untyped JSON first, because the layout of `data` + // depends on the `version` beside it. + let response: VersionedResponse = self.get(&path).await?; let fork = ForkName::parse(&response.version.to_ascii_lowercase()).ok_or_else(|| { Error::InconsistentResponse(format!( "aggregate names fork {}, which this client does not know", @@ -595,18 +858,14 @@ impl BeaconNodeApi for HttpBeaconNode { // Electra widened `Attestation` with `committee_bits`, so an earlier // fork's bytes are a different shape and this client has no container // for them. Named explicitly rather than `fork < ForkName::Electra`, - // so a fork added after fulu is not silently waved through as - // electra-shaped. - match fork { - // Gloas's own attestation shape is not this client's electra one - // (EIP-7549 continues to change under EIP-7688), so it is - // refused by name too rather than mis-decoded. + // so a fork added after gloas is not silently waved through as + // gloas-shaped. + let attestation = match fork { ForkName::Phase0 | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb - | ForkName::Gloas | ForkName::Lean => { return Err(Error::InconsistentResponse(format!( "node produced a {} aggregate, which this client does not publish; electra \ @@ -614,19 +873,29 @@ impl BeaconNodeApi for HttpBeaconNode { fork.as_str() ))); } - ForkName::Electra | ForkName::Fulu => {} - } - let attestation = Attestation::try_from(&response.data)?; + ForkName::Electra | ForkName::Fulu => { + let dto: AttestationDto = serde_json::from_value(response.data) + .map_err(|err| Error::Decode(format!("electra aggregate: {err}")))?; + AggregateKind::Electra(Attestation::try_from(&dto)?) + } + // Gloas's attestation is its own container, with its own hash tree + // root, so it is decoded as one and signed over as one. + ForkName::Gloas => { + let attestation: gloas::Attestation = serde_json::from_value(response.data) + .map_err(|err| Error::Decode(format!("gloas aggregate: {err}")))?; + AggregateKind::Gloas(attestation) + } + }; // The same contract the other two fetches carry, enforced here so a // node answering about the wrong slot is failed over from rather than // wrapped in a signature. `committee_index` is deliberately not checked // against `committee_bits`: which committees an aggregate covers is the // node's answer to the question, and electra's gossip rules already // require exactly one. - if attestation.data.slot != slot { + if attestation.data().slot != slot { return Err(Error::InconsistentResponse(format!( "requested an aggregate for slot {slot}, node answered for slot {}", - attestation.data.slot + attestation.data().slot ))); } Ok(AggregateAttestation { fork, attestation }) @@ -637,18 +906,23 @@ impl BeaconNodeApi for HttpBeaconNode { async fn publish_aggregates( &self, fork: ForkName, - aggregates: &[SignedAggregateAndProof], + aggregates: &SignedAggregates, ) -> Result<()> { - let body: Vec = aggregates - .iter() - .map(SignedAggregateAndProofOutDto::from) - .collect(); - self.post_no_content( - "/eth/v2/validator/aggregate_and_proofs", - &body, - Some(fork.as_str()), - ) - .await + const PATH: &str = "/eth/v2/validator/aggregate_and_proofs"; + match aggregates { + SignedAggregates::Electra(list) => { + let body: Vec = list + .iter() + .map(SignedAggregateAndProofOutDto::from) + .collect(); + self.post_no_content(PATH, &body, Some(fork.as_str())).await + } + // The gloas container serialises as the endpoint's JSON itself + // (hex bitfields, quoted integers), so there is no second DTO. + SignedAggregates::Gloas(list) => { + self.post_no_content(PATH, list, Some(fork.as_str())).await + } + } } async fn prepare_beacon_proposer(&self, preparations: &[ProposerPreparationDto]) -> Result<()> { @@ -772,4 +1046,40 @@ mod tests { assert!(path.contains(parameter), "{parameter} missing from {path}"); } } + + /// Before gloas the query still names committee 0, exactly as it always + /// has; from gloas it is omitted. + #[test] + fn the_committee_index_is_omitted_from_gloas_on() { + assert_eq!( + HttpBeaconNode::attestation_data_path(96, ForkName::Fulu), + "/eth/v1/validator/attestation_data?slot=96&committee_index=0" + ); + assert_eq!( + HttpBeaconNode::attestation_data_path(96, ForkName::Gloas), + "/eth/v1/validator/attestation_data?slot=96" + ); + } + + /// The body `produceBlockV4` requires: self-build only, so no builders. + #[test] + fn the_builder_config_is_the_documented_self_build_body() { + let value: serde_json::Value = serde_json::from_str(BUILDER_CONFIG).expect("valid json"); + assert_eq!(value["min_bid"], "0"); + assert_eq!(value["builder_boost_factor"], "0"); + assert_eq!(value["builders"], serde_json::json!([])); + } + + #[test] + fn a_boolean_header_is_read_strictly() { + assert_eq!( + HttpBeaconNode::bool_header(Some("True".into()), "Included").expect("ok"), + Some(true) + ); + assert_eq!( + HttpBeaconNode::bool_header(None, "Included").expect("ok"), + None + ); + HttpBeaconNode::bool_header(Some("yes".into()), "Included").expect_err("not a boolean"); + } } diff --git a/crates/validator/src/beacon_node/mock.rs b/crates/validator/src/beacon_node/mock.rs index 9e1370304..34d01f57b 100644 --- a/crates/validator/src/beacon_node/mock.rs +++ b/crates/validator/src/beacon_node/mock.rs @@ -7,22 +7,38 @@ use std::sync::Mutex; use async_trait::async_trait; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::electra::{Attestation, SignedAggregateAndProof}; +use ethlambda_types::beacon::containers::gloas; use ethlambda_types::beacon::containers::shared::AttestationData; use ethlambda_types::beacon::containers::shared::Checkpoint; use ethlambda_types::beacon::fork::ForkName; use ethlambda_types::beacon::primitives::{BlsPubkey, Epoch, Root, Slot, ValidatorIndex}; -use crate::beacon_node::block_contents::{Contents, ProducedBlock, empty_block_for}; +use crate::beacon_node::block_contents::{ + Contents, GloasPayload, ProducedBlock, empty_block_for, empty_gloas_block_for, + empty_gloas_envelope_for, +}; use crate::beacon_node::dto::{ - AttesterDutyDto, CommitteeSubscriptionDto, ProposerDutyDto, ProposerPreparationDto, + AttesterDutyDto, CommitteeSubscriptionDto, ProposerDutyDto, ProposerPreparationDto, PtcDutyDto, SingleAttestationDto, }; use crate::beacon_node::{ - AggregateAttestation, AttesterDuties, BeaconNodeApi, BlockRequest, Genesis, ProposerDuties, - Published, ValidatorEntry, + AggregateAttestation, AggregateKind, AttesterDuties, BeaconNodeApi, BlockRequest, Genesis, + ProposerDuties, PtcDuties, Published, SignedAggregates, ValidatorEntry, }; use crate::error::{BeaconNodeFailure, Error, Result}; +/// What the mock produces at a gloas slot. +#[derive(Debug, Clone, Copy)] +pub struct GloasBlockSpec { + pub slot: Slot, + pub proposer_index: ValidatorIndex, + /// The builder the bid names; `BUILDER_INDEX_SELF_BUILD` for a self-build. + pub builder_index: u64, + /// Whether `produce_block` returns the envelope with the block, as + /// `include_payload=true` does. + pub include_payload: bool, +} + /// A `BeaconNodeApi` driven entirely by fields and methods a test controls, /// rather than by a network or a real beacon node. #[derive(Default)] @@ -49,6 +65,15 @@ pub struct MockBeaconNode { /// The slot and proposer the mock will produce a block for. `None` makes /// `produce_block` fail, the way an unset `attestation_data` does. pub produces_block: Option<(Slot, ValidatorIndex)>, + /// What the mock produces when asked for a gloas block. + pub produces_gloas_block: Option, + /// When set, the envelope the mock serves is for this root instead of the + /// produced block's, so a test can present a wrong one. + pub envelope_root_override: Option, + /// Every envelope body published, with `Eth-Blob-Data-Included`. + pub published_envelopes: Mutex, bool)>>, + /// Every `execution_payload_envelope` request, as (slot, block root). + pub envelope_requests: Mutex>, /// What `publish_block` reports. Defaults to `Imported`; a test that cares /// about the 202 path sets it. pub publish_outcome: Option, @@ -71,6 +96,18 @@ pub struct MockBeaconNode { /// When set, `aggregate_attestation` answers with an aggregate over this /// data. `None` makes it fail the way a node with nothing to fold does. pub aggregate: Option, + /// The fork the mock names for its aggregate. Defaults to electra. + pub aggregate_fork: Option, + /// Every published gloas aggregate batch. + pub published_gloas_aggregates: Mutex)>>, + pub ptc: Mutex>, + pub ptc_calls: Mutex, + /// What `payload_attestation_data` answers with. `None` is the 204. + pub payload_attestation_data: Option, + pub payload_attestation_data_calls: Mutex, + pub submitted_payload_attestations: Mutex>, + /// The fork each `attestation_data` call named, in call order. + pub attestation_data_forks: Mutex>, pub duties_calls: Mutex, pub validator_indices_calls: Mutex, pub proposer_duties_calls: Mutex, @@ -201,6 +238,117 @@ impl MockBeaconNode { self } + /// Answer `produce_block` at a gloas slot with a bare gloas block whose + /// bid names `builder_index`, with its self-built envelope attached when + /// `include_payload` is set. + pub fn with_gloas_block( + mut self, + slot: Slot, + proposer_index: ValidatorIndex, + builder_index: u64, + include_payload: bool, + ) -> Self { + self.produces_gloas_block = Some(GloasBlockSpec { + slot, + proposer_index, + builder_index, + include_payload, + }); + self + } + + /// Serve an envelope for `root` whatever block was produced. + pub fn with_envelope_for(mut self, root: Root) -> Self { + self.envelope_root_override = Some(root); + self + } + + /// Every envelope body published, with `Eth-Blob-Data-Included`. + pub fn published_envelopes(&self) -> Vec<(Vec, bool)> { + self.published_envelopes.lock().expect("lock").clone() + } + + /// Every envelope fetch, as (slot, block root). + pub fn envelope_requests(&self) -> Vec<(Slot, Root)> { + self.envelope_requests.lock().expect("lock").clone() + } + + /// Answer `aggregate_attestation` with a gloas aggregate over `data`. + pub fn with_gloas_aggregate(mut self, data: AttestationData) -> Self { + self.aggregate = Some(data); + self.aggregate_fork = Some(ForkName::Gloas); + self + } + + /// Every gloas aggregate batch published so far. + pub fn published_gloas_aggregates( + &self, + ) -> Vec<(ForkName, Vec)> { + self.published_gloas_aggregates + .lock() + .expect("lock") + .clone() + } + + /// Give `attestation_data` answers a nonzero `index`, the way a gloas node + /// signals a full payload. + pub fn with_attestation_index(mut self, index: u64) -> Self { + if let Some(data) = self.attestation_data.as_mut() { + data.index = index; + } + self + } + + /// The forks `attestation_data` was asked under, in call order. + pub fn attestation_data_forks(&self) -> Vec { + self.attestation_data_forks.lock().expect("lock").clone() + } + + pub fn with_ptc_duties( + self, + epoch: Epoch, + dependent_root: Root, + duties: Vec, + ) -> Self { + self.set_ptc_duties(epoch, dependent_root, duties); + self + } + + /// Replace the PTC duties stored for `epoch`, the way `set_duties` does. + pub fn set_ptc_duties(&self, epoch: Epoch, dependent_root: Root, duties: Vec) { + let mut stored = self.ptc.lock().expect("lock"); + let entry = PtcDuties { + dependent_root, + duties, + }; + match stored.iter_mut().find(|(held, _)| *held == epoch) { + Some((_, existing)) => *existing = entry, + None => stored.push((epoch, entry)), + } + } + + pub fn ptc_call_count(&self) -> usize { + *self.ptc_calls.lock().expect("lock") + } + + /// Answer `payload_attestation_data` with `data`. + pub fn with_payload_attestation_data(mut self, data: gloas::PayloadAttestationData) -> Self { + self.payload_attestation_data = Some(data); + self + } + + pub fn payload_attestation_data_call_count(&self) -> usize { + *self.payload_attestation_data_calls.lock().expect("lock") + } + + /// Every payload attestation message submitted so far. + pub fn submitted_payload_attestations(&self) -> Vec { + self.submitted_payload_attestations + .lock() + .expect("lock") + .clone() + } + pub fn with_publish_outcome(mut self, outcome: Published) -> Self { self.publish_outcome = Some(outcome); self @@ -377,9 +525,10 @@ impl BeaconNodeApi for MockBeaconNode { }) } - async fn attestation_data(&self, slot: Slot) -> Result { + async fn attestation_data(&self, slot: Slot, fork: ForkName) -> Result { self.guard("attestation_data")?; *self.attestation_data_calls.lock().expect("lock") += 1; + self.attestation_data_forks.lock().expect("lock").push(fork); let data = self.attestation_data.ok_or_else(|| Error::BeaconNode { url: "mock".to_string(), failure: BeaconNodeFailure::Request, @@ -405,6 +554,28 @@ impl BeaconNodeApi for MockBeaconNode { .lock() .expect("lock") .push(request.clone()); + if request.fork >= ForkName::Gloas { + let spec = self.produces_gloas_block.ok_or_else(|| Error::BeaconNode { + url: "mock".to_string(), + failure: BeaconNodeFailure::Request, + detail: "no gloas block set".into(), + })?; + let block = empty_gloas_block_for(spec.slot, spec.proposer_index, spec.builder_index); + let payload = spec.include_payload.then(|| { + use ethlambda_types::beacon::primitives::HashTreeRoot as _; + Box::new(GloasPayload { + envelope: empty_gloas_envelope_for(block.hash_tree_root(), spec.builder_index), + kzg_proofs: Default::default(), + blobs: Default::default(), + }) + }); + let block = ProducedBlock { + fork: ForkName::Gloas, + contents: Contents::Gloas { block, payload }, + }; + crate::beacon_node::validate_produced_block(request, &block)?; + return Ok(block); + } let (slot, proposer_index) = self.produces_block.ok_or_else(|| Error::BeaconNode { url: "mock".to_string(), failure: BeaconNodeFailure::Request, @@ -434,6 +605,92 @@ impl BeaconNodeApi for MockBeaconNode { Ok(self.publish_outcome.unwrap_or(Published::Imported)) } + async fn execution_payload_envelope( + &self, + slot: Slot, + block_root: Root, + ) -> Result { + self.guard("execution_payload_envelope")?; + self.envelope_requests + .lock() + .expect("lock") + .push((slot, block_root)); + let spec = self + .produces_gloas_block + .ok_or_else(|| Error::BeaconNodeStatus { + status: 404, + body: "no envelope cached".to_string(), + })?; + let root = self.envelope_root_override.unwrap_or(block_root); + let envelope = empty_gloas_envelope_for(root, spec.builder_index); + // Honours the contract the real implementation does. + if envelope.beacon_block_root != block_root { + return Err(Error::InconsistentResponse( + "the envelope is for another block".to_string(), + )); + } + Ok(envelope) + } + + async fn publish_execution_payload_envelope( + &self, + body: &[u8], + blob_data_included: bool, + ) -> Result<()> { + self.guard("publish_execution_payload_envelope")?; + self.published_envelopes + .lock() + .expect("lock") + .push((body.to_vec(), blob_data_included)); + Ok(()) + } + + async fn ptc_duties(&self, epoch: Epoch, _indices: &[ValidatorIndex]) -> Result { + self.guard("ptc_duties")?; + *self.ptc_calls.lock().expect("lock") += 1; + self.ptc + .lock() + .expect("lock") + .iter() + .find(|(stored, _)| *stored == epoch) + .map(|(_, duties)| duties.clone()) + .ok_or_else(|| Error::BeaconNode { + url: "mock".to_string(), + failure: BeaconNodeFailure::Request, + detail: format!("no ptc duties for epoch {epoch}"), + }) + } + + async fn payload_attestation_data( + &self, + slot: Slot, + ) -> Result> { + self.guard("payload_attestation_data")?; + *self.payload_attestation_data_calls.lock().expect("lock") += 1; + let Some(data) = self.payload_attestation_data else { + return Ok(None); + }; + if data.slot != slot { + return Err(Error::InconsistentResponse(format!( + "requested payload attestation data for slot {slot}, node answered for slot {}", + data.slot + ))); + } + Ok(Some(data)) + } + + async fn submit_payload_attestations( + &self, + messages: &[gloas::PayloadAttestationMessage], + ) -> Result { + self.guard("submit_payload_attestations")?; + self.submitted_payload_attestations + .lock() + .expect("lock") + .extend_from_slice(messages); + Ok(messages.len()) + } + async fn submit_attestations( &self, attestations: &[SingleAttestationDto], @@ -476,27 +733,46 @@ impl BeaconNodeApi for MockBeaconNode { data.slot ))); } + if self.aggregate_fork == Some(ForkName::Gloas) { + return Ok(AggregateAttestation { + fork: ForkName::Gloas, + attestation: AggregateKind::Gloas(gloas::Attestation { + aggregation_bits: Default::default(), + data, + signature: Default::default(), + committee_bits: Default::default(), + }), + }); + } Ok(AggregateAttestation { fork: ForkName::Electra, - attestation: Attestation { + attestation: AggregateKind::Electra(Attestation { aggregation_bits: Default::default(), data, signature: Default::default(), committee_bits: Default::default(), - }, + }), }) } async fn publish_aggregates( &self, fork: ForkName, - aggregates: &[SignedAggregateAndProof], + aggregates: &SignedAggregates, ) -> Result<()> { self.guard("publish_aggregates")?; - self.published_aggregates - .lock() - .expect("lock") - .push((fork, aggregates.to_vec())); + match aggregates { + SignedAggregates::Electra(list) => self + .published_aggregates + .lock() + .expect("lock") + .push((fork, list.clone())), + SignedAggregates::Gloas(list) => self + .published_gloas_aggregates + .lock() + .expect("lock") + .push((fork, list.clone())), + } Ok(()) } diff --git a/crates/validator/src/beacon_node/mod.rs b/crates/validator/src/beacon_node/mod.rs index 06d68e29a..9fa5f1ab3 100644 --- a/crates/validator/src/beacon_node/mod.rs +++ b/crates/validator/src/beacon_node/mod.rs @@ -16,11 +16,12 @@ use ethlambda_types::beacon::signing::compute_epoch_at_slot; use crate::beacon_node::block_contents::ProducedBlock; use crate::beacon_node::dto::{ - AttesterDutyDto, CommitteeSubscriptionDto, ProposerDutyDto, ProposerPreparationDto, + AttesterDutyDto, CommitteeSubscriptionDto, ProposerDutyDto, ProposerPreparationDto, PtcDutyDto, SingleAttestationDto, }; use crate::error::Result; -use ethlambda_types::beacon::containers::electra::{Attestation, SignedAggregateAndProof}; +use ethlambda_types::beacon::containers::electra; +use ethlambda_types::beacon::containers::gloas; pub mod block_contents; pub mod dto; @@ -94,6 +95,12 @@ pub struct Duties { /// means here. pub type AttesterDuties = Duties; +/// One epoch's payload timeliness committee duties for the indices asked +/// about. See [`Duties`] for what `dependent_root` means; for these it is the +/// same kind of root the attester schedule uses, since committee membership is +/// derived from the same shuffling. +pub type PtcDuties = Duties; + /// Every proposer for one epoch, not only this client's. The endpoint takes no /// validator list, so the caller filters. See [`Duties`] for what /// `dependent_root` means here, and why it is the more fragile of the two. @@ -111,6 +118,10 @@ pub type ProposerDuties = Duties; #[derive(Debug, Clone)] pub struct BlockRequest { pub slot: Slot, + /// The fork `slot` is in, by this client's own schedule. Selects the + /// endpoint version: `produceBlockV3` before gloas, `produceBlockV4` from + /// it on. + pub fork: ForkName, /// The validator this client believes proposes `slot`. pub proposer_index: ValidatorIndex, /// This proposer's reveal for the slot's epoch, which the node needs @@ -150,17 +161,19 @@ pub enum Published { /// else is one this client's key can only sign uselessly, while still burning /// the guard's record for that slot. pub fn validate_produced_block(request: &BlockRequest, block: &ProducedBlock) -> Result<()> { - let block = block.block(); - if block.slot != request.slot { + if block.slot() != request.slot { return Err(crate::error::Error::InconsistentResponse(format!( "requested a block for slot {}, node produced one for slot {}", - request.slot, block.slot + request.slot, + block.slot() ))); } - if block.proposer_index != request.proposer_index { + if block.proposer_index() != request.proposer_index { return Err(crate::error::Error::InconsistentResponse(format!( "block for slot {} names proposer {}, expected {}", - request.slot, block.proposer_index, request.proposer_index + request.slot, + block.proposer_index(), + request.proposer_index ))); } Ok(()) @@ -174,7 +187,49 @@ pub fn validate_produced_block(request: &BlockRequest, block: &ProducedBlock) -> #[derive(Debug, Clone)] pub struct AggregateAttestation { pub fork: ForkName, - pub attestation: Attestation, + pub attestation: AggregateKind, +} + +/// The attestation container an aggregate is, by fork. +/// +/// Gloas's `Attestation` is not electra's: its `aggregation_bits` is the +/// unbounded progressive bitlist, so its hash tree root differs and an +/// `AggregateAndProof` wrapping it must be signed over the gloas root. +#[derive(Debug, Clone)] +pub enum AggregateKind { + /// Electra and fulu share one layout. + Electra(electra::Attestation), + Gloas(gloas::Attestation), +} + +impl AggregateKind { + /// The data the aggregate votes on, whichever layout carries it. + pub fn data(&self) -> &AttestationData { + match self { + Self::Electra(attestation) => &attestation.data, + Self::Gloas(attestation) => &attestation.data, + } + } +} + +/// A batch of signed aggregates, all of one fork. +#[derive(Debug, Clone)] +pub enum SignedAggregates { + Electra(Vec), + Gloas(Vec), +} + +impl SignedAggregates { + pub fn len(&self) -> usize { + match self { + Self::Electra(list) => list.len(), + Self::Gloas(list) => list.len(), + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } } /// A validator's index and status, as resolved from its public key. @@ -252,11 +307,15 @@ pub trait BeaconNodeApi: Send + Sync { /// treats the two the same way will act on a stale proposer schedule. async fn proposer_duties(&self, epoch: Epoch) -> Result; - /// Produce the attestation data for `slot`. + /// Produce the attestation data for `slot`, which is in `fork`. /// - /// No committee index: the specification deprecated that parameter, from - /// Electra on `AttestationData.index` must be zero, and beacon nodes ignore - /// whatever is supplied. Keeping it out means a caller cannot get it wrong. + /// No committee index is ever a parameter here. Before gloas the query + /// still carries `committee_index=0`, which the specification deprecated + /// from electra on and nodes ignore, so a caller cannot get it wrong. From + /// gloas it is omitted, because the answer's `index` is no longer a + /// committee but the payload signal (0 for a same-slot block, 1 when the + /// attested block's payload was seen as full), and the caller must pass it + /// through unchanged: it is signed. /// /// # Contract: the answer is for `slot`, or this is an `Err` /// @@ -273,10 +332,14 @@ pub trait BeaconNodeApi: Send + Sync { /// returned, so a node stuck on a stale head is seen as a success every /// slot and the next node is never consulted. Enforced here, a wrong /// answer is an `Err` that failover treats like any other and moves past. - async fn attestation_data(&self, slot: Slot) -> Result; + async fn attestation_data(&self, slot: Slot, fork: ForkName) -> Result; /// Ask the node to build a block for `request.slot`. /// + /// From gloas the node is asked to include the self-built payload's + /// envelope in the answer, which saves a round trip inside the proposal's + /// budget. + /// /// Returns the block and whatever travelled with it, decoded from SSZ. /// JSON is not used here: see [`block_contents`] for why a block, alone /// among everything this client handles, cannot go through the JSON path. @@ -308,6 +371,56 @@ pub trait BeaconNodeApi: Send + Sync { /// proposal, whether or not the first one succeeded. async fn publish_block(&self, fork: ForkName, body: &[u8]) -> Result; + /// The envelope the node cached for the self-built payload of the block + /// whose root is `block_root`, proposed at `slot`. + /// + /// Only needed when `produce_block` came back without its payload + /// (`Eth-Execution-Payload-Included: false`). An answer whose + /// `beacon_block_root` is not `block_root` is an `Err`, for the reason + /// [`validate_produced_block`] is enforced in the implementations. + async fn execution_payload_envelope( + &self, + slot: Slot, + block_root: Root, + ) -> Result; + + /// Publish a signed envelope, given its SSZ body. + /// + /// `blob_data_included` is the `Eth-Blob-Data-Included` header: `true` + /// when the body is `SignedExecutionPayloadEnvelopeContents`, `false` when + /// it is the bare signed envelope and the node attaches the blobs it + /// cached at production time. + async fn publish_execution_payload_envelope( + &self, + body: &[u8], + blob_data_included: bool, + ) -> Result<()>; + + /// The payload timeliness committee duties for `indices` in `epoch`. + /// + /// An epoch before gloas answers with no duties rather than an error. + async fn ptc_duties(&self, epoch: Epoch, indices: &[ValidatorIndex]) -> Result; + + /// What a payload timeliness committee member should vote on at `slot`, or + /// `None` when the node knows no block for it (204), which is an ordinary + /// outcome and not an error. + /// + /// # Contract: the answer is for `slot`, or this is an `Err` + /// + /// For the reason [`Self::attestation_data`] states. + async fn payload_attestation_data( + &self, + slot: Slot, + ) -> Result>; + + /// Submit signed payload attestation messages to the node's pool. Returns + /// how many it accepted, with the same partial-success reading as + /// [`Self::submit_attestations`]. + async fn submit_payload_attestations( + &self, + messages: &[gloas::PayloadAttestationMessage], + ) -> Result; + /// Submits signed attestations to the node's pool, so they reach gossip. /// `fork_name` names the fork the attestations were produced under, since /// the endpoint requires it as a header rather than inferring it. @@ -353,11 +466,8 @@ pub trait BeaconNodeApi: Send + Sync { /// hand-written mapping this crate avoids, which is why the two differ. /// /// The endpoint takes a list even for one aggregate, so this does too. - async fn publish_aggregates( - &self, - fork: ForkName, - aggregates: &[SignedAggregateAndProof], - ) -> Result<()>; + async fn publish_aggregates(&self, fork: ForkName, aggregates: &SignedAggregates) + -> Result<()>; /// Tells the node where to pay each validator's execution-layer block /// rewards, so it has somewhere to send them when it builds a payload. diff --git a/crates/validator/src/metrics.rs b/crates/validator/src/metrics.rs index d8455d95c..a47da8598 100644 --- a/crates/validator/src/metrics.rs +++ b/crates/validator/src/metrics.rs @@ -139,6 +139,42 @@ static BLOCK_PROPOSAL_FAILURES_TOTAL: LazyLock = LazyLock::new(|| { .unwrap() }); +/// Envelopes of self-built gloas payloads the node accepted. +static ENVELOPES_PUBLISHED_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter!( + "ethlambda_validator_envelopes_published_total", + "Execution payload envelopes published for blocks this client proposed" + ) + .unwrap() +}); + +/// Gloas proposals whose block went out but whose envelope did not: the slot's +/// payload is then withheld, which costs the proposer its execution rewards and +/// the chain a payload. +static ENVELOPE_FAILURES_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter!( + "ethlambda_validator_envelope_failures_total", + "Proposed blocks whose self-built envelope was not published" + ) + .unwrap() +}); + +static PAYLOAD_ATTESTATIONS_PUBLISHED_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter!( + "ethlambda_validator_payload_attestations_published_total", + "Payload timeliness committee votes the beacon node accepted" + ) + .unwrap() +}); + +static PAYLOAD_ATTESTATION_FAILURES_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter!( + "ethlambda_validator_payload_attestation_failures_total", + "Slots whose payload timeliness committee duty did not result in published votes" + ) + .unwrap() +}); + /// End to end, from the slot's start to the node accepting the block. /// /// The buckets are tighter at the low end than the attestation histogram's, @@ -242,6 +278,10 @@ pub fn init() { LazyLock::force(&BLOCKS_BROADCAST_NOT_IMPORTED_TOTAL); LazyLock::force(&BLOCKS_REFUSED_TOTAL); LazyLock::force(&BLOCK_PROPOSAL_FAILURES_TOTAL); + LazyLock::force(&ENVELOPES_PUBLISHED_TOTAL); + LazyLock::force(&ENVELOPE_FAILURES_TOTAL); + LazyLock::force(&PAYLOAD_ATTESTATIONS_PUBLISHED_TOTAL); + LazyLock::force(&PAYLOAD_ATTESTATION_FAILURES_TOTAL); LazyLock::force(&BLOCK_PUBLICATION_DELAY_SECONDS); LazyLock::force(&AGGREGATES_PUBLISHED_TOTAL); LazyLock::force(&AGGREGATION_FAILURES_TOTAL); @@ -299,6 +339,22 @@ pub fn inc_block_proposal_failures() { BLOCK_PROPOSAL_FAILURES_TOTAL.inc(); } +pub fn inc_envelopes_published() { + ENVELOPES_PUBLISHED_TOTAL.inc(); +} + +pub fn inc_envelope_failures() { + ENVELOPE_FAILURES_TOTAL.inc(); +} + +pub fn inc_payload_attestations_published(count: u64) { + PAYLOAD_ATTESTATIONS_PUBLISHED_TOTAL.inc_by(count); +} + +pub fn inc_payload_attestation_failures() { + PAYLOAD_ATTESTATION_FAILURES_TOTAL.inc(); +} + /// Record how long after a slot's start its block was accepted. pub fn observe_block_publication_delay(seconds: f64) { BLOCK_PUBLICATION_DELAY_SECONDS.observe(seconds); diff --git a/crates/validator/src/proposal.rs b/crates/validator/src/proposal.rs index 91993f7ce..af61f58e7 100644 --- a/crates/validator/src/proposal.rs +++ b/crates/validator/src/proposal.rs @@ -29,6 +29,16 @@ //! deadline safe: a proposal abandoned between signing and publishing cannot be //! re-signed, because the guard already counts that slot as proposed. //! +//! # Gloas: the block, then its envelope +//! +//! From gloas the block commits to a bid and the payload is revealed in an +//! envelope the builder signs. This client only self-builds: when the bid names +//! the self-build sentinel, the proposer's own key signs the envelope under the +//! builder domain, and it is published right after the block, in the same +//! budget. A bid naming anyone else is a builder's to reveal, and is left alone. +//! +//! The envelope is not slashable, so the guard is untouched by it. +//! //! # Do not retry this call within a slot //! //! For the same reason [`crate::attestation::AttestationService::attest`] must @@ -40,13 +50,15 @@ use std::sync::Arc; +use ethlambda_types::beacon::constants::BUILDER_INDEX_SELF_BUILD; use ethlambda_types::beacon::fork::ForkName; -use ethlambda_types::beacon::primitives::{Bytes32, ExecutionAddress, HashTreeRoot as _, Slot}; +use ethlambda_types::beacon::primitives::{BlsPubkey, Bytes32, ExecutionAddress, Root, Slot}; use ethlambda_types::beacon::signing::compute_epoch_at_slot; use tokio::sync::RwLock; use tracing::{info, warn}; -use crate::beacon_node::dto::{ProposerDutyDto, parse_pubkey}; +use crate::beacon_node::block_contents::{GloasPayload, signed_envelope_ssz}; +use crate::beacon_node::dto::{ProposerDutyDto, encode_hex, parse_pubkey}; use crate::beacon_node::{BeaconNodeApi, BlockRequest, Published, validate_produced_block}; use crate::error::{Error, Result}; use crate::keys::ValidatorStore; @@ -124,7 +136,9 @@ impl ProposalService { let Some(expected) = self.fee_recipient else { return; }; - let actual = produced.block().body.execution_payload.fee_recipient; + // The payload's before gloas, the bid's from it on: the same address + // the execution layer is told to pay, read where the block keeps it. + let actual = produced.fee_recipient(); if actual != expected { tracing::error!( %slot, @@ -195,11 +209,12 @@ impl ProposalService { let request = BlockRequest { slot, + fork: self.context.config.fork_at_epoch(epoch), proposer_index: duty.validator_index, randao_reveal, graffiti: self.graffiti, }; - let produced = self.beacon_node.produce_block(&request).await?; + let mut produced = self.beacon_node.produce_block(&request).await?; // Checked again here, having already been checked by the // implementation this call went through. Not redundant, and the @@ -249,7 +264,7 @@ impl ProposalService { // The root is taken from the decoded container, never from anything // reassembled here, and it is taken once: the same value is what the // guard's slot is recorded against and what the signature covers. - let block_root = produced.block().hash_tree_root(); + let block_root = produced.block_root(); let signature = { let store = store.read().await; @@ -278,14 +293,111 @@ impl ProposalService { self.context.sign_block(&store, &pubkey, block_root, slot)? }; + // Taken out before the block is signed into its body: the envelope is + // published on its own, after the block, and the bare gloas block body + // has no place for it. + let payload = produced.take_gloas_payload(); + let builder_index = produced.builder_index(); + let body = produced.into_signed_ssz(signature); let published = self .publish(fork, &body, slot, duty.validator_index) .await?; crate::metrics::inc_blocks_proposed(); + + // Only a self-built payload is this client's to reveal. + if builder_index == Some(BUILDER_INDEX_SELF_BUILD) { + // Logged and counted rather than propagated. The block is out and + // the proposal did happen; what failed is the reveal, which has its + // own series because it is the failure that costs the payload. + let _ = self + .publish_envelope( + slot, + duty.validator_index, + &pubkey, + block_root, + payload, + store, + ) + .await + .inspect(|()| crate::metrics::inc_envelopes_published()) + .inspect_err(|err| { + tracing::error!( + %slot, + validator = duty.validator_index, + block_root = %ethlambda_types::ShortRoot(&block_root.0), + %err, + "Block published but its envelope was not; the slot's payload is withheld" + ); + crate::metrics::inc_envelope_failures(); + }); + } else if let Some(builder_index) = builder_index { + info!( + %slot, + builder_index, + "Block commits to a builder's bid; leaving the payload reveal to that builder" + ); + } Ok(published) } + /// Reveal the self-built payload of the block just published. + /// + /// `payload` is what `produceBlockV4` returned with the block, or `None` + /// when the node did not include it, in which case it is fetched by + /// `(slot, block_root)` from the node's cache. Either way it is checked to + /// be for this block and this self-build before it is signed: the signature + /// covers the envelope as it stands, so one for another block would be a + /// valid signature on the wrong payload. + async fn publish_envelope( + &self, + slot: Slot, + validator: u64, + pubkey: &BlsPubkey, + block_root: Root, + payload: Option, + store: &RwLock, + ) -> Result<()> { + let included = payload.is_some(); + let envelope = match &payload { + Some(payload) => payload.envelope.clone(), + None => { + self.beacon_node + .execution_payload_envelope(slot, block_root) + .await? + } + }; + if envelope.beacon_block_root != block_root { + return Err(Error::InconsistentResponse(format!( + "envelope for slot {slot} reveals the payload of block {}, not of the block \ + this client proposed ({})", + encode_hex(&envelope.beacon_block_root.0), + encode_hex(&block_root.0) + ))); + } + if envelope.builder_index != BUILDER_INDEX_SELF_BUILD { + return Err(Error::InconsistentResponse(format!( + "envelope for slot {slot} names builder {}, but the bid was a self-build", + envelope.builder_index + ))); + } + + let signature = { + let store = store.read().await; + self.context + .sign_execution_payload_envelope(&store, pubkey, &envelope, slot)? + }; + let body = match payload { + Some(payload) => payload.into_signed_contents_ssz(signature), + None => signed_envelope_ssz(envelope, signature), + }; + self.beacon_node + .publish_execution_payload_envelope(&body, included) + .await?; + info!(%slot, validator, bytes = body.len(), blob_data_included = included, "Envelope published"); + Ok(()) + } + /// Send the signed body and report what the node made of it. /// /// Split out so the 202 case has one place to be explained rather than @@ -328,6 +440,7 @@ mod tests { use crate::beacon_node::block_contents::SignedBlockContents; use crate::beacon_node::mock::MockBeaconNode; use ethlambda_types::beacon::config::Config; + use ethlambda_types::beacon::primitives::HashTreeRoot as _; use ethlambda_types::beacon::primitives::{BlsPubkey, H160, Root}; use libssz::SszDecode as _; @@ -641,3 +754,249 @@ mod tests { assert_eq!(node.published_blocks().len(), 1); } } + +#[cfg(test)] +mod gloas_tests { + use super::*; + use crate::beacon_node::block_contents::SignedExecutionPayloadEnvelopeContents; + use crate::beacon_node::mock::MockBeaconNode; + use ethlambda_types::beacon::config::Config; + use ethlambda_types::beacon::containers::gloas; + use ethlambda_types::beacon::preset::SLOTS_PER_EPOCH; + use ethlambda_types::beacon::primitives::{BlsPubkey, HashTreeRoot as _}; + use libssz::SszDecode as _; + + const GLOAS_EPOCH: u64 = 10; + const DST: &[u8] = b"BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_"; + + fn secret() -> [u8; 32] { + hex::decode("000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f") + .expect("valid hex") + .try_into() + .expect("32 bytes") + } + + fn context() -> Arc { + Arc::new(SigningContext { + config: Config::mainnet().with_fork_epoch(ForkName::Gloas, GLOAS_EPOCH), + genesis_validators_root: Root::ZERO, + }) + } + + fn store() -> (RwLock, BlsPubkey) { + let mut store = ValidatorStore::new(); + let pubkey = store.insert_secret("test", &secret()).expect("inserts"); + (RwLock::new(store), pubkey) + } + + fn slot() -> Slot { + GLOAS_EPOCH * SLOTS_PER_EPOCH + 5 + } + + fn duty(pubkey: &BlsPubkey) -> ProposerDutyDto { + ProposerDutyDto { + pubkey: encode_hex(&pubkey.0), + validator_index: 7, + slot: slot(), + } + } + + fn service(node: Arc) -> ProposalService { + ProposalService::new(node, context(), Bytes32::repeat_byte(0xab), None) + } + + fn verifies(pubkey: &BlsPubkey, signature: &[u8; 96], root: Root) -> bool { + use blst::min_pk::{PublicKey, Signature}; + let pk = PublicKey::from_bytes(&pubkey.0).expect("valid pubkey"); + let sig = Signature::from_bytes(signature).expect("valid signature"); + sig.verify(true, root.as_slice(), DST, &[], &pk, true) == blst::BLST_ERROR::BLST_SUCCESS + } + + /// The block goes out bare, under the gloas header, signed over the gloas + /// block's root, and then the envelope follows with the blobs, signed under + /// the builder domain by the proposer's own key. + #[tokio::test] + async fn a_self_built_block_is_published_and_then_its_envelope() { + let (store, pubkey) = store(); + let node = Arc::new(MockBeaconNode::new().with_gloas_block( + slot(), + 7, + BUILDER_INDEX_SELF_BUILD, + true, + )); + + let published = service(node.clone()) + .propose(slot(), &duty(&pubkey), &store) + .await + .expect("proposes"); + assert_eq!(published, Published::Imported); + + let request = node.block_requests().remove(0); + assert_eq!(request.fork, ForkName::Gloas); + + let blocks = node.published_blocks(); + assert_eq!(blocks.len(), 1); + assert_eq!(blocks[0].0, ForkName::Gloas); + let signed = gloas::SignedBeaconBlock::from_ssz_bytes(&blocks[0].1) + .expect("the body is a bare gloas signed block"); + let block_root = signed.message.hash_tree_root(); + let block_signing_root = context().block_signing_root(block_root, slot()); + assert!(verifies(&pubkey, &signed.signature.0, block_signing_root)); + + let envelopes = node.published_envelopes(); + assert_eq!(envelopes.len(), 1); + let (body, blob_data_included) = &envelopes[0]; + assert!(*blob_data_included, "the contents form carries the blobs"); + let contents = SignedExecutionPayloadEnvelopeContents::from_ssz_bytes(body) + .expect("the body is the signed envelope contents"); + let envelope = &contents.signed_execution_payload_envelope; + assert_eq!(envelope.message.beacon_block_root, block_root); + assert!( + verifies( + &pubkey, + &envelope.signature.0, + context().envelope_signing_root(&envelope.message, slot()) + ), + "the envelope must be signed under the builder domain" + ); + assert!( + node.envelope_requests().is_empty(), + "an included payload needs no fetch" + ); + } + + /// The builder domain is not the proposer domain: an envelope signature + /// must not verify as a block signature over the same root. + #[tokio::test] + async fn the_envelope_signature_is_under_the_builder_domain_not_the_proposers() { + let (store, pubkey) = store(); + let node = Arc::new(MockBeaconNode::new().with_gloas_block( + slot(), + 7, + BUILDER_INDEX_SELF_BUILD, + true, + )); + service(node.clone()) + .propose(slot(), &duty(&pubkey), &store) + .await + .expect("proposes"); + + let (body, _) = node.published_envelopes().remove(0); + let contents = SignedExecutionPayloadEnvelopeContents::from_ssz_bytes(&body).expect("ok"); + let envelope = contents.signed_execution_payload_envelope; + let as_proposer = context().block_signing_root(envelope.message.hash_tree_root(), slot()); + assert!(!verifies(&pubkey, &envelope.signature.0, as_proposer)); + } + + /// Without `Eth-Execution-Payload-Included` the envelope is fetched by + /// (slot, block root) and published bare, with the blob header false. + #[tokio::test] + async fn an_envelope_the_node_did_not_include_is_fetched_and_published_bare() { + let (store, pubkey) = store(); + let node = Arc::new(MockBeaconNode::new().with_gloas_block( + slot(), + 7, + BUILDER_INDEX_SELF_BUILD, + false, + )); + + service(node.clone()) + .propose(slot(), &duty(&pubkey), &store) + .await + .expect("proposes"); + + let signed = gloas::SignedBeaconBlock::from_ssz_bytes(&node.published_blocks()[0].1) + .expect("bare block"); + let block_root = signed.message.hash_tree_root(); + assert_eq!(node.envelope_requests(), vec![(slot(), block_root)]); + + let (body, blob_data_included) = node.published_envelopes().remove(0); + assert!(!blob_data_included); + let envelope = + gloas::SignedExecutionPayloadEnvelope::from_ssz_bytes(&body).expect("bare envelope"); + assert_eq!(envelope.message.beacon_block_root, block_root); + assert!(verifies( + &pubkey, + &envelope.signature.0, + context().envelope_signing_root(&envelope.message, slot()) + )); + } + + /// A bid naming a builder is that builder's to reveal; this client signs + /// nothing for it. + #[tokio::test] + async fn a_builders_bid_gets_no_envelope_from_this_client() { + let (store, pubkey) = store(); + let node = Arc::new(MockBeaconNode::new().with_gloas_block(slot(), 7, 3, true)); + + service(node.clone()) + .propose(slot(), &duty(&pubkey), &store) + .await + .expect("proposes"); + assert_eq!(node.published_blocks().len(), 1); + assert!(node.published_envelopes().is_empty()); + assert!(node.envelope_requests().is_empty()); + } + + /// An envelope for another block must never be signed. The block is out + /// already, so the proposal still reports it, but nothing is revealed. + #[tokio::test] + async fn an_envelope_for_another_block_is_not_signed() { + let (store, pubkey) = store(); + let node = Arc::new( + MockBeaconNode::new() + .with_gloas_block(slot(), 7, BUILDER_INDEX_SELF_BUILD, false) + .with_envelope_for(Root::repeat_byte(9)), + ); + + let result = service(node.clone()) + .propose(slot(), &duty(&pubkey), &store) + .await; + assert!(result.is_ok(), "the block was published"); + assert_eq!(node.published_blocks().len(), 1); + assert!(node.published_envelopes().is_empty()); + } + + /// The guard's semantics are unchanged: a second proposal for the slot is + /// refused before asking for anything, and reveals no second envelope. + #[tokio::test] + async fn a_repeated_gloas_slot_is_refused_and_reveals_nothing_more() { + let (store, pubkey) = store(); + let node = Arc::new(MockBeaconNode::new().with_gloas_block( + slot(), + 7, + BUILDER_INDEX_SELF_BUILD, + true, + )); + let service = service(node.clone()); + service + .propose(slot(), &duty(&pubkey), &store) + .await + .expect("first"); + let err = service + .propose(slot(), &duty(&pubkey), &store) + .await + .expect_err("a second block for one slot must be refused"); + assert!(matches!(err, Error::ProposalRefused { .. }), "got {err:?}"); + assert_eq!(node.block_requests().len(), 1); + assert_eq!(node.published_envelopes().len(), 1); + } + + /// The block is out by the time the envelope fails, so the proposal is + /// still reported as published; the failure is logged and counted. + #[tokio::test] + async fn a_failed_envelope_publication_does_not_fail_the_proposal() { + let (store, pubkey) = store(); + let node = Arc::new( + MockBeaconNode::new() + .with_gloas_block(slot(), 7, BUILDER_INDEX_SELF_BUILD, true) + .failing_call("publish_execution_payload_envelope", "node is unhappy"), + ); + let published = service(node.clone()) + .propose(slot(), &duty(&pubkey), &store) + .await + .expect("the block was published"); + assert_eq!(published, Published::Imported); + assert!(node.published_envelopes().is_empty()); + } +} diff --git a/crates/validator/src/signing.rs b/crates/validator/src/signing.rs index 533b55473..305634818 100644 --- a/crates/validator/src/signing.rs +++ b/crates/validator/src/signing.rs @@ -7,8 +7,11 @@ use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::constants::{ - DOMAIN_AGGREGATE_AND_PROOF, DOMAIN_BEACON_ATTESTER, DOMAIN_BEACON_PROPOSER, DOMAIN_RANDAO, - DOMAIN_SELECTION_PROOF, + DOMAIN_AGGREGATE_AND_PROOF, DOMAIN_BEACON_ATTESTER, DOMAIN_BEACON_BUILDER, + DOMAIN_BEACON_PROPOSER, DOMAIN_PTC_ATTESTER, DOMAIN_RANDAO, DOMAIN_SELECTION_PROOF, +}; +use ethlambda_types::beacon::containers::gloas::{ + ExecutionPayloadEnvelope, PayloadAttestationData, }; use ethlambda_types::beacon::containers::shared::AttestationData; use ethlambda_types::beacon::primitives::{ @@ -124,6 +127,28 @@ impl SigningContext { compute_signing_root(root, domain) } + /// The root an execution payload envelope's signature is computed over. + /// + /// `slot` is the slot of the block the envelope reveals the payload for, + /// not anything read out of the envelope: the domain is the builder's, at + /// the epoch containing that slot (`get_domain(state, DOMAIN_BEACON_BUILDER, + /// compute_epoch_at_slot(state.slot))` in the specification's + /// `process_execution_payload`, where the state is the block's own). + pub fn envelope_signing_root(&self, envelope: &ExecutionPayloadEnvelope, slot: Slot) -> Root { + let domain = self.domain(DOMAIN_BEACON_BUILDER, compute_epoch_at_slot(slot)); + compute_signing_root(envelope.hash_tree_root(), domain) + } + + /// The root a payload timeliness committee member's vote is computed over. + /// + /// The domain is taken at the epoch of `data.slot`, and the message is the + /// data alone: the validator index travels beside the signature in the + /// message wrapper and is deliberately not signed over. + pub fn payload_attestation_signing_root(&self, data: &PayloadAttestationData) -> Root { + let domain = self.domain(DOMAIN_PTC_ATTESTER, compute_epoch_at_slot(data.slot)); + compute_signing_root(data.hash_tree_root(), domain) + } + /// Sign an already-computed signing root on behalf of `pubkey`. /// /// Every public signing method funnels through here, so there is one place @@ -211,6 +236,38 @@ impl SigningContext { ) } + /// Sign the envelope revealing the payload of the block proposed for `slot` + /// on behalf of `pubkey`, which for a self-built payload is the proposer's + /// own key. + /// + /// Not slashable and not guarded. The specification attaches no slashing + /// condition to an envelope: signing two for one slot is at worst a + /// payload the network ignores one of. The block it belongs to is the + /// guarded signature. + pub fn sign_execution_payload_envelope( + &self, + store: &ValidatorStore, + pubkey: &BlsPubkey, + envelope: &ExecutionPayloadEnvelope, + slot: Slot, + ) -> Result { + self.sign_root(store, pubkey, self.envelope_signing_root(envelope, slot)) + } + + /// Sign a payload timeliness committee vote on behalf of `pubkey`. + /// + /// Not slashable by the specification, and deduplicated per validator and + /// slot by [`crate::payload_attestation`] only to avoid publishing the same + /// vote twice. + pub fn sign_payload_attestation( + &self, + store: &ValidatorStore, + pubkey: &BlsPubkey, + data: &PayloadAttestationData, + ) -> Result { + self.sign_root(store, pubkey, self.payload_attestation_signing_root(data)) + } + /// Sign the block whose root is `block_root`, proposed for `slot`, on /// behalf of `pubkey`. /// From a4f629435778de8c43f1131c3955311af526c6b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:53:49 -0300 Subject: [PATCH 086/132] feat(validator): attest and aggregate at gloas slots Gloas repurposes AttestationData.index as the payload signal, so the client must neither ask about committee 0 nor reset the answer to zero: the query now omits committee_index from the first gloas slot on (the fork is passed from the client's own schedule), and the index the node answers is submitted and signed unchanged. A test pins that an index of 1 reaches both the submission and the signature, and that a zeroed copy would be a different message. Aggregates are fork-tagged: the node's answer is parsed into gloas's own Attestation, the wrapper is the gloas AggregateAndProof signed over its own root (the two attestation layouts hash differently), and the batch goes out as gloas JSON under the gloas header. The doc claims that the index is always zero are corrected. --- crates/validator/src/aggregation.rs | 76 +++++++++++++++++++++ crates/validator/src/attestation.rs | 90 +++++++++++++++++++++++++ crates/validator/src/beacon_node/dto.rs | 39 +++++++++++ crates/validator/src/beacon_node/mod.rs | 5 +- 4 files changed, 208 insertions(+), 2 deletions(-) diff --git a/crates/validator/src/aggregation.rs b/crates/validator/src/aggregation.rs index eca92222d..3a0a32dcd 100644 --- a/crates/validator/src/aggregation.rs +++ b/crates/validator/src/aggregation.rs @@ -529,4 +529,80 @@ mod tests { assert_eq!(published, 0); assert!(node.aggregate_requests().is_empty()); } + + fn gloas_context() -> Arc { + Arc::new(SigningContext { + config: Config::mainnet().with_fork_epoch(ForkName::Gloas, 100), + genesis_validators_root: Root::ZERO, + }) + } + + /// A slot in the gloas era at which this key is selected under the gloas + /// fork version, found rather than written down for the reason `slots` is. + fn gloas_selected_slot(pubkey: &BlsPubkey, store: &ValidatorStore) -> Slot { + let context = gloas_context(); + (3_200..3_600) + .find(|slot| { + let proof = context + .sign_selection_proof(store, pubkey, *slot) + .expect("signs"); + is_aggregator(128, &proof) + }) + .expect("some slot selects this key") + } + + /// At a gloas slot the aggregate is gloas's container, and the wrapper is + /// signed over the gloas `AggregateAndProof`'s root, not electra's: the two + /// attestations hash differently, so the wrong root signs a message no node + /// can verify. + #[tokio::test] + async fn a_gloas_aggregate_is_wrapped_signed_and_published_as_gloas() { + use blst::min_pk::{PublicKey, Signature}; + + let (store, pubkey) = store(); + let slot = { + let guard = store.read().await; + gloas_selected_slot(&pubkey, &guard) + }; + let node = Arc::new(MockBeaconNode::new().with_gloas_aggregate(data(slot))); + + let published = AggregationService::new(node.clone(), gloas_context()) + .aggregate(slot, &data(slot), &[duty(&pubkey, 1, slot, 2)], &store) + .await + .expect("aggregates"); + assert_eq!(published, 1); + + assert!( + node.published_aggregates().is_empty(), + "not the electra form" + ); + let (fork, list) = node.published_gloas_aggregates().remove(0); + assert_eq!(fork, ForkName::Gloas); + let entry = &list[0]; + assert_eq!(entry.message.aggregator_index, 1); + + let root = + gloas_context().aggregate_and_proof_signing_root(entry.message.hash_tree_root(), slot); + let pk = PublicKey::from_bytes(&pubkey.0).expect("valid pubkey"); + let sig = Signature::from_bytes(&entry.signature.0).expect("valid signature"); + assert_eq!( + sig.verify( + true, + root.as_slice(), + b"BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_", + &[], + &pk, + true + ), + blst::BLST_ERROR::BLST_SUCCESS + ); + + // The same wrapper built from electra's container hashes differently. + let as_electra = electra::AggregateAndProof { + aggregator_index: 1, + aggregate: electra::Attestation::try_from(&entry.message.aggregate).expect("converts"), + selection_proof: entry.message.selection_proof, + }; + assert_ne!(as_electra.hash_tree_root(), entry.message.hash_tree_root()); + } } diff --git a/crates/validator/src/attestation.rs b/crates/validator/src/attestation.rs index a9080561d..a5984e590 100644 --- a/crates/validator/src/attestation.rs +++ b/crates/validator/src/attestation.rs @@ -660,4 +660,94 @@ mod tests { Some("altair") ); } + + fn gloas_context() -> Arc { + Arc::new(SigningContext { + config: Config::mainnet() + .with_fork_epoch(ethlambda_types::beacon::fork::ForkName::Gloas, 10), + genesis_validators_root: Root::ZERO, + }) + } + + /// From gloas `data.index` is the payload signal, so a node answering 1 + /// must see it submitted as 1 and signed as 1: resetting it to zero, as + /// the electra rule would suggest, signs a vote the node never made. + #[tokio::test] + async fn a_gloas_index_of_one_is_submitted_and_signed_unchanged() { + use blst::min_pk::{PublicKey, Signature}; + use ethlambda_types::beacon::fork::ForkName; + + let mut store = ValidatorStore::new(); + let pubkey = store.insert_secret("test", &secret()).expect("inserts"); + let pubkey_hex = encode_hex(&pubkey.0); + let store = RwLock::new(store); + + let slot = 10 * preset::SLOTS_PER_EPOCH + 3; + let node = Arc::new( + MockBeaconNode::new() + .with_attestation_data(slot) + .with_attestation_index(1), + ); + let attested = AttestationService::new(node.clone(), gloas_context()) + .attest(slot, &[duty(&pubkey_hex, 1337, slot, 3)], &store) + .await + .expect("attests"); + + assert_eq!(attested.data.expect("data").index, 1); + let submitted = node.submitted(); + assert_eq!(submitted.len(), 1); + assert_eq!(submitted[0].data.index, 1, "the index must pass through"); + assert_eq!(node.last_submitted_fork_name().as_deref(), Some("gloas")); + assert_eq!(node.attestation_data_forks(), vec![ForkName::Gloas]); + + // The signature is over the data with index 1, not over a zeroed copy. + let mut signed_over = node.attestation_data.expect("set"); + assert_eq!(signed_over.index, 1); + let root = gloas_context().attestation_signing_root(&signed_over); + let signature_bytes: [u8; 96] = + hex::decode(submitted[0].signature.trim_start_matches("0x")) + .expect("hex") + .try_into() + .expect("96 bytes"); + let pk = PublicKey::from_bytes(&pubkey.0).expect("valid pubkey"); + let sig = Signature::from_bytes(&signature_bytes).expect("valid signature"); + let verify = |root: Root| { + sig.verify( + true, + root.as_slice(), + b"BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_", + &[], + &pk, + true, + ) + }; + assert_eq!(verify(root), blst::BLST_ERROR::BLST_SUCCESS); + + signed_over.index = 0; + assert_ne!( + verify(gloas_context().attestation_signing_root(&signed_over)), + blst::BLST_ERROR::BLST_SUCCESS, + "a vote for index 0 is a different message" + ); + } + + /// Slots before gloas are asked about under their own fork, so the HTTP + /// client keeps sending `committee_index=0` there. + #[tokio::test] + async fn a_pre_gloas_slot_is_asked_about_under_its_own_fork() { + use ethlambda_types::beacon::fork::ForkName; + + let mut store = ValidatorStore::new(); + let pubkey = store.insert_secret("test", &secret()).expect("inserts"); + let pubkey_hex = encode_hex(&pubkey.0); + let store = RwLock::new(store); + + let slot = 9 * preset::SLOTS_PER_EPOCH + 3; + let node = Arc::new(MockBeaconNode::new().with_attestation_data(slot)); + AttestationService::new(node.clone(), gloas_context()) + .attest(slot, &[duty(&pubkey_hex, 1337, slot, 3)], &store) + .await + .expect("attests"); + assert_eq!(node.attestation_data_forks(), vec![ForkName::Phase0]); + } } diff --git a/crates/validator/src/beacon_node/dto.rs b/crates/validator/src/beacon_node/dto.rs index 22dd42ddb..e62c24e30 100644 --- a/crates/validator/src/beacon_node/dto.rs +++ b/crates/validator/src/beacon_node/dto.rs @@ -241,6 +241,10 @@ pub struct SingleAttestationDto { pub signature: String, } +/// `index` is whatever the beacon node answered, never rewritten: it is part of +/// the signed message. It is zero from electra, and from gloas it carries the +/// payload signal (1 when the attested block's payload was seen as full), which +/// is why it must not be normalised on the way out. #[derive(Debug, Clone, Serialize)] pub struct AttestationDataOutDto { #[serde(with = "quoted_u64")] @@ -1019,4 +1023,39 @@ mod tests { let err = config_from_spec_response(&response).expect_err("must reject"); assert!(matches!(err, Error::Decode(_)), "got {err:?}"); } + + /// A gloas aggregate parses straight into the gloas container, bitfields + /// included, and serialises back to the JSON the electra DTO produces for + /// the same vote, which is what the endpoint takes. + #[test] + fn a_gloas_aggregate_round_trips_through_its_json() { + use ethlambda_types::beacon::containers::gloas; + + let electra_attestation = Attestation { + aggregation_bits: AggregationBits::from_ssz_bytes(&[0x3f]).expect("bits"), + data: AttestationData { + slot: 5, + index: 1, + beacon_block_root: Root::repeat_byte(3), + source: Checkpoint { + epoch: 0, + root: Root::ZERO, + }, + target: Checkpoint { + epoch: 0, + root: Root::repeat_byte(4), + }, + }, + signature: BlsSignature([9; 96]), + committee_bits: CommitteeBits::from_ssz_bytes(&[0x01, 0, 0, 0, 0, 0, 0, 0]) + .expect("bits"), + }; + let json = serde_json::to_value(AttestationOutDto::from(&electra_attestation)) + .expect("serialises"); + + let parsed: gloas::Attestation = serde_json::from_value(json.clone()).expect("parses"); + assert_eq!(parsed, gloas::Attestation::from(&electra_attestation)); + assert_eq!(parsed.data.index, 1, "the payload signal survives parsing"); + assert_eq!(serde_json::to_value(&parsed).expect("serialises"), json); + } } diff --git a/crates/validator/src/beacon_node/mod.rs b/crates/validator/src/beacon_node/mod.rs index 9fa5f1ab3..f14824c96 100644 --- a/crates/validator/src/beacon_node/mod.rs +++ b/crates/validator/src/beacon_node/mod.rs @@ -440,8 +440,9 @@ pub trait BeaconNodeApi: Send + Sync { /// `attestation_data_root` at `slot`. /// /// `committee_index` is separate from the root and not derivable from it. - /// From electra on, `AttestationData.index` is required to be zero, so the - /// root no longer distinguishes one committee's votes from another's in the + /// From electra on, `AttestationData.index` no longer names a committee (it + /// is zero, and from gloas it is the payload signal, 0 or 1), so the root + /// no longer distinguishes one committee's votes from another's in the /// same slot; the committee moved into the attestation's `committee_bits`. /// That is exactly why the v1 form of this endpoint was removed rather than /// deprecated: it had no way to ask the question. From e75554b57446575b79ac7bbdf43e9bbaa2acf6e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:56:57 -0300 Subject: [PATCH 087/132] feat(validator): vote on payload timeliness from the committee From gloas a small committee is drawn each slot to attest whether the payload arrived. DutiesService now keeps that schedule for the current and next epoch on the attester schedule's terms (replaced when the dependent root changes, narrowed to this client's indices), and refresh_epoch fetches it for gloas epochs only, gating each epoch on its own fork since the lookahead can be the first gloas epoch. PayloadAttestationService asks the node for the slot's data, signs it under the PTC domain at the data's epoch and submits it. A 204 means no block is known and is not a vote against the payload. One vote per validator and slot is recorded at signing time, so a duty dropped at its deadline is lost rather than re-signed. The duty loop no longer skips a slot that has no attester duty: committee seats are drawn independently, so the old early continue would have dropped exactly those votes. The per-slot work moves into serve_slot_duties, which sleeps to the payload deadline (three quarters in) and is bounded by the end of the slot like the other duties. --- crates/validator/src/duties.rs | 161 +++++++- crates/validator/src/lib.rs | 429 +++++++++++++++++--- crates/validator/src/payload_attestation.rs | 379 +++++++++++++++++ 3 files changed, 908 insertions(+), 61 deletions(-) create mode 100644 crates/validator/src/payload_attestation.rs diff --git a/crates/validator/src/duties.rs b/crates/validator/src/duties.rs index 5f05c7ab6..6b9744df9 100644 --- a/crates/validator/src/duties.rs +++ b/crates/validator/src/duties.rs @@ -19,6 +19,13 @@ //! dropped as they come in rather than at every lookup. //! - **A change here is not a change for subscriptions.** Subnet subscriptions //! follow committees, and a proposer has none. +//! +//! Payload timeliness committee (PTC) duties, which exist from gloas, are held +//! in a third map on the attester schedule's terms: the current epoch and the +//! next, replaced when the `dependent_root` changes. They are narrowed to this +//! client's indices on arrival, and are not part of subnet subscriptions +//! either. Whether an epoch is a gloas one is the caller's to decide, since +//! this service holds no fork schedule. use std::collections::HashMap; use std::sync::Arc; @@ -28,7 +35,7 @@ use ethlambda_types::beacon::primitives::{Epoch, Root, Slot, ValidatorIndex}; use tracing::{info, warn}; use crate::beacon_node::BeaconNodeApi; -use crate::beacon_node::dto::{AttesterDutyDto, ProposerDutyDto}; +use crate::beacon_node::dto::{AttesterDutyDto, ProposerDutyDto, PtcDutyDto}; use crate::error::Result; /// One epoch's schedule of some kind, and the block root it is derived from. @@ -46,6 +53,8 @@ pub struct DutiesService { /// Proposer duties, already narrowed to `indices`. See the module doc for /// why this is a second map rather than another field on the first. proposers: HashMap>, + /// Payload timeliness committee duties, already narrowed to `indices`. + ptc: HashMap>, /// Whether the "no validator indices resolved" warning has already fired. /// Without this, an idle client would repeat it every epoch forever; with /// it, the operator still gets exactly one signal that duties are not @@ -60,6 +69,7 @@ impl DutiesService { indices, by_epoch: HashMap::new(), proposers: HashMap::new(), + ptc: HashMap::new(), warned_no_indices: false, } } @@ -197,6 +207,74 @@ impl DutiesService { Ok(()) } + /// Fetch `epoch`'s payload timeliness committee duties for this client's + /// indices, replacing anything held for it if the `dependent_root` has + /// changed. + /// + /// Only to be called for a gloas epoch: an earlier one answers with no + /// duties, which is harmless but a request wasted every epoch. + pub async fn refresh_ptc(&mut self, epoch: Epoch) -> Result<()> { + if self.indices.is_empty() { + return Ok(()); + } + + let fetched = self.beacon_node.ptc_duties(epoch, &self.indices).await?; + + let held = self.ptc.get(&epoch); + if held.is_some_and(|held| held.dependent_root == fetched.dependent_root) { + return Ok(()); + } + if held.is_some() { + warn!( + %epoch, + "Payload timeliness committee duties invalidated by a reorg; replacing the schedule" + ); + } + + // The node was asked about these indices only, but a node that answers + // for more must not make this client vote for validators it does not + // hold. + let mine: Vec = fetched + .duties + .into_iter() + .filter(|duty| self.indices.contains(&duty.validator_index)) + .collect(); + if mine.is_empty() { + tracing::debug!(%epoch, "No payload timeliness committee duties this epoch"); + } else { + info!(%epoch, count = mine.len(), "Payload timeliness committee duties updated"); + } + self.ptc.insert( + epoch, + EpochDuties { + dependent_root: fetched.dependent_root, + duties: mine, + }, + ); + Ok(()) + } + + /// Forget payload timeliness committee duties for epochs before `epoch`. + pub fn prune_ptc_before(&mut self, epoch: Epoch) { + self.ptc.retain(|held, _| *held >= epoch); + } + + /// This client's committee duties at `slot`. More than one is ordinary: a + /// client holding several validators can have several seats in one slot's + /// committee. + pub fn ptc_at_slot(&self, slot: Slot, epoch: Epoch) -> Vec { + self.ptc + .get(&epoch) + .map(|held| { + held.duties + .iter() + .filter(|duty| duty.slot == slot) + .cloned() + .collect() + }) + .unwrap_or_default() + } + /// This client's proposer duty for `slot`, if it has one. /// /// At most one, and not because of any filtering here: a slot has exactly @@ -605,4 +683,85 @@ mod tests { service.refresh_proposers(3).await.expect("no-op"); assert_eq!(node.proposer_duties_call_count(), 0); } + + fn ptc_duty(validator_index: ValidatorIndex, slot: Slot) -> PtcDutyDto { + PtcDutyDto { + pubkey: "0x00".to_string(), + validator_index, + slot, + } + } + + #[tokio::test] + async fn payload_committee_duties_are_fetched_filtered_and_indexed_by_slot() { + let node = Arc::new(MockBeaconNode::new().with_ptc_duties( + 12, + Root::repeat_byte(1), + // Validator 99 is not ours and must be dropped on arrival. + vec![ptc_duty(7, 390), ptc_duty(99, 390), ptc_duty(7, 395)], + )); + let mut service = DutiesService::new(node.clone(), vec![7]); + + service.refresh_ptc(12).await.expect("refreshes"); + + let at_390 = service.ptc_at_slot(390, 12); + assert_eq!(at_390.len(), 1); + assert_eq!(at_390[0].validator_index, 7); + assert_eq!(service.ptc_at_slot(395, 12).len(), 1); + assert!(service.ptc_at_slot(391, 12).is_empty()); + assert!( + service.ptc_at_slot(390, 13).is_empty(), + "scoped to the epoch" + ); + } + + /// A reorg past the dependent root replaces the schedule; an unchanged + /// root keeps it, which is what stops a refresh every epoch from churning. + #[tokio::test] + async fn a_changed_ptc_dependent_root_replaces_the_schedule() { + let node = Arc::new(MockBeaconNode::new().with_ptc_duties( + 12, + Root::repeat_byte(1), + vec![ptc_duty(7, 390)], + )); + let mut service = DutiesService::new(node.clone(), vec![7]); + service.refresh_ptc(12).await.expect("first"); + + node.set_ptc_duties(12, Root::repeat_byte(1), vec![ptc_duty(7, 391)]); + service.refresh_ptc(12).await.expect("same root"); + assert_eq!( + service.ptc_at_slot(390, 12).len(), + 1, + "unchanged root keeps it" + ); + + node.set_ptc_duties(12, Root::repeat_byte(2), vec![ptc_duty(7, 391)]); + service.refresh_ptc(12).await.expect("new root"); + assert!(service.ptc_at_slot(390, 12).is_empty()); + assert_eq!(service.ptc_at_slot(391, 12).len(), 1); + } + + #[tokio::test] + async fn pruning_forgets_earlier_payload_committee_epochs() { + let node = Arc::new( + MockBeaconNode::new() + .with_ptc_duties(12, Root::repeat_byte(1), vec![ptc_duty(7, 390)]) + .with_ptc_duties(13, Root::repeat_byte(1), vec![ptc_duty(7, 420)]), + ); + let mut service = DutiesService::new(node, vec![7]); + service.refresh_ptc(12).await.expect("12"); + service.refresh_ptc(13).await.expect("13"); + + service.prune_ptc_before(13); + assert!(service.ptc_at_slot(390, 12).is_empty()); + assert_eq!(service.ptc_at_slot(420, 13).len(), 1); + } + + #[tokio::test] + async fn no_indices_means_no_payload_committee_request() { + let node = Arc::new(MockBeaconNode::new()); + let mut service = DutiesService::new(node.clone(), Vec::new()); + service.refresh_ptc(12).await.expect("idles"); + assert_eq!(node.ptc_call_count(), 0); + } } diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index a54a39fe6..95c2c9cb1 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -39,6 +39,7 @@ pub mod error; pub mod http_api; pub mod keys; pub mod metrics; +pub mod payload_attestation; pub mod proposal; pub mod proposal_guard; pub(crate) mod secure_fs; @@ -64,6 +65,7 @@ use crate::beacon_node::fallback::FallbackBeaconNode; use crate::beacon_node::http::HttpBeaconNode; use crate::duties::DutiesService; use crate::keys::ValidatorStore; +use crate::payload_attestation::PayloadAttestationService; use crate::proposal::ProposalService; use crate::signing::SigningContext; use crate::slot_clock::SlotClock; @@ -176,6 +178,7 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { let mut duties = DutiesService::new(beacon_node.clone(), Vec::new()); let attestation = AttestationService::new(beacon_node.clone(), context.clone()); let aggregation = AggregationService::new(beacon_node.clone(), context.clone()); + let payload_attestation = PayloadAttestationService::new(beacon_node.clone(), context.clone()); let proposal = ProposalService::new( beacon_node.clone(), context.clone(), @@ -292,72 +295,167 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { tokio::time::sleep(clock.until_attestation(slot, SystemTime::now())).await; let slot_duties = duties.at_slot(slot, epoch); - if slot_duties.is_empty() { - continue; + let committee = duties.ptc_at_slot(slot, epoch); + serve_slot_duties( + &attestation, + &aggregation, + &payload_attestation, + &clock, + slot, + &slot_duties, + &committee, + &store, + ) + .await; + } +} + +/// Everything a slot owes after its proposal: attest and aggregate, then the +/// payload timeliness committee vote. +/// +/// The two halves are independent on purpose. Committee membership is drawn +/// separately from attester duties, so a client can sit on a slot's committee +/// with no attester duty in it at all, and an early return for "no attester +/// duties" would skip exactly those votes. The committee half is also gated on +/// the slot being a gloas one: before the fork there is no committee and no +/// payload to vote on. +/// +/// Three quarters into the slot the committee sleeps until its own deadline, +/// then runs bounded by the end of the slot like the other duties. +#[allow(clippy::too_many_arguments)] +async fn serve_slot_duties( + attestation: &AttestationService, + aggregation: &AggregationService, + payload_attestation: &PayloadAttestationService, + clock: &SlotClock, + slot: u64, + slot_duties: &[crate::beacon_node::dto::AttesterDutyDto], + committee: &[crate::beacon_node::dto::PtcDutyDto], + store: &RwLock, +) { + if !slot_duties.is_empty() { + attest_and_aggregate(attestation, aggregation, clock, slot, slot_duties, store).await; + } + + if clock.is_gloas(slot) && !committee.is_empty() { + tokio::time::sleep(clock.until_payload_attestation(slot, SystemTime::now())).await; + payload_attest(payload_attestation, clock, slot, committee, store).await; + } +} + +/// Run one slot's attestation duty and, when it produced data, the aggregation +/// duty that follows it. +/// +/// Split out of the loop so the loop can go on to the payload committee duty +/// whether or not this client has an attester duty in the slot. +async fn attest_and_aggregate( + attestation: &AttestationService, + aggregation: &AggregationService, + clock: &SlotClock, + slot: u64, + slot_duties: &[crate::beacon_node::dto::AttesterDutyDto], + store: &RwLock, +) { + // Bound the slot's work by what is left of the slot. + // + // Nothing else does. The per-request timeout in `HttpBeaconNode` is 8 + // seconds, failover tries each node in turn, and `attest` makes two + // calls, so one hung node can carry a 12-second slot's duty well past + // the slot itself and into the next one, whose duty is then late in + // turn. An attestation that misses its slot is worth little; one that + // also delays the next slot's is worth less than nothing. + // + // Dropping the future mid-flight is safe here specifically because of + // `AttestationGuard`: it records at signing time, so a duty abandoned + // between signing and submission cannot be re-signed next slot under + // the same target. That is the intended outcome and matches `attest`'s + // own "do not retry within a slot" rule; the attestation is simply + // lost. + let budget = clock.remaining_in(slot, SystemTime::now()); + let attempt = tokio::time::timeout(budget, attestation.attest(slot, slot_duties, store)); + match attempt.await.unwrap_or_else(|_| { + warn!( + %slot, + budget_ms = budget.as_millis() as u64, + "Attestation duty ran past the end of its slot and was abandoned" + ); + metrics::inc_attestation_deadline_missed(); + Err(Error::AttestationDeadline { slot }) + }) { + // `attest` scopes its own read guard away from every await (see + // its doc comment), so passing the shared `store` straight + // through here holds nothing across this call. + Ok(attested) => { + if attested.published > 0 { + let elapsed = SystemTime::now() + .duration_since(clock.start_of(slot)) + .unwrap_or_default(); + metrics::observe_publication_delay(elapsed.as_secs_f64()); + } + + // Aggregation, for whichever of this slot's duties this client + // was selected for. + // + // Reached only when the attestation duty produced data, since + // that data is what names the aggregate to ask for. It is not + // gated on anything having been *published*, though: an + // aggregator collects the whole committee's votes, so the duty + // is still owed when this client's own signatures were refused. + if let Some(data) = attested.data { + let delay = clock.until_aggregation(slot, SystemTime::now()); + tokio::time::sleep(delay).await; + aggregate(aggregation, clock, slot, &data, slot_duties, store).await; + } + } + Err(err) => { + error!(%slot, %err, "Failed to publish attestations for this slot"); + metrics::inc_attestation_failures(); + // Every error `attest` can return originates from the beacon + // node (a failed fetch, a bad or mismatched response, a + // failed submission), so a failure here is exactly the signal + // this gauge exists for: `refresh_epoch` only runs once an + // epoch and would otherwise leave it reporting stale + // availability for up to that long. + metrics::set_beacon_node_available(false); } + } +} - // Bound the slot's work by what is left of the slot. - // - // Nothing else does. The per-request timeout in `HttpBeaconNode` is 8 - // seconds, failover tries each node in turn, and `attest` makes two - // calls, so one hung node can carry a 12-second slot's duty well past - // the slot itself and into the next one, whose duty is then late in - // turn. An attestation that misses its slot is worth little; one that - // also delays the next slot's is worth less than nothing. - // - // Dropping the future mid-flight is safe here specifically because of - // `AttestationGuard`: it records at signing time, so a duty abandoned - // between signing and submission cannot be re-signed next slot under - // the same target. That is the intended outcome and matches `attest`'s - // own "do not retry within a slot" rule; the attestation is simply - // lost. - let budget = clock.remaining_in(slot, SystemTime::now()); - let attempt = tokio::time::timeout(budget, attestation.attest(slot, &slot_duties, &store)); - match attempt.await.unwrap_or_else(|_| { +/// Run one slot's payload timeliness committee duty, bounded by what is left of +/// the slot. +/// +/// The budget is the end of the slot, like the aggregation's and for the same +/// reason: this is the last duty in its slot, so the only thing past its +/// deadline is the next slot's work. A vote that arrives after its slot is +/// worthless, since the committee's attestations are only counted for the slot +/// they are about. +/// +/// Abandoning it is safe: the service records per validator at signing time, so +/// a duty dropped mid-flight is lost rather than re-signed. Failures are logged +/// and counted rather than propagated, like every other duty's. +async fn payload_attest( + service: &PayloadAttestationService, + clock: &SlotClock, + slot: u64, + committee: &[crate::beacon_node::dto::PtcDutyDto], + store: &RwLock, +) { + let budget = clock.remaining_in(slot, SystemTime::now()); + let attempt = tokio::time::timeout(budget, service.attest(slot, committee, store)); + match attempt.await { + Ok(Ok(_)) => {} + Ok(Err(err)) => { + error!(%slot, %err, "Failed to publish payload attestations for this slot"); + metrics::inc_payload_attestation_failures(); + metrics::set_beacon_node_available(false); + } + Err(_) => { warn!( %slot, budget_ms = budget.as_millis() as u64, - "Attestation duty ran past the end of its slot and was abandoned" + "Payload attestation duty ran past the end of its slot and was abandoned" ); - metrics::inc_attestation_deadline_missed(); - Err(Error::AttestationDeadline { slot }) - }) { - // `attest` scopes its own read guard away from every await (see - // its doc comment), so passing the shared `store` straight - // through here holds nothing across this call. - Ok(attested) => { - if attested.published > 0 { - let elapsed = SystemTime::now() - .duration_since(clock.start_of(slot)) - .unwrap_or_default(); - metrics::observe_publication_delay(elapsed.as_secs_f64()); - } - - // Aggregation, two thirds in, for whichever of this slot's - // duties this client was selected for. - // - // Reached only when the attestation duty produced data, since - // that data is what names the aggregate to ask for. It is not - // gated on anything having been *published*, though: an - // aggregator collects the whole committee's votes, so the duty - // is still owed when this client's own signatures were refused. - if let Some(data) = attested.data { - let delay = clock.until_aggregation(slot, SystemTime::now()); - tokio::time::sleep(delay).await; - aggregate(&aggregation, &clock, slot, &data, &slot_duties, &store).await; - } - } - Err(err) => { - error!(%slot, %err, "Failed to publish attestations for this slot"); - metrics::inc_attestation_failures(); - // Every error `attest` can return originates from the beacon - // node (a failed fetch, a bad or mismatched response, a - // failed submission), so a failure here is exactly the signal - // this gauge exists for: `refresh_epoch` only runs once an - // epoch and would otherwise leave it reporting stale - // availability for up to that long. - metrics::set_beacon_node_available(false); - } + metrics::inc_payload_attestation_failures(); } } } @@ -539,6 +637,7 @@ async fn refresh_epoch( } let changed = duties.refresh_around(epoch).await?; + refresh_ptc(duties, epoch, context).await; metrics::set_duties_held(duties.all().len() as u64); if changed { subscriptions::subscribe(beacon_node, &duties.all(), store, context).await?; @@ -547,6 +646,34 @@ async fn refresh_epoch( Ok(()) } +/// Fetch the payload timeliness committee schedule for this epoch and the next, +/// each only if it is a gloas epoch. +/// +/// Gated per epoch rather than once, because the fork can activate between the +/// two: the lookahead is the epoch the first gloas slots belong to when the +/// current one is the last before the fork. A failed fetch is logged and +/// shrugged off, like the proposer and lookahead ones: it must not cost this +/// epoch's attester schedule, which is already in hand. +async fn refresh_ptc( + duties: &mut DutiesService, + epoch: u64, + context: &SigningContext, +) { + for target in [epoch, epoch + 1] { + if context.config.fork_at_epoch(target) < ethlambda_types::beacon::fork::ForkName::Gloas { + continue; + } + if let Err(err) = duties.refresh_ptc(target).await { + warn!( + epoch = target, + %err, + "Payload committee duties fetch failed; this client may miss its votes" + ); + } + } + duties.prune_ptc_before(epoch); +} + #[cfg(test)] mod tests { use super::*; @@ -727,4 +854,186 @@ mod tests { refresh(node.clone(), Some(H160([0xab; 20]))).await; assert!(node.preparations().is_empty()); } + + // ---- gloas ---------------------------------------------------------- + + fn gloas_context(epoch: u64) -> SigningContext { + SigningContext { + config: ethlambda_types::beacon::config::Config::mainnet() + .with_fork_epoch(ethlambda_types::beacon::fork::ForkName::Gloas, epoch), + genesis_validators_root: Root::ZERO, + } + } + + fn ptc_duty(index: u64, slot: u64) -> crate::beacon_node::dto::PtcDutyDto { + crate::beacon_node::dto::PtcDutyDto { + pubkey: crate::beacon_node::dto::encode_hex(&[0u8; 48]), + validator_index: index, + slot, + } + } + + /// The committee schedule is fetched for gloas epochs only, so a chain that + /// has not forked is never asked, and the lookahead is gated on its own + /// epoch: the last pre-gloas epoch looks ahead into the first gloas one. + #[tokio::test] + async fn the_committee_schedule_is_fetched_for_gloas_epochs_only() { + let node = Arc::new( + node() + .with_ptc_duties(4, Root::repeat_byte(1), vec![ptc_duty(11, 130)]) + .with_ptc_duties(5, Root::repeat_byte(1), Vec::new()), + ); + let mut duties = DutiesService::new(node.clone(), Vec::new()); + let keys = [pubkey(1), pubkey(2)]; + + // Epoch 3 is the last before gloas at epoch 4: only the lookahead asks. + node.set_duties(3, Root::repeat_byte(1), Vec::new()); + refresh_epoch( + &node, + &mut duties, + &keys, + 3, + None, + &empty_store(), + &gloas_context(4), + ) + .await + .expect("refreshes"); + assert_eq!( + node.ptc_call_count(), + 1, + "only epoch 4, the first gloas one" + ); + assert_eq!(duties.ptc_at_slot(130, 4).len(), 1); + + // A chain with no gloas scheduled is never asked at all. + let node = Arc::new(node_without_ptc()); + let mut duties = DutiesService::new(node.clone(), Vec::new()); + refresh_epoch( + &node, + &mut duties, + &keys, + 3, + None, + &empty_store(), + &context(), + ) + .await + .expect("refreshes"); + assert_eq!(node.ptc_call_count(), 0); + } + + fn node_without_ptc() -> MockBeaconNode { + node() + } + + /// A failed committee fetch must not cost the epoch's attester schedule. + #[tokio::test] + async fn a_failed_committee_fetch_does_not_fail_the_refresh() { + let node = Arc::new(node().failing_call("ptc_duties", "node is unhappy")); + let mut duties = DutiesService::new(node.clone(), Vec::new()); + let keys = [pubkey(1), pubkey(2)]; + refresh_epoch( + &node, + &mut duties, + &keys, + 3, + None, + &empty_store(), + &gloas_context(0), + ) + .await + .expect("the refresh survives it"); + assert_eq!(duties.indices(), &[11, 22]); + } + + /// The restructured loop: a client on a slot's committee with no attester + /// duty at all must still vote. This is the case the old early `continue` + /// on "no attester duties" would have skipped. + #[tokio::test] + async fn a_committee_member_with_no_attester_duty_still_votes() { + use crate::beacon_node::mock::MockBeaconNode; + use ethlambda_types::beacon::containers::gloas::PayloadAttestationData; + + let mut config = ethlambda_types::beacon::config::Config::mainnet() + .with_fork_epoch(ethlambda_types::beacon::fork::ForkName::Gloas, 0); + config.slot_duration_ms = 1_000; + let context = Arc::new(SigningContext { + config: config.clone(), + genesis_validators_root: Root::ZERO, + }); + let now_secs = SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("after the epoch") + .as_secs(); + let clock = SlotClock::from_config(now_secs - 100, &config); + let slot = clock.now().expect("after genesis") + 1; + + let mut store = ValidatorStore::new(); + let secret: [u8; 32] = + hex::decode("000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f") + .expect("hex") + .try_into() + .expect("32 bytes"); + let key = store.insert_secret("test", &secret).expect("inserts"); + let store = RwLock::new(store); + + let node = Arc::new(MockBeaconNode::new().with_payload_attestation_data( + PayloadAttestationData { + beacon_block_root: Root::repeat_byte(5), + slot, + payload_present: true, + blob_data_available: true, + }, + )); + let committee = [crate::beacon_node::dto::PtcDutyDto { + pubkey: crate::beacon_node::dto::encode_hex(&key.0), + validator_index: 3, + slot, + }]; + + serve_slot_duties( + &AttestationService::new(node.clone(), context.clone()), + &AggregationService::new(node.clone(), context.clone()), + &PayloadAttestationService::new(node.clone(), context.clone()), + &clock, + slot, + &[], + &committee, + &store, + ) + .await; + + assert_eq!(node.attestation_data_call_count(), 0, "no attester duty"); + assert_eq!(node.submitted_payload_attestations().len(), 1); + } + + /// Before gloas there is no committee: even a stale schedule entry is not + /// acted on. + #[tokio::test] + async fn no_committee_vote_is_cast_before_gloas() { + use crate::beacon_node::mock::MockBeaconNode; + + let config = ethlambda_types::beacon::config::Config::mainnet(); + let context = Arc::new(SigningContext { + config: config.clone(), + genesis_validators_root: Root::ZERO, + }); + let clock = SlotClock::from_config(0, &config); + let node = Arc::new(MockBeaconNode::new()); + let store = empty_store(); + + serve_slot_duties( + &AttestationService::new(node.clone(), context.clone()), + &AggregationService::new(node.clone(), context.clone()), + &PayloadAttestationService::new(node.clone(), context.clone()), + &clock, + 5, + &[], + &[ptc_duty(3, 5)], + &store, + ) + .await; + assert_eq!(node.payload_attestation_data_call_count(), 0); + } } diff --git a/crates/validator/src/payload_attestation.rs b/crates/validator/src/payload_attestation.rs new file mode 100644 index 000000000..0870c038a --- /dev/null +++ b/crates/validator/src/payload_attestation.rs @@ -0,0 +1,379 @@ +//! Voting on whether a slot's payload arrived: the payload timeliness +//! committee (PTC) duty that gloas adds. +//! +//! A small committee is drawn each slot, and its members attest to two facts +//! about the block they saw: whether the builder revealed the payload on time +//! (`payload_present`) and whether the blob data was available +//! (`blob_data_available`). The beacon node assembles those facts from its own +//! view, so this client does not judge them: it asks for the data, checks it is +//! about the right slot, signs it and submits it, which is the same division of +//! labour the attestation duty has. +//! +//! # Timing +//! +//! Due at `PAYLOAD_ATTESTATION_DUE_BPS` of the slot, three quarters of the way +//! in, which is when the node's view of the payload is as complete as it will +//! be. See [`crate::slot_clock::SlotClock::until_payload_attestation`]. +//! +//! # No block, no vote +//! +//! A node that knows no block for the slot answers 204. That is not an error +//! and not a vote against the payload: there is nothing to attest to, so +//! nothing is signed. Voting `payload_present = false` for a block this client +//! never saw would be a claim about a block it cannot name. +//! +//! # Duplicates +//! +//! The specification attaches no slashing condition to a PTC vote, but a second, +//! different vote from one member for one slot is equivocation the network +//! penalises in gossip. So the slot is recorded per validator at signing time, +//! the way the attestation guard records, and a duty abandoned between signing +//! and submission is lost rather than re-signed. In memory only: this is not +//! slashing protection, and does not pretend to be. + +use std::collections::HashSet; +use std::sync::Arc; + +use ethlambda_types::beacon::containers::gloas::PayloadAttestationMessage; +use ethlambda_types::beacon::primitives::{Slot, ValidatorIndex}; +use tokio::sync::RwLock; +use tracing::{error, info, warn}; + +use crate::beacon_node::BeaconNodeApi; +use crate::beacon_node::dto::{PtcDutyDto, parse_pubkey}; +use crate::error::Result; +use crate::keys::ValidatorStore; +use crate::signing::SigningContext; + +/// How many slots of dedup history are kept. Far more than the one slot a duty +/// is ever owed for; the margin only absorbs a clock that steps back a little. +const HISTORY_SLOTS: u64 = 64; + +pub struct PayloadAttestationService { + beacon_node: Arc, + context: Arc, + /// Which (validator, slot) pairs already produced a signature. + /// + /// A `std::sync::Mutex` for the reason the attestation service's guard is: + /// the critical section is a lookup and an insert with no await inside it. + done: std::sync::Mutex>, +} + +impl PayloadAttestationService { + pub fn new(beacon_node: Arc, context: Arc) -> Self { + Self { + beacon_node, + context, + done: std::sync::Mutex::new(HashSet::new()), + } + } + + /// Sign and submit this client's committee votes for `slot`. + /// + /// Returns how many votes the node accepted: zero when the node has no + /// block for the slot, or when every member was already served. + /// + /// # Do not retry this call within a slot + /// + /// For the reason the attestation duty's is not retried: a second call + /// fetches the data again, and a head that moved in between would give a + /// different vote from the same member. The dedup refuses it, so a retry is + /// merely useless, but callers should not rely on it. + pub async fn attest( + &self, + slot: Slot, + duties: &[PtcDutyDto], + store: &RwLock, + ) -> Result { + if duties.is_empty() { + return Ok(0); + } + + // Cheap pre-check, so a slot already served costs the node no request. + let pending: Vec<&PtcDutyDto> = { + let done = self.lock(); + duties + .iter() + .filter(|duty| !done.contains(&(duty.validator_index, slot))) + .collect() + }; + if pending.is_empty() { + return Ok(0); + } + + let Some(data) = self.beacon_node.payload_attestation_data(slot).await? else { + info!(%slot, "No block known for this slot; not voting on its payload"); + return Ok(0); + }; + + // Checked again here, having already been checked by the + // implementation (see `BeaconNodeApi::payload_attestation_data`), for + // the reason the attestation duty does: the trait is public, and this + // is the last point before the data becomes a signature. + if data.slot != slot { + return Err(crate::error::Error::InconsistentResponse(format!( + "requested payload attestation data for slot {slot}, node answered for slot {}", + data.slot + ))); + } + + let mut messages = Vec::with_capacity(pending.len()); + { + let _timing = crate::metrics::time_signing(); + let store = store.read().await; + for duty in pending { + let pubkey = match parse_pubkey(&duty.pubkey) { + Ok(pubkey) => pubkey, + Err(err) => { + error!(%slot, validator = duty.validator_index, %err, "Duty carried an unreadable pubkey"); + continue; + } + }; + + // Recorded before signing, so a signature that exists is one + // the record already knows about. + if !self.record(duty.validator_index, slot) { + continue; + } + + match self + .context + .sign_payload_attestation(&store, &pubkey, &data) + { + Ok(signature) => messages.push(PayloadAttestationMessage { + validator_index: duty.validator_index, + data, + signature, + }), + Err(err) => { + error!(%slot, validator = duty.validator_index, %err, "Failed to sign payload attestation"); + crate::metrics::inc_signing_failures(); + } + } + } + } + + if messages.is_empty() { + return Ok(0); + } + + let submitted = messages.len(); + let published = self + .beacon_node + .submit_payload_attestations(&messages) + .await?; + if published < submitted { + warn!( + %slot, + published, + submitted, + "Some payload attestations in this slot's batch were rejected by the beacon node" + ); + } + info!( + %slot, + count = published, + payload_present = data.payload_present, + blob_data_available = data.blob_data_available, + "Published payload attestations" + ); + crate::metrics::inc_payload_attestations_published(published as u64); + Ok(published) + } + + /// Record that `validator` is being served for `slot`. `false` when it + /// already was, or when the record is unusable, which is treated as a + /// refusal for the reason the other guards' poisoned locks are. + fn record(&self, validator: ValidatorIndex, slot: Slot) -> bool { + let Ok(mut done) = self.done.lock() else { + error!(%slot, "Payload attestation record is poisoned; refusing to sign"); + return false; + }; + done.retain(|(_, held)| held + HISTORY_SLOTS > slot); + done.insert((validator, slot)) + } + + fn lock(&self) -> std::sync::MutexGuard<'_, HashSet<(ValidatorIndex, Slot)>> { + // A poisoned set is still a set: the worst a panic mid-insert could + // leave is a missing entry, and the signing path re-checks through + // `record`, which refuses on poison. + self.done + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::beacon_node::dto::encode_hex; + use crate::beacon_node::mock::MockBeaconNode; + use crate::error::Error; + use ethlambda_types::beacon::config::Config; + use ethlambda_types::beacon::containers::gloas::PayloadAttestationData; + use ethlambda_types::beacon::fork::ForkName; + use ethlambda_types::beacon::preset::SLOTS_PER_EPOCH; + use ethlambda_types::beacon::primitives::{BlsPubkey, Root}; + + const GLOAS_EPOCH: u64 = 10; + + fn secret() -> [u8; 32] { + hex::decode("000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f") + .expect("valid hex") + .try_into() + .expect("32 bytes") + } + + fn context() -> Arc { + Arc::new(SigningContext { + config: Config::mainnet().with_fork_epoch(ForkName::Gloas, GLOAS_EPOCH), + genesis_validators_root: Root::ZERO, + }) + } + + fn store() -> (RwLock, BlsPubkey) { + let mut store = ValidatorStore::new(); + let pubkey = store.insert_secret("test", &secret()).expect("inserts"); + (RwLock::new(store), pubkey) + } + + fn slot() -> Slot { + GLOAS_EPOCH * SLOTS_PER_EPOCH + 4 + } + + fn data(slot: Slot) -> PayloadAttestationData { + PayloadAttestationData { + beacon_block_root: Root::repeat_byte(5), + slot, + payload_present: true, + blob_data_available: false, + } + } + + fn duty(pubkey: &BlsPubkey, validator_index: u64, slot: Slot) -> PtcDutyDto { + PtcDutyDto { + pubkey: encode_hex(&pubkey.0), + validator_index, + slot, + } + } + + #[tokio::test] + async fn a_committee_member_signs_and_submits_the_nodes_data() { + use blst::min_pk::{PublicKey, Signature}; + + let (store, pubkey) = store(); + let node = Arc::new(MockBeaconNode::new().with_payload_attestation_data(data(slot()))); + let service = PayloadAttestationService::new(node.clone(), context()); + + let published = service + .attest(slot(), &[duty(&pubkey, 9, slot())], &store) + .await + .expect("attests"); + assert_eq!(published, 1); + + let messages = node.submitted_payload_attestations(); + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].validator_index, 9); + assert_eq!(messages[0].data, data(slot())); + + // Over the data's root alone, under the PTC domain at the data's epoch. + let root = context().payload_attestation_signing_root(&messages[0].data); + let pk = PublicKey::from_bytes(&pubkey.0).expect("valid pubkey"); + let sig = Signature::from_bytes(&messages[0].signature.0).expect("valid signature"); + assert_eq!( + sig.verify( + true, + root.as_slice(), + b"BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_", + &[], + &pk, + true + ), + blst::BLST_ERROR::BLST_SUCCESS + ); + } + + /// A 204 is an ordinary answer: nothing is signed and nothing fails. + #[tokio::test] + async fn no_block_for_the_slot_means_no_vote() { + let (store, pubkey) = store(); + let node = Arc::new(MockBeaconNode::new()); + let service = PayloadAttestationService::new(node.clone(), context()); + + let published = service + .attest(slot(), &[duty(&pubkey, 9, slot())], &store) + .await + .expect("a 204 is not an error"); + assert_eq!(published, 0); + assert_eq!(node.payload_attestation_data_call_count(), 1); + assert!(node.submitted_payload_attestations().is_empty()); + } + + /// One vote per (validator, slot): the second call neither asks the node + /// nor signs again. + #[tokio::test] + async fn a_member_is_served_once_per_slot() { + let (store, pubkey) = store(); + let node = Arc::new(MockBeaconNode::new().with_payload_attestation_data(data(slot()))); + let service = PayloadAttestationService::new(node.clone(), context()); + let duties = [duty(&pubkey, 9, slot())]; + + service + .attest(slot(), &duties, &store) + .await + .expect("first"); + let again = service + .attest(slot(), &duties, &store) + .await + .expect("second"); + + assert_eq!(again, 0); + assert_eq!(node.submitted_payload_attestations().len(), 1); + assert_eq!(node.payload_attestation_data_call_count(), 1); + } + + /// A new slot is a new vote, and a second member of the same slot is + /// served even after the first. + #[tokio::test] + async fn dedup_is_per_validator_and_per_slot() { + let (store, pubkey) = store(); + let node = Arc::new(MockBeaconNode::new().with_payload_attestation_data(data(slot()))); + let service = PayloadAttestationService::new(node.clone(), context()); + + service + .attest(slot(), &[duty(&pubkey, 9, slot())], &store) + .await + .expect("first member"); + service + .attest(slot(), &[duty(&pubkey, 10, slot())], &store) + .await + .expect("second member"); + assert_eq!(node.submitted_payload_attestations().len(), 2); + } + + /// Data about another slot is never signed. + #[tokio::test] + async fn data_for_another_slot_is_rejected() { + let (store, pubkey) = store(); + let node = Arc::new(MockBeaconNode::new().with_payload_attestation_data(data(slot() - 1))); + let service = PayloadAttestationService::new(node.clone(), context()); + + let err = service + .attest(slot(), &[duty(&pubkey, 9, slot())], &store) + .await + .expect_err("must reject"); + assert!(matches!(err, Error::InconsistentResponse(_)), "got {err:?}"); + assert!(node.submitted_payload_attestations().is_empty()); + } + + #[tokio::test] + async fn no_duties_asks_the_node_for_nothing() { + let (store, _) = store(); + let node = Arc::new(MockBeaconNode::new().with_payload_attestation_data(data(slot()))); + let service = PayloadAttestationService::new(node.clone(), context()); + + assert_eq!(service.attest(slot(), &[], &store).await.expect("no-op"), 0); + assert_eq!(node.payload_attestation_data_call_count(), 0); + } +} From 1ee2374eac644872c5c06b24ca31f85ed3e55997 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:59:58 -0300 Subject: [PATCH 088/132] feat(beacon): serve duties and attestation data at gloas epochs A validator client needs proposer and attester duties and attestation data from the first gloas epoch on, and all three were refused by name. Proposer duties read the lookahead of a fulu or a gloas state: gloas keeps fulu's field and `upgrade_to_gloas` carries it over, so a fulu head can already answer the first gloas epoch. Attestation data at a gloas slot sets `index` as the payload-present flag: 0 when the head block is from the slot itself, else 1 iff fork choice holds the head's payload as FULL (a pre-gloas head counts as FULL, the boundary rule fork choice applies). `committee_index` is optional in the query, as gloas's beacon-APIs have it. --- crates/net/rpc/src/beacon/validator.rs | 215 ++++++++++++++++++------- 1 file changed, 156 insertions(+), 59 deletions(-) diff --git a/crates/net/rpc/src/beacon/validator.rs b/crates/net/rpc/src/beacon/validator.rs index 2d827de73..d1bca404c 100644 --- a/crates/net/rpc/src/beacon/validator.rs +++ b/crates/net/rpc/src/beacon/validator.rs @@ -21,6 +21,8 @@ use ethlambda_types::{ BeaconState, shared::{AttestationData, Checkpoint}, }, + fork::ForkName, + fork_choice::PayloadStatus, preset, primitives::{BlsPubkey, CommitteeIndex, Epoch, ExecutionAddress, Slot, ValidatorIndex}, signing::{compute_epoch_at_slot, compute_start_slot_at_epoch}, @@ -31,12 +33,12 @@ use serde::{Deserialize, Serialize}; use ethlambda_network_api::RpcToP2PRef; use ethlambda_state_transition::beacon::{ - fork_choice::checkpoint_state, + fork_choice::{checkpoint_state, get_head_node}, gossip::attestation::compute_subnet_for_attestation, helpers::accessors::{CommitteeCacheExt as _, get_block_root_at_slot}, }; -use crate::beacon::{ApiError, refuse_validator_duties_from_gloas}; +use crate::beacon::ApiError; pub(crate) fn routes() -> Router { Router::new() @@ -190,7 +192,7 @@ struct ProposerDuty { /// `GET /eth/v1/validator/duties/proposer/{epoch}`. /// -/// Read from fulu's `proposer_lookahead`, which the state keeps for its own +/// Read from the `proposer_lookahead` fulu introduced and gloas keeps, which the state keeps for its own /// epoch and the next `MIN_SEED_LOOKAHEAD` epochs, so any epoch in that window /// is answered without advancing a state. Any other epoch is refused. /// @@ -206,17 +208,18 @@ async fn get_proposer_duties(Path(epoch): Path, State(store): State Result { let epoch = parse_epoch(epoch)?; - refuse_validator_duties_from_gloas( - &store.config(), - epoch, - "gloas proposer duties are not supported", - )?; let (head_root, state) = head(store)?; - let BeaconState::Fulu(fulu) = state.as_ref() else { - return Err(ApiError::BadRequest( - "proposer duties are served from fulu's proposer lookahead only", - )); + // Gloas keeps fulu's lookahead as it is, and `upgrade_to_gloas` carries it + // over, so a fulu head already holds the first gloas epoch's proposers. + let proposer_lookahead = match state.as_ref() { + BeaconState::Fulu(fulu) => &fulu.proposer_lookahead, + BeaconState::Gloas(gloas) => &gloas.proposer_lookahead, + _ => { + return Err(ApiError::BadRequest( + "proposer duties are served from the proposer lookahead of fulu and gloas only", + )); + } }; let state_epoch = compute_epoch_at_slot(state.slot()); let offset = epoch @@ -228,7 +231,7 @@ fn proposer_duties(store: &Store, epoch: &str) -> Result()) .collect::, _>>() .map_err(|_| ApiError::BadRequest("invalid validator index"))?; - refuse_validator_duties_from_gloas( - &store.config(), - epoch, - "gloas attester duties are not supported", - )?; let (head_root, state) = head(store)?; let state_epoch = compute_epoch_at_slot(state.slot()); @@ -363,11 +361,11 @@ fn attester_duties( #[derive(Debug, Deserialize)] struct AttestationDataQuery { slot: Slot, - /// Required by the endpoint, and ignored: from electra on the committee - /// travels outside `AttestationData`, whose `index` is always zero, so - /// every committee of a slot attests to the same data. + /// Ignored, and optional since gloas's beacon-APIs dropped it: from electra + /// on the committee travels outside `AttestationData`, whose `index` no + /// longer names it, so every committee of a slot attests to the same data. #[allow(dead_code)] - committee_index: CommitteeIndex, + committee_index: Option, } /// `GET /eth/v1/validator/attestation_data?slot&committee_index`. @@ -384,7 +382,8 @@ struct AttestationDataQuery { /// - `target` is `slot`'s epoch and its boundary block: the head itself when /// no block has filled the boundary slot since, else the root the state /// recorded there. -/// - `index` is zero, as electra requires. +/// - `index` is zero, as electra requires. At a gloas slot it is the +/// payload-present flag instead; see [`payload_present_index`]. /// /// A slot before the head's, or more than one slot past the wall clock, is /// refused: neither is a slot a validator is asked to attest to. @@ -399,11 +398,6 @@ async fn get_attestation_data( } fn attestation_data(store: &Store, slot: Slot) -> Result { - refuse_validator_duties_from_gloas( - &store.config(), - compute_epoch_at_slot(slot), - "gloas attestation data is not supported", - )?; let (head_root, state) = head(store)?; if slot < state.slot() { return Err(ApiError::BadRequest("slot is before the head block")); @@ -430,15 +424,56 @@ fn attestation_data(store: &Store, slot: Slot) -> Result Result { + if head_state.slot() == slot { + return Ok(0); + } + if !matches!(head_state, BeaconState::Gloas(_)) { + return Ok(1); + } + let status = match store.head_payload_status() { + Some(status) => status, + None => { + get_head_node(store, &store.config()) + .map_err(|_| ApiError::Internal("fork choice head lookup failed"))? + .payload_status + } + }; + Ok(u64::from(status == PayloadStatus::Full)) +} + #[cfg(test)] mod tests { use super::*; @@ -451,6 +486,7 @@ mod tests { fulu::{get_beacon_proposer_indices, initialize_proposer_lookahead}, test_state::with_signing_validators_at, }; + use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::fork::ForkName; use http_body_util::BodyExt as _; use tower::ServiceExt as _; @@ -848,52 +884,113 @@ mod tests { } #[tokio::test] - async fn attestation_data_is_answered_up_to_the_last_fulu_slot_and_refused_after() { - let (status, _) = - with_gloas_next_epoch(|_, head_slot| attestation_data_request(head_slot)).await; - assert_eq!(status, StatusCode::OK); - - let (status, json) = with_gloas_next_epoch(|head_epoch, _| { - attestation_data_request(compute_start_slot_at_epoch(head_epoch + 1)) + async fn a_fulu_head_answers_the_first_gloas_epoch_for_every_duty() { + let (status, _) = with_gloas_next_epoch(|head_epoch, _| { + get_request(format!( + "/eth/v1/validator/duties/proposer/{}", + head_epoch + 1 + )) }) .await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(json["message"], "gloas attestation data is not supported"); - } + assert_eq!(status, StatusCode::OK); - #[tokio::test] - async fn proposer_duties_are_answered_for_the_last_fulu_epoch_and_refused_after() { let (status, _) = with_gloas_next_epoch(|head_epoch, _| { - get_request(format!("/eth/v1/validator/duties/proposer/{head_epoch}")) + Request::post(format!( + "/eth/v1/validator/duties/attester/{}", + head_epoch + 1 + )) + .header("content-type", "application/json") + .body(Body::from("[\"0\"]")) + .unwrap() }) .await; assert_eq!(status, StatusCode::OK); + // The head is a fulu block, which counts as FULL at the boundary. let (status, json) = with_gloas_next_epoch(|head_epoch, _| { - get_request(format!( - "/eth/v1/validator/duties/proposer/{}", - head_epoch + 1 - )) + attestation_data_request(compute_start_slot_at_epoch(head_epoch + 1)) }) .await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(json["message"], "gloas proposer duties are not supported"); + assert_eq!(status, StatusCode::OK); + assert_eq!(json["data"]["index"], "1"); } #[tokio::test] - async fn attester_duties_are_answered_for_the_last_fulu_epoch_and_refused_after() { - let attester = |epoch: Epoch| { - Request::post(format!("/eth/v1/validator/duties/attester/{epoch}")) - .header("content-type", "application/json") - .body(Body::from("[\"0\"]")) - .unwrap() + async fn the_gloas_proposer_duties_match_the_lookahead_of_a_gloas_state() { + let mut state = with_signing_validators_at(ForkName::Gloas, COUNT); + let lookahead = initialize_proposer_lookahead(&state).unwrap(); + let BeaconState::Gloas(gloas) = &mut state else { + unreachable!("built as gloas") }; - let (status, _) = with_gloas_next_epoch(|head_epoch, _| attester(head_epoch)).await; - assert_eq!(status, StatusCode::OK); + gloas.proposer_lookahead = lookahead.clone().try_into().unwrap(); + let epoch = compute_epoch_at_slot(state.slot()); + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 0); + let (store, _) = crate::test_utils::beacon_store_with_config(state, config); + let request = get_request(format!("/eth/v1/validator/duties/proposer/{epoch}")); + let response = routes().with_state(store).oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = response.into_body().collect().await.unwrap().to_bytes(); + let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let proposers: Vec = json["data"] + .as_array() + .unwrap() + .iter() + .map(|duty| duty["validator_index"].as_str().unwrap().parse().unwrap()) + .collect(); + assert_eq!(proposers, lookahead[..preset::SLOTS_PER_EPOCH as usize]); + } + + /// A store whose head is a gloas state `slots_past_boundary` into its + /// epoch, under a schedule with gloas from epoch 0, and the payload status + /// fork choice recorded for that head. + fn gloas_head( + slots_past_boundary: u64, + recorded: Option, + ) -> (Store, BeaconState) { + let mut state = with_signing_validators_at(ForkName::Gloas, COUNT); + let BeaconState::Gloas(gloas) = &mut state else { + unreachable!("built as gloas") + }; + gloas.slot += slots_past_boundary; + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 0); + let (store, head_root) = crate::test_utils::beacon_store_with_config(state.clone(), config); + if let Some(status) = recorded { + store.set_head_payload_status(head_root, status); + } + (store, state) + } - let (status, json) = with_gloas_next_epoch(|head_epoch, _| attester(head_epoch + 1)).await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(json["message"], "gloas attester duties are not supported"); + async fn gloas_index(store: Store, slot: Slot) -> (StatusCode, serde_json::Value) { + // No `committee_index`: gloas's request omits it. + let request = get_request(format!("/eth/v1/validator/attestation_data?slot={slot}")); + let response = routes().with_state(store).oneshot(request).await.unwrap(); + let status = response.status(); + let body = response.into_body().collect().await.unwrap().to_bytes(); + (status, serde_json::from_slice(&body).unwrap()) + } + + #[tokio::test] + async fn a_gloas_vote_in_the_head_blocks_own_slot_has_index_zero() { + let (store, state) = gloas_head(1, Some(PayloadStatus::Full)); + let (status, json) = gloas_index(store, state.slot()).await; + assert_eq!(status, StatusCode::OK, "{json}"); + assert_eq!(json["data"]["index"], "0"); + } + + #[tokio::test] + async fn a_gloas_vote_for_an_earlier_full_head_has_index_one() { + let (store, state) = gloas_head(1, Some(PayloadStatus::Full)); + let (status, json) = gloas_index(store, state.slot() + 1).await; + assert_eq!(status, StatusCode::OK, "{json}"); + assert_eq!(json["data"]["index"], "1"); + } + + #[tokio::test] + async fn a_gloas_vote_for_an_earlier_empty_head_has_index_zero() { + let (store, state) = gloas_head(1, Some(PayloadStatus::Empty)); + let (status, json) = gloas_index(store, state.slot() + 1).await; + assert_eq!(status, StatusCode::OK, "{json}"); + assert_eq!(json["data"]["index"], "0"); } #[tokio::test] From 5f5b94a551ed6d7d940e8814c4f99ca31f164323 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:59:58 -0300 Subject: [PATCH 089/132] feat(beacon): accept gloas attestations and aggregates through the pool endpoints The submit endpoints refused the gloas header and the aggregate query refused gloas slots. A gloas slot now needs the gloas header (and an older one an electra or fulu header), a vote's `data.index` is 0 or 1 and passes the gossip payload-status rule (reused, not reimplemented), and aggregates are served and accepted in gloas's container. Submitted and gossip gloas aggregates are pooled by converting them to the pool's electra shape, so block production keeps one pool. The refusal helper has no caller left and is removed. --- .../state_transition/src/beacon/gossip/mod.rs | 2 +- crates/net/p2p/src/beacon/verdict.rs | 54 +- crates/net/rpc/src/beacon/mod.rs | 43 -- crates/net/rpc/src/beacon/pool.rs | 477 ++++++++++++++---- 4 files changed, 427 insertions(+), 149 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs index 945255d5f..187f612b7 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs @@ -497,7 +497,7 @@ pub(crate) fn is_gloas_slot(config: &Config, slot: Slot) -> bool { /// /// The specification lets a vote on an unseen envelope be queued and asks the /// node to request it by root; neither is done here, so it is an `IGNORE`. -pub(crate) fn verify_attestation_payload_status( +pub fn verify_attestation_payload_status( store: &Store, data: &AttestationData, ) -> Result<(), Outcome> { diff --git a/crates/net/p2p/src/beacon/verdict.rs b/crates/net/p2p/src/beacon/verdict.rs index ebebdef92..3cdabc22a 100644 --- a/crates/net/p2p/src/beacon/verdict.rs +++ b/crates/net/p2p/src/beacon/verdict.rs @@ -18,7 +18,7 @@ use ethlambda_network_api::{AggregateArrival, BlockArrival, BlockSource}; use ethlambda_state_transition::beacon::gossip::{self, IgnoreReason, Outcome}; use ethlambda_state_transition::beacon::helpers::accessors::CommitteeCacheExt as _; use ethlambda_storage::{CacheKey, Store}; -use ethlambda_types::beacon::containers::electra::SingleAttestation; +use ethlambda_types::beacon::containers::electra::{self, SingleAttestation}; use ethlambda_types::beacon::containers::gloas::{ PayloadAttestationMessage, SignedExecutionPayloadEnvelope, }; @@ -268,18 +268,26 @@ impl Validated { /// block fails the whole block. Pooling on arrival also means a slot's /// aggregates, published two thirds of the way through it, are in the pool /// when the next slot's block is asked for at its start, rather than -/// waiting for the chain actor's next tick. Only electra's shape is pooled, -/// since the pool holds electra attestations and electra is the earliest fork -/// this node produces blocks for. +/// waiting for the chain actor's next tick. The pool holds electra's shape, so +/// a gloas aggregate is converted to it first. fn pool_gossip_aggregate(server: &P2PServer, aggregate: &SignedAggregateAndProof) { - let SignedAggregateAndProof::Electra(signed) = aggregate else { - return; + let pooled = match aggregate { + SignedAggregateAndProof::Electra(signed) => signed.message.aggregate.clone(), + // Gloas's aggregate converts to the pool's electra shape; the bits + // only fail to fit electra's bound for a size no committee reaches. + SignedAggregateAndProof::Gloas(signed) => { + let Ok(converted) = electra::Attestation::try_from(&signed.message.aggregate) else { + return; + }; + converted + } + SignedAggregateAndProof::Phase0(_) => return, }; server .attestation_pool .lock() .expect("attestation pool lock poisoned") - .insert_aggregate(signed.message.aggregate.clone()); + .insert_aggregate(pooled); } /// Pool an accepted subnet attestation for a validator client's aggregator. @@ -507,7 +515,9 @@ mod tests { use ethlambda_state_transition::beacon::gossip::{QueueReason, RejectReason}; use ethlambda_types::attestation::{SignedAggregatedAttestation, SignedAttestation}; use ethlambda_types::beacon::config::Config; - use ethlambda_types::beacon::containers::{AttestationData, Checkpoint, electra, phase0}; + use ethlambda_types::beacon::containers::{ + AttestationData, Checkpoint, electra, gloas, phase0, + }; use spawned_concurrency::error::ActorError; use super::*; @@ -826,6 +836,34 @@ mod tests { ); } + /// A gloas aggregate is pooled as the electra-shaped aggregate it converts + /// to, since the pool holds electra's shape. + #[tokio::test] + async fn an_accepted_gloas_aggregate_is_pooled_in_the_pools_shape() { + let server = unconnected_beacon_server(Config::mainnet(), 0).await; + let SignedAggregateAndProof::Electra(electra_signed) = electra_aggregate(5, 1) else { + unreachable!("built as electra") + }; + let expected = electra_signed.message.aggregate; + let gloas_signed = gloas::SignedAggregateAndProof { + message: gloas::AggregateAndProof { + aggregator_index: 1, + aggregate: gloas::Attestation::from(&expected), + selection_proof: Default::default(), + }, + signature: Default::default(), + }; + Validated::Aggregate { + aggregate: Box::new(SignedAggregateAndProof::Gloas(gloas_signed)), + attesting_indices: Vec::new(), + } + .forward(&server, Instant::now(), Outcome::Accept); + assert_eq!( + server.attestation_pool.lock().unwrap().block_candidates(), + vec![expected] + ); + } + /// Only `Accept` means the signatures were verified; anything else must /// stay out of the pool, since one unverified attestation fails the /// whole block it is packed into. diff --git a/crates/net/rpc/src/beacon/mod.rs b/crates/net/rpc/src/beacon/mod.rs index a28b3beaf..2da230150 100644 --- a/crates/net/rpc/src/beacon/mod.rs +++ b/crates/net/rpc/src/beacon/mod.rs @@ -11,7 +11,6 @@ use axum::{ response::{IntoResponse, Response}, }; use ethlambda_storage::Store; -use ethlambda_types::beacon::{config::Config, fork::ForkName, primitives::Epoch}; use serde::Serialize; use crate::shared::block_id::IdError; @@ -63,36 +62,6 @@ pub(crate) enum ApiError { ServiceUnavailable(&'static str), } -/// Refuses `epoch` when the node's schedule places it at gloas: validator -/// duties are not supported from that fork. -/// -/// The node follows gloas but only as a follower. It has no builder bid -/// handling, no payload-timeliness committee duty and no gloas proposer -/// flow, and its attestation pool holds electra-shaped votes. An answer for a -/// gloas epoch would carry fulu's shapes under gloas's fork (a vote a -/// validator client would sign as valid, an aggregate of the wrong shape), so -/// it is refused by name, as the submit endpoints refuse the gloas header. -/// Deliberately not [`ethlambda_types::beacon::fork::ForkName::is_followed`]: -/// following a fork and serving its validator duties are different questions. -pub(crate) fn refuse_validator_duties_from_gloas( - config: &Config, - epoch: Epoch, - message: &'static str, -) -> Result<(), ApiError> { - match config.fork_at_epoch(epoch) { - ForkName::Phase0 - | ForkName::Altair - | ForkName::Bellatrix - | ForkName::Capella - | ForkName::Deneb - | ForkName::Electra - | ForkName::Fulu => Ok(()), - // `Config::fork_at_epoch` never returns Lean; refused like gloas - // rather than answered. - ForkName::Gloas | ForkName::Lean => Err(ApiError::BadRequest(message)), - } -} - impl From for ApiError { fn from(err: IdError) -> Self { match err { @@ -156,18 +125,6 @@ mod tests { assert_eq!(json["message"], "block not found"); } - #[test] - fn validator_duties_are_refused_from_the_gloas_epoch_even_though_gloas_is_followed() { - let mut config = Config::mainnet(); - config.gloas_fork_epoch = 10; - assert!(ForkName::Gloas.is_followed()); - let refuse = |epoch| refuse_validator_duties_from_gloas(&config, epoch, "refused"); - assert!(refuse(0).is_ok()); - assert!(refuse(9).is_ok()); - assert!(matches!(refuse(10), Err(ApiError::BadRequest("refused")))); - assert!(matches!(refuse(11), Err(ApiError::BadRequest("refused")))); - } - #[test] fn a_malformed_id_is_a_400_and_an_absent_one_a_404() { assert_eq!( diff --git a/crates/net/rpc/src/beacon/pool.rs b/crates/net/rpc/src/beacon/pool.rs index 9ce9ecb04..88ad1bab5 100644 --- a/crates/net/rpc/src/beacon/pool.rs +++ b/crates/net/rpc/src/beacon/pool.rs @@ -4,7 +4,8 @@ //! `POST /eth/v2/validator/aggregate_and_proofs`, how it publishes the result. //! //! Each attestation is checked against the electra `beacon_attestation_{subnet_id}` -//! gossip conditions (p2p-interface) that can be evaluated here, then +//! gossip conditions (p2p-interface) that can be evaluated here (gloas's +//! payload-status rule on `data.index` too, at a gloas slot), then //! published on its subnet. Validating before publishing is not optional: a //! peer that relays invalid attestations has its gossipsub score cut, and //! enough of that disconnects it. @@ -29,7 +30,7 @@ use ethlambda_state_transition::beacon::{ attestation_pool::SharedAttestationPool, bls, gossip::attestation::compute_subnet_for_attestation, - gossip::{Outcome, aggregate}, + gossip::{IgnoreReason, Outcome, RejectReason, aggregate, verify_attestation_payload_status}, helpers::accessors::{CommitteeCacheExt as _, get_domain}, }; use ethlambda_storage::Store; @@ -39,6 +40,7 @@ use ethlambda_types::{ containers::{ BeaconState, SignedAggregateAndProof, electra::{self, SingleAttestation}, + gloas, }, fork::ForkName, primitives::{CommitteeIndex, Epoch, Root, Slot}, @@ -49,7 +51,7 @@ use ethlambda_types::{ use serde::{Deserialize, Serialize}; use tracing::{debug, warn}; -use crate::beacon::{ApiError, refuse_validator_duties_from_gloas, validator::head}; +use crate::beacon::{ApiError, validator::head}; pub(crate) fn routes() -> Router { Router::new() @@ -90,9 +92,10 @@ async fn post_pool_attestations( headers: HeaderMap, body: Bytes, ) -> Response { - if let Err(err) = require_electra_or_fulu(&headers) { - return err.into_response(); - } + let header_fork = match submission_fork(&headers) { + Ok(fork) => fork, + Err(err) => return err.into_response(), + }; let Ok(attestations) = serde_json::from_slice::>(&body) else { return ApiError::BadRequest("invalid request body").into_response(); }; @@ -109,7 +112,11 @@ async fn post_pool_attestations( for (index, attestation) in attestations.into_iter().enumerate() { let slot = attestation.data.slot; let validator = attestation.attester_index; - let checked = validate(&store, &state, &attestation, now_ms); + let checked = if header_matches_slot(&store, header_fork, slot) { + validate(&store, &state, &attestation, now_ms) + } else { + Err("Eth-Consensus-Version does not match the attestation slot's fork") + }; // Pooled as well as published: gossip never delivers a node its own // messages, so without this an aggregator served by this node would // be missing its own validator client's votes. @@ -172,8 +179,14 @@ fn validate( return Err("attestation is older than the previous epoch"); } - // [REJECT] data.index == 0, and the target epoch is the slot's. - if data.index != 0 { + // [REJECT] data.index == 0 (electra), or at a gloas slot 0 or 1, the + // payload-present flag; the target epoch is the slot's. + let at_gloas = config.fork_at_epoch(attestation_epoch) == ForkName::Gloas; + if at_gloas { + if data.index > 1 { + return Err("data.index must be 0 or 1 from gloas on"); + } + } else if data.index != 0 { return Err("data.index must be zero from electra on"); } if data.target.epoch != attestation_epoch { @@ -195,6 +208,11 @@ fn validate( { return Err("target root is not the voted block's checkpoint"); } + // [New in Gloas:EIP7732] The payload flag agrees with what this node knows + // of the voted block's payload. + if at_gloas { + verify_attestation_payload_status(store, data).map_err(payload_status_failure)?; + } // [REJECT] The committee index is in range, and the attester is in it. let epoch_committees = store.committee_cache().committees(state, data.target.epoch); @@ -235,22 +253,35 @@ fn validate( }) } -/// The endpoints here take electra's containers, which exist from that fork -/// on and which fulu keeps; `Eth-Consensus-Version` is required to say so. -/// Gloas changed the attestation (a progressive list, and `data.index` as the -/// payload-availability signal) and the pool rules that go with it, neither of -/// which this node models, so it is refused by name rather than read as -/// electra. -fn require_electra_or_fulu(headers: &HeaderMap) -> Result<(), ApiError> { +/// Why a gloas vote failed [`verify_attestation_payload_status`]. The gossip +/// rule's `IGNORE`s are failures here too: a vote this node would not +/// propagate must not be published on a validator client's say-so. +fn payload_status_failure(outcome: Outcome) -> &'static str { + match outcome { + Outcome::Reject(RejectReason::SameSlotPayloadFlag) => { + "a same-slot attestation cannot vote for the payload" + } + Outcome::Ignore(IgnoreReason::PayloadEnvelopeUnseen) => { + "the voted block's payload has not been seen and verified" + } + Outcome::Ignore(IgnoreReason::PayloadOptimistic) => { + "the voted block's payload is optimistic" + } + _ => "the voted block's payload status could not be checked", + } +} + +/// The fork a submit endpoint's `Eth-Consensus-Version` names. The endpoints +/// take electra's containers, which fulu keeps, and gloas's, which are +/// electra's own for a `SingleAttestation` and a new shape for an aggregate; +/// anything earlier is refused. +fn submission_fork(headers: &HeaderMap) -> Result { let fork = headers .get("eth-consensus-version") .and_then(|value| value.to_str().ok()) .and_then(ForkName::parse); match fork { - Some(ForkName::Electra | ForkName::Fulu) => Ok(()), - Some(ForkName::Gloas) => Err(ApiError::BadRequest( - "Eth-Consensus-Version gloas is not supported", - )), + Some(fork @ (ForkName::Electra | ForkName::Fulu | ForkName::Gloas)) => Ok(fork), // `ForkName::parse` never returns Lean: it is absent from `ALL`. Some( ForkName::Phase0 @@ -261,11 +292,20 @@ fn require_electra_or_fulu(headers: &HeaderMap) -> Result<(), ApiError> { | ForkName::Lean, ) | None => Err(ApiError::BadRequest( - "Eth-Consensus-Version must name electra or fulu", + "Eth-Consensus-Version must name electra, fulu or gloas", )), } } +/// Whether `header` is a version the fork at `slot` accepts: gloas's slots need +/// the gloas header, and electra's and fulu's, whose containers are the same, +/// accept either of theirs. +fn header_matches_slot(store: &Store, header: ForkName, slot: Slot) -> bool { + let slot_is_gloas = + store.config().fork_at_epoch(compute_epoch_at_slot(slot)) == ForkName::Gloas; + slot_is_gloas == (header == ForkName::Gloas) +} + /// `200` when nothing failed, else the Beacon API's `IndexedErrorMessage` /// naming each failed item by position; the rest were still published. fn batch_response(failures: Vec, message: &'static str) -> Response { @@ -295,11 +335,27 @@ async fn post_aggregate_and_proofs( headers: HeaderMap, body: Bytes, ) -> Response { - if let Err(err) = require_electra_or_fulu(&headers) { - return err.into_response(); - } - let Ok(aggregates) = serde_json::from_slice::>(&body) - else { + let header_fork = match submission_fork(&headers) { + Ok(fork) => fork, + Err(err) => return err.into_response(), + }; + // Gloas's aggregate is its own container, so the header picks the decoder. + let aggregates = if header_fork == ForkName::Gloas { + serde_json::from_slice::>(&body).map(|signed| { + signed + .into_iter() + .map(SignedAggregateAndProof::Gloas) + .collect() + }) + } else { + serde_json::from_slice::>(&body).map(|signed| { + signed + .into_iter() + .map(SignedAggregateAndProof::Electra) + .collect() + }) + }; + let Ok(aggregates): Result, _> = aggregates else { return ApiError::BadRequest("invalid request body").into_response(); }; @@ -310,27 +366,38 @@ async fn post_aggregate_and_proofs( let capacity = std::num::NonZeroUsize::MIN; let mut failures = Vec::new(); for (index, aggregate) in aggregates.into_iter().enumerate() { - let inner = aggregate.message.aggregate.clone(); - let aggregate = SignedAggregateAndProof::Electra(aggregate); let slot = aggregate.slot(); let aggregator = aggregate.aggregator_index(); let seen = aggregate::SeenAggregates::new(capacity, capacity); - let checked = aggregate::cheap_checks(&seen, &store, &aggregate, now_ms) - .and_then(|()| aggregate::stateful_checks(&store, &aggregate).map(|_| ())); - let published = checked - .map_err(|outcome: Outcome| { - warn!(%slot, aggregator, ?outcome, "Refused a submitted aggregate"); - "aggregate failed validation" - }) - .and_then(|()| { - // Recorded for block production, which packs the aggregates - // this node has validated. + let checked = if header_matches_slot(&store, header_fork, slot) { + aggregate::cheap_checks(&seen, &store, &aggregate, now_ms) + .and_then(|()| aggregate::stateful_checks(&store, &aggregate).map(|_| ())) + .map_err(|outcome: Outcome| { + warn!(%slot, aggregator, ?outcome, "Refused a submitted aggregate"); + "aggregate failed validation" + }) + } else { + Err("Eth-Consensus-Version does not match the aggregate slot's fork") + }; + let published = checked.and_then(|()| { + // Recorded for block production, which packs the aggregates this + // node has validated. The pool holds electra's shape, which a + // gloas aggregate converts to. + let pooled = match &aggregate { + SignedAggregateAndProof::Electra(signed) => Some(signed.message.aggregate.clone()), + SignedAggregateAndProof::Gloas(signed) => { + electra::Attestation::try_from(&signed.message.aggregate).ok() + } + SignedAggregateAndProof::Phase0(_) => None, + }; + if let Some(pooled) = pooled { pool.lock() .expect("attestation pool lock poisoned") - .insert_aggregate(inner); - p2p.publish_beacon_aggregate(aggregate) - .map_err(|_| "the network actor is not running") - }); + .insert_aggregate(pooled); + } + p2p.publish_beacon_aggregate(aggregate) + .map_err(|_| "the network actor is not running") + }); match published { Ok(()) => debug!(%slot, aggregator, "Accepted aggregate for gossip"), Err(message) => failures.push(Failure { index, message }), @@ -361,16 +428,6 @@ async fn get_aggregate_attestation( let fork = store .config() .fork_at_epoch(compute_epoch_at_slot(query.slot)); - // The pool holds electra-shaped votes, and the response is labelled with - // the slot's fork, so a gloas slot would be served as gloas with a body of - // the wrong shape. - if let Err(err) = refuse_validator_duties_from_gloas( - &store.config(), - compute_epoch_at_slot(query.slot), - "gloas aggregates are not supported", - ) { - return err.into_response(); - } let aggregate = pool .lock() .expect("attestation pool lock poisoned") @@ -382,9 +439,16 @@ async fn get_aggregate_attestation( let Some(aggregate) = aggregate else { return ApiError::NotFound("no matching attestations to aggregate").into_response(); }; + // The pool holds electra-shaped votes; a gloas slot is served the same + // vote in gloas's container, whose JSON is the same shape. + let data = if fork == ForkName::Gloas { + serde_json::json!(gloas::Attestation::from(&aggregate)) + } else { + serde_json::json!(aggregate) + }; let response = crate::json_response(serde_json::json!({ "version": fork.as_str(), - "data": aggregate, + "data": data, })); crate::shared::content::with_consensus_version(response, fork) } @@ -408,7 +472,7 @@ mod tests { use std::sync::Arc; use super::*; - use crate::test_utils::{RecordingNetwork, beacon_store_at}; + use crate::test_utils::RecordingNetwork; use axum::{body::Body, http::Request}; use ethlambda_state_transition::beacon::helpers::{ accessors::get_beacon_committee, @@ -425,27 +489,47 @@ mod tests { head_root: Root, network: Arc, pool: SharedAttestationPool, + /// The `Eth-Consensus-Version` the fixture's fork submits under. + version: &'static str, } /// A fulu head state, stored in the current wall-clock epoch so the /// submitted attestations are neither future nor stale. fn fixture() -> Fixture { - let mut state = with_signing_validators_at(ForkName::Fulu, 64); - let BeaconState::Fulu(fulu) = &mut state else { - unreachable!("built as fulu") + fixture_at(ForkName::Fulu, 0) + } + + /// A head of `fork`'s state (fulu, or gloas under a schedule with gloas + /// from epoch 0) `slots_back` slots before the first slot of the wall + /// clock's epoch. At zero the head is its own epoch's checkpoint block and + /// the target root of a vote at its slot; further back, a vote in the + /// wall-clock epoch's first slot is for an earlier block. + fn fixture_at(fork: ForkName, slots_back: u64) -> Fixture { + let mut state = with_signing_validators_at(fork, 64); + let config = match fork { + ForkName::Gloas => Config::mainnet().with_fork_epoch(ForkName::Gloas, 0), + _ => Config::mainnet(), }; - // At the first slot of the wall clock's epoch, so the head is its own - // epoch's checkpoint block and the attestation's target root. - let (probe, _) = beacon_store_at(BeaconState::Fulu(fulu.clone())); + let (probe, _) = crate::test_utils::beacon_store_with_config(state.clone(), config.clone()); let wall_epoch = compute_epoch_at_slot(crate::beacon::node::wall_slot(&probe)); - fulu.slot = compute_start_slot_at_epoch(wall_epoch); - let (store, head_root) = beacon_store_at(state.clone()); + let head_slot = compute_start_slot_at_epoch(wall_epoch) - slots_back; + match &mut state { + BeaconState::Fulu(fulu) => fulu.slot = head_slot, + BeaconState::Gloas(gloas) => gloas.slot = head_slot, + _ => unreachable!("built as fulu or gloas"), + } + let (store, head_root) = crate::test_utils::beacon_store_with_config(state.clone(), config); Fixture { store, state, head_root, network: Arc::new(RecordingNetwork::default()), pool: SharedAttestationPool::default(), + version: if fork == ForkName::Gloas { + "gloas" + } else { + "fulu" + }, } } @@ -453,15 +537,35 @@ mod tests { /// voting for the head at the head's own slot. fn attestation(fixture: &Fixture, committee_index: u64, position: usize) -> SingleAttestation { let slot = fixture.state.slot(); + attestation_for( + fixture, + slot, + (fixture.head_root, fixture.head_root), + 0, + committee_index, + position, + ) + } + + /// A correctly signed vote at `slot` for the block and target roots in + /// `(block, target)`, carrying `index` as its `data.index`. + fn attestation_for( + fixture: &Fixture, + slot: u64, + (block, target): (Root, Root), + index: u64, + committee_index: u64, + position: usize, + ) -> SingleAttestation { let epoch = compute_epoch_at_slot(slot); let data = AttestationData { slot, - index: 0, - beacon_block_root: fixture.head_root, + index, + beacon_block_root: block, source: fixture.state.current_justified_checkpoint(), target: Checkpoint { epoch, - root: fixture.head_root, + root: target, }, }; let committee = get_beacon_committee(&fixture.state, slot, committee_index).unwrap(); @@ -479,6 +583,14 @@ mod tests { async fn submit( fixture: &Fixture, attestations: &[SingleAttestation], + ) -> (StatusCode, serde_json::Value) { + submit_as(fixture, attestations, fixture.version).await + } + + async fn submit_as( + fixture: &Fixture, + attestations: &[SingleAttestation], + version: &str, ) -> (StatusCode, serde_json::Value) { let network: RpcToP2PRef = fixture.network.clone(); let app = routes() @@ -487,7 +599,7 @@ mod tests { .layer(Extension(fixture.pool.clone())); let request = Request::post("/eth/v2/beacon/pool/attestations") .header("content-type", "application/json") - .header("eth-consensus-version", "fulu") + .header("eth-consensus-version", version) .body(Body::from(serde_json::to_vec(attestations).unwrap())) .unwrap(); let response = app.oneshot(request).await.unwrap(); @@ -771,40 +883,211 @@ mod tests { } #[tokio::test] - async fn an_aggregate_query_at_a_gloas_slot_is_refused_by_name() { - let mut fixture = fixture(); - let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 0); - let (store, _) = crate::test_utils::beacon_store_with_config(fixture.state.clone(), config); - fixture.store = store; - let (status, json) = get_aggregate(&fixture, Root::ZERO, fixture.state.slot(), 0).await; + async fn the_gloas_header_needs_a_gloas_slot_and_the_older_ones_a_pre_gloas_slot() { + let gloas = fixture_at(ForkName::Gloas, 0); + let vote = attestation(&gloas, 0, 0); + let (status, json) = submit_as(&gloas, std::slice::from_ref(&vote), "fulu").await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!( + json["failures"][0]["message"], + "Eth-Consensus-Version does not match the attestation slot's fork" + ); + assert!(gloas.network.published.lock().unwrap().is_empty()); + + let fulu = fixture(); + let vote = attestation(&fulu, 0, 0); + let (status, _) = submit_as(&fulu, std::slice::from_ref(&vote), "gloas").await; assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(json["message"], "gloas aggregates are not supported"); + assert!(fulu.network.published.lock().unwrap().is_empty()); } #[tokio::test] - async fn a_gloas_fork_header_is_refused_by_name() { - for path in [ - "/eth/v2/beacon/pool/attestations", - "/eth/v2/validator/aggregate_and_proofs", - ] { - let fixture = fixture(); - let network: RpcToP2PRef = fixture.network.clone(); - let app = routes() - .with_state(fixture.store.clone()) - .layer(Extension(network)) - .layer(Extension(fixture.pool.clone())); - let request = Request::post(path) - .header("eth-consensus-version", "gloas") - .body(Body::from("[]")) - .unwrap(); - let response = app.oneshot(request).await.unwrap(); - assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{path}"); - let body = response.into_body().collect().await.unwrap().to_bytes(); - let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); - assert_eq!( - json["message"], "Eth-Consensus-Version gloas is not supported", - "{path}" - ); - } + async fn a_gloas_vote_in_the_voted_blocks_own_slot_is_published_with_index_zero() { + let fixture = fixture_at(ForkName::Gloas, 0); + let vote = attestation(&fixture, 0, 0); + let (status, json) = submit(&fixture, std::slice::from_ref(&vote)).await; + assert_eq!(status, StatusCode::OK, "{json}"); + assert_eq!(fixture.network.published.lock().unwrap().len(), 1); + } + + #[tokio::test] + async fn a_same_slot_gloas_vote_for_the_payload_is_refused() { + let fixture = fixture_at(ForkName::Gloas, 0); + let slot = fixture.state.slot(); + let roots = (fixture.head_root, fixture.head_root); + let vote = attestation_for(&fixture, slot, roots, 1, 0, 0); + let (status, json) = submit(&fixture, &[vote]).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!( + json["failures"][0]["message"], + "a same-slot attestation cannot vote for the payload" + ); + assert!(fixture.network.published.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn a_gloas_index_past_one_is_refused() { + let fixture = fixture_at(ForkName::Gloas, 3); + let slot = fixture.state.slot() + 3; + let roots = (fixture.head_root, fixture.head_root); + let vote = attestation_for(&fixture, slot, roots, 2, 0, 0); + let (status, json) = submit(&fixture, &[vote]).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!( + json["failures"][0]["message"], + "data.index must be 0 or 1 from gloas on" + ); + } + + /// The voted block is the last pre-gloas one, whose payload this node + /// treats as FULL, so a vote for it with `index` 1 is valid. + #[tokio::test] + async fn a_gloas_vote_for_an_earlier_pre_gloas_block_may_name_its_payload() { + let fixture = fixture_at(ForkName::Gloas, 3); + let slot = fixture.state.slot() + 3; + let roots = (fixture.head_root, fixture.head_root); + let vote = attestation_for(&fixture, slot, roots, 1, 0, 0); + let (status, json) = submit(&fixture, &[vote]).await; + assert_eq!(status, StatusCode::OK, "{json}"); + assert_eq!(fixture.network.published.lock().unwrap().len(), 1); + } + + #[tokio::test] + async fn a_gloas_vote_for_a_payload_never_seen_is_refused() { + let mut fixture = fixture_at(ForkName::Gloas, 3); + let head_slot = fixture.state.slot(); + let block = crate::test_utils::gloas_beacon_block( + head_slot + 1, + fixture.head_root, + Root::repeat_byte(1), + Root::repeat_byte(2), + ); + let block_root = block.message_hash_tree_root(); + fixture + .store + .insert_signed_block(block_root, block) + .expect("insert gloas block"); + let slot = head_slot + 3; + let roots = (block_root, block_root); + let empty = attestation_for(&fixture, slot, roots, 0, 0, 0); + let full = attestation_for(&fixture, slot, roots, 1, 0, 1); + let (status, json) = submit(&fixture, &[empty, full]).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{json}"); + let failures = json["failures"].as_array().unwrap(); + assert_eq!(failures.len(), 1); + assert_eq!(failures[0]["index"], 1); + assert_eq!( + failures[0]["message"], + "the voted block's payload has not been seen and verified" + ); + fixture.network = Arc::new(RecordingNetwork::default()); + } + + /// `aggregate` as a gloas message, signed like [`signed_aggregate`]. + fn signed_gloas_aggregate( + fixture: &Fixture, + aggregator: u64, + aggregate: &electra::Attestation, + ) -> gloas::SignedAggregateAndProof { + use ethlambda_types::beacon::constants::{ + DOMAIN_AGGREGATE_AND_PROOF, DOMAIN_SELECTION_PROOF, + }; + let slot = aggregate.data.slot; + let epoch = compute_epoch_at_slot(slot); + let selection_domain = get_domain(&fixture.state, DOMAIN_SELECTION_PROOF, Some(epoch)); + let selection_proof = sign_for( + aggregator as usize, + compute_signing_root(slot.hash_tree_root(), selection_domain), + ); + let message = gloas::AggregateAndProof { + aggregator_index: aggregator, + aggregate: gloas::Attestation::from(aggregate), + selection_proof, + }; + let domain = get_domain(&fixture.state, DOMAIN_AGGREGATE_AND_PROOF, Some(epoch)); + let signature = sign_for( + aggregator as usize, + compute_signing_root(message.hash_tree_root(), domain), + ); + gloas::SignedAggregateAndProof { message, signature } + } + + /// A gloas aggregator's slot: votes in, the aggregate out as gloas's + /// container, and the signed gloas aggregate published and pooled. + #[tokio::test] + async fn a_gloas_aggregator_can_fetch_and_publish_its_aggregate() { + let fixture = fixture_at(ForkName::Gloas, 0); + let slot = fixture.state.slot(); + let committee = get_beacon_committee(&fixture.state, slot, 0).unwrap(); + let votes: Vec = (0..committee.len()) + .map(|position| attestation(&fixture, 0, position)) + .collect(); + let (status, json) = submit(&fixture, &votes).await; + assert_eq!(status, StatusCode::OK, "{json}"); + + let data = votes[0].data; + let (status, json) = get_aggregate(&fixture, data.hash_tree_root(), slot, 0).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(json["version"], "gloas"); + let served: gloas::Attestation = serde_json::from_value(json["data"].clone()).unwrap(); + let pooled = fixture + .pool + .lock() + .unwrap() + .aggregate(data.hash_tree_root(), slot, 0) + .unwrap(); + assert_eq!(served, gloas::Attestation::from(&pooled)); + + let signed = signed_gloas_aggregate(&fixture, committee[0], &pooled); + let network: RpcToP2PRef = fixture.network.clone(); + let app = routes() + .with_state(fixture.store.clone()) + .layer(Extension(network)) + .layer(Extension(fixture.pool.clone())); + let request = Request::post("/eth/v2/validator/aggregate_and_proofs") + .header("content-type", "application/json") + .header("eth-consensus-version", "gloas") + .body(Body::from( + serde_json::to_vec(std::slice::from_ref(&signed)).unwrap(), + )) + .unwrap(); + let response = app.oneshot(request).await.unwrap(); + let status = response.status(); + let body = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(status, StatusCode::OK, "{body:?}"); + let published = fixture.network.aggregates.lock().unwrap(); + assert_eq!(published.len(), 1); + assert_eq!(published[0], SignedAggregateAndProof::Gloas(signed)); + } + + #[tokio::test] + async fn a_gloas_aggregate_with_a_forged_signature_is_refused() { + let fixture = fixture_at(ForkName::Gloas, 0); + let slot = fixture.state.slot(); + let committee = get_beacon_committee(&fixture.state, slot, 0).unwrap(); + let votes: Vec = (0..committee.len()) + .map(|position| attestation(&fixture, 0, position)) + .collect(); + submit(&fixture, &votes).await; + let pooled = fixture + .pool + .lock() + .unwrap() + .aggregate(votes[0].data.hash_tree_root(), slot, 0) + .unwrap(); + let mut forged = signed_gloas_aggregate(&fixture, committee[0], &pooled); + forged.signature = signed_gloas_aggregate(&fixture, committee[1], &pooled).signature; + let network: RpcToP2PRef = fixture.network.clone(); + let app = routes() + .with_state(fixture.store.clone()) + .layer(Extension(network)) + .layer(Extension(fixture.pool.clone())); + let request = Request::post("/eth/v2/validator/aggregate_and_proofs") + .header("eth-consensus-version", "gloas") + .body(Body::from(serde_json::to_vec(&[forged]).unwrap())) + .unwrap(); + let response = app.oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert!(fixture.network.aggregates.lock().unwrap().is_empty()); } } From 450bc0a1047e004374e774b7cf557e22f199b91d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:43:32 -0300 Subject: [PATCH 090/132] feat(beacon): key the payload attestation pool by slot, data and validator The placeholder vector scanned every held message on each query. Keying by slot first makes pruning a split and the per-slot reads touch only their own slot; dedup per (validator, data) is a map lookup. --- .../src/beacon/payload_attestation_pool.rs | 128 +++++++++++++++--- 1 file changed, 107 insertions(+), 21 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/payload_attestation_pool.rs b/crates/blockchain/state_transition/src/beacon/payload_attestation_pool.rs index 0105bb5e3..f7cbfc058 100644 --- a/crates/blockchain/state_transition/src/beacon/payload_attestation_pool.rs +++ b/crates/blockchain/state_transition/src/beacon/payload_attestation_pool.rs @@ -11,16 +11,18 @@ //! signatures safe: one bad share fails the whole aggregate, and with it the //! block that carries it. //! -//! The method signatures below are the shared contract between the tasks -//! that fill the pool and block production, which only calls -//! [`messages_for`]. The storage behind them is a placeholder. +//! The method signatures below are the shared contract between what fills the +//! pool and block production, which only calls [`messages_for`]. //! //! [`messages_for`]: PayloadAttestationPool::messages_for -use std::sync::{Arc, Mutex}; +use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, +}; use ethlambda_types::beacon::{ - containers::gloas::PayloadAttestationMessage, + containers::gloas::{PayloadAttestationData, PayloadAttestationMessage}, primitives::{Root, Slot}, }; @@ -30,10 +32,20 @@ use ethlambda_types::beacon::{ /// There must be exactly one per node. pub type SharedPayloadAttestationPool = Arc>; +/// Votes sharing one `PayloadAttestationData`, by voting validator. The data +/// has no `Hash`/`Ord`, and a slot holds only a handful of distinct values +/// (the committee splits at most a few ways), so a linear scan over a `Vec` +/// is both simple and fast. +type VotesByData = Vec<( + PayloadAttestationData, + BTreeMap, +)>; + #[derive(Debug, Default)] pub struct PayloadAttestationPool { - /// Placeholder storage; the PTC task may restructure it freely. - messages: Vec, + /// slot -> data -> validator index -> message. Keyed by slot first so + /// pruning and the per-slot queries never visit unrelated slots. + messages: BTreeMap, } impl PayloadAttestationPool { @@ -43,13 +55,19 @@ impl PayloadAttestationPool { /// message's own. pub fn insert(&mut self, message: PayloadAttestationMessage) -> bool { self.prune_before(message.data.slot.saturating_sub(1)); - let held = self.messages.iter().any(|held| { - held.validator_index == message.validator_index && held.data == message.data - }); - if !held { - self.messages.push(message); + let by_data = self.messages.entry(message.data.slot).or_default(); + let votes = match by_data.iter().position(|(data, _)| *data == message.data) { + Some(position) => &mut by_data[position].1, + None => { + by_data.push((message.data, BTreeMap::new())); + &mut by_data.last_mut().expect("just pushed").1 + } + }; + if votes.contains_key(&message.validator_index) { + return false; } - !held + votes.insert(message.validator_index, message); + true } /// Every held message voting on `beacon_block_root` at `slot`, in any @@ -60,11 +78,11 @@ impl PayloadAttestationPool { beacon_block_root: Root, ) -> Vec { self.messages - .iter() - .filter(|held| { - held.data.slot == slot && held.data.beacon_block_root == beacon_block_root - }) - .cloned() + .get(&slot) + .into_iter() + .flatten() + .filter(|(data, _)| data.beacon_block_root == beacon_block_root) + .flat_map(|(_, votes)| votes.values().cloned()) .collect() } @@ -72,13 +90,81 @@ impl PayloadAttestationPool { pub fn all(&self, slot: Option) -> Vec { self.messages .iter() - .filter(|held| slot.is_none_or(|slot| held.data.slot == slot)) - .cloned() + .filter(|(held, _)| slot.is_none_or(|slot| **held == slot)) + .flat_map(|(_, by_data)| by_data.iter()) + .flat_map(|(_, votes)| votes.values().cloned()) .collect() } /// Drop every message for a slot before `slot`. pub fn prune_before(&mut self, slot: Slot) { - self.messages.retain(|held| held.data.slot >= slot); + self.messages = self.messages.split_off(&slot); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ethlambda_types::beacon::primitives::BlsSignature; + + fn message( + validator_index: u64, + slot: Slot, + root: u8, + present: bool, + ) -> PayloadAttestationMessage { + PayloadAttestationMessage { + validator_index, + data: PayloadAttestationData { + beacon_block_root: Root::from([root; 32]), + slot, + payload_present: present, + blob_data_available: true, + }, + signature: BlsSignature::default(), + } + } + + #[test] + fn insert_dedups_per_validator_and_data() { + let mut pool = PayloadAttestationPool::default(); + assert!(pool.insert(message(1, 10, 1, true))); + assert!(!pool.insert(message(1, 10, 1, true))); + // Same validator, different data: an equivocation, kept as its own entry. + assert!(pool.insert(message(1, 10, 1, false))); + assert!(pool.insert(message(2, 10, 1, true))); + assert_eq!(pool.all(Some(10)).len(), 3); + } + + #[test] + fn messages_for_filters_by_slot_and_root() { + let mut pool = PayloadAttestationPool::default(); + pool.insert(message(1, 10, 1, true)); + pool.insert(message(2, 10, 2, true)); + pool.insert(message(3, 11, 1, true)); + let held = pool.messages_for(10, Root::from([1; 32])); + assert_eq!(held.len(), 1); + assert_eq!(held[0].validator_index, 1); + assert!(pool.messages_for(12, Root::from([1; 32])).is_empty()); + } + + #[test] + fn insert_prunes_to_current_and_previous_slot() { + let mut pool = PayloadAttestationPool::default(); + pool.insert(message(1, 8, 1, true)); + pool.insert(message(1, 9, 1, true)); + pool.insert(message(1, 10, 1, true)); + let slots: Vec = pool.all(None).iter().map(|m| m.data.slot).collect(); + assert_eq!(slots, vec![9, 10]); + } + + #[test] + fn prune_before_drops_older_slots() { + let mut pool = PayloadAttestationPool::default(); + pool.insert(message(1, 5, 1, true)); + pool.insert(message(1, 6, 1, true)); + pool.prune_before(6); + assert_eq!(pool.all(None).len(), 1); + assert!(pool.all(Some(5)).is_empty()); } } From 3e069740ccc17aadacd3ed289c3cfb97c5763a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:58:41 -0300 Subject: [PATCH 091/132] feat(beacon): share one payload attestation pool across p2p and the Beacon API Gossip-accepted payload_attestation_message votes now land in the pool block production packs from, and the Beacon API router gets the same pool as an Extension for its pool endpoints. --- bin/ethlambda/src/main.rs | 6 ++++ crates/net/p2p/src/beacon/verdict.rs | 41 +++++++++++++++++++++++++ crates/net/p2p/src/lib.rs | 12 +++++++- crates/net/p2p/src/req_resp/handlers.rs | 1 + crates/net/rpc/src/lib.rs | 8 ++++- 5 files changed, 66 insertions(+), 2 deletions(-) diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index 2dfa5206e..df76685d1 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -718,12 +718,17 @@ async fn run_node(options: Options) -> eyre::Result<()> { // read by the aggregate endpoint and block production; unused on lean. let attestation_pool = ethlambda_state_transition::beacon::attestation_pool::SharedAttestationPool::default(); + // Filled by gossip and the Beacon API's payload attestation endpoint, read + // by block production and `GET .../pool/payload_attestations`. + let payload_attestation_pool = + ethlambda_state_transition::beacon::payload_attestation_pool::SharedPayloadAttestationPool::default(); let p2p = P2P::spawn( built, setup.store.clone(), setup.node_names, discovery, attestation_pool.clone(), + payload_attestation_pool.clone(), ) .await .wrap_err("failed to start discv5 discovery")?; @@ -758,6 +763,7 @@ async fn run_node(options: Options) -> eyre::Result<()> { ethlambda_rpc::BeaconApiHandles { p2p: rpc_p2p, attestation_pool: attestation_pool.clone(), + payload_attestation_pool: payload_attestation_pool.clone(), engine: rpc_engine, }, local_peer_id, diff --git a/crates/net/p2p/src/beacon/verdict.rs b/crates/net/p2p/src/beacon/verdict.rs index 3cdabc22a..5d820eed1 100644 --- a/crates/net/p2p/src/beacon/verdict.rs +++ b/crates/net/p2p/src/beacon/verdict.rs @@ -199,6 +199,17 @@ impl Validated { { pool_aggregator_attestation(server, attestation); } + if let Self::PayloadAttestation(message) = &self + && outcome == Outcome::Accept + { + // Block production packs these, and a node with no chain actor + // attached still serves `GET .../pool/payload_attestations`. + server + .payload_attestation_pool + .lock() + .expect("payload attestation pool lock") + .insert(message.clone()); + } let Some(blockchain) = &server.blockchain else { return; }; @@ -1068,6 +1079,36 @@ mod tests { assert_eq!(received.try_recv(), Ok(Sent::PayloadAttestation)); } + /// An accepted payload attestation is pooled for block production, with + /// or without a chain actor attached; any other outcome stays out, since + /// one unverified share fails the aggregate it is packed into. + #[tokio::test] + async fn an_accepted_payload_attestation_is_pooled_and_others_are_not() { + let server = unconnected_beacon_server(Config::mainnet(), 0).await; + let forward = |outcome| { + Validated::PayloadAttestation(payload_attestation(5, 1)).forward( + &server, + Instant::now(), + outcome, + ) + }; + forward(Outcome::Ignore(IgnoreReason::Overloaded)); + forward(Outcome::Reject(RejectReason::NotInPtc)); + assert!( + server + .payload_attestation_pool + .lock() + .unwrap() + .all(None) + .is_empty() + ); + forward(Outcome::Accept); + assert_eq!( + server.payload_attestation_pool.lock().unwrap().all(None), + vec![payload_attestation(5, 1)] + ); + } + /// The pool an aggregate or a subnet attestation draws its stateful-check /// permit from is not the pool a block or a column draws from: exhausting /// one must leave the other untouched, or a burst on this topic could diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index 053638ecd..655b5a927 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -50,12 +50,14 @@ use ethlambda_network_api::{ }, }; use ethlambda_state_transition::beacon::aggregate::MAX_AGGREGATES_PER_SLOT; -use ethlambda_state_transition::beacon::attestation_pool::SharedAttestationPool; use ethlambda_state_transition::beacon::gossip::{ SeenBlockColumns, SeenBlocks, SeenColumns, aggregate::SeenAggregates, attestation::SeenAttestations, envelope::SeenEnvelopes, payload_attestation::SeenPayloadAttestations, }; +use ethlambda_state_transition::beacon::{ + attestation_pool::SharedAttestationPool, payload_attestation_pool::SharedPayloadAttestationPool, +}; use ethlambda_storage::{Chain, Store}; use ethlambda_types::beacon::preset::{MAX_VALIDATORS_PER_COMMITTEE, SLOTS_PER_EPOCH}; use ethlambda_types::primitives::H256; @@ -1095,6 +1097,7 @@ impl P2P { node_names: HashMap, discovery: Option, attestation_pool: SharedAttestationPool, + payload_attestation_pool: SharedPayloadAttestationPool, ) -> Result { let discovery = match discovery { Some(config) => Some(spawn_discovery(config).await?), @@ -1157,6 +1160,7 @@ impl P2P { ATTESTATION_VALIDATION_PERMITS, )), attestation_pool, + payload_attestation_pool, aggregator_subnets: HashMap::new(), }; let discovery_enabled = server.discovery.is_some(); @@ -1293,6 +1297,11 @@ pub struct P2PServer { /// touches it. pub(crate) attestation_pool: SharedAttestationPool, + /// Accepted `payload_attestation_message` votes, shared with the Beacon + /// API that serves and fills the same pool. Filled by `verdict::forward`; + /// lean never touches it. + pub(crate) payload_attestation_pool: SharedPayloadAttestationPool, + /// The attestation subnets joined for a validator client's aggregators, /// each with the last slot it is needed for. Short-lived by design: never /// advertised in `attnets`, and left once the slot has passed. The @@ -2780,6 +2789,7 @@ pub(crate) mod test_support { crate::ATTESTATION_VALIDATION_PERMITS, )), attestation_pool: Default::default(), + payload_attestation_pool: Default::default(), aggregator_subnets: HashMap::new(), } } diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index 4aa6dd8ab..4db3f424b 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -2810,6 +2810,7 @@ pub(crate) mod tests { crate::ATTESTATION_VALIDATION_PERMITS, )), attestation_pool: Default::default(), + payload_attestation_pool: Default::default(), aggregator_subnets: HashMap::new(), } } diff --git a/crates/net/rpc/src/lib.rs b/crates/net/rpc/src/lib.rs index 4c312d8f0..a51a6aeb1 100644 --- a/crates/net/rpc/src/lib.rs +++ b/crates/net/rpc/src/lib.rs @@ -3,7 +3,9 @@ use std::net::{IpAddr, SocketAddr}; use axum::{Extension, Router}; use ethlambda_blockchain::{EventBus, SyncStatusController}; use ethlambda_network_api::RpcToP2PRef; -use ethlambda_state_transition::beacon::attestation_pool::SharedAttestationPool; +use ethlambda_state_transition::beacon::{ + attestation_pool::SharedAttestationPool, payload_attestation_pool::SharedPayloadAttestationPool, +}; use ethlambda_storage::Store; use ethlambda_types::aggregator::AggregatorController; use tokio_util::sync::CancellationToken; @@ -192,6 +194,9 @@ pub struct BeaconApiHandles { /// Filled by the attestation pool endpoint and the aggregator subnets, /// read by the aggregate endpoint and block production. pub attestation_pool: SharedAttestationPool, + /// Filled by gossip and the payload attestation pool endpoint, read by + /// block production and the pool's GET. + pub payload_attestation_pool: SharedPayloadAttestationPool, /// The execution client block production builds payloads with; `None` /// makes it answer 503. pub engine: Option, @@ -215,6 +220,7 @@ pub async fn start_beacon_rpc_server( .layer(Extension(sync_status)) .layer(Extension(handles.p2p)) .layer(Extension(handles.attestation_pool)) + .layer(Extension(handles.payload_attestation_pool)) .layer(Extension(beacon::validator::FeeRecipients::default())) .layer(Extension(handles.engine)); start_http_servers(config, Some(api_router), shutdown).await From 7e9d3d9ff452a097973beb7a86c375a805c2b9ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:58:44 -0300 Subject: [PATCH 092/132] feat(beacon): gossip payload attestations a validator client signed Gossipsub never delivers a node its own message, so the publish path also hands the vote to the chain actor and marks the seen cache, which keeps a peer echo from being validated and forwarded a second time. --- crates/net/p2p/src/gossipsub/handler.rs | 49 ++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/crates/net/p2p/src/gossipsub/handler.rs b/crates/net/p2p/src/gossipsub/handler.rs index bc9b100b0..ae7be1be4 100644 --- a/crates/net/p2p/src/gossipsub/handler.rs +++ b/crates/net/p2p/src/gossipsub/handler.rs @@ -810,16 +810,53 @@ pub async fn publish_execution_payload_envelope( /// over by the Beacon API, on `payload_attestation_message`, and pass it to /// the chain actor. /// -/// STUB: owned by the PTC task (see the contract), which replaces this body. +/// The API validated it with the checks gossip would apply, so it goes out +/// as is. Gossipsub never delivers a node its own message, so the chain actor +/// is handed it here to count the vote in its fork choice. The seen cache is +/// marked as well: a peer that echoes the vote back would otherwise pass +/// triage and be validated and forwarded a second time. pub async fn publish_payload_attestation_message( - _server: &mut P2PServer, + server: &mut P2PServer, message: ethlambda_types::beacon::containers::gloas::PayloadAttestationMessage, ) { - error!( - slot = message.data.slot, - validator = message.validator_index, - "Publishing payload attestation messages is not implemented; dropping it" + let slot = message.data.slot; + let validator = message.validator_index; + let Some(beacon) = server.wire.beacon() else { + error!( + slot, + "A payload attestation reached a lean node; dropping it" + ); + return; + }; + let Some(digest) = beacon.publish_digest(slot) else { + warn!( + slot, + "No held fork digest covers this payload attestation's slot; not publishing" + ); + return; + }; + let topic = IdentTopic::new(beacon_topics::topic_name( + digest, + beacon_topics::PAYLOAD_ATTESTATION_MESSAGE, + )); + server + .swarm_handle + .publish(topic, compress_message(&message.to_ssz())); + server.seen_payload_attestations.record(slot, validator); + info!( + slot, + validator, + block_root = %ShortRoot(&message.data.beacon_block_root.0), + payload_present = message.data.payload_present, + "Published payload attestation to gossipsub" ); + if let Some(ref blockchain) = server.blockchain { + let _ = blockchain + .new_payload_attestation_message(message, BlockArrival::now()) + .inspect_err( + |err| error!(%err, "Failed to hand the published payload attestation to the chain"), + ); + } } /// The beacon wall-clock slot, from the wire's genesis and slot duration. From 43412348d9f990ec7ec6de0d8f9209a6070a355a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:01:06 -0300 Subject: [PATCH 093/132] feat(beacon): record when a block's envelope arrived for payload_present The committee votes payload_present when an envelope was seen before get_payload_due_ms() into the slot, and the store only knew whether a payload was verified, not when it arrived. Record the arrival time (not the verification time, since envelopes can be held for columns or the engine) in an in-memory map pruned with the el-hash cache, and add get_payload_due_ms. --- crates/blockchain/src/beacon_envelope.rs | 5 ++ .../src/beacon/fork_choice.rs | 7 +++ crates/storage/src/store.rs | 60 ++++++++++++++++++- 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/crates/blockchain/src/beacon_envelope.rs b/crates/blockchain/src/beacon_envelope.rs index ca2046014..a893a651c 100644 --- a/crates/blockchain/src/beacon_envelope.rs +++ b/crates/blockchain/src/beacon_envelope.rs @@ -562,6 +562,11 @@ impl BlockChainServer { ); self.store .insert_beacon_block_payload_status(root, slot, status); + // When it arrived, not when it finished verifying: a held envelope is + // verified late, and the committee's `payload_present` asks whether it + // was seen on time. + let seen_ms = held.event_ms.saturating_sub(config.genesis_time_ms()); + self.store.insert_beacon_envelope_seen(root, slot, seen_ms); // Lets `latestValidHash` find this FULL node. self.store.insert_beacon_el_block_hash( root, diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs index bf0df95e0..a04859745 100644 --- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs +++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs @@ -3530,6 +3530,13 @@ pub fn get_payload_attestation_due_ms(config: &Config) -> u64 { get_slot_component_duration_ms(config.payload_attestation_due_bps, config) } +/// `get_payload_due_ms` (gloas `fork-choice.md`): how far into a slot, in +/// milliseconds, an execution payload envelope must have been seen for the +/// payload timeliness committee to vote `payload_present`. +pub fn get_payload_due_ms(config: &Config) -> u64 { + get_slot_component_duration_ms(config.payload_due_bps, config) +} + // --------------------------------------------------------------------------- // Merge transition helpers (bellatrix) // --------------------------------------------------------------------------- diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 37499f6c7..ec8be43e8 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -784,6 +784,15 @@ pub(crate) struct BeaconScratch { /// specification's `block_payload_statuses`; an absent root reads as /// `NOT_VALIDATED`. pub(crate) block_payload_statuses: HashMap, + /// Gloas: beacon root to `(slot, milliseconds past genesis the block's + /// envelope reached this node)`. What the payload timeliness committee's + /// `payload_present` is decided from: the specification's vote is "an + /// envelope was seen before `get_payload_due_ms()` into the slot", and + /// nothing else this store keeps records *when*. A cache like + /// `el_block_hashes`, not chain history, and pruned with it by + /// `prune_beacon_el_block_hashes`; a restart forgets it, which can only + /// turn a late `payload_present` into `false` for the one slot in flight. + pub(crate) envelope_seen_ms: HashMap, } /// Encode a LiveChain key (slot, root) to bytes. @@ -3986,8 +3995,34 @@ impl Store { .insert(root, (slot, block_hash)); } + /// Records when `root`'s envelope reached this node, in milliseconds past + /// genesis. The earliest arrival wins: a later copy says nothing about + /// whether the envelope was on time. + pub fn insert_beacon_envelope_seen(&mut self, root: H256, slot: u64, seen_ms: u64) { + self.beacon + .lock() + .unwrap() + .envelope_seen_ms + .entry(root) + .and_modify(|(_, held)| *held = (*held).min(seen_ms)) + .or_insert((slot, seen_ms)); + } + + /// When `root`'s envelope reached this node, in milliseconds past genesis, + /// if it has. + pub fn beacon_envelope_seen_ms(&self, root: &H256) -> Option { + self.beacon + .lock() + .unwrap() + .envelope_seen_ms + .get(root) + .map(|(_, seen_ms)| *seen_ms) + } + /// Drops cached hashes strictly below `finalized_slot`, always keeping - /// `keep`. + /// `keep`. The envelope arrival times go with them, below the same bound + /// (they are only ever read for a slot in flight, so `keep` needs no + /// exemption there). /// /// Strictly below, not at or below: the justified and head blocks /// `forkchoiceUpdated` reads are at or above that slot, so this bound keeps @@ -4009,6 +4044,11 @@ impl Store { .unwrap() .el_block_hashes .retain(|root, (slot, _hash)| *slot >= finalized_slot || *root == keep); + self.beacon + .lock() + .unwrap() + .envelope_seen_ms + .retain(|_, (slot, _)| *slot >= finalized_slot); } /// Returns `root`'s unrealized justification, if this store has computed @@ -7985,6 +8025,24 @@ mod tests { ); } + #[test] + fn envelope_arrival_keeps_the_earliest_and_is_pruned_with_the_hash_cache() { + let mut store = Store::test_store(); + let old = H256::repeat_byte(1); + let fresh = H256::repeat_byte(2); + store.insert_beacon_envelope_seen(old, 4, 100); + store.insert_beacon_envelope_seen(fresh, 6, 900); + store.insert_beacon_envelope_seen(fresh, 6, 2_000); + store.insert_beacon_envelope_seen(fresh, 6, 500); + + assert_eq!(store.beacon_envelope_seen_ms(&fresh), Some(500)); + assert_eq!(store.beacon_envelope_seen_ms(&H256::repeat_byte(9)), None); + + store.prune_beacon_el_block_hashes(5, H256::repeat_byte(7)); + assert_eq!(store.beacon_envelope_seen_ms(&old), None); + assert_eq!(store.beacon_envelope_seen_ms(&fresh), Some(500)); + } + #[test] fn el_block_hashes_prune_strictly_below_the_finalized_slot() { let mut store = Store::test_store(); From 6a32f5dbec3fac6eab46491462f79df9de39eab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:12:23 -0300 Subject: [PATCH 094/132] feat(beacon): serve PTC duties, payload attestation data and the pool A gloas validator client needs three things from its beacon node to sit on the payload timeliness committee: its seat (POST duties/ptc/{epoch}), the data to sign (GET payload_attestation_data) and somewhere to submit the vote (POST pool/payload_attestations). Submitted votes go through the same stateful checks gossip applies to a peer's, then into the pool block production packs from, and out on gossip. The first gloas epoch is answered from a copy of a fulu head state advanced across the upgrade on a blocking thread, since get_ptc only reads the cached window of a gloas state. --- bin/ethlambda/src/main.rs | 7 + .../src/beacon/helpers/gloas.rs | 26 + crates/net/rpc/src/beacon/ptc.rs | 1091 ++++++++++++++++- crates/net/rpc/src/lib.rs | 9 + 4 files changed, 1128 insertions(+), 5 deletions(-) diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index df76685d1..c31645461 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -742,6 +742,12 @@ async fn run_node(options: Options) -> eyre::Result<()> { let rpc_p2p = p2p.actor_ref().to_rpc_to_p2p_ref(); // Block production builds its payloads with the same execution client the // chain actor validates them with. + let rpc_custody_columns = match &setup.chain { + ChainActor::Beacon { + custody_columns, .. + } => ethlambda_rpc::CustodyColumns(custody_columns.clone()), + ChainActor::Lean(..) => ethlambda_rpc::CustodyColumns::default(), + }; let rpc_engine = match &setup.chain { ChainActor::Beacon { engine, .. } => engine.clone(), ChainActor::Lean(..) => None, @@ -764,6 +770,7 @@ async fn run_node(options: Options) -> eyre::Result<()> { p2p: rpc_p2p, attestation_pool: attestation_pool.clone(), payload_attestation_pool: payload_attestation_pool.clone(), + custody_columns: rpc_custody_columns, engine: rpc_engine, }, local_peer_id, diff --git a/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs b/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs index a8f97f9d9..23f7fd84d 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/gloas.rs @@ -715,6 +715,32 @@ pub fn get_ptc( }) } +/// The slot of `epoch` in which each of `validators` sits in the payload +/// timeliness committee, from `get_ptc_assignment` (gloas `validator.md`). +/// +/// The specification answers one validator at a time, each call walking the +/// epoch's slots; this walks them once for the whole set. A validator with +/// several seats gets its first slot, as `get_ptc_assignment` returns, and one +/// in no committee of the epoch is absent. `epoch` must be one [`get_ptc`] +/// can answer from `state`. +pub fn get_ptc_assignments( + state: &BeaconState, + epoch: Epoch, + validators: &std::collections::HashSet, + config: &Config, +) -> Result> { + let mut assignments = std::collections::HashMap::new(); + let first_slot = compute_start_slot_at_epoch(epoch); + for slot in first_slot..first_slot + preset::SLOTS_PER_EPOCH { + for validator_index in get_ptc(state, slot, config)? { + if validators.contains(&validator_index) { + assignments.entry(validator_index).or_insert(slot); + } + } + } + Ok(assignments) +} + /// The same payload attestation with its attesters named rather than /// bit-encoded, the payload-attestation counterpart of /// [`super::electra::get_indexed_attestation`]. diff --git a/crates/net/rpc/src/beacon/ptc.rs b/crates/net/rpc/src/beacon/ptc.rs index 181616235..67a912a3c 100644 --- a/crates/net/rpc/src/beacon/ptc.rs +++ b/crates/net/rpc/src/beacon/ptc.rs @@ -1,13 +1,1094 @@ //! The payload timeliness committee's validator-facing endpoints. //! -//! STUB: owned by the PTC task (see the contract), which fills in -//! `POST /eth/v1/validator/duties/ptc/{epoch}`, -//! `GET /eth/v1/validator/payload_attestation_data` and -//! `GET`/`POST /eth/v1/beacon/pool/payload_attestations`. +//! - `POST /eth/v1/validator/duties/ptc/{epoch}`: which slot of an epoch each +//! requested validator sits in the committee (`get_ptc_assignment`). +//! - `GET /eth/v1/validator/payload_attestation_data`: what a committee member +//! signs for a slot. +//! - `POST /eth/v1/beacon/pool/payload_attestations`: a member's signed vote, +//! validated as gossip would, pooled for block production and gossiped. +//! - `GET /eth/v1/beacon/pool/payload_attestations`: what the pool holds. -use axum::Router; +use std::collections::HashSet; +use std::sync::Arc; + +use axum::{ + Extension, Router, + body::Bytes, + extract::{Path, Query, State}, + http::{HeaderMap, StatusCode, header}, + response::{IntoResponse, Response}, + routing::{get, post}, +}; +use ethlambda_blockchain::SyncStatusController; +use ethlambda_blockchain::metrics::SyncStatus; +use ethlambda_network_api::RpcToP2PRef; +use ethlambda_state_transition::beacon::{ + fork_choice::{get_current_slot, get_payload_due_ms}, + gossip::{ + Outcome, + payload_attestation::{SeenPayloadAttestations, cheap_checks, stateful_checks}, + }, + helpers::{accessors::get_block_root_at_slot, gloas::get_ptc_assignments}, + payload_attestation_pool::SharedPayloadAttestationPool, + preset, + stf::process_slots, +}; use ethlambda_storage::Store; +use ethlambda_types::{ + beacon::{ + containers::{ + BeaconState, SignedBeaconBlock, + gloas::{PayloadAttestationData, PayloadAttestationMessage}, + }, + fork::ForkName, + primitives::{BlsPubkey, Epoch, Slot, ValidatorIndex}, + signing::{compute_epoch_at_slot, compute_start_slot_at_epoch}, + }, + primitives::H256, +}; +use libssz::SszEncode; +use serde::{Deserialize, Serialize}; +use tracing::{debug, warn}; + +use crate::{ + CustodyColumns, + beacon::{ApiError, validator::head}, + shared::content::{Encoding, ssz_response, with_consensus_version}, +}; pub(crate) fn routes() -> Router { Router::new() + .route( + "/eth/v1/validator/duties/ptc/{epoch}", + post(post_ptc_duties), + ) + .route( + "/eth/v1/validator/payload_attestation_data", + get(get_payload_attestation_data), + ) + .route( + "/eth/v1/beacon/pool/payload_attestations", + get(get_pool_payload_attestations).post(post_pool_payload_attestations), + ) +} + +// --------------------------------------------------------------------------- +// Duties +// --------------------------------------------------------------------------- + +#[derive(Debug, Serialize)] +struct PtcDuty { + pubkey: BlsPubkey, + #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")] + validator_index: ValidatorIndex, + #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")] + slot: Slot, +} + +/// `POST /eth/v1/validator/duties/ptc/{epoch}`. +/// +/// The body is a JSON array of quoted validator indices. Each answers with the +/// first slot of `epoch` whose committee holds it: a validator with several +/// seats still gets one duty per epoch, and one in no committee gets none, as +/// does an unknown index. +/// +/// `epoch` may be at most one past the current one. A gloas epoch is answered +/// from the head state's `ptc_window`, which `get_ptc` can read for the +/// state's epoch, the one before and the next one. Anything else (the first +/// gloas epoch while the head is still fulu, or an epoch the head has not +/// caught up to) is answered from a copy of the head state advanced to the +/// epoch's first slot, crossing the gloas upgrade where it falls. That runs on +/// a blocking thread: on a large registry the epoch transitions are seconds +/// of work, not something to hold a runtime worker for. +/// +/// An epoch before gloas has no committee, so the answer is empty rather than +/// an error: a validator client asks every epoch and reads the empty list as +/// "no duty". +/// +/// `dependent_root` is the block root at `compute_start_slot_at_epoch(epoch - 1) - 1` +/// (the genesis block's at the start), the same definition the attester duties +/// use: the committee window is fixed by the state at the end of the epoch +/// before the previous one. +async fn post_ptc_duties( + Path(epoch): Path, + State(store): State, + body: Bytes, +) -> Response { + let Ok(epoch) = epoch.parse::() else { + return ApiError::BadRequest("invalid epoch").into_response(); + }; + let wanted = match parse_validator_indices(&body) { + Ok(wanted) => wanted, + Err(err) => return err.into_response(), + }; + let computed = tokio::task::spawn_blocking(move || ptc_duties(&store, epoch, &wanted)).await; + match computed { + Ok(Ok(body)) => crate::json_response(body), + Ok(Err(err)) => err.into_response(), + Err(_) => ApiError::Internal("computing the duties failed").into_response(), + } +} + +fn parse_validator_indices(body: &[u8]) -> Result, ApiError> { + let indices = serde_json::from_slice::>(body) + .map_err(|_| ApiError::BadRequest("invalid request body"))?; + if indices.is_empty() { + return Err(ApiError::BadRequest("no validator indices")); + } + indices + .iter() + .map(|index| index.parse::()) + .collect::>() + .map_err(|_| ApiError::BadRequest("invalid validator index")) +} + +fn ptc_duties( + store: &Store, + epoch: Epoch, + wanted: &HashSet, +) -> Result { + let config = store.config(); + let (head_root, head_state) = head(store)?; + let state_epoch = compute_epoch_at_slot(head_state.slot()); + // The wall clock can be ahead of the head (a node that missed slots) and, + // in a follower's store, behind it; either way an epoch more than one past + // the later of the two is not one a validator is asked about yet. + let clock_epoch = compute_epoch_at_slot(get_current_slot(store, &config)); + if epoch > state_epoch.max(clock_epoch) + 1 { + return Err(ApiError::BadRequest( + "epoch is more than one past the current", + )); + } + + let dependent_slot = compute_start_slot_at_epoch(epoch.saturating_sub(1)).saturating_sub(1); + let dependent_root = if dependent_slot >= head_state.slot() { + head_root + } else { + get_block_root_at_slot(&head_state, dependent_slot) + .map_err(|_| ApiError::Internal("dependent slot is outside the state's root window"))? + }; + let respond = |duties: Vec| { + serde_json::json!({ + "dependent_root": dependent_root, + "execution_optimistic": store.is_beacon_optimistic(head_root), + "data": duties, + }) + }; + + if epoch < config.gloas_fork_epoch { + return Ok(respond(Vec::new())); + } + if epoch + 1 < state_epoch { + return Err(ApiError::BadRequest( + "epoch is more than one before the head state's", + )); + } + + let readable_from_head = matches!(*head_state, BeaconState::Gloas(_)) + && epoch <= state_epoch + preset::MIN_SEED_LOOKAHEAD; + let state: Arc = if readable_from_head { + head_state + } else { + let start_slot = compute_start_slot_at_epoch(epoch); + let mut advanced = (*head_state).clone(); + process_slots(&mut advanced, start_slot, &config) + .map_err(|_| ApiError::Internal("advancing the head state failed"))?; + Arc::new(advanced) + }; + + let assignments = get_ptc_assignments(&state, epoch, wanted, &config) + .map_err(|_| ApiError::Internal("the payload timeliness committee is unavailable"))?; + let mut duties: Vec = assignments + .into_iter() + .filter_map(|(validator_index, slot)| { + let pubkey = state.validator(validator_index).ok()?.pubkey; + Some(PtcDuty { + pubkey, + validator_index, + slot, + }) + }) + .collect(); + duties.sort_by_key(|duty| (duty.slot, duty.validator_index)); + Ok(respond(duties)) +} + +// --------------------------------------------------------------------------- +// Payload attestation data +// --------------------------------------------------------------------------- + +#[derive(Debug, Deserialize)] +struct PayloadAttestationDataQuery { + slot: Slot, +} + +/// `GET /eth/v1/validator/payload_attestation_data?slot`. +/// +/// For the canonical block of `slot`, meaning the block of that very slot on +/// the head's chain (the head itself when it is one). A slot with no such block +/// answers 204, which tells the validator client to cast no vote: a member +/// votes on the block of its slot, and a slot that went empty has none. +/// +/// - `payload_present`: an envelope for the block reached this node before +/// `get_payload_due_ms()` into the slot, the specification's rule. A late +/// envelope, or one this node never saw, is `false`. +/// - `blob_data_available`: the bid commits to no blobs, or the payload is +/// already verified (which required them), or every column this node +/// custodies for the block is stored. Columns are verified on arrival, so +/// holding them is the evidence `is_data_available` asks for. +async fn get_payload_attestation_data( + Query(query): Query, + State(store): State, + Extension(custody): Extension, + Extension(sync_status): Extension, + headers: HeaderMap, +) -> Response { + if sync_status.get() == SyncStatus::Syncing { + return ApiError::ServiceUnavailable("node is syncing").into_response(); + } + let data = match payload_attestation_data(&store, &custody.0, query.slot) { + Ok(Some(data)) => data, + Ok(None) => return StatusCode::NO_CONTENT.into_response(), + Err(err) => return err.into_response(), + }; + let accept = headers.get(header::ACCEPT).and_then(|v| v.to_str().ok()); + let response = match Encoding::from_accept(accept) { + Encoding::Ssz => ssz_response(data.to_ssz()), + Encoding::Json => crate::json_response(serde_json::json!({ + "version": ForkName::Gloas.as_str(), + "data": data, + })), + }; + with_consensus_version(response, ForkName::Gloas) +} + +/// The number of blob commitments in a gloas block's bid. +fn bid_commitment_count(block: &SignedBeaconBlock) -> Option { + match block { + SignedBeaconBlock::Gloas(inner) => Some( + inner + .message + .body + .signed_execution_payload_bid + .message + .blob_kzg_commitments + .len(), + ), + _ => None, + } +} + +/// The block of `slot` on the head's chain, if there is one. +fn canonical_block_at(store: &Store, slot: Slot) -> Result, ApiError> { + let (head_slot, head_root) = store + .beacon_head() + .ok_or(ApiError::Internal("no head block"))?; + if slot > head_slot { + return Ok(None); + } + if slot == head_slot { + return Ok(Some(head_root)); + } + let (_, state) = head(store)?; + // The state records the latest block at or before the slot, which is the + // block of a slot only when that slot was not skipped. + let Ok(root) = get_block_root_at_slot(&state, slot) else { + return Ok(None); + }; + Ok(store + .block_slot_and_state_root(&root) + .filter(|(block_slot, _)| *block_slot == slot) + .map(|_| root)) +} + +fn payload_attestation_data( + store: &Store, + custody_columns: &[u64], + slot: Slot, +) -> Result, ApiError> { + let config = store.config(); + if compute_epoch_at_slot(slot) < config.gloas_fork_epoch { + return Err(ApiError::BadRequest("slot is before the gloas fork")); + } + let Some(root) = canonical_block_at(store, slot)? else { + return Ok(None); + }; + let block = store + .get_signed_block(&root) + .map_err(|_| ApiError::Internal("store read failed"))?; + let Some(commitment_count) = block.as_ref().and_then(bid_commitment_count) else { + return Ok(None); + }; + + let due_ms = slot + .saturating_mul(config.slot_duration_ms) + .saturating_add(get_payload_due_ms(&config)); + let payload_present = store + .beacon_envelope_seen_ms(&root) + .is_some_and(|seen_ms| seen_ms < due_ms); + + let blob_data_available = commitment_count == 0 || store.has_verified_payload(&root) || { + let present = store + .data_column_indices_for(slot, &root) + .map_err(|_| ApiError::Internal("store read failed"))?; + custody_columns + .iter() + .all(|column| present.contains(column)) + }; + + Ok(Some(PayloadAttestationData { + beacon_block_root: root, + slot, + payload_present, + blob_data_available, + })) +} + +// --------------------------------------------------------------------------- +// Pool +// --------------------------------------------------------------------------- + +/// One rejected message, in the Beacon API's `IndexedErrorMessage` shape: its +/// position in the submitted array, and why. +#[derive(Debug, Serialize)] +struct Failure { + index: usize, + message: String, +} + +/// `POST /eth/v1/beacon/pool/payload_attestations`. +/// +/// Each message goes through the checks gossip applies to a peer's +/// (`validate_payload_attestation_message_gossip`: the current slot, a known +/// block at the slot, committee membership, the signature), since a vote that +/// fails them is one every peer would score this node down for relaying. +/// A valid one is pooled for block production, gossiped, and handed to the +/// chain actor by the publish path (gossip never delivers a node its own +/// messages), the others are reported by position and the rest still go out. +async fn post_pool_payload_attestations( + State(store): State, + Extension(p2p): Extension, + Extension(pool): Extension, + headers: HeaderMap, + body: Bytes, +) -> Response { + if let Err(err) = require_gloas_or_absent(&headers) { + return err.into_response(); + } + let Ok(messages) = serde_json::from_slice::>(&body) else { + return ApiError::BadRequest("invalid request body").into_response(); + }; + + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| elapsed.as_millis() as u64) + .unwrap_or(0); + // Not P2P's seen cache: that holds what peers sent, and says nothing about + // what this node's own validators submit. The pool's own dedup covers a + // repeat. + let seen = SeenPayloadAttestations::new(std::num::NonZeroUsize::MIN); + let mut failures = Vec::new(); + for (index, message) in messages.into_iter().enumerate() { + let slot = message.data.slot; + let validator = message.validator_index; + let verdict = match cheap_checks(&seen, &store, &message, now_ms) { + Ok(()) => stateful_checks(&store, &message), + Err(outcome) => outcome, + }; + if verdict != Outcome::Accept { + let (outcome, reason) = verdict.labels(); + warn!(%slot, validator, outcome, reason, "Refused a submitted payload attestation"); + failures.push(Failure { + index, + message: format!("{outcome}: {reason}"), + }); + continue; + } + let first = pool + .lock() + .expect("payload attestation pool lock poisoned") + .insert(message.clone()); + if !first { + debug!(%slot, validator, "Payload attestation already pooled; not republishing"); + continue; + } + match p2p.publish_payload_attestation_message(message) { + Ok(()) => debug!(%slot, validator, "Accepted payload attestation for gossip"), + Err(_) => failures.push(Failure { + index, + message: "the network actor is not running".to_string(), + }), + } + } + + if failures.is_empty() { + return StatusCode::OK.into_response(); + } + let body = serde_json::json!({ + "code": 400, + "message": "some payload attestations failed validation and were not published", + "failures": failures, + }); + let mut response = crate::json_response(body); + *response.status_mut() = StatusCode::BAD_REQUEST; + response +} + +/// `Eth-Consensus-Version` is optional here, and must name gloas when given: +/// the container exists from that fork alone. +fn require_gloas_or_absent(headers: &HeaderMap) -> Result<(), ApiError> { + let Some(value) = headers.get("eth-consensus-version") else { + return Ok(()); + }; + match value.to_str().ok().and_then(ForkName::parse) { + Some(ForkName::Gloas) => Ok(()), + _ => Err(ApiError::BadRequest( + "Eth-Consensus-Version must name gloas", + )), + } +} + +#[derive(Debug, Deserialize)] +struct PoolQuery { + slot: Option, +} + +/// `GET /eth/v1/beacon/pool/payload_attestations?slot=`: the votes this node +/// holds, those of `slot` only when given. +/// +/// The messages themselves, not the aggregates the specification's response +/// names: the pool keeps them unaggregated so block production can combine +/// the ones it needs, and a consumer wanting the aggregate can build it. +async fn get_pool_payload_attestations( + Query(query): Query, + Extension(pool): Extension, +) -> Response { + let held = pool + .lock() + .expect("payload attestation pool lock poisoned") + .all(query.slot); + with_consensus_version( + crate::json_response(serde_json::json!({ + "version": ForkName::Gloas.as_str(), + "data": held, + })), + ForkName::Gloas, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{RecordingNetwork, gloas_beacon_block}; + use axum::{body::Body, http::Request}; + use ethlambda_state_transition::beacon::{ + constants::DOMAIN_PTC_ATTESTER, + helpers::{ + accessors::get_domain, + misc::compute_signing_root, + test_state::{sign_for, with_signing_validators_at}, + }, + }; + use ethlambda_storage::{ForkCheckpoints, backend::InMemoryBackend}; + use ethlambda_types::beacon::{config::Config, primitives::KzgCommitment}; + use ethlambda_types::checkpoint::Checkpoint; + use ethlambda_types::primitives::HashTreeRoot as _; + use http_body_util::BodyExt as _; + use libssz::SszDecode; + use tower::ServiceExt as _; + + const COUNT: usize = preset::PTC_WINDOW_LENGTH; + + fn now_secs() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() + } + + /// A gloas state with every `ptc_window` entry naming its own window index + /// as the whole committee, so the validator a slot's committee holds is + /// readable straight off the window layout. + fn gloas_state() -> BeaconState { + let mut state = with_signing_validators_at(ForkName::Gloas, COUNT); + let BeaconState::Gloas(inner) = &mut state else { + unreachable!("built as gloas") + }; + // Distinct roots per slot, so a `dependent_root` read from the wrong + // slot shows up. + for (slot, root) in inner.block_roots.iter_mut().enumerate() { + *root = H256::repeat_byte(slot as u8 + 1); + } + for index in 0..preset::PTC_WINDOW_LENGTH { + inner.ptc_window[index] = vec![index as u64; preset::PTC_SIZE] + .try_into() + .expect("built at exactly PTC_SIZE"); + } + state + } + + /// The window index (and so the validator, see [`gloas_state`]) of the + /// `offset`th slot of `epoch`, for a state in `state_epoch`. + fn window_index(state_epoch: Epoch, epoch: Epoch, offset: u64) -> u64 { + (epoch + 1 - state_epoch) * preset::SLOTS_PER_EPOCH + offset + } + + /// A store whose head is a gloas block at `state`'s slot with + /// `commitments` blob commitments in its bid, on a clock that has that + /// slot running now (the gossip checks only take the current slot). + fn store_with_head(state: BeaconState, config: Config, commitments: usize) -> (Store, H256) { + let slot = state.slot(); + let mut block = gloas_beacon_block(slot, H256::ZERO, H256::ZERO, H256::repeat_byte(1)); + let SignedBeaconBlock::Gloas(inner) = &mut block else { + unreachable!("built as gloas") + }; + inner + .message + .body + .signed_execution_payload_bid + .message + .blob_kzg_commitments = vec![KzgCommitment::default(); commitments] + .try_into() + .expect("a few commitments fit"); + let root = block.message_hash_tree_root(); + let slot_secs = config.slot_duration_ms / 1000; + let genesis = now_secs() - slot * slot_secs - 1; + let mut store = Store::init_beacon( + Arc::new(InMemoryBackend::default()), + genesis, + config, + root, + Checkpoint { root, slot }, + slot, + ); + store.insert_signed_block(root, block).unwrap(); + store.insert_state(root, state).unwrap(); + store + .update_checkpoints(ForkCheckpoints::head_only(root)) + .unwrap(); + (store, root) + } + + fn gloas_config() -> Config { + Config::mainnet().with_fork_epoch(ForkName::Gloas, 0) + } + + struct Reply { + status: StatusCode, + headers: HeaderMap, + body: Bytes, + } + + impl Reply { + fn json(&self) -> serde_json::Value { + serde_json::from_slice(&self.body).unwrap() + } + } + + async fn send( + store: Store, + pool: SharedPayloadAttestationPool, + network: Arc, + custody: Vec, + sync: SyncStatusController, + request: Request, + ) -> Reply { + let p2p: RpcToP2PRef = network; + let response = routes() + .with_state(store) + .layer(Extension(p2p)) + .layer(Extension(pool)) + .layer(Extension(CustodyColumns(custody))) + .layer(Extension(sync)) + .oneshot(request) + .await + .unwrap(); + let status = response.status(); + let headers = response.headers().clone(); + let body = response.into_body().collect().await.unwrap().to_bytes(); + Reply { + status, + headers, + body, + } + } + + async fn request(store: Store, request: Request) -> Reply { + send( + store, + Default::default(), + Default::default(), + Vec::new(), + Default::default(), + request, + ) + .await + } + + fn duties_request(epoch: u64, body: &str) -> Request { + Request::post(format!("/eth/v1/validator/duties/ptc/{epoch}")) + .body(Body::from(body.to_string())) + .unwrap() + } + + // ----- duties ---------------------------------------------------------- + + #[tokio::test] + async fn duties_are_read_from_the_window_for_the_previous_current_and_next_epoch() { + let state = gloas_state(); + let state_epoch = compute_epoch_at_slot(state.slot()); + let head_slot = state.slot(); + let (store, head_root) = store_with_head(state, gloas_config(), 0); + for epoch in [state_epoch - 1, state_epoch, state_epoch + 1] { + let validator = window_index(state_epoch, epoch, 3); + let reply = request( + store.clone(), + duties_request(epoch, &format!(r#"["{validator}","9999"]"#)), + ) + .await; + assert_eq!(reply.status, StatusCode::OK); + let json = reply.json(); + // The unknown validator has no duty; the other sits in its slot. + assert_eq!(json["data"].as_array().unwrap().len(), 1); + assert_eq!(json["data"][0]["validator_index"], validator.to_string()); + assert_eq!( + json["data"][0]["slot"], + (compute_start_slot_at_epoch(epoch) + 3).to_string() + ); + assert!( + json["data"][0]["pubkey"] + .as_str() + .unwrap() + .starts_with("0x") + ); + assert_eq!(json["execution_optimistic"], false); + // The block before the previous epoch's start; the head itself + // when that slot is not before the head's. + let dependent_slot = + compute_start_slot_at_epoch(epoch.saturating_sub(1)).saturating_sub(1); + let expected = if dependent_slot >= head_slot { + head_root + } else { + H256::repeat_byte(dependent_slot as u8 + 1) + }; + assert_eq!(json["dependent_root"], format!("{expected:?}")); + } + } + + #[tokio::test] + async fn a_validator_with_several_seats_gets_its_first_slot_once() { + let mut state = gloas_state(); + let state_epoch = compute_epoch_at_slot(state.slot()); + let BeaconState::Gloas(inner) = &mut state else { + unreachable!("built as gloas") + }; + let early = window_index(state_epoch, state_epoch, 5) as usize; + let late = window_index(state_epoch, state_epoch, 9) as usize; + inner.ptc_window[early][1] = 77; + inner.ptc_window[late][0] = 77; + inner.ptc_window[late][1] = 77; + let (store, _) = store_with_head(state, gloas_config(), 0); + + let reply = request(store, duties_request(state_epoch, r#"["77"]"#)).await; + + let json = reply.json(); + assert_eq!(json["data"].as_array().unwrap().len(), 1); + assert_eq!( + json["data"][0]["slot"], + (compute_start_slot_at_epoch(state_epoch) + 5).to_string() + ); + } + + #[tokio::test] + async fn malformed_bodies_and_far_epochs_are_a_400() { + let state = gloas_state(); + let state_epoch = compute_epoch_at_slot(state.slot()); + let (store, _) = store_with_head(state, gloas_config(), 0); + for body in ["[]", "not json", r#"["x"]"#, r#"[1]"#, r#"{"a":1}"#] { + let reply = request(store.clone(), duties_request(state_epoch, body)).await; + assert_eq!(reply.status, StatusCode::BAD_REQUEST, "{body}"); + } + let reply = request(store.clone(), duties_request(state_epoch + 2, r#"["1"]"#)).await; + assert_eq!(reply.status, StatusCode::BAD_REQUEST); + let reply = request(store, duties_request(u64::MAX, r#"["1"]"#)).await; + assert_eq!(reply.status, StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn a_pre_gloas_epoch_is_answered_with_no_duties() { + let state = gloas_state(); + let state_epoch = compute_epoch_at_slot(state.slot()); + // Gloas starts after the head's epoch, so the head's own epoch has no + // committee even though the state happens to be gloas-shaped. + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, state_epoch + 1); + let (store, _) = store_with_head(state, config, 0); + + let reply = request(store, duties_request(state_epoch, r#"["1","2"]"#)).await; + + assert_eq!(reply.status, StatusCode::OK); + assert_eq!(reply.json()["data"], serde_json::json!([])); + assert!(reply.json()["dependent_root"].is_string()); + } + + /// While the head is still fulu, the first gloas epoch is answered from a + /// copy of the head state advanced across the upgrade. + #[tokio::test] + async fn the_first_gloas_epoch_is_computed_from_a_fulu_head() { + use ethlambda_state_transition::beacon::helpers::fulu::initialize_proposer_lookahead; + let mut state = with_signing_validators_at(ForkName::Fulu, COUNT); + let lookahead = initialize_proposer_lookahead(&state).unwrap(); + let BeaconState::Fulu(fulu) = &mut state else { + unreachable!("built as fulu") + }; + fulu.proposer_lookahead = lookahead.try_into().unwrap(); + let head_epoch = compute_epoch_at_slot(state.slot()); + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, head_epoch + 1); + let (store, _root) = crate::test_utils::beacon_store_with_config(state, config); + let everyone = + serde_json::to_string(&(0..COUNT as u64).map(|i| i.to_string()).collect::>()) + .unwrap(); + + let reply = request(store.clone(), duties_request(head_epoch + 1, &everyone)).await; + + assert_eq!(reply.status, StatusCode::OK, "{:?}", reply.body); + let duties = reply.json()["data"].as_array().unwrap().clone(); + assert!( + !duties.is_empty(), + "the committee seats some of the registry" + ); + let first_slot = compute_start_slot_at_epoch(head_epoch + 1); + for duty in &duties { + let slot: u64 = duty["slot"].as_str().unwrap().parse().unwrap(); + assert!((first_slot..first_slot + preset::SLOTS_PER_EPOCH).contains(&slot)); + } + // The fulu epoch itself has none. + let reply = request(store, duties_request(head_epoch, &everyone)).await; + assert_eq!(reply.json()["data"], serde_json::json!([])); + } + + // ----- payload attestation data ---------------------------------------- + + fn data_request(slot: u64, accept: Option<&str>) -> Request { + let mut request = Request::get(format!( + "/eth/v1/validator/payload_attestation_data?slot={slot}" + )); + if let Some(accept) = accept { + request = request.header("accept", accept); + } + request.body(Body::empty()).unwrap() + } + + fn due_ms(store: &Store, slot: u64) -> u64 { + let config = store.config(); + slot * config.slot_duration_ms + get_payload_due_ms(&config) + } + + #[tokio::test] + async fn the_data_names_the_head_block_and_defaults_to_nothing_seen() { + let state = gloas_state(); + let slot = state.slot(); + let (store, root) = store_with_head(state, gloas_config(), 0); + + let reply = request(store, data_request(slot, None)).await; + + assert_eq!(reply.status, StatusCode::OK); + assert_eq!(reply.headers["eth-consensus-version"], "gloas"); + let json = reply.json(); + assert_eq!(json["version"], "gloas"); + assert_eq!(json["data"]["beacon_block_root"], format!("{root:?}")); + assert_eq!(json["data"]["slot"], slot.to_string()); + assert_eq!(json["data"]["payload_present"], false); + // No commitments, so there is no blob data to wait for. + assert_eq!(json["data"]["blob_data_available"], true); + } + + #[tokio::test] + async fn payload_present_needs_an_envelope_seen_before_the_payload_deadline() { + let slot = gloas_state().slot(); + let present_when_seen_at = |offset: i64| async move { + let (mut store, root) = store_with_head(gloas_state(), gloas_config(), 0); + let seen = (due_ms(&store, slot) as i64 + offset) as u64; + store.insert_beacon_envelope_seen(root, slot, seen); + request(store, data_request(slot, None)).await.json()["data"]["payload_present"].clone() + }; + + assert_eq!(present_when_seen_at(-1).await, true); + // The deadline itself is already late. + assert_eq!(present_when_seen_at(0).await, false); + assert_eq!(present_when_seen_at(1_000).await, false); + } + + #[tokio::test] + async fn blob_availability_follows_the_custody_columns_unless_the_payload_is_verified() { + let slot = gloas_state().slot(); + let ask = |store: Store| async move { + send( + store, + Default::default(), + Default::default(), + vec![0, 1], + Default::default(), + data_request(slot, None), + ) + .await + .json()["data"]["blob_data_available"] + .clone() + }; + let (store, root) = store_with_head(gloas_state(), gloas_config(), 1); + + assert_eq!(ask(store.clone()).await, false); + store + .put_data_column_sidecar(slot, &root, 0, vec![1]) + .unwrap(); + assert_eq!(ask(store.clone()).await, false); + store + .put_data_column_sidecar(slot, &root, 1, vec![1]) + .unwrap(); + assert_eq!(ask(store.clone()).await, true); + + // A verified payload had its data, whatever columns are stored. + let (mut store, root) = store_with_head(gloas_state(), gloas_config(), 1); + let envelope = crate::test_utils::gloas_envelope(root, slot); + store.insert_verified_payload(slot, &envelope); + assert_eq!(ask(store).await, true); + } + + #[tokio::test] + async fn a_slot_without_a_block_is_a_204_and_a_pre_gloas_slot_a_400() { + let state = gloas_state(); + let slot = state.slot(); + let (store, _) = store_with_head(state, gloas_config(), 0); + for missing in [slot + 1, slot - 1] { + let reply = request(store.clone(), data_request(missing, None)).await; + assert_eq!(reply.status, StatusCode::NO_CONTENT, "slot {missing}"); + assert!(reply.body.is_empty()); + } + + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 5); + let (store, _) = store_with_head(gloas_state(), config, 0); + let reply = request(store, data_request(slot, None)).await; + assert_eq!(reply.status, StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn the_data_is_served_as_ssz_on_request_and_refused_while_syncing() { + let state = gloas_state(); + let slot = state.slot(); + let (store, root) = store_with_head(state, gloas_config(), 0); + + let reply = request( + store.clone(), + data_request(slot, Some("application/octet-stream")), + ) + .await; + assert_eq!(reply.status, StatusCode::OK); + assert_eq!(reply.headers["eth-consensus-version"], "gloas"); + assert_eq!(reply.headers["content-type"], "application/octet-stream"); + let data = PayloadAttestationData::from_ssz_bytes(&reply.body).unwrap(); + assert_eq!(data.beacon_block_root, root); + assert_eq!(data.slot, slot); + + let reply = send( + store, + Default::default(), + Default::default(), + Vec::new(), + SyncStatusController::new(SyncStatus::Syncing), + data_request(slot, None), + ) + .await; + assert_eq!(reply.status, StatusCode::SERVICE_UNAVAILABLE); + } + + // ----- pool ------------------------------------------------------------ + + /// A vote by `validator` on the head block at the head's slot, signed under + /// the state's PTC domain unless `sign` is false. + fn vote( + state: &BeaconState, + root: H256, + validator: u64, + sign: bool, + ) -> PayloadAttestationMessage { + let data = PayloadAttestationData { + beacon_block_root: root, + slot: state.slot(), + payload_present: true, + blob_data_available: true, + }; + let domain = get_domain( + state, + DOMAIN_PTC_ATTESTER, + Some(compute_epoch_at_slot(data.slot)), + ); + let signing_root = compute_signing_root(data.hash_tree_root(), domain); + PayloadAttestationMessage { + validator_index: validator, + data, + signature: if sign { + sign_for(validator as usize, signing_root) + } else { + Default::default() + }, + } + } + + fn submit(messages: &[PayloadAttestationMessage], version: Option<&str>) -> Request { + let mut request = Request::post("/eth/v1/beacon/pool/payload_attestations"); + if let Some(version) = version { + request = request.header("eth-consensus-version", version); + } + request + .body(Body::from(serde_json::to_vec(messages).unwrap())) + .unwrap() + } + + async fn submit_to( + store: Store, + messages: &[PayloadAttestationMessage], + version: Option<&str>, + ) -> (Reply, SharedPayloadAttestationPool, Arc) { + let pool = SharedPayloadAttestationPool::default(); + let network = Arc::new(RecordingNetwork::default()); + let reply = send( + store, + pool.clone(), + network.clone(), + Vec::new(), + Default::default(), + submit(messages, version), + ) + .await; + (reply, pool, network) + } + + #[tokio::test] + async fn a_valid_vote_is_pooled_and_published_once() { + let state = gloas_state(); + let state_epoch = compute_epoch_at_slot(state.slot()); + let member = window_index(state_epoch, state_epoch, 0); + let (store, root) = store_with_head(state.clone(), gloas_config(), 0); + let message = vote(&state, root, member, true); + + let (reply, pool, network) = + submit_to(store, &[message.clone(), message.clone()], Some("gloas")).await; + + assert_eq!(reply.status, StatusCode::OK, "{:?}", reply.body); + assert_eq!(pool.lock().unwrap().all(None), vec![message.clone()]); + // The repeat is neither re-pooled nor re-gossiped. + assert_eq!(*network.payload_attestations.lock().unwrap(), vec![message]); + } + + #[tokio::test] + async fn a_bad_vote_is_reported_by_position_and_the_rest_still_go_out() { + let state = gloas_state(); + let state_epoch = compute_epoch_at_slot(state.slot()); + let member = window_index(state_epoch, state_epoch, 0); + let (store, root) = store_with_head(state.clone(), gloas_config(), 0); + let good = vote(&state, root, member, true); + let unsigned = vote(&state, root, member, false); + // Signed, but its sender sits in no committee of the slot. + let outsider = vote(&state, root, 5, true); + let mut unknown_block = vote(&state, root, member, true); + unknown_block.data.beacon_block_root = H256::repeat_byte(0xee); + + let (reply, pool, network) = submit_to( + store, + &[unsigned, good.clone(), outsider, unknown_block], + Some("gloas"), + ) + .await; + + assert_eq!(reply.status, StatusCode::BAD_REQUEST); + let json = reply.json(); + assert_eq!(json["code"], 400); + let positions: Vec = json["failures"] + .as_array() + .unwrap() + .iter() + .map(|failure| failure["index"].as_u64().unwrap()) + .collect(); + assert_eq!(positions, vec![0, 2, 3]); + assert_eq!(pool.lock().unwrap().all(None), vec![good.clone()]); + assert_eq!(*network.payload_attestations.lock().unwrap(), vec![good]); + } + + #[tokio::test] + async fn a_vote_for_another_slot_is_refused() { + let state = gloas_state(); + let state_epoch = compute_epoch_at_slot(state.slot()); + let member = window_index(state_epoch, state_epoch, 0); + let (store, root) = store_with_head(state.clone(), gloas_config(), 0); + let mut stale = vote(&state, root, member, true); + stale.data.slot -= 1; + + let (reply, pool, _) = submit_to(store, &[stale], None).await; + + assert_eq!(reply.status, StatusCode::BAD_REQUEST); + assert!(pool.lock().unwrap().all(None).is_empty()); + } + + #[tokio::test] + async fn the_consensus_version_must_be_gloas_when_given() { + let state = gloas_state(); + let (store, root) = store_with_head(state.clone(), gloas_config(), 0); + let message = vote(&state, root, 0, false); + for version in ["electra", "fulu", "nonsense"] { + let (reply, pool, _) = + submit_to(store.clone(), std::slice::from_ref(&message), Some(version)).await; + assert_eq!(reply.status, StatusCode::BAD_REQUEST, "{version}"); + assert!(pool.lock().unwrap().all(None).is_empty()); + } + let reply = request( + store, + Request::post("/eth/v1/beacon/pool/payload_attestations") + .body(Body::from("not json")) + .unwrap(), + ) + .await; + assert_eq!(reply.status, StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn the_pool_is_listed_by_slot() { + let state = gloas_state(); + let state_epoch = compute_epoch_at_slot(state.slot()); + let member = window_index(state_epoch, state_epoch, 0); + let (store, root) = store_with_head(state.clone(), gloas_config(), 0); + let pool = SharedPayloadAttestationPool::default(); + let message = vote(&state, root, member, true); + let mut older = message.clone(); + older.data.slot -= 1; + pool.lock().unwrap().insert(older); + pool.lock().unwrap().insert(message.clone()); + let list = |uri: String| { + let store = store.clone(); + let pool = pool.clone(); + async move { + send( + store, + pool, + Default::default(), + Vec::new(), + Default::default(), + Request::get(uri).body(Body::empty()).unwrap(), + ) + .await + } + }; + + let all = list("/eth/v1/beacon/pool/payload_attestations".to_string()).await; + assert_eq!(all.status, StatusCode::OK); + assert_eq!(all.headers["eth-consensus-version"], "gloas"); + assert_eq!(all.json()["version"], "gloas"); + assert_eq!(all.json()["data"].as_array().unwrap().len(), 2); + + let slot = message.data.slot; + let one = list(format!( + "/eth/v1/beacon/pool/payload_attestations?slot={slot}" + )) + .await; + assert_eq!(one.json()["data"].as_array().unwrap().len(), 1); + assert_eq!(one.json()["data"][0]["validator_index"], member.to_string()); + } } diff --git a/crates/net/rpc/src/lib.rs b/crates/net/rpc/src/lib.rs index a51a6aeb1..ca9c94bad 100644 --- a/crates/net/rpc/src/lib.rs +++ b/crates/net/rpc/src/lib.rs @@ -186,6 +186,11 @@ pub fn build_beacon_api_router(store: Store, version: &'static str, peer_id: Str .with_state(store) } +/// The data columns this node custodies, as the chain actor's availability +/// gate reads them. Empty on a node with nothing to custody. +#[derive(Clone, Debug, Default)] +pub struct CustodyColumns(pub Vec); + /// What the Beacon API's validator endpoints reach beyond the store. pub struct BeaconApiHandles { /// Through which the pool, aggregate and block endpoints gossip what a @@ -197,6 +202,9 @@ pub struct BeaconApiHandles { /// Filled by gossip and the payload attestation pool endpoint, read by /// block production and the pool's GET. pub payload_attestation_pool: SharedPayloadAttestationPool, + /// The columns this node samples, which `payload_attestation_data` checks a + /// block's blob availability against. + pub custody_columns: CustodyColumns, /// The execution client block production builds payloads with; `None` /// makes it answer 503. pub engine: Option, @@ -221,6 +229,7 @@ pub async fn start_beacon_rpc_server( .layer(Extension(handles.p2p)) .layer(Extension(handles.attestation_pool)) .layer(Extension(handles.payload_attestation_pool)) + .layer(Extension(handles.custody_columns)) .layer(Extension(beacon::validator::FeeRecipients::default())) .layer(Extension(handles.engine)); start_http_servers(config, Some(api_router), shutdown).await From 0648151a1cb412f89d8f62ee92acdcbc9979048a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:40:37 -0300 Subject: [PATCH 095/132] feat(engine): build gloas payloads through forkchoiceUpdatedV4 and getPayloadV6 A self-built gloas block needs the execution client to start a build with PayloadAttributesV4 (slot number and target gas limit) and to hand back an ExecutionPayloadV4 with its block access list and BlobsBundleV2. Both methods are advertised in the capabilities handshake. --- crates/net/engine/src/building.rs | 262 ++++++++++++++++++++++++++---- crates/net/engine/src/client.rs | 36 +++- crates/net/engine/src/lib.rs | 1 + 3 files changed, 262 insertions(+), 37 deletions(-) diff --git a/crates/net/engine/src/building.rs b/crates/net/engine/src/building.rs index e7493d9e9..b407c9326 100644 --- a/crates/net/engine/src/building.rs +++ b/crates/net/engine/src/building.rs @@ -11,7 +11,7 @@ //! The response is decoded straight into `ethlambda-types`' own containers, so //! the block producer never sees a wire shape. -use ethlambda_types::beacon::containers::{bellatrix, capella, deneb}; +use ethlambda_types::beacon::containers::{bellatrix, capella, deneb, gloas}; use ethlambda_types::beacon::primitives::{ Bytes32, ExecutionAddress, H160, H256, KzgCommitment, KzgProof, Root, U256, Uint256, }; @@ -60,6 +60,36 @@ impl Serialize for PayloadAttributesV3 { } } +/// `PayloadAttributesV4` (Amsterdam): `PayloadAttributesV3` plus the slot the +/// payload is built for and the gas limit the execution client should steer +/// toward. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PayloadAttributesV4 { + /// V3's five fields, serialized flat alongside the two below. + pub v3: PayloadAttributesV3, + /// The payload's `slotNumber` (EIP-7843): the beacon slot, not derived + /// from `timestamp` by the execution client. + pub slot_number: u64, + /// The `gasLimit` the payload should move toward, which the execution + /// client steps to within the protocol's per-block bound. + pub target_gas_limit: u64, +} + +impl Serialize for PayloadAttributesV4 { + fn serialize(&self, serializer: S) -> Result { + let mut value = serde_json::to_value(&self.v3).map_err(serde::ser::Error::custom)?; + let object = value + .as_object_mut() + .expect("PayloadAttributesV3 serializes as an object"); + object.insert("slotNumber".into(), quantity(self.slot_number).into()); + object.insert( + "targetGasLimit".into(), + quantity(self.target_gas_limit).into(), + ); + value.serialize(serializer) + } +} + /// The execution client's name for one build process, `DATA` of eight bytes. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct PayloadId(pub [u8; 8]); @@ -90,6 +120,18 @@ pub struct BuiltPayload { pub execution_requests: Vec>, } +/// `engine_getPayloadV6`'s answer, decoded: [`BuiltPayload`] with Amsterdam's +/// `ExecutionPayloadV4`, which gloas's envelope carries. +#[derive(Debug, Clone)] +pub struct BuiltGloasPayload { + pub execution_payload: gloas::ExecutionPayload, + /// What the payload pays its fee recipient, in wei. + pub block_value: Uint256, + pub blobs_bundle: BlobsBundle, + /// The EIP-7685 request list, each entry its type byte then its data. + pub execution_requests: Vec>, +} + /// `BlobsBundleV2`: the payload's blobs, their commitments, and every blob's /// `CELLS_PER_EXT_BLOB` cell proofs, flattened blob by blob. #[derive(Debug, Clone, Default)] @@ -109,6 +151,17 @@ pub(crate) struct GetPayloadV5Response { execution_requests: Vec, } +/// `engine_getPayloadV6`'s answer: V5's, with `ExecutionPayloadV4`. +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct GetPayloadV6Response { + execution_payload: ExecutionPayloadJson, + block_value: String, + blobs_bundle: BlobsBundleJson, + #[serde(default)] + execution_requests: Vec, +} + #[derive(Deserialize)] #[serde(rename_all = "camelCase")] struct ExecutionPayloadJson { @@ -129,6 +182,12 @@ struct ExecutionPayloadJson { withdrawals: Vec, blob_gas_used: String, excess_blob_gas: String, + /// `ExecutionPayloadV4` only (Amsterdam). + #[serde(default)] + block_access_list: Option, + /// `ExecutionPayloadV4` only (Amsterdam). + #[serde(default)] + slot_number: Option, } #[derive(Deserialize)] @@ -160,18 +219,7 @@ impl TryFrom for BuiltPayload { .map_err(|_| decode("a transaction exceeds MAX_BYTES_PER_TRANSACTION")) }) .collect::, _>>()?; - let withdrawals = payload - .withdrawals - .iter() - .map(|withdrawal| { - Ok(capella::Withdrawal { - index: parse_quantity(&withdrawal.index)?, - validator_index: parse_quantity(&withdrawal.validator_index)?, - address: H160(parse_fixed(&withdrawal.address)?), - amount: parse_quantity(&withdrawal.amount)?, - }) - }) - .collect::, EngineError>>()?; + let withdrawals = decode_withdrawals(&payload.withdrawals)?; let execution_payload = deneb::ExecutionPayload { parent_hash: H256(parse_fixed(&payload.parent_hash)?), @@ -199,38 +247,110 @@ impl TryFrom for BuiltPayload { excess_blob_gas: parse_quantity(&payload.excess_blob_gas)?, }; - let bundle = response.blobs_bundle; - let blobs_bundle = BlobsBundle { - commitments: bundle - .commitments - .iter() - .map(|text| parse_fixed(text).map(KzgCommitment)) - .collect::>()?, - proofs: bundle - .proofs - .iter() - .map(|text| parse_fixed(text).map(KzgProof)) - .collect::>()?, - blobs: bundle - .blobs - .iter() - .map(|text| parse_data(text)) - .collect::>()?, - }; + let blobs_bundle = decode_blobs_bundle(response.blobs_bundle)?; Ok(BuiltPayload { execution_payload, block_value: parse_uint256(&response.block_value)?, blobs_bundle, - execution_requests: response - .execution_requests - .iter() - .map(|text| parse_data(text)) - .collect::>()?, + execution_requests: decode_requests(&response.execution_requests)?, }) } } +impl TryFrom for BuiltGloasPayload { + type Error = EngineError; + + fn try_from(response: GetPayloadV6Response) -> Result { + let payload = response.execution_payload; + let transactions = payload + .transactions + .iter() + .map(|transaction| parse_data(transaction).map(gloas::Transaction::from)) + .collect::, _>>()?; + let withdrawals = decode_withdrawals(&payload.withdrawals)?; + let block_access_list = payload + .block_access_list + .as_deref() + .ok_or_else(|| decode("ExecutionPayloadV4 carries no blockAccessList"))?; + let slot_number = payload + .slot_number + .as_deref() + .ok_or_else(|| decode("ExecutionPayloadV4 carries no slotNumber"))?; + + let execution_payload = gloas::ExecutionPayload { + parent_hash: H256(parse_fixed(&payload.parent_hash)?), + fee_recipient: H160(parse_fixed(&payload.fee_recipient)?), + state_root: H256(parse_fixed(&payload.state_root)?), + receipts_root: H256(parse_fixed(&payload.receipts_root)?), + logs_bloom: bellatrix::LogsBloom::try_from(parse_data(&payload.logs_bloom)?) + .map_err(|_| decode("logsBloom is not BYTES_PER_LOGS_BLOOM long"))?, + prev_randao: H256(parse_fixed(&payload.prev_randao)?), + block_number: parse_quantity(&payload.block_number)?, + gas_limit: parse_quantity(&payload.gas_limit)?, + gas_used: parse_quantity(&payload.gas_used)?, + timestamp: parse_quantity(&payload.timestamp)?, + extra_data: bellatrix::ExtraData::try_from(parse_data(&payload.extra_data)?) + .map_err(|_| decode("extraData exceeds MAX_EXTRA_DATA_BYTES"))?, + base_fee_per_gas: parse_uint256(&payload.base_fee_per_gas)?, + block_hash: H256(parse_fixed(&payload.block_hash)?), + transactions: transactions.into(), + withdrawals: withdrawals.into(), + blob_gas_used: parse_quantity(&payload.blob_gas_used)?, + excess_blob_gas: parse_quantity(&payload.excess_blob_gas)?, + block_access_list: parse_data(block_access_list)?.into(), + slot_number: parse_quantity(slot_number)?, + }; + + Ok(BuiltGloasPayload { + execution_payload, + block_value: parse_uint256(&response.block_value)?, + blobs_bundle: decode_blobs_bundle(response.blobs_bundle)?, + execution_requests: decode_requests(&response.execution_requests)?, + }) + } +} + +fn decode_withdrawals( + withdrawals: &[WithdrawalJson], +) -> Result, EngineError> { + withdrawals + .iter() + .map(|withdrawal| { + Ok(capella::Withdrawal { + index: parse_quantity(&withdrawal.index)?, + validator_index: parse_quantity(&withdrawal.validator_index)?, + address: H160(parse_fixed(&withdrawal.address)?), + amount: parse_quantity(&withdrawal.amount)?, + }) + }) + .collect() +} + +fn decode_blobs_bundle(bundle: BlobsBundleJson) -> Result { + Ok(BlobsBundle { + commitments: bundle + .commitments + .iter() + .map(|text| parse_fixed(text).map(KzgCommitment)) + .collect::>()?, + proofs: bundle + .proofs + .iter() + .map(|text| parse_fixed(text).map(KzgProof)) + .collect::>()?, + blobs: bundle + .blobs + .iter() + .map(|text| parse_data(text)) + .collect::>()?, + }) +} + +fn decode_requests(requests: &[String]) -> Result>, EngineError> { + requests.iter().map(|text| parse_data(text)).collect() +} + fn decode(message: &str) -> EngineError { EngineError::Decode(message.to_string()) } @@ -375,4 +495,74 @@ mod tests { assert_eq!(uint256(&parse_uint256(&max).unwrap()), max); assert!(parse_uint256(&format!("0x{}", "f".repeat(65))).is_err()); } + + fn v6_json() -> serde_json::Value { + let mut json = response_json(); + json["executionPayload"]["blockAccessList"] = "0xc180".into(); + json["executionPayload"]["slotNumber"] = "0x1000".into(); + json["blobsBundle"] = serde_json::json!({ + "commitments": [format!("0x{}", "aa".repeat(48))], + "proofs": [format!("0x{}", "bb".repeat(48)), format!("0x{}", "cc".repeat(48))], + "blobs": [format!("0x{}", "01".repeat(8))], + }); + json["executionRequests"] = serde_json::json!(["0x0011", "0x02ff"]); + json + } + + #[test] + fn a_get_payload_v6_answer_decodes_into_the_gloas_containers() { + let response: GetPayloadV6Response = serde_json::from_value(v6_json()).unwrap(); + let built = BuiltGloasPayload::try_from(response).unwrap(); + let payload = &built.execution_payload; + assert_eq!(payload.slot_number, 0x1000); + assert_eq!(payload.block_access_list.as_ref(), &[0xc1, 0x80]); + assert_eq!(payload.block_hash, H256([0x55; 32])); + assert_eq!(payload.transactions.len(), 1); + assert_eq!(payload.withdrawals[0].amount, 3); + assert_eq!(uint256(&built.block_value), "0x1bc16d674ec80000"); + assert_eq!(built.blobs_bundle.commitments.len(), 1); + assert_eq!(built.blobs_bundle.proofs.len(), 2); + assert_eq!(built.blobs_bundle.blobs, vec![vec![1u8; 8]]); + assert_eq!( + built.execution_requests, + vec![vec![0x00, 0x11], vec![0x02, 0xff]] + ); + } + + #[test] + fn a_v6_answer_without_a_slot_number_is_a_decode_error() { + let mut json = v6_json(); + json["executionPayload"] + .as_object_mut() + .unwrap() + .remove("slotNumber"); + let response: GetPayloadV6Response = serde_json::from_value(json).unwrap(); + assert!(matches!( + BuiltGloasPayload::try_from(response), + Err(EngineError::Decode(_)) + )); + } + + #[test] + fn payload_attributes_v4_append_the_slot_and_target_gas_limit() { + let attributes = PayloadAttributesV4 { + v3: PayloadAttributesV3 { + timestamp: 12, + prev_randao: H256([1; 32]), + suggested_fee_recipient: H160([2; 20]), + withdrawals: vec![], + parent_beacon_block_root: Root::repeat_byte(4), + }, + slot_number: 4096, + target_gas_limit: 60_000_000, + }; + let json = serde_json::to_value(&attributes).unwrap(); + assert_eq!(json["timestamp"], "0xc"); + assert_eq!(json["slotNumber"], "0x1000"); + assert_eq!(json["targetGasLimit"], "0x3938700"); + assert_eq!( + json["parentBeaconBlockRoot"], + format!("0x{}", "04".repeat(32)) + ); + } } diff --git a/crates/net/engine/src/client.rs b/crates/net/engine/src/client.rs index 25a4db2e9..fb16ba6db 100644 --- a/crates/net/engine/src/client.rs +++ b/crates/net/engine/src/client.rs @@ -311,6 +311,36 @@ impl EngineClient { response.try_into() } + /// `engine_forkchoiceUpdatedV4` with `payloadAttributes`: Amsterdam's fork + /// choice notification plus a request to build a payload for the slot the + /// attributes name, with the consensus client's custody set. + /// + /// Returns the head's status and the build process's id; the id is `None` + /// when the execution client declined to build. + pub async fn forkchoice_updated_v4_with_attributes( + &self, + state: &ForkchoiceStateV1, + attributes: &crate::building::PayloadAttributesV4, + custody_columns: Option, + ) -> Result<(PayloadStatusV1, Option), EngineError> { + let params = json!([state, attributes, custody_columns]); + let response: ForkchoiceUpdatedResponse = + self.call("engine_forkchoiceUpdatedV4", params).await?; + Ok((response.payload_status, response.payload_id)) + } + + /// `engine_getPayloadV6`: [`Self::get_payload`] for Amsterdam, answering + /// with `ExecutionPayloadV4` (`blockAccessList` and `slotNumber` included). + pub async fn get_payload_v6( + &self, + payload_id: crate::building::PayloadId, + ) -> Result { + let response: crate::building::GetPayloadV6Response = self + .call("engine_getPayloadV6", json!([payload_id])) + .await?; + response.try_into() + } + /// `engine_exchangeCapabilities`. Returns what the execution client says it /// supports. pub async fn exchange_capabilities(&self, ours: &[&str]) -> Result, EngineError> { @@ -355,7 +385,11 @@ impl EngineClient { // network that does not schedule gloas never needs it, and an // execution client that predates Amsterdam is right not to offer it. if gloas_scheduled { - for required in ["engine_newPayloadV5", "engine_forkchoiceUpdatedV4"] { + for required in [ + "engine_newPayloadV5", + "engine_forkchoiceUpdatedV4", + "engine_getPayloadV6", + ] { if !theirs.iter().any(|method| method == required) { warn!( method = required, diff --git a/crates/net/engine/src/lib.rs b/crates/net/engine/src/lib.rs index f3cf1d737..742cd0bbf 100644 --- a/crates/net/engine/src/lib.rs +++ b/crates/net/engine/src/lib.rs @@ -44,5 +44,6 @@ pub const ETHLAMBDA_ENGINE_CAPABILITIES: &[&str] = &[ "engine_forkchoiceUpdatedV3", "engine_forkchoiceUpdatedV4", "engine_getPayloadV5", + "engine_getPayloadV6", "engine_getClientVersionV1", ]; From 85a54ba73f22c6b630408776a20a1024c00d2eac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:57:54 -0300 Subject: [PATCH 096/132] feat(beacon): assemble self-built gloas blocks and their payload envelopes Gloas moves the payload out of the block: the body commits to a bid and the payload is revealed in an envelope. Block production therefore needs its own assembly: the payload inputs that depend on whether the parent's payload is built on, the five-kind request list, gloas attestation packing (payload-availability index rules), payload attestation aggregation by PTC seat, the self-build bid, the state root, and the unsigned envelope. The envelope is checked against the block's post-state before it is handed out, so a malformed build fails at the producer. Column sidecars for the blobs are built from the cells and the execution client's cell proofs. --- .../src/beacon/gloas_block_production.rs | 1006 +++++++++++++++++ .../state_transition/src/beacon/mod.rs | 1 + 2 files changed, 1007 insertions(+) create mode 100644 crates/blockchain/state_transition/src/beacon/gloas_block_production.rs diff --git a/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs b/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs new file mode 100644 index 000000000..0f78ee9f1 --- /dev/null +++ b/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs @@ -0,0 +1,1006 @@ +//! Assembling a self-built gloas block and the payload envelope that fulfills +//! it, per gloas `validator.md` ("Constructing the `BeaconBlockBody`") and +//! `builder.md` ("Constructing the `SignedExecutionPayloadBid`" and +//! "Constructing the `SignedExecutionPayloadEnvelope`"). +//! +//! Gloas splits what fulu's [`super::block_production`] assembles in one piece: +//! the block commits to a bid, and the payload travels afterward in an +//! envelope. This node only ever builds for itself (`BUILDER_INDEX_SELF_BUILD`), +//! so the bid is its own: zero value, the G2 point at infinity for a +//! signature, and every field read off the payload its execution client built. +//! The proposer then owes the network the envelope (signed with its own key, +//! see `process_execution_payload_bid`'s self-build branch) and the data +//! columns of the blobs the bid commits to; [`assemble_gloas_block`] returns +//! the unsigned envelope next to the block. +//! +//! Everything the execution client supplies arrives already built. What this +//! module decides is what the consensus layer decides: which parent payload +//! the build extends ([`gloas_payload_inputs`]), which attestations and +//! payload attestations ride in the body, and the state root, which comes from +//! running the block through gloas's `process_block` on a copy of the state. + +use std::collections::BTreeMap; + +use super::block_production::{empty_sync_aggregate, pack_attestations}; +use super::bls; +use super::config::Config; +use super::error::{Error, Result, verify}; +use super::helpers::accessors::{ + CommitteeCache, get_beacon_proposer_index, get_current_epoch, get_randao_mix, +}; +use super::helpers::gloas::{ + get_indexed_payload_attestation, get_ptc, gloas_state_ref, is_attestation_same_slot, + is_valid_indexed_payload_attestation, +}; +use super::stf; +use ethlambda_types::beacon::{ + constants, + containers::{ + BeaconState, + capella::Withdrawal, + electra, + gloas::{ + self, BeaconBlock, BeaconBlockBody, ExecutionPayload, ExecutionPayloadBid, + ExecutionPayloadEnvelope, ExecutionRequests, PayloadAttestation, + PayloadAttestationData, PayloadAttestationMessage, SignedExecutionPayloadBid, + }, + }, + preset, + primitives::{ + BlsSignature, Bytes32, ExecutionBlockHash, HashTreeRoot as _, KzgCommitment, Root, Slot, + ValidatorIndex, + }, +}; + +/// What the execution client needs to build the payload for the slot `state` +/// has been advanced to: `PayloadAttributesV4`, less the fee recipient, plus +/// the execution block the build extends. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GloasPayloadInputs { + pub timestamp: u64, + pub prev_randao: Bytes32, + pub withdrawals: Vec, + /// The execution block the payload must extend: the head of the + /// `forkchoiceUpdated` that starts the build. + pub head_block_hash: ExecutionBlockHash, + /// `hash_tree_root(state.latest_block_header)`. + pub parent_beacon_block_root: Root, + pub slot_number: u64, + pub target_gas_limit: u64, +} + +/// The payload inputs for the block at `state`'s slot, from `prepare_execution_payload` +/// (gloas `validator.md`). +/// +/// `build_on_full` is `should_build_on_full(store, head, current_slot)`: the +/// proposer builds on the parent's revealed payload, so the parent's requests +/// (`parent_requests`, empty when the parent is pre-gloas) are applied to a +/// copy of the state first, and the withdrawals are those of that copy. Else +/// the parent's payload is skipped and the withdrawals are the ones the state +/// already expects. The target gas limit is the parent bid's own (contract +/// decision: this node expresses no preference of its own). +pub fn gloas_payload_inputs( + state: &BeaconState, + build_on_full: bool, + parent_requests: &ExecutionRequests, + config: &Config, +) -> Result { + let inner = gloas_state_ref(state, "gloas_payload_inputs")?; + let parent_bid = &inner.latest_execution_payload_bid; + let (withdrawals, head_block_hash) = if build_on_full { + let mut applied = state.clone(); + stf::gloas::apply_parent_execution_payload(&mut applied, parent_requests, config)?; + ( + stf::gloas::get_expected_withdrawals(&applied)?.withdrawals, + parent_bid.block_hash, + ) + } else { + ( + inner.payload_expected_withdrawals.iter().cloned().collect(), + parent_bid.parent_block_hash, + ) + }; + Ok(GloasPayloadInputs { + timestamp: stf::bellatrix::compute_timestamp_at_slot(state, state.slot(), config), + prev_randao: get_randao_mix(state, get_current_epoch(state)), + withdrawals, + head_block_hash, + parent_beacon_block_root: state.latest_block_header().hash_tree_root(), + slot_number: state.slot(), + target_gas_limit: parent_bid.gas_limit, + }) +} + +/// `get_execution_requests` (gloas `validator.md`): the execution client's +/// EIP-7685 request list into gloas's five-kind `ExecutionRequests`. +/// +/// The types must be strictly ascending and no element empty, which is +/// checked rather than trusted, since a list this node accepted would go into +/// a bid its peers then reject. +pub fn parse_gloas_execution_requests(list: &[Vec]) -> Result { + let mut requests = ExecutionRequests::default(); + let mut previous_type: Option = None; + for element in list { + let (&request_type, data) = element + .split_first() + .ok_or(Error::SpecAssert("an execution request carries its type"))?; + verify(!data.is_empty(), "an execution request is not empty")?; + verify( + previous_type.is_none_or(|previous| request_type > previous), + "execution request types are strictly ascending", + )?; + previous_type = Some(request_type); + let malformed = |_| Error::SpecAssert("an execution request list decodes as SSZ"); + match request_type { + constants::DEPOSIT_REQUEST_TYPE => { + requests.deposits = libssz::SszDecode::from_ssz_bytes(data).map_err(malformed)?; + } + constants::WITHDRAWAL_REQUEST_TYPE => { + requests.withdrawals = + libssz::SszDecode::from_ssz_bytes(data).map_err(malformed)?; + } + constants::CONSOLIDATION_REQUEST_TYPE => { + requests.consolidations = + libssz::SszDecode::from_ssz_bytes(data).map_err(malformed)?; + } + constants::BUILDER_DEPOSIT_REQUEST_TYPE => { + requests.builder_deposits = + libssz::SszDecode::from_ssz_bytes(data).map_err(malformed)?; + } + constants::BUILDER_EXIT_REQUEST_TYPE => { + requests.builder_exits = + libssz::SszDecode::from_ssz_bytes(data).map_err(malformed)?; + } + _ => return Err(Error::SpecAssert("a known execution request type")), + } + } + Ok(requests) +} + +/// Pick the attestations for a gloas block at `state`'s slot from the +/// electra-shaped pool `candidates`. +/// +/// Everything [`pack_attestations`] enforces, plus gloas `process_attestation`'s +/// payload-availability rule: `data.index` is `0` or `1`, and `0` when the +/// vote is for the block of its own slot (`is_attestation_same_slot`), which +/// `get_attestation_participation_flag_indices` asserts. A vote failing that +/// would fail the whole block, so it is left out here. +pub fn pack_gloas_attestations( + state: &BeaconState, + candidates: Vec, +) -> Vec { + let includable = |attestation: &electra::Attestation| { + let data = &attestation.data; + data.index < 2 + && is_attestation_same_slot(state, data) + .is_ok_and(|same_slot| !same_slot || data.index == 0) + }; + pack_attestations(state, candidates.into_iter().filter(includable).collect()) + .iter() + .map(gloas::Attestation::from) + .collect() +} + +/// The payload attestations for a block at `state`'s slot, aggregated from the +/// pool's `messages` (gloas `validator.md`, "Payload attestations"). +/// +/// Only the parent block's own slot is attested to: the votes must be for +/// `parent_root`, and the parent must sit exactly one slot before this block. +/// Votes sharing one `PayloadAttestationData` become one aggregate. Its bit +/// `i` is set for every position `i` of `get_ptc(state, parent_slot)` whose +/// validator has a vote on that data, and that vote's signature is aggregated +/// once per such position: a validator holding two seats sets two bits and +/// contributes twice, which is what `get_indexed_payload_attestation` followed +/// by `FastAggregateVerify` over the repeated key expects. +/// +/// An aggregate that does not verify against `state` is dropped, since one bad +/// operation fails the whole block. At most `MAX_PAYLOAD_ATTESTATIONS` are +/// returned, the ones covering the most seats first. +pub fn pack_payload_attestations( + state: &BeaconState, + parent_root: Root, + parent_slot: Slot, + messages: Vec, + config: &Config, +) -> Vec { + if parent_slot.checked_add(1) != Some(state.slot()) { + return Vec::new(); + } + let Ok(ptc) = get_ptc(state, parent_slot, config) else { + return Vec::new(); + }; + + // One entry per distinct data: the first signature each validator sent. + let mut groups: BTreeMap< + Root, + ( + PayloadAttestationData, + BTreeMap, + ), + > = BTreeMap::new(); + for message in messages { + if message.data.beacon_block_root != parent_root || message.data.slot != parent_slot { + continue; + } + groups + .entry(message.data.hash_tree_root()) + .or_insert_with(|| (message.data, BTreeMap::new())) + .1 + .entry(message.validator_index) + .or_insert(message.signature); + } + + let mut packed: Vec<(usize, PayloadAttestation)> = groups + .into_values() + .filter_map(|(data, votes)| { + let mut aggregation_bits = gloas::PayloadTimelinessCommitteeBits::default(); + let mut signatures = Vec::new(); + for (position, validator) in ptc.iter().enumerate() { + if let Some(signature) = votes.get(validator) { + aggregation_bits.set(position, true).ok()?; + signatures.push(*signature); + } + } + let seats = signatures.len(); + let attestation = PayloadAttestation { + aggregation_bits, + data, + signature: bls::aggregate(&signatures).ok()?, + }; + let indexed = get_indexed_payload_attestation(state, &attestation, config).ok()?; + is_valid_indexed_payload_attestation(state, &indexed).then_some((seats, attestation)) + }) + .collect(); + packed.sort_by_key(|(seats, _)| std::cmp::Reverse(*seats)); + packed + .into_iter() + .map(|(_, attestation)| attestation) + .take(preset::MAX_PAYLOAD_ATTESTATIONS as usize) + .collect() +} + +/// `get_data_column_sidecars` (gloas `builder.md`): the `NUMBER_OF_COLUMNS` +/// column sidecars of the blobs a payload carries, named by the block they +/// belong to rather than by a signed header and an inclusion proof. +/// +/// `blobs` are the raw blobs and `cell_proofs` the execution client's +/// flattened cell proofs, `CELLS_PER_EXT_BLOB` per blob in blob order +/// (`BlobsBundleV2`). The cells are recomputed from the blobs, since the +/// bundle carries only proofs. Fails when the proof count does not match, or +/// a blob is not valid. +pub fn gloas_data_column_sidecars( + beacon_block_root: Root, + slot: Slot, + blobs: &[Vec], + cell_proofs: &[ethlambda_types::beacon::primitives::KzgProof], +) -> Result> { + verify( + cell_proofs.len() == blobs.len() * preset::CELLS_PER_EXT_BLOB, + "a blob carries CELLS_PER_EXT_BLOB cell proofs", + )?; + let per_blob_cells = blobs + .iter() + .map(|blob| super::kzg::compute_cells(blob)) + .collect::>>()?; + + (0..preset::NUMBER_OF_COLUMNS) + .map(|column_index| { + let mut column = Vec::with_capacity(blobs.len()); + let mut kzg_proofs = Vec::with_capacity(blobs.len()); + for (blob, cells) in per_blob_cells.iter().enumerate() { + let cell = ethlambda_types::beacon::containers::fulu::Cell::try_from( + cells[column_index].to_bytes().to_vec(), + ) + .map_err(|_| Error::SpecAssert("len(cell) == BYTES_PER_CELL"))?; + column.push(cell); + kzg_proofs.push(cell_proofs[blob * preset::CELLS_PER_EXT_BLOB + column_index]); + } + Ok(gloas::DataColumnSidecar { + index: column_index as u64, + column: column.into(), + kzg_proofs: kzg_proofs.into(), + slot, + beacon_block_root, + }) + }) + .collect() +} + +/// What a gloas block body carries beyond what this module derives from the +/// state. +#[derive(Debug, Clone)] +pub struct GloasBlockInputs { + pub randao_reveal: BlsSignature, + pub graffiti: Bytes32, + pub attestations: Vec, + pub payload_attestations: Vec, + /// The parent envelope's requests when building on its full payload and + /// the parent is gloas; empty otherwise. + pub parent_execution_requests: ExecutionRequests, + /// The payload `engine_getPayloadV6` built, which the envelope reveals. + pub execution_payload: ExecutionPayload, + /// The blobs bundle's commitments, which the bid commits to. + pub blob_kzg_commitments: Vec, + /// The payload's own requests, which the bid commits to by root and the + /// envelope carries. + pub execution_requests: ExecutionRequests, +} + +/// A produced block and the unsigned envelope that reveals its payload. +#[derive(Debug, Clone)] +pub struct GloasProduced { + pub block: BeaconBlock, + pub envelope: ExecutionPayloadEnvelope, +} + +/// The unsigned self-built block for the slot `state` has been advanced to, with +/// its state root computed, and the unsigned envelope for its payload. +/// +/// The body votes the state's own `eth1_data`, carries no slashings, exits or +/// credential changes (this node pools none) and an empty sync aggregate. The +/// block is run through gloas's `process_block` on a copy of `state`, which +/// is also what rejects a body the network would, before anything is signed. +/// The envelope is then checked against that post-state with every condition +/// `verify_execution_payload_envelope` holds it to except the signature, so a +/// malformed build fails here and not on the network. +pub fn assemble_gloas_block( + state: &BeaconState, + inputs: GloasBlockInputs, + config: &Config, +) -> Result { + verify( + matches!(state, BeaconState::Gloas(_)), + "gloas block production runs on a gloas state", + )?; + let payload = &inputs.execution_payload; + let bid = ExecutionPayloadBid { + parent_block_hash: payload.parent_hash, + parent_block_root: state.latest_block_header().hash_tree_root(), + block_hash: payload.block_hash, + prev_randao: payload.prev_randao, + fee_recipient: payload.fee_recipient, + gas_limit: payload.gas_limit, + builder_index: constants::BUILDER_INDEX_SELF_BUILD, + slot: state.slot(), + value: 0, + execution_payment: 0, + blob_kzg_commitments: inputs.blob_kzg_commitments.into(), + execution_requests_root: inputs.execution_requests.hash_tree_root(), + }; + let body = BeaconBlockBody { + randao_reveal: inputs.randao_reveal, + eth1_data: state.eth1_data().clone(), + graffiti: inputs.graffiti, + attestations: inputs.attestations.into(), + sync_aggregate: empty_sync_aggregate(), + signed_execution_payload_bid: SignedExecutionPayloadBid { + message: bid, + signature: BlsSignature(bls::G2_POINT_AT_INFINITY), + }, + payload_attestations: inputs.payload_attestations.into(), + parent_execution_requests: inputs.parent_execution_requests, + ..BeaconBlockBody::empty() + }; + let mut block = BeaconBlock { + slot: state.slot(), + proposer_index: get_beacon_proposer_index(state)?, + // `process_slots` filled the header's state root on the way out of + // the parent's slot, so this is the parent block's root. + parent_root: state.latest_block_header().hash_tree_root(), + state_root: Root::ZERO, + body, + }; + + let mut post = state.clone(); + let BeaconState::Gloas(_) = &post else { + unreachable!("checked above") + }; + stf::gloas::process_block(&mut post, &block, config, &CommitteeCache::default())?; + block.state_root = post.hash_tree_root(); + + let envelope = ExecutionPayloadEnvelope { + payload: inputs.execution_payload, + execution_requests: inputs.execution_requests, + builder_index: constants::BUILDER_INDEX_SELF_BUILD, + beacon_block_root: block.hash_tree_root(), + parent_beacon_block_root: block.parent_root, + }; + check_envelope(&post, &envelope, config)?; + Ok(GloasProduced { block, envelope }) +} + +/// `verify_execution_payload_envelope` (gloas `fork-choice.md`) without its +/// signature check and its engine call, against `post`, the state after the +/// block the envelope belongs to. +/// +/// Mirrors [`stf::gloas::verify_execution_payload_envelope`] condition for +/// condition rather than calling it, because that function verifies the +/// signature first and the producer has none to give yet: the proposer signs +/// the envelope after this node hands it over. +fn check_envelope( + post: &BeaconState, + envelope: &ExecutionPayloadEnvelope, + config: &Config, +) -> Result<()> { + let payload = &envelope.payload; + let mut header = post.latest_block_header().clone(); + header.state_root = post.compute_state_root(); + verify( + envelope.beacon_block_root == header.hash_tree_root(), + "envelope.beacon_block_root == hash_tree_root(header)", + )?; + verify( + envelope.parent_beacon_block_root == post.latest_block_header().parent_root, + "envelope.parent_beacon_block_root == state.latest_block_header.parent_root", + )?; + let inner = gloas_state_ref(post, "check_envelope")?; + let bid = &inner.latest_execution_payload_bid; + verify( + envelope.builder_index == bid.builder_index, + "envelope.builder_index == bid.builder_index", + )?; + verify( + payload.prev_randao == bid.prev_randao, + "payload.prev_randao == bid.prev_randao", + )?; + verify( + payload.gas_limit == bid.gas_limit, + "payload.gas_limit == bid.gas_limit", + )?; + verify( + payload.block_hash == bid.block_hash, + "payload.block_hash == bid.block_hash", + )?; + verify( + envelope.execution_requests.hash_tree_root() == bid.execution_requests_root, + "hash_tree_root(envelope.execution_requests) == bid.execution_requests_root", + )?; + verify( + payload.slot_number == post.slot(), + "payload.slot_number == state.slot", + )?; + verify( + payload.parent_hash == inner.latest_block_hash, + "payload.parent_hash == state.latest_block_hash", + )?; + verify( + payload.timestamp == stf::bellatrix::compute_timestamp_at_slot(post, post.slot(), config), + "payload.timestamp == compute_time_at_slot(state, state.slot)", + )?; + verify( + payload.withdrawals.hash_tree_root() == inner.payload_expected_withdrawals.hash_tree_root(), + "hash_tree_root(payload.withdrawals) == hash_tree_root(state.payload_expected_withdrawals)", + )?; + Ok(()) +} + +#[cfg(test)] +mod gloas_block_production_tests { + use super::super::stf::ExecutionEngine; + use super::*; + use crate::beacon::ForkName; + use crate::beacon::block_production::advance_to_slot; + use crate::beacon::helpers::accessors::get_domain; + use crate::beacon::helpers::fulu::initialize_proposer_lookahead; + use crate::beacon::helpers::gloas::compute_ptc; + use crate::beacon::helpers::misc::compute_signing_root; + use crate::beacon::helpers::test_state::{sign_for, with_signing_validators_at}; + use ethlambda_types::beacon::containers::bellatrix::{ExtraData, LogsBloom}; + use ethlambda_types::beacon::primitives::Uint256; + + const PARENT_BLOCK_HASH: u8 = 0x11; + const GRANDPARENT_BLOCK_HASH: u8 = 0x10; + + fn config() -> Config { + Config::mainnet().with_fork_epoch(ForkName::Gloas, 0) + } + + /// A gloas state at slot 33 whose parent block (slot 32) revealed a payload, + /// its PTC for slot 32 filled from the real registry, with the lookahead + /// and sync committee a real registry would give it. + fn state_to_build_on() -> BeaconState { + let mut state = with_signing_validators_at(ForkName::Gloas, 64); + let lookahead = initialize_proposer_lookahead(&state).unwrap(); + let sync_committee = + crate::beacon::helpers::altair::get_next_sync_committee(&state).unwrap(); + let ptc = compute_ptc(&state, state.slot(), &CommitteeCache::default()).unwrap(); + let BeaconState::Gloas(inner) = &mut state else { + unreachable!("built as gloas") + }; + inner.proposer_lookahead = lookahead.try_into().unwrap(); + inner.current_sync_committee = sync_committee.clone(); + inner.next_sync_committee = sync_committee; + inner.latest_block_header.slot = inner.slot; + inner.latest_block_hash = ExecutionBlockHash::repeat_byte(GRANDPARENT_BLOCK_HASH); + inner.latest_execution_payload_bid = ExecutionPayloadBid { + parent_block_hash: ExecutionBlockHash::repeat_byte(GRANDPARENT_BLOCK_HASH), + block_hash: ExecutionBlockHash::repeat_byte(PARENT_BLOCK_HASH), + gas_limit: 30_000_000, + slot: inner.slot, + builder_index: constants::BUILDER_INDEX_SELF_BUILD, + execution_requests_root: ExecutionRequests::default().hash_tree_root(), + ..Default::default() + }; + // The window entry `get_ptc` reads for a slot of the state's own epoch. + let window_index = + (preset::SLOTS_PER_EPOCH + inner.slot % preset::SLOTS_PER_EPOCH) as usize; + inner.ptc_window[window_index] = ptc; + let slot = state.slot() + 1; + advance_to_slot(&state, slot, &config()).unwrap() + } + + fn payload_for(inputs: &GloasPayloadInputs) -> ExecutionPayload { + ExecutionPayload { + parent_hash: inputs.head_block_hash, + fee_recipient: Default::default(), + state_root: Bytes32::repeat_byte(1), + receipts_root: Bytes32::repeat_byte(2), + logs_bloom: LogsBloom::try_from(vec![0u8; preset::BYTES_PER_LOGS_BLOOM]).unwrap(), + prev_randao: inputs.prev_randao, + block_number: 7, + gas_limit: inputs.target_gas_limit, + gas_used: 0, + timestamp: inputs.timestamp, + extra_data: ExtraData::default(), + base_fee_per_gas: Uint256::from_u128(7), + block_hash: ExecutionBlockHash::repeat_byte(0x22), + transactions: Default::default(), + withdrawals: inputs.withdrawals.clone().into(), + blob_gas_used: 0, + excess_blob_gas: 0, + block_access_list: Default::default(), + slot_number: inputs.slot_number, + } + } + + fn randao_reveal(state: &BeaconState) -> BlsSignature { + let proposer = get_beacon_proposer_index(state).unwrap(); + let epoch = get_current_epoch(state); + let domain = get_domain(state, constants::DOMAIN_RANDAO, Some(epoch)); + sign_for( + proposer as usize, + compute_signing_root(epoch.hash_tree_root(), domain), + ) + } + + fn produce( + state: &BeaconState, + build_on_full: bool, + payload_attestations: Vec, + ) -> Result { + let requests = ExecutionRequests::default(); + let inputs = gloas_payload_inputs(state, build_on_full, &requests, &config())?; + assemble_gloas_block( + state, + GloasBlockInputs { + randao_reveal: randao_reveal(state), + graffiti: Bytes32::repeat_byte(7), + attestations: Vec::new(), + payload_attestations, + parent_execution_requests: requests, + execution_payload: payload_for(&inputs), + blob_kzg_commitments: vec![KzgCommitment([5; 48])], + execution_requests: ExecutionRequests::default(), + }, + &config(), + ) + } + + /// A vote on the parent block (slot 32) signed by `validator`. + fn ptc_message( + state: &BeaconState, + validator: ValidatorIndex, + data: PayloadAttestationData, + ) -> PayloadAttestationMessage { + let domain = get_domain( + state, + constants::DOMAIN_PTC_ATTESTER, + Some(ethlambda_types::beacon::signing::compute_epoch_at_slot( + data.slot, + )), + ); + PayloadAttestationMessage { + validator_index: validator, + data, + signature: sign_for( + validator as usize, + compute_signing_root(data.hash_tree_root(), domain), + ), + } + } + + fn parent_vote(state: &BeaconState, present: bool) -> PayloadAttestationData { + PayloadAttestationData { + beacon_block_root: state.latest_block_header().hash_tree_root(), + slot: state.slot() - 1, + payload_present: present, + blob_data_available: true, + } + } + + #[test] + fn a_self_built_block_passes_the_stf_with_a_real_signature_and_its_envelope_verifies() { + let state = state_to_build_on(); + let produced = produce(&state, true, Vec::new()).unwrap(); + let GloasProduced { block, envelope } = &produced; + + // Signed by the proposer, as a validator client would. + let signed = gloas::SignedBeaconBlock { + message: block.clone(), + signature: { + let domain = get_domain(&state, constants::DOMAIN_BEACON_PROPOSER, None); + sign_for( + block.proposer_index as usize, + compute_signing_root(block.hash_tree_root(), domain), + ) + }, + }; + let wrapped = ethlambda_types::beacon::containers::SignedBeaconBlock::Gloas(signed); + assert!(stf::verify_block_signature(&state, &wrapped)); + + // Applying it again, the way a peer importing it would, lands on the + // state root it claims. + let mut post = state.clone(); + stf::block::process_block( + &mut post, + &wrapped, + &config(), + &ExecutionEngine::valid(), + &CommitteeCache::default(), + ) + .unwrap(); + assert_eq!(block.state_root, post.hash_tree_root()); + + // The envelope, signed by the same proposer under DOMAIN_BEACON_BUILDER, + // passes the full import-side check. + let domain = get_domain(&post, constants::DOMAIN_BEACON_BUILDER, None); + let signed_envelope = gloas::SignedExecutionPayloadEnvelope { + message: envelope.clone(), + signature: sign_for( + block.proposer_index as usize, + compute_signing_root(envelope.hash_tree_root(), domain), + ), + }; + stf::gloas::verify_execution_payload_envelope( + &post, + &signed_envelope, + &config(), + &ExecutionEngine::valid(), + ) + .unwrap(); + assert_eq!(envelope.beacon_block_root, block.hash_tree_root()); + assert_eq!(envelope.parent_beacon_block_root, block.parent_root); + } + + #[test] + fn the_bid_is_a_zero_value_self_build_over_the_payload() { + let state = state_to_build_on(); + let produced = produce(&state, true, Vec::new()).unwrap(); + let signed_bid = &produced.block.body.signed_execution_payload_bid; + let bid = &signed_bid.message; + let payload = &produced.envelope.payload; + assert_eq!(bid.builder_index, constants::BUILDER_INDEX_SELF_BUILD); + assert_eq!((bid.value, bid.execution_payment), (0, 0)); + assert_eq!( + signed_bid.signature, + BlsSignature(bls::G2_POINT_AT_INFINITY) + ); + assert_eq!(bid.slot, state.slot()); + assert_eq!(bid.parent_block_hash, payload.parent_hash); + assert_eq!(bid.block_hash, payload.block_hash); + assert_eq!(bid.prev_randao, payload.prev_randao); + assert_eq!(bid.gas_limit, payload.gas_limit); + assert_eq!( + bid.parent_block_root, + state.latest_block_header().hash_tree_root() + ); + assert_eq!(bid.blob_kzg_commitments.len(), 1); + assert_eq!( + bid.execution_requests_root, + produced.envelope.execution_requests.hash_tree_root() + ); + } + + #[test] + fn building_on_full_extends_the_parent_payload_and_on_empty_skips_it() { + let state = state_to_build_on(); + let requests = ExecutionRequests::default(); + let full = gloas_payload_inputs(&state, true, &requests, &config()).unwrap(); + let empty = gloas_payload_inputs(&state, false, &requests, &config()).unwrap(); + assert_eq!( + full.head_block_hash, + ExecutionBlockHash::repeat_byte(PARENT_BLOCK_HASH) + ); + assert_eq!( + empty.head_block_hash, + ExecutionBlockHash::repeat_byte(GRANDPARENT_BLOCK_HASH) + ); + assert_eq!(full.slot_number, state.slot()); + assert_eq!(full.target_gas_limit, 30_000_000); + assert_eq!( + full.parent_beacon_block_root, + state.latest_block_header().hash_tree_root() + ); + + // Both are valid blocks, and the empty one's bid names the older hash. + let on_full = produce(&state, true, Vec::new()).unwrap(); + let on_empty = produce(&state, false, Vec::new()).unwrap(); + let parent_hash = |produced: &GloasProduced| { + produced + .block + .body + .signed_execution_payload_bid + .message + .parent_block_hash + }; + assert_eq!(parent_hash(&on_full), full.head_block_hash); + assert_eq!(parent_hash(&on_empty), empty.head_block_hash); + } + + #[test] + fn a_payload_on_the_wrong_parent_is_refused_before_signing() { + let state = state_to_build_on(); + let requests = ExecutionRequests::default(); + let inputs = gloas_payload_inputs(&state, true, &requests, &config()).unwrap(); + let mut payload = payload_for(&inputs); + payload.parent_hash = ExecutionBlockHash::repeat_byte(9); + let result = assemble_gloas_block( + &state, + GloasBlockInputs { + randao_reveal: randao_reveal(&state), + graffiti: Bytes32::ZERO, + attestations: Vec::new(), + payload_attestations: Vec::new(), + parent_execution_requests: requests, + execution_payload: payload, + blob_kzg_commitments: Vec::new(), + execution_requests: ExecutionRequests::default(), + }, + &config(), + ); + assert!(result.is_err()); + } + + #[test] + fn a_withdrawal_set_the_state_does_not_expect_is_refused() { + let state = state_to_build_on(); + let requests = ExecutionRequests::default(); + let inputs = gloas_payload_inputs(&state, true, &requests, &config()).unwrap(); + let mut payload = payload_for(&inputs); + payload.withdrawals = vec![Withdrawal { + index: 99, + validator_index: 1, + address: Default::default(), + amount: 1, + }] + .into(); + let produced = assemble_gloas_block( + &state, + GloasBlockInputs { + randao_reveal: randao_reveal(&state), + graffiti: Bytes32::ZERO, + attestations: Vec::new(), + payload_attestations: Vec::new(), + parent_execution_requests: requests, + execution_payload: payload, + blob_kzg_commitments: Vec::new(), + execution_requests: ExecutionRequests::default(), + }, + &config(), + ); + assert!(produced.is_err()); + } + + #[test] + fn payload_attestations_set_every_seat_and_aggregate_every_signature() { + let state = state_to_build_on(); + let ptc = get_ptc(&state, state.slot() - 1, &config()).unwrap(); + let first = ptc[0]; + let second = *ptc.iter().find(|validator| **validator != first).unwrap(); + let data = parent_vote(&state, true); + let messages = vec![ + ptc_message(&state, first, data), + ptc_message(&state, second, data), + // Another view of the payload: its own aggregate. + ptc_message(&state, first, parent_vote(&state, false)), + ]; + let parent_root = state.latest_block_header().hash_tree_root(); + let packed = + pack_payload_attestations(&state, parent_root, state.slot() - 1, messages, &config()); + + assert_eq!(packed.len(), 2); + let aggregate = &packed[0]; + assert_eq!(aggregate.data, data); + let seats = |validator: ValidatorIndex| ptc.iter().filter(|v| **v == validator).count(); + let expected_bits: Vec = ptc + .iter() + .enumerate() + .filter(|(_, validator)| **validator == first || **validator == second) + .map(|(position, _)| position) + .collect(); + let set: Vec = (0..preset::PTC_SIZE) + .filter(|&position| aggregate.aggregation_bits.get(position).unwrap_or(false)) + .collect(); + assert_eq!(set, expected_bits); + assert_eq!(set.len(), seats(first) + seats(second)); + + // It rides in a block that passes the full STF. + let produced = produce(&state, true, packed).unwrap(); + assert_eq!(produced.block.body.payload_attestations.len(), 2); + } + + #[test] + fn payload_attestations_for_another_parent_or_a_distant_slot_are_left_out() { + let state = state_to_build_on(); + let ptc = get_ptc(&state, state.slot() - 1, &config()).unwrap(); + let data = parent_vote(&state, true); + let message = ptc_message(&state, ptc[0], data); + let parent_root = state.latest_block_header().hash_tree_root(); + + // Not the parent block. + assert!( + pack_payload_attestations( + &state, + Root::repeat_byte(9), + state.slot() - 1, + vec![message.clone()], + &config() + ) + .is_empty() + ); + // The parent is two slots back: nothing to attest to. + assert!( + pack_payload_attestations( + &state, + parent_root, + state.slot() - 2, + vec![message], + &config() + ) + .is_empty() + ); + // A forged signature does not verify, so the aggregate is dropped. + let mut forged = ptc_message(&state, ptc[0], data); + forged.signature = ptc_message(&state, ptc[0], parent_vote(&state, false)).signature; + assert!( + pack_payload_attestations( + &state, + parent_root, + state.slot() - 1, + vec![forged], + &config() + ) + .is_empty() + ); + } + + /// A single-committee vote at slot 32 signed by committee 0's first member. + fn vote(state: &BeaconState, beacon_block_root: Root, index: u64) -> electra::Attestation { + use crate::beacon::helpers::accessors::{get_beacon_committee, get_block_root}; + let epoch = get_current_epoch(state); + let data = ethlambda_types::beacon::containers::shared::AttestationData { + slot: state.slot() - 1, + index, + beacon_block_root, + source: state.current_justified_checkpoint(), + target: ethlambda_types::beacon::containers::shared::Checkpoint { + epoch, + root: get_block_root(state, epoch).unwrap(), + }, + }; + let members = get_beacon_committee(state, data.slot, 0).unwrap(); + let domain = get_domain(state, constants::DOMAIN_BEACON_ATTESTER, Some(epoch)); + let mut aggregation_bits = electra::AggregationBits::with_length(members.len()).unwrap(); + aggregation_bits.set(0, true).unwrap(); + let mut committee_bits = electra::CommitteeBits::default(); + committee_bits.set(0, true).unwrap(); + electra::Attestation { + aggregation_bits, + data, + signature: sign_for( + members[0] as usize, + compute_signing_root(data.hash_tree_root(), domain), + ), + committee_bits, + } + } + + #[test] + fn attestations_honor_the_gloas_payload_index_rules() { + let state = state_to_build_on(); + let parent_root = state.latest_block_header().hash_tree_root(); + // A vote for an older block may name either payload status. + let elsewhere = Root::repeat_byte(1); + assert_eq!( + pack_gloas_attestations(&state, vec![vote(&state, elsewhere, 1)]).len(), + 1 + ); + // A vote for the block of its own slot must say 0, and none says 2. + assert!(pack_gloas_attestations(&state, vec![vote(&state, parent_root, 1)]).is_empty()); + assert_eq!( + pack_gloas_attestations(&state, vec![vote(&state, parent_root, 0)]).len(), + 1 + ); + assert!(pack_gloas_attestations(&state, vec![vote(&state, elsewhere, 2)]).is_empty()); + + // The packed vote rides in a block that passes the full STF. + let packed = pack_gloas_attestations(&state, vec![vote(&state, parent_root, 0)]); + let requests = ExecutionRequests::default(); + let inputs = gloas_payload_inputs(&state, true, &requests, &config()).unwrap(); + assemble_gloas_block( + &state, + GloasBlockInputs { + randao_reveal: randao_reveal(&state), + graffiti: Bytes32::ZERO, + attestations: packed, + payload_attestations: Vec::new(), + parent_execution_requests: requests, + execution_payload: payload_for(&inputs), + blob_kzg_commitments: Vec::new(), + execution_requests: ExecutionRequests::default(), + }, + &config(), + ) + .unwrap(); + } + + #[test] + fn column_sidecars_carry_every_cell_and_verify_against_the_commitments() { + // A blob of small field elements, valid and cheap to commit to. + let mut blob = vec![0u8; preset::BYTES_PER_BLOB]; + for (i, element) in blob.chunks_mut(32).enumerate() { + element[31] = (i % 200) as u8; + } + let commitment = crate::beacon::kzg::blob_to_kzg_commitment(&blob).unwrap(); + let (_, proofs) = crate::beacon::kzg::compute_cells_and_kzg_proofs(&blob).unwrap(); + let root = Root::repeat_byte(3); + let sidecars = gloas_data_column_sidecars(root, 33, &[blob], &proofs[..]).unwrap(); + + assert_eq!(sidecars.len(), preset::NUMBER_OF_COLUMNS); + for (index, sidecar) in sidecars.iter().enumerate() { + assert_eq!((sidecar.index, sidecar.slot), (index as u64, 33)); + assert_eq!(sidecar.beacon_block_root, root); + assert!( + crate::beacon::fork_choice::gloas_verify_data_column_sidecar( + sidecar, + &[commitment] + ) + ); + } + for index in [0, 77, preset::NUMBER_OF_COLUMNS - 1] { + assert!( + crate::beacon::fork_choice::gloas_verify_data_column_sidecar_kzg_proofs( + &sidecars[index], + &[commitment] + ) + .unwrap() + ); + } + // A proof count that does not fit the blobs is refused. + assert!( + gloas_data_column_sidecars(root, 33, &[vec![0; preset::BYTES_PER_BLOB]], &[]).is_err() + ); + } + + #[test] + fn a_gloas_request_list_round_trips_and_a_malformed_one_is_refused() { + let requests = ExecutionRequests { + builder_exits: vec![gloas::BuilderExitRequest::default()].into(), + ..Default::default() + }; + let list = stf::gloas::get_execution_requests_list(&requests); + assert_eq!(parse_gloas_execution_requests(&list).unwrap(), requests); + assert_eq!( + parse_gloas_execution_requests(&[]).unwrap(), + ExecutionRequests::default() + ); + let exit = vec![constants::BUILDER_EXIT_REQUEST_TYPE, 1]; + let deposit = vec![constants::DEPOSIT_REQUEST_TYPE, 1]; + assert!(parse_gloas_execution_requests(&[exit, deposit]).is_err()); + assert!( + parse_gloas_execution_requests(&[vec![constants::BUILDER_DEPOSIT_REQUEST_TYPE]]) + .is_err() + ); + assert!(parse_gloas_execution_requests(&[vec![0x7f, 0]]).is_err()); + } +} diff --git a/crates/blockchain/state_transition/src/beacon/mod.rs b/crates/blockchain/state_transition/src/beacon/mod.rs index bae1ac018..1a4471fa6 100644 --- a/crates/blockchain/state_transition/src/beacon/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/mod.rs @@ -75,6 +75,7 @@ pub mod bls; pub mod das; pub mod fork_choice; pub mod genesis; +pub mod gloas_block_production; pub mod gossip; pub mod hash; pub mod helpers; From 34d4935362564d165de7143def03ee6d30732f62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:09:51 -0300 Subject: [PATCH 097/132] feat(beacon): serve gloas block production and envelope publication A validator client needs three things from the node to propose at gloas: a self-built block (produceBlockV4), a way to publish it, and a way to publish the payload envelope that reveals it together with the data columns of its blobs, which a self-building proposer owes the network. produceBlockV4 builds on the head's recorded payload branch, asks the execution client through forkchoiceUpdatedV4 and getPayloadV6, and caches the envelope, blobs and cell proofs for the current and previous slot so include_payload=false and the stateful publish form work. publishBlockV2 takes a bare gloas block. The envelope endpoint waits briefly for a block the chain actor has not imported yet, since a validator client publishes the envelope as soon as the block is handed to the network. The node's custody set is handed to the RPC server for the build request. --- bin/ethlambda/src/main.rs | 4 +- .../src/beacon/gloas_block_production.rs | 78 +- crates/net/rpc/src/beacon/gloas_proposal.rs | 1433 ++++++++++++++++- crates/net/rpc/src/beacon/proposal.rs | 143 +- crates/net/rpc/src/lib.rs | 6 +- 5 files changed, 1640 insertions(+), 24 deletions(-) diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index c31645461..c86b692ad 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -740,14 +740,14 @@ async fn run_node(options: Options) -> eyre::Result<()> { let rpc_sync_status = sync_status.clone(); let rpc_events = events.clone(); let rpc_p2p = p2p.actor_ref().to_rpc_to_p2p_ref(); - // Block production builds its payloads with the same execution client the - // chain actor validates them with. let rpc_custody_columns = match &setup.chain { ChainActor::Beacon { custody_columns, .. } => ethlambda_rpc::CustodyColumns(custody_columns.clone()), ChainActor::Lean(..) => ethlambda_rpc::CustodyColumns::default(), }; + // Block production builds its payloads with the same execution client the + // chain actor validates them with. let rpc_engine = match &setup.chain { ChainActor::Beacon { engine, .. } => engine.clone(), ChainActor::Lean(..) => None, diff --git a/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs b/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs index 0f78ee9f1..ccb4bd9e5 100644 --- a/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs +++ b/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs @@ -474,9 +474,11 @@ fn check_envelope( Ok(()) } -#[cfg(test)] -mod gloas_block_production_tests { - use super::super::stf::ExecutionEngine; +/// Gloas test fixtures shared by this module's tests and by the Beacon API's +/// (`test-utils` feature): a gloas state a block can be built on, and the pieces +/// an execution client would hand over. +#[cfg(any(test, feature = "test-utils"))] +pub mod test_support { use super::*; use crate::beacon::ForkName; use crate::beacon::block_production::advance_to_slot; @@ -488,17 +490,18 @@ mod gloas_block_production_tests { use ethlambda_types::beacon::containers::bellatrix::{ExtraData, LogsBloom}; use ethlambda_types::beacon::primitives::Uint256; - const PARENT_BLOCK_HASH: u8 = 0x11; - const GRANDPARENT_BLOCK_HASH: u8 = 0x10; + pub const PARENT_BLOCK_HASH: u8 = 0x11; + pub const GRANDPARENT_BLOCK_HASH: u8 = 0x10; - fn config() -> Config { + /// The schedule these fixtures assume: gloas from epoch 0. + pub fn config() -> Config { Config::mainnet().with_fork_epoch(ForkName::Gloas, 0) } - /// A gloas state at slot 33 whose parent block (slot 32) revealed a payload, - /// its PTC for slot 32 filled from the real registry, with the lookahead - /// and sync committee a real registry would give it. - fn state_to_build_on() -> BeaconState { + /// A gloas state at slot 32, the post-state of a parent block whose + /// payload was revealed, its PTC for slot 32 filled from the real registry, + /// with the lookahead and sync committee a real registry would give it. + pub fn parent_state() -> BeaconState { let mut state = with_signing_validators_at(ForkName::Gloas, 64); let lookahead = initialize_proposer_lookahead(&state).unwrap(); let sync_committee = @@ -525,11 +528,18 @@ mod gloas_block_production_tests { let window_index = (preset::SLOTS_PER_EPOCH + inner.slot % preset::SLOTS_PER_EPOCH) as usize; inner.ptc_window[window_index] = ptc; + state + } + + /// [`parent_state`] advanced to slot 33, the state a block is built on. + pub fn state_to_build_on() -> BeaconState { + let state = parent_state(); let slot = state.slot() + 1; advance_to_slot(&state, slot, &config()).unwrap() } - fn payload_for(inputs: &GloasPayloadInputs) -> ExecutionPayload { + /// The payload an execution client would build for `inputs`. + pub fn payload_for(inputs: &GloasPayloadInputs) -> ExecutionPayload { ExecutionPayload { parent_hash: inputs.head_block_hash, fee_recipient: Default::default(), @@ -553,7 +563,8 @@ mod gloas_block_production_tests { } } - fn randao_reveal(state: &BeaconState) -> BlsSignature { + /// The proposer's real RANDAO reveal for `state`'s slot. + pub fn randao_reveal(state: &BeaconState) -> BlsSignature { let proposer = get_beacon_proposer_index(state).unwrap(); let epoch = get_current_epoch(state); let domain = get_domain(state, constants::DOMAIN_RANDAO, Some(epoch)); @@ -563,7 +574,8 @@ mod gloas_block_production_tests { ) } - fn produce( + /// A block and envelope for `state` carrying one blob commitment. + pub fn produce( state: &BeaconState, build_on_full: bool, payload_attestations: Vec, @@ -586,6 +598,46 @@ mod gloas_block_production_tests { ) } + /// The state after `block` is applied to `state`, as the chain stores it. + pub fn post_state(state: &BeaconState, block: &BeaconBlock) -> BeaconState { + let mut post = state.clone(); + stf::gloas::process_block(&mut post, block, &config(), &CommitteeCache::default()).unwrap(); + post + } + + /// The proposer's signature over `block`, as a validator client gives it. + pub fn sign_block(state: &BeaconState, block: &BeaconBlock) -> BlsSignature { + let domain = get_domain(state, constants::DOMAIN_BEACON_PROPOSER, None); + sign_for( + block.proposer_index as usize, + compute_signing_root(block.hash_tree_root(), domain), + ) + } + + /// The proposer's signature over `envelope` under `DOMAIN_BEACON_BUILDER`, + /// against the block's post-state. + pub fn sign_envelope( + post: &BeaconState, + proposer: ValidatorIndex, + envelope: &ExecutionPayloadEnvelope, + ) -> BlsSignature { + let domain = get_domain(post, constants::DOMAIN_BEACON_BUILDER, None); + sign_for( + proposer as usize, + compute_signing_root(envelope.hash_tree_root(), domain), + ) + } +} + +#[cfg(test)] +mod gloas_block_production_tests { + use super::super::stf::ExecutionEngine; + use super::test_support::*; + use super::*; + use crate::beacon::helpers::accessors::get_domain; + use crate::beacon::helpers::misc::compute_signing_root; + use crate::beacon::helpers::test_state::sign_for; + /// A vote on the parent block (slot 32) signed by `validator`. fn ptc_message( state: &BeaconState, diff --git a/crates/net/rpc/src/beacon/gloas_proposal.rs b/crates/net/rpc/src/beacon/gloas_proposal.rs index 846086c33..951587597 100644 --- a/crates/net/rpc/src/beacon/gloas_proposal.rs +++ b/crates/net/rpc/src/beacon/gloas_proposal.rs @@ -1,13 +1,1438 @@ //! Gloas block production and payload envelope publication. //! -//! STUB: owned by the block-production task (see the contract), which fills -//! in `POST /eth/v4/validator/blocks/{slot}`, +//! `POST /eth/v4/validator/blocks/{slot}` (`produceBlockV4`) builds a +//! self-built block with this node's own execution client and hands the +//! validator client the block, and, with `include_payload`, the unsigned +//! envelope that reveals its payload plus the blobs and cell proofs. The three +//! pieces come back separately because a gloas proposer signs the block and +//! the envelope on their own: the block goes out first through +//! `POST /eth/v2/beacon/blocks` (see `proposal.rs`), the envelope after it +//! through `POST /eth/v1/beacon/execution_payload_envelopes`, which also +//! publishes the data columns of the blobs (a builder's duty in gloas, which a +//! self-building proposer therefore takes on). +//! //! `GET /eth/v1/validator/execution_payload_envelopes/{slot}/{beacon_block_root}` -//! and `POST /eth/v1/beacon/execution_payload_envelopes`. +//! serves the envelope back for a validator client that asked for the block +//! with `include_payload=false`. +//! +//! This node never takes a builder's bid: the `BuilderConfig` request body is +//! decoded, as the specification requires of a body that cannot be, and +//! otherwise ignored. +//! +//! What `produceBlockV4` builds is kept in a small cache, keyed by slot and +//! block root and holding the current and the previous slot only, so the +//! envelope and the blobs can be served and published without the caller +//! sending them back (`Eth-Blob-Data-Included: false`). + +use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, + time::{Duration, Instant}, +}; -use axum::Router; +use axum::{ + Extension, Router, + body::Bytes, + extract::{Path, Query, State, rejection::QueryRejection}, + http::{HeaderMap, HeaderValue, StatusCode, header}, + response::{IntoResponse, Response}, + routing::{get, post}, +}; +use ethlambda_blockchain::checkpoint_hash; +use ethlambda_engine::{ + CustodyColumns, EngineClient, ForkchoiceStateV1, + building::{BuiltGloasPayload, PayloadAttributesV3, PayloadAttributesV4}, +}; +use ethlambda_network_api::RpcToP2PRef; +use ethlambda_state_transition::beacon::{ + attestation_pool::SharedAttestationPool, + block_production::advance_to_slot, + bls, + fork_choice::{ + get_head_node, gloas_verify_data_column_sidecar, + gloas_verify_data_column_sidecar_kzg_proofs, should_build_on_full, + }, + gloas_block_production::{ + GloasBlockInputs, GloasPayloadInputs, GloasProduced, assemble_gloas_block, + gloas_data_column_sidecars, gloas_payload_inputs, pack_gloas_attestations, + pack_payload_attestations, parse_gloas_execution_requests, + }, + helpers::{ + accessors::{get_beacon_proposer_index, get_domain}, + misc::compute_signing_root, + }, + payload_attestation_pool::SharedPayloadAttestationPool, + stf::gloas::verify_execution_payload_envelope_signature, +}; use ethlambda_storage::Store; +use ethlambda_types::{ + beacon::{ + config::Config, + constants, + containers::{ + self, BeaconState, DataColumnSidecar, + deneb::Blob, + gloas::{ + BeaconBlock, ExecutionPayloadEnvelope, ExecutionRequests, + SignedExecutionPayloadEnvelope, + }, + }, + fork::ForkName, + fork_choice::{ForkChoiceNode, PayloadStatus}, + primitives::{BlsSignature, Bytes32, ExecutionAddress, HashTreeRoot as _, KzgProof, Slot}, + signing::compute_epoch_at_slot, + }, + primitives::H256, +}; +use libssz::{SszDecode as _, SszEncode as _}; +use libssz_derive::{SszDecode, SszEncode}; +use serde::Deserialize; +use tracing::{debug, info, warn}; + +use crate::beacon::{ + ApiError, + proposal::{Blobs, CellKzgProofs, decimal, require_gloas_slot}, + validator::{FeeRecipients, head}, +}; +use crate::shared::content::{Encoding, ssz_response, with_consensus_version}; +// The node's custody set as the Beacon API's handles carry it, renamed so it +// cannot be confused with the engine's bitfield of the same name. +use crate::CustodyColumns as NodeCustodyColumns; + +/// How long `publishExecutionPayloadEnvelope` waits for a block this node has +/// not imported yet. +/// +/// A validator client publishes the envelope right after `publishBlockV2` +/// returns, and that returns once the block is handed to the network actor, not +/// once the chain actor has imported it, so the envelope routinely arrives +/// first. +const BLOCK_WAIT: Duration = Duration::from_secs(4); +const BLOCK_POLL_INTERVAL: Duration = Duration::from_millis(50); + +/// What `produceBlockV4` built for one block, kept for the envelope endpoints. +#[derive(Debug, Clone)] +struct CachedPayload { + envelope: ExecutionPayloadEnvelope, + blobs: Vec>, + cell_proofs: Vec, +} + +/// The current and previous slots' productions, by `(slot, block root)`. +#[derive(Debug, Clone, Default)] +pub(crate) struct PayloadCache(Arc>>); + +impl PayloadCache { + fn insert(&self, slot: Slot, block_root: H256, payload: CachedPayload) { + let mut entries = self.0.lock().expect("payload cache lock poisoned"); + entries.insert((slot, block_root), payload); + let keep_from = slot.saturating_sub(1); + entries.retain(|(held, _), _| *held >= keep_from); + } + + fn get(&self, slot: Slot, block_root: H256) -> Option { + self.0 + .lock() + .expect("payload cache lock poisoned") + .get(&(slot, block_root)) + .cloned() + } +} + +/// Gloas's `BlockContents`, the Beacon API's envelope for a self-built block +/// with its payload: not a consensus container, so it lives with the API. +#[derive(Debug, Clone, PartialEq, SszEncode, SszDecode, serde::Serialize)] +pub(crate) struct GloasBlockContents { + pub(crate) block: BeaconBlock, + pub(crate) execution_payload_envelope: ExecutionPayloadEnvelope, + #[serde(serialize_with = "ethlambda_types::beacon::serde_helpers::seq::serialize")] + pub(crate) kzg_proofs: CellKzgProofs, + #[serde(serialize_with = "ethlambda_types::beacon::serde_helpers::ssz_hex_seq::serialize")] + pub(crate) blobs: Blobs, +} + +/// Gloas's `SignedExecutionPayloadEnvelopeContents`, what the stateless form of +/// `publishExecutionPayloadEnvelope` receives. +#[derive(Debug, Clone, PartialEq, SszEncode, SszDecode)] +pub(crate) struct GloasSignedEnvelopeContents { + pub(crate) signed_execution_payload_envelope: SignedExecutionPayloadEnvelope, + pub(crate) kzg_proofs: CellKzgProofs, + pub(crate) blobs: Blobs, +} pub(crate) fn routes() -> Router { Router::new() + .route("/eth/v4/validator/blocks/{slot}", post(post_produce_block)) + .route( + "/eth/v1/validator/execution_payload_envelopes/{slot}/{beacon_block_root}", + get(get_envelope), + ) + .route( + "/eth/v1/beacon/execution_payload_envelopes", + post(post_envelope), + ) + .layer(Extension(PayloadCache::default())) +} + +fn is_ssz(headers: &HeaderMap) -> bool { + headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.starts_with(crate::SSZ_CONTENT_TYPE)) +} + +fn consensus_version(headers: &HeaderMap) -> Option { + headers + .get("eth-consensus-version") + .and_then(|value| value.to_str().ok()) + .and_then(ForkName::parse) +} + +/// `BuilderConfig` as it arrives as JSON. Only decoded: this node takes no +/// builder bids, so nothing past the shape is read. +#[derive(Debug, Deserialize)] +struct BuilderConfigJson { + #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")] + #[allow(dead_code)] + min_bid: u64, + #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")] + #[allow(dead_code)] + builder_boost_factor: u64, + builders: Vec, +} + +/// Decodes the request body as a `BuilderConfig`, returning how many builder +/// entries it names. A body that does not decode is invalid per the +/// specification. +/// +/// The SSZ form is `min_bid`, `builder_boost_factor` and the offset of the +/// `builders` list, so a decodable body is at least that long and the offset +/// points just past those three fields; the entries themselves are not parsed. +fn decode_builder_config(headers: &HeaderMap, body: &[u8]) -> Result { + if is_ssz(headers) { + let offset = body + .get(16..20) + .map(|bytes| u32::from_le_bytes(bytes.try_into().expect("four bytes"))); + return match offset { + Some(20) => Ok(usize::from(body.len() > 20)), + _ => Err(ApiError::BadRequest("the body is not a BuilderConfig")), + }; + } + serde_json::from_slice::(body) + .map(|config| config.builders.len()) + .map_err(|_| ApiError::BadRequest("the body is not a BuilderConfig")) +} + +#[derive(Debug, Deserialize)] +struct ProduceQuery { + randao_reveal: BlsSignature, + #[serde(default)] + graffiti: Option, + include_payload: bool, + /// Accepted for compatibility. Not honored: the block's state root comes + /// from running it through the state transition, which verifies the reveal. + #[serde(default)] + #[allow(dead_code)] + skip_randao_verification: Option, +} + +/// `POST /eth/v4/validator/blocks/{slot}`. +#[allow(clippy::too_many_arguments)] +async fn post_produce_block( + Path(slot): Path, + query: Result, QueryRejection>, + State(store): State, + Extension(engine): Extension>, + Extension(pool): Extension, + Extension(ptc_pool): Extension, + Extension(fee_recipients): Extension, + Extension(custody): Extension, + Extension(cache): Extension, + headers: HeaderMap, + body: Bytes, +) -> Response { + let Ok(slot) = slot.parse::() else { + return ApiError::BadRequest("invalid slot").into_response(); + }; + let Ok(Query(query)) = query else { + return ApiError::BadRequest( + "randao_reveal and include_payload are required, graffiti is a 32-byte hex string", + ) + .into_response(); + }; + if consensus_version(&headers).is_some_and(|fork| fork != ForkName::Gloas) { + return ApiError::BadRequest("Eth-Consensus-Version must be gloas").into_response(); + } + // Ahead of the engine check: a slot this node cannot build is a 400 on any + // node, not a 503 on one with no execution client. + if let Err(err) = require_gloas_slot( + &store.config(), + slot, + "this endpoint serves gloas slots only", + ) { + return err.into_response(); + } + let builders = match decode_builder_config(&headers, &body) { + Ok(builders) => builders, + Err(err) => return err.into_response(), + }; + if builders > 0 { + debug!(%slot, builders, "Ignoring the builders of the block production request"); + } + let Some(engine) = engine else { + return ApiError::ServiceUnavailable( + "no execution client configured to build a payload with", + ) + .into_response(); + }; + + let graffiti = query.graffiti.unwrap_or(Bytes32::ZERO); + let produced = produce( + &store, + &engine, + &pool, + &ptc_pool, + &fee_recipients, + &custody, + slot, + query.randao_reveal, + graffiti, + ) + .await; + let produced = match produced { + Ok(produced) => produced, + Err(err) => return err.into_response(), + }; + let Produced { + built, + block, + envelope, + payload_value, + } = produced; + + let block_root = block.hash_tree_root(); + cache.insert( + slot, + block_root, + CachedPayload { + envelope: envelope.clone(), + blobs: built.blobs_bundle.blobs.clone(), + cell_proofs: built.blobs_bundle.proofs.clone(), + }, + ); + + let accept = headers.get(header::ACCEPT).and_then(|v| v.to_str().ok()); + let encoding = Encoding::from_accept(accept); + let included = query.include_payload; + let mut response = if included { + let (Ok(kzg_proofs), Ok(blobs)) = ( + CellKzgProofs::try_from(built.blobs_bundle.proofs), + blobs_list(built.blobs_bundle.blobs), + ) else { + return ApiError::Internal("the blobs bundle exceeds the block's bounds") + .into_response(); + }; + let contents = GloasBlockContents { + block, + execution_payload_envelope: envelope, + kzg_proofs, + blobs, + }; + match encoding { + Encoding::Ssz => ssz_response(contents.to_ssz()), + Encoding::Json => block_json(&payload_value, true, &contents), + } + } else { + match encoding { + Encoding::Ssz => ssz_response(block.to_ssz()), + Encoding::Json => block_json(&payload_value, false, &block), + } + }; + let response_headers = response.headers_mut(); + response_headers.insert( + "eth-execution-payload-included", + HeaderValue::from_static(if included { "true" } else { "false" }), + ); + if let Ok(value) = HeaderValue::from_str(&payload_value) { + response_headers.insert("eth-execution-payload-value", value); + } + // Not computed: nothing here reads it, and the builder comparison it + // exists for does not happen on this node. + response_headers.insert("eth-consensus-block-value", HeaderValue::from_static("0")); + with_consensus_version(response, ForkName::Gloas) +} + +fn block_json(payload_value: &str, included: bool, data: &T) -> Response { + crate::json_response(serde_json::json!({ + "version": ForkName::Gloas.as_str(), + "consensus_block_value": "0", + "execution_payload_value": payload_value, + "execution_payload_included": included, + "data": data, + })) +} + +fn blobs_list(blobs: Vec>) -> Result { + let blobs = blobs + .into_iter() + .map(|blob| Blob::try_from(blob).map_err(|_| ())) + .collect::, ()>>()?; + Blobs::try_from(blobs).map_err(|_| ()) +} + +/// Everything `produceBlockV4` built. +struct Produced { + built: BuiltGloasPayload, + block: BeaconBlock, + envelope: ExecutionPayloadEnvelope, + payload_value: String, +} + +/// What the build needs from the chain, read and advanced off the runtime. +struct Prepared { + state: BeaconState, + inputs: GloasPayloadInputs, + proposer: u64, + head_root: H256, + head_slot: Slot, + /// The parent envelope's requests when building on its full payload and + /// the parent is gloas; empty otherwise. + parent_requests: ExecutionRequests, +} + +#[allow(clippy::too_many_arguments)] +async fn produce( + store: &Store, + engine: &EngineClient, + pool: &SharedAttestationPool, + ptc_pool: &SharedPayloadAttestationPool, + fee_recipients: &FeeRecipients, + custody: &NodeCustodyColumns, + slot: Slot, + randao_reveal: BlsSignature, + graffiti: Bytes32, +) -> Result { + let prepare_store = store.clone(); + let prepared = + tokio::task::spawn_blocking(move || prepare(&prepare_store, slot, randao_reveal)) + .await + .map_err(|_| ApiError::Internal("preparing the block failed"))??; + + let built = build_payload(store, engine, fee_recipients, custody, &prepared).await?; + let bundle = &built.blobs_bundle; + if bundle.commitments.len() != bundle.blobs.len() + || bundle.proofs.len() + != bundle.blobs.len() * ethlambda_types::beacon::preset::CELLS_PER_EXT_BLOB + { + warn!(%slot, "The execution client's blobs bundle is inconsistent"); + return Err(ApiError::ServiceUnavailable( + "the execution client returned an inconsistent blobs bundle", + )); + } + let execution_requests = parse_gloas_execution_requests(&built.execution_requests) + .map_err(|_| ApiError::Internal("the execution client's request list is malformed"))?; + let payload_value = decimal(&built.block_value); + + let candidates = pool + .lock() + .expect("attestation pool lock poisoned") + .block_candidates(); + let messages = ptc_pool + .lock() + .expect("payload attestation pool lock poisoned") + .messages_for(slot.saturating_sub(1), prepared.head_root); + let config = store.config(); + let assembled = { + let built = built.clone(); + tokio::task::spawn_blocking(move || { + assemble( + &config, + prepared, + built, + execution_requests, + candidates, + messages, + randao_reveal, + graffiti, + ) + }) + .await + .map_err(|_| ApiError::Internal("assembling the block failed"))?? + }; + let GloasProduced { block, envelope } = assembled; + info!( + %slot, + proposer = block.proposer_index, + attestations = block.body.attestations.len(), + payload_attestations = block.body.payload_attestations.len(), + blobs = built.blobs_bundle.blobs.len(), + "Produced gloas block" + ); + Ok(Produced { + built, + block, + envelope, + payload_value, + }) +} + +/// The chain-side half of production: pick the parent payload branch, advance +/// the head state to `slot` and derive the payload inputs from it. +fn prepare(store: &Store, slot: Slot, randao_reveal: BlsSignature) -> Result { + let config = store.config(); + let (head_root, head_state) = head(store)?; + let head_slot = head_state.slot(); + if slot <= head_slot { + return Err(ApiError::BadRequest("slot is not after the head block")); + } + + // The payload branch of the head: the one fork choice recorded, or, when + // it has not been recorded for this head yet, a fresh walk. + let payload_status = match store.head_payload_status() { + Some(status) => status, + None => { + get_head_node(store, &config) + .map_err(|_| ApiError::Internal("fork choice could not name the head"))? + .payload_status + } + }; + let node = ForkChoiceNode { + root: head_root, + payload_status, + }; + let build_on_full = should_build_on_full(store, node, slot) + .map_err(|_| ApiError::Internal("could not decide which parent payload to build on"))?; + debug_assert_ne!(payload_status, PayloadStatus::Pending); + + let head_is_gloas = matches!( + store.get_signed_block(&head_root), + Ok(Some(containers::SignedBeaconBlock::Gloas(_))) + ); + let parent_requests = if build_on_full && head_is_gloas { + store + .get_execution_payload_envelope(&head_root) + .map_err(|_| ApiError::Internal("store read failed"))? + .ok_or(ApiError::ServiceUnavailable( + "the parent's payload envelope is not held to build on", + ))? + .message + .execution_requests + } else { + ExecutionRequests::default() + }; + + let state = advance_to_slot(&head_state, slot, &config) + .map_err(|_| ApiError::Internal("advancing the head state failed"))?; + let proposer = get_beacon_proposer_index(&state) + .map_err(|_| ApiError::Internal("no proposer for the slot"))?; + verify_randao_reveal(&state, proposer, randao_reveal)?; + let inputs = gloas_payload_inputs(&state, build_on_full, &parent_requests, &config) + .map_err(|_| ApiError::Internal("computing the payload attributes failed"))?; + Ok(Prepared { + state, + inputs, + proposer, + head_root, + head_slot, + parent_requests, + }) +} + +/// A reveal that does not verify would fail the block's own state transition +/// later with an opaque error; here it is the caller's mistake and says so. +fn verify_randao_reveal( + state: &BeaconState, + proposer: u64, + randao_reveal: BlsSignature, +) -> Result<(), ApiError> { + let epoch = compute_epoch_at_slot(state.slot()); + let domain = get_domain(state, constants::DOMAIN_RANDAO, Some(epoch)); + let signing_root = compute_signing_root(epoch.hash_tree_root(), domain); + let pubkey = state + .validator(proposer) + .map_err(|_| ApiError::Internal("the proposer is not in the registry"))? + .pubkey; + if bls::verify(&pubkey, signing_root, &randao_reveal) { + Ok(()) + } else { + Err(ApiError::BadRequest("invalid randao_reveal")) + } +} + +/// Ask the execution client to build on the chosen parent payload for the +/// prepared slot, then collect what it built. +/// +/// Collected straight away rather than after waiting, as the fulu path does: the +/// execution client starts with a valid payload and improves it, so an early +/// `getPayload` always answers, and the validator client asks at the start of +/// the slot, when the block is due. +async fn build_payload( + store: &Store, + engine: &EngineClient, + fee_recipients: &FeeRecipients, + custody: &NodeCustodyColumns, + prepared: &Prepared, +) -> Result { + let inputs = &prepared.inputs; + let fee_recipient = fee_recipients + .lock() + .expect("fee recipient lock poisoned") + .get(&prepared.proposer) + .copied() + .unwrap_or_else(|| { + warn!( + proposer = prepared.proposer, + "No fee recipient prepared for the proposer; using the zero address" + ); + ExecutionAddress::ZERO + }); + let forkchoice = ForkchoiceStateV1 { + head_block_hash: inputs.head_block_hash, + safe_block_hash: checkpoint_hash(store, store.beacon_justified_checkpoint().root), + finalized_block_hash: checkpoint_hash(store, store.beacon_finalized_checkpoint().root), + }; + let attributes = PayloadAttributesV4 { + v3: PayloadAttributesV3 { + timestamp: inputs.timestamp, + prev_randao: inputs.prev_randao, + suggested_fee_recipient: fee_recipient, + withdrawals: inputs.withdrawals.clone(), + parent_beacon_block_root: inputs.parent_beacon_block_root, + }, + slot_number: inputs.slot_number, + target_gas_limit: inputs.target_gas_limit, + }; + let custody_columns = CustodyColumns::from_indices(custody.0.iter().copied()); + let (status, payload_id) = engine + .forkchoice_updated_v4_with_attributes(&forkchoice, &attributes, custody_columns) + .await + .map_err(|err| { + warn!(%err, "forkchoiceUpdatedV4 with payload attributes failed"); + ApiError::ServiceUnavailable("the execution client did not start building") + })?; + let Some(payload_id) = payload_id else { + warn!(status = ?status.status, "The execution client declined to build a payload"); + return Err(ApiError::ServiceUnavailable( + "the execution client declined to build a payload", + )); + }; + engine.get_payload_v6(payload_id).await.map_err(|err| { + warn!(%err, "getPayloadV6 failed"); + ApiError::ServiceUnavailable("the execution client did not return a payload") + }) +} + +/// The pure half: the pooled operations and the built payload into a block and +/// its envelope. +#[allow(clippy::too_many_arguments)] +fn assemble( + config: &Config, + prepared: Prepared, + built: BuiltGloasPayload, + execution_requests: ExecutionRequests, + candidates: Vec, + messages: Vec, + randao_reveal: BlsSignature, + graffiti: Bytes32, +) -> Result { + let Prepared { + state, + head_root, + head_slot, + parent_requests, + .. + } = prepared; + let attestations = pack_gloas_attestations(&state, candidates); + let payload_attestations = + pack_payload_attestations(&state, head_root, head_slot, messages, config); + let commitments = built.blobs_bundle.commitments; + let inputs = |attestations, payload_attestations| GloasBlockInputs { + randao_reveal, + graffiti, + attestations, + payload_attestations, + parent_execution_requests: parent_requests.clone(), + execution_payload: built.execution_payload.clone(), + blob_kzg_commitments: commitments.clone(), + execution_requests: execution_requests.clone(), + }; + let operations = attestations.len() + payload_attestations.len(); + match assemble_gloas_block(&state, inputs(attestations, payload_attestations), config) { + Ok(produced) => Ok(produced), + // The packers check every operation's signature against this state, so + // this should not happen; but a block without them still earns the + // proposal, and one that fails to build earns nothing. + Err(err) if operations > 0 => { + warn!(slot = state.slot(), %err, "Block with operations failed to build; retrying without"); + assemble_gloas_block(&state, inputs(Vec::new(), Vec::new()), config) + .map_err(|_| ApiError::Internal("the block failed to build")) + } + Err(_) => Err(ApiError::Internal("the block failed to build")), + } +} + +/// `GET /eth/v1/validator/execution_payload_envelopes/{slot}/{beacon_block_root}`. +async fn get_envelope( + Path((slot, block_root)): Path<(String, String)>, + Extension(cache): Extension, + headers: HeaderMap, +) -> Response { + let Ok(slot) = slot.parse::() else { + return ApiError::BadRequest("invalid slot").into_response(); + }; + let Some(block_root) = parse_root(&block_root) else { + return ApiError::BadRequest("invalid beacon_block_root").into_response(); + }; + let Some(cached) = cache.get(slot, block_root) else { + return ApiError::NotFound("execution payload envelope not available for the slot") + .into_response(); + }; + let accept = headers.get(header::ACCEPT).and_then(|v| v.to_str().ok()); + let response = match Encoding::from_accept(accept) { + Encoding::Ssz => ssz_response(cached.envelope.to_ssz()), + Encoding::Json => crate::json_response(serde_json::json!({ + "version": ForkName::Gloas.as_str(), + "data": cached.envelope, + })), + }; + with_consensus_version(response, ForkName::Gloas) +} + +fn parse_root(text: &str) -> Option { + let bytes: [u8; 32] = hex::decode(text.strip_prefix("0x")?) + .ok()? + .try_into() + .ok()?; + Some(H256(bytes)) +} + +/// The block and its post-state for `root`, waiting up to `timeout` for the +/// chain actor to import them. +async fn wait_for_block( + store: &Store, + root: H256, + timeout: Duration, +) -> Option<(containers::SignedBeaconBlock, Arc)> { + let deadline = Instant::now() + timeout; + loop { + if let (Ok(Some(block)), Ok(Some(state))) = + (store.get_signed_block(&root), store.get_state(&root)) + { + return Some((block, state)); + } + if Instant::now() >= deadline { + return None; + } + tokio::time::sleep(BLOCK_POLL_INTERVAL).await; + } +} + +/// `POST /eth/v1/beacon/execution_payload_envelopes`, SSZ only. +/// +/// The envelope must fulfill the bid of a gloas block this node holds (waiting +/// briefly for the block, see [`BLOCK_WAIT`]), carry the proposer's signature +/// under `DOMAIN_BEACON_BUILDER`, and come with the blobs the bid commits to, +/// from the body (`Eth-Blob-Data-Included: true`) or from what +/// `produceBlockV4` cached (`false`). The data columns are built from the +/// blobs, checked against the bid's commitments, and gossiped with the +/// envelope. +async fn post_envelope( + State(store): State, + Extension(p2p): Extension, + Extension(cache): Extension, + headers: HeaderMap, + body: Bytes, +) -> Response { + post_envelope_waiting(&store, &p2p, &cache, &headers, &body, BLOCK_WAIT).await +} + +async fn post_envelope_waiting( + store: &Store, + p2p: &RpcToP2PRef, + cache: &PayloadCache, + headers: &HeaderMap, + body: &[u8], + block_wait: Duration, +) -> Response { + if consensus_version(headers) != Some(ForkName::Gloas) { + return ApiError::BadRequest("Eth-Consensus-Version must be gloas").into_response(); + } + if !is_ssz(headers) { + return ( + StatusCode::UNSUPPORTED_MEDIA_TYPE, + "envelopes are accepted as application/octet-stream only", + ) + .into_response(); + } + let blob_data_included = match headers + .get("eth-blob-data-included") + .and_then(|value| value.to_str().ok()) + { + Some("true") => true, + Some("false") => false, + _ => { + return ApiError::BadRequest("Eth-Blob-Data-Included must be true or false") + .into_response(); + } + }; + let (signed, supplied) = if blob_data_included { + match GloasSignedEnvelopeContents::from_ssz_bytes(body) { + Ok(contents) => ( + contents.signed_execution_payload_envelope, + Some(( + contents + .blobs + .iter() + .map(|blob| blob.to_vec()) + .collect::>(), + contents.kzg_proofs.to_vec(), + )), + ), + Err(_) => { + return ApiError::BadRequest( + "the body is not a gloas SignedExecutionPayloadEnvelopeContents", + ) + .into_response(); + } + } + } else { + match SignedExecutionPayloadEnvelope::from_ssz_bytes(body) { + Ok(signed) => (signed, None), + Err(_) => { + return ApiError::BadRequest( + "the body is not a gloas SignedExecutionPayloadEnvelope", + ) + .into_response(); + } + } + }; + + let block_root = signed.message.beacon_block_root; + let Some((block, post_state)) = wait_for_block(store, block_root, block_wait).await else { + return ApiError::BadRequest("unknown block: the envelope's block is not held") + .into_response(); + }; + let containers::SignedBeaconBlock::Gloas(block) = block else { + return ApiError::BadRequest("the envelope's block is not a gloas block").into_response(); + }; + let block_slot = block.message.slot; + let bid = block.message.body.signed_execution_payload_bid.message; + if signed.message.builder_index != bid.builder_index + || signed.message.payload.block_hash != bid.block_hash + { + return ApiError::BadRequest("the envelope does not fulfill the block's bid") + .into_response(); + } + + let (blobs, cell_proofs) = match supplied { + Some(supplied) => supplied, + None => match cache.get(block_slot, block_root) { + Some(cached) => (cached.blobs, cached.cell_proofs), + None if bid.blob_kzg_commitments.is_empty() => (Vec::new(), Vec::new()), + None => { + return ApiError::BadRequest( + "no cached blobs for the block: send them with Eth-Blob-Data-Included: true", + ) + .into_response(); + } + }, + }; + if blobs.len() != bid.blob_kzg_commitments.len() { + return ApiError::BadRequest("the blobs do not match the bid's commitments") + .into_response(); + } + + // Signature and the columns' proofs are CPU-bound; off the runtime. + let commitments: Vec<_> = bid.blob_kzg_commitments.iter().copied().collect(); + let envelope_for_check = signed.clone(); + let checked = tokio::task::spawn_blocking(move || { + if !verify_execution_payload_envelope_signature(&post_state, &envelope_for_check) + .map_err(|_| ApiError::BadRequest("the envelope's signer is unknown"))? + { + return Err(ApiError::BadRequest("invalid envelope signature")); + } + let sidecars = if blobs.is_empty() { + Vec::new() + } else { + let sidecars = gloas_data_column_sidecars(block_root, block_slot, &blobs, &cell_proofs) + .map_err(|_| { + ApiError::BadRequest("the blobs and proofs do not form valid columns") + })?; + let verified = sidecars.iter().all(|sidecar| { + gloas_verify_data_column_sidecar(sidecar, &commitments) + && gloas_verify_data_column_sidecar_kzg_proofs(sidecar, &commitments) + .unwrap_or(false) + }); + if !verified { + return Err(ApiError::BadRequest( + "the data columns do not verify against the bid's commitments", + )); + } + sidecars + }; + Ok(sidecars) + }) + .await; + let sidecars = match checked { + Ok(Ok(sidecars)) => sidecars, + Ok(Err(err)) => return err.into_response(), + Err(_) => return ApiError::Internal("verifying the envelope failed").into_response(), + }; + + let sidecars: Vec = + sidecars.into_iter().map(DataColumnSidecar::Gloas).collect(); + if p2p + .publish_execution_payload_envelope(Box::new(signed), sidecars) + .is_err() + { + return ApiError::Internal("the network actor is not running").into_response(); + } + StatusCode::OK.into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{RecordingNetwork, beacon_store_with_config}; + use axum::{body::Body, http::Request}; + use ethlambda_state_transition::beacon::gloas_block_production::test_support::{ + config, post_state, produce as produce_block, sign_block, sign_envelope, state_to_build_on, + }; + use ethlambda_state_transition::beacon::helpers::test_state::with_signing_validators_at; + use http_body_util::BodyExt as _; + use tower::ServiceExt as _; + + async fn respond(app: Router, request: Request) -> (StatusCode, serde_json::Value) { + let response = app.oneshot(request).await.unwrap(); + let status = response.status(); + let body = response.into_body().collect().await.unwrap().to_bytes(); + (status, serde_json::from_slice(&body).unwrap_or_default()) + } + + /// A store holding a fulu head, under a schedule that puts every slot at + /// gloas: the head is behind the fork, as on a node running across it. + fn gloas_scheduled_store() -> (Store, Slot) { + let state = with_signing_validators_at(ForkName::Fulu, 64); + let head_slot = state.slot(); + let (store, _root) = beacon_store_with_config(state, config()); + (store, head_slot) + } + + fn produce_app(store: Store, engine: Option) -> Router { + routes() + .with_state(store) + .layer(Extension(engine)) + .layer(Extension(SharedAttestationPool::default())) + .layer(Extension(SharedPayloadAttestationPool::default())) + .layer(Extension(FeeRecipients::default())) + .layer(Extension(NodeCustodyColumns::default())) + } + + fn produce_request(slot: Slot, query: &str, body: &str) -> Request { + Request::post(format!( + "/eth/v4/validator/blocks/{slot}?randao_reveal=0x{}{query}", + "00".repeat(96) + )) + .header("eth-consensus-version", "gloas") + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(body.to_string())) + .unwrap() + } + + const EMPTY_CONFIG: &str = r#"{"min_bid":"0","builder_boost_factor":"0","builders":[]}"#; + + #[tokio::test] + async fn producing_a_pre_gloas_slot_is_a_400_even_with_no_execution_client() { + let state = with_signing_validators_at(ForkName::Fulu, 64); + let head_slot = state.slot(); + // The default schedule never reaches gloas. + let (store, _root) = beacon_store_with_config(state, Config::mainnet()); + let app = produce_app(store, None); + let (status, json) = respond( + app, + produce_request(head_slot + 1, "&include_payload=true", EMPTY_CONFIG), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(json["message"], "this endpoint serves gloas slots only"); + } + + #[tokio::test] + async fn producing_without_an_execution_client_is_a_503() { + let (store, head_slot) = gloas_scheduled_store(); + let app = produce_app(store, None); + let (status, _) = respond( + app, + produce_request(head_slot + 1, "&include_payload=true", EMPTY_CONFIG), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + } + + #[tokio::test] + async fn a_missing_include_payload_or_an_undecodable_body_is_a_400() { + let (store, head_slot) = gloas_scheduled_store(); + let (status, _) = respond( + produce_app(store.clone(), None), + produce_request(head_slot + 1, "", EMPTY_CONFIG), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + + let (status, json) = respond( + produce_app(store.clone(), None), + produce_request(head_slot + 1, "&include_payload=true", "not a config"), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(json["message"], "the body is not a BuilderConfig"); + + // A wrong fork header is refused too. + let request = Request::post(format!( + "/eth/v4/validator/blocks/{}?randao_reveal=0x{}&include_payload=true", + head_slot + 1, + "00".repeat(96) + )) + .header("eth-consensus-version", "fulu") + .body(Body::from(EMPTY_CONFIG)) + .unwrap(); + let (status, _) = respond(produce_app(store, None), request).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + } + + #[test] + fn a_builder_config_decodes_as_json_or_ssz() { + let mut headers = HeaderMap::new(); + assert_eq!( + decode_builder_config(&headers, EMPTY_CONFIG.as_bytes()).unwrap(), + 0 + ); + assert!(decode_builder_config(&headers, b"{}").is_err()); + + headers.insert( + header::CONTENT_TYPE, + HeaderValue::from_static(crate::SSZ_CONTENT_TYPE), + ); + // min_bid, boost factor, then the offset of an empty builders list. + let mut ssz = vec![0u8; 16]; + ssz.extend_from_slice(&20u32.to_le_bytes()); + assert_eq!(decode_builder_config(&headers, &ssz).unwrap(), 0); + assert!(decode_builder_config(&headers, &ssz[..19]).is_err()); + } + + #[test] + fn the_cache_keeps_the_current_and_previous_slot_only() { + let cache = PayloadCache::default(); + let state = state_to_build_on(); + let produced = produce_block(&state, true, Vec::new()).unwrap(); + let entry = CachedPayload { + envelope: produced.envelope, + blobs: Vec::new(), + cell_proofs: Vec::new(), + }; + for slot in [10, 11, 12] { + cache.insert(slot, H256::repeat_byte(slot as u8), entry.clone()); + } + assert!(cache.get(10, H256::repeat_byte(10)).is_none()); + assert!(cache.get(11, H256::repeat_byte(11)).is_some()); + assert!(cache.get(12, H256::repeat_byte(12)).is_some()); + // The root is part of the key: a re-orged block gets no envelope. + assert!(cache.get(12, H256::repeat_byte(99)).is_none()); + } + + /// A store holding the built block and its post-state (the way the chain + /// actor stores an import), with the gloas schedule. + fn store_with_block( + state: &BeaconState, + produced: &GloasProduced, + signature: BlsSignature, + ) -> (Store, H256, BeaconState) { + let (mut store, _anchor) = beacon_store_with_config(state.clone(), config()); + let root = produced.block.hash_tree_root(); + let post = post_state(state, &produced.block); + store + .insert_signed_block( + root, + containers::SignedBeaconBlock::Gloas(containers::gloas::SignedBeaconBlock { + message: produced.block.clone(), + signature, + }), + ) + .unwrap(); + store.insert_state(root, post.clone()).unwrap(); + (store, root, post) + } + + fn envelope_request(included: Option<&str>, body: Vec) -> (HeaderMap, Vec) { + let mut headers = HeaderMap::new(); + headers.insert("eth-consensus-version", HeaderValue::from_static("gloas")); + headers.insert( + header::CONTENT_TYPE, + HeaderValue::from_static(crate::SSZ_CONTENT_TYPE), + ); + if let Some(included) = included { + headers.insert( + "eth-blob-data-included", + HeaderValue::from_str(included).unwrap(), + ); + } + (headers, body) + } + + /// A produced block with no blobs, so the envelope needs no columns. + fn produced_without_blobs(state: &BeaconState) -> GloasProduced { + use ethlambda_state_transition::beacon::gloas_block_production::{ + GloasBlockInputs, assemble_gloas_block, gloas_payload_inputs, + test_support::{payload_for, randao_reveal}, + }; + let requests = ExecutionRequests::default(); + let inputs = gloas_payload_inputs(state, true, &requests, &config()).unwrap(); + assemble_gloas_block( + state, + GloasBlockInputs { + randao_reveal: randao_reveal(state), + graffiti: Bytes32::ZERO, + attestations: Vec::new(), + payload_attestations: Vec::new(), + parent_execution_requests: requests, + execution_payload: payload_for(&inputs), + blob_kzg_commitments: Vec::new(), + execution_requests: ExecutionRequests::default(), + }, + &config(), + ) + .unwrap() + } + + fn signed_envelope( + post: &BeaconState, + produced: &GloasProduced, + ) -> SignedExecutionPayloadEnvelope { + SignedExecutionPayloadEnvelope { + message: produced.envelope.clone(), + signature: sign_envelope(post, produced.block.proposer_index, &produced.envelope), + } + } + + #[tokio::test] + async fn a_signed_envelope_for_a_held_block_is_gossiped() { + let state = state_to_build_on(); + let produced = produced_without_blobs(&state); + let signature = sign_block(&state, &produced.block); + let (store, _root, post) = store_with_block(&state, &produced, signature); + let signed = signed_envelope(&post, &produced); + + let network = Arc::new(RecordingNetwork::default()); + let p2p: RpcToP2PRef = network.clone(); + let (headers, body) = envelope_request(Some("false"), signed.to_ssz()); + let response = post_envelope_waiting( + &store, + &p2p, + &PayloadCache::default(), + &headers, + &body, + Duration::from_millis(100), + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let published = network.envelopes.lock().unwrap(); + assert_eq!(published.len(), 1); + assert_eq!(published[0].0, signed); + assert!(published[0].1.is_empty()); + } + + #[tokio::test] + async fn an_envelope_for_a_block_that_arrives_late_waits_for_it() { + let state = state_to_build_on(); + let produced = produced_without_blobs(&state); + let signature = sign_block(&state, &produced.block); + let (store, root, post) = store_with_block(&state, &produced, signature); + let signed = signed_envelope(&post, &produced); + // The block is imported only after the request is already waiting: + // an empty store to start with, filled in by a second task. + let (empty_store, _anchor) = beacon_store_with_config(state.clone(), config()); + let mut late_store = empty_store.clone(); + let block = store.get_signed_block(&root).unwrap().unwrap(); + let importer = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(200)).await; + late_store.insert_signed_block(root, block).unwrap(); + late_store.insert_state(root, post).unwrap(); + }); + + let network = Arc::new(RecordingNetwork::default()); + let p2p: RpcToP2PRef = network.clone(); + let (headers, body) = envelope_request(Some("false"), signed.to_ssz()); + let response = post_envelope_waiting( + &empty_store, + &p2p, + &PayloadCache::default(), + &headers, + &body, + Duration::from_secs(5), + ) + .await; + importer.await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(network.envelopes.lock().unwrap().len(), 1); + } + + #[tokio::test] + async fn an_envelope_for_an_unknown_block_is_a_400_after_the_wait() { + let state = state_to_build_on(); + let produced = produced_without_blobs(&state); + let post = post_state(&state, &produced.block); + let signed = signed_envelope(&post, &produced); + let (store, _anchor) = beacon_store_with_config(state, config()); + + let network = Arc::new(RecordingNetwork::default()); + let p2p: RpcToP2PRef = network.clone(); + let (headers, body) = envelope_request(Some("false"), signed.to_ssz()); + let started = Instant::now(); + let response = post_envelope_waiting( + &store, + &p2p, + &PayloadCache::default(), + &headers, + &body, + Duration::from_millis(150), + ) + .await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert!(started.elapsed() >= Duration::from_millis(150)); + assert!(network.envelopes.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn a_bad_signature_a_wrong_bid_and_a_wrong_media_type_are_refused() { + let state = state_to_build_on(); + let produced = produced_without_blobs(&state); + let signature = sign_block(&state, &produced.block); + let (store, _root, post) = store_with_block(&state, &produced, signature); + let network = Arc::new(RecordingNetwork::default()); + let p2p: RpcToP2PRef = network.clone(); + let cache = PayloadCache::default(); + let wait = Duration::from_millis(100); + + // Signed by someone else. + let mut forged = signed_envelope(&post, &produced); + forged.signature = + sign_envelope(&post, produced.block.proposer_index + 1, &produced.envelope); + let (headers, body) = envelope_request(Some("false"), forged.to_ssz()); + let response = post_envelope_waiting(&store, &p2p, &cache, &headers, &body, wait).await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + + // A payload other than the one the bid commits to. + let mut other = signed_envelope(&post, &produced); + other.message.payload.block_hash = H256::repeat_byte(0x77); + let (headers, body) = envelope_request(Some("false"), other.to_ssz()); + let response = post_envelope_waiting(&store, &p2p, &cache, &headers, &body, wait).await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + + // JSON is not accepted. + let signed = signed_envelope(&post, &produced); + let (mut headers, body) = envelope_request(Some("false"), signed.to_ssz()); + headers.insert( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ); + let response = post_envelope_waiting(&store, &p2p, &cache, &headers, &body, wait).await; + assert_eq!(response.status(), StatusCode::UNSUPPORTED_MEDIA_TYPE); + + // The blob-data header is required. + let (headers, body) = envelope_request(None, signed.to_ssz()); + let response = post_envelope_waiting(&store, &p2p, &cache, &headers, &body, wait).await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert!(network.envelopes.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn the_cached_envelope_is_served_for_its_slot_and_root_only() { + let state = state_to_build_on(); + let produced = produced_without_blobs(&state); + let root = produced.block.hash_tree_root(); + let slot = produced.block.slot; + let (store, _anchor) = beacon_store_with_config(state, config()); + let app = routes().with_state(store); + // The route layer's own cache cannot be reached from here, so the + // handler is exercised directly through a router with a seeded one. + let cache = PayloadCache::default(); + cache.insert( + slot, + root, + CachedPayload { + envelope: produced.envelope.clone(), + blobs: Vec::new(), + cell_proofs: Vec::new(), + }, + ); + let seeded = Router::new() + .route( + "/eth/v1/validator/execution_payload_envelopes/{slot}/{beacon_block_root}", + get(get_envelope), + ) + .layer(Extension(cache)); + let uri = format!( + "/eth/v1/validator/execution_payload_envelopes/{slot}/0x{}", + hex::encode(root.0) + ); + let (status, json) = respond( + seeded.clone(), + Request::get(&uri).body(Body::empty()).unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(json["version"], "gloas"); + assert_eq!( + json["data"]["builder_index"], + constants::BUILDER_INDEX_SELF_BUILD.to_string() + ); + + let ssz_request = Request::get(&uri) + .header(header::ACCEPT, "application/octet-stream") + .body(Body::empty()) + .unwrap(); + let response = seeded.clone().oneshot(ssz_request).await.unwrap(); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(bytes.as_ref(), produced.envelope.to_ssz().as_slice()); + + let other = format!( + "/eth/v1/validator/execution_payload_envelopes/{slot}/0x{}", + "11".repeat(32) + ); + let (status, _) = respond(seeded, Request::get(other).body(Body::empty()).unwrap()).await; + assert_eq!(status, StatusCode::NOT_FOUND); + // The unseeded production router answers 404 too. + let (status, _) = respond(app, Request::get(&uri).body(Body::empty()).unwrap()).await; + assert_eq!(status, StatusCode::NOT_FOUND); + } + + /// A block carrying one real blob, its signed envelope, and the body the + /// stateless form of the publish endpoint takes. + struct WithBlob { + state: BeaconState, + produced: GloasProduced, + blob: Vec, + proofs: Vec, + } + + fn with_blob() -> WithBlob { + use ethlambda_state_transition::beacon::{ + gloas_block_production::test_support::{payload_for, randao_reveal}, + kzg::{blob_to_kzg_commitment, compute_cells_and_kzg_proofs}, + }; + let state = state_to_build_on(); + let mut blob = vec![0u8; ethlambda_types::beacon::preset::BYTES_PER_BLOB]; + for (i, element) in blob.chunks_mut(32).enumerate() { + element[31] = (i % 100) as u8; + } + let commitment = blob_to_kzg_commitment(&blob).unwrap(); + let (_, proofs) = compute_cells_and_kzg_proofs(&blob).unwrap(); + let requests = ExecutionRequests::default(); + let inputs = gloas_payload_inputs(&state, true, &requests, &config()).unwrap(); + let produced = assemble_gloas_block( + &state, + GloasBlockInputs { + randao_reveal: randao_reveal(&state), + graffiti: Bytes32::ZERO, + attestations: Vec::new(), + payload_attestations: Vec::new(), + parent_execution_requests: requests, + execution_payload: payload_for(&inputs), + blob_kzg_commitments: vec![commitment], + execution_requests: ExecutionRequests::default(), + }, + &config(), + ) + .unwrap(); + WithBlob { + state, + produced, + blob, + proofs: proofs.to_vec(), + } + } + + #[tokio::test] + async fn an_envelope_with_blobs_gossips_every_column_and_a_tampered_blob_is_refused() { + let WithBlob { + state, + produced, + blob, + proofs, + } = with_blob(); + let signature = sign_block(&state, &produced.block); + let (store, root, post) = store_with_block(&state, &produced, signature); + let signed = signed_envelope(&post, &produced); + let wait = Duration::from_millis(100); + let contents = |blob: &[u8]| GloasSignedEnvelopeContents { + signed_execution_payload_envelope: signed.clone(), + kzg_proofs: CellKzgProofs::try_from(proofs.clone()).unwrap(), + blobs: blobs_list(vec![blob.to_vec()]).unwrap(), + }; + + // Stateless: the blobs travel with the envelope. + let network = Arc::new(RecordingNetwork::default()); + let p2p: RpcToP2PRef = network.clone(); + let (headers, body) = envelope_request(Some("true"), contents(&blob).to_ssz()); + let response = post_envelope_waiting( + &store, + &p2p, + &PayloadCache::default(), + &headers, + &body, + wait, + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + { + let published = network.envelopes.lock().unwrap(); + assert_eq!(published.len(), 1); + assert_eq!( + published[0].1.len(), + ethlambda_types::beacon::preset::NUMBER_OF_COLUMNS + ); + } + + // A different blob under the same proofs does not verify. + let mut tampered = blob.clone(); + tampered[63] ^= 1; + let rejected = Arc::new(RecordingNetwork::default()); + let p2p: RpcToP2PRef = rejected.clone(); + let (headers, body) = envelope_request(Some("true"), contents(&tampered).to_ssz()); + let response = post_envelope_waiting( + &store, + &p2p, + &PayloadCache::default(), + &headers, + &body, + wait, + ) + .await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + + // Stateful: nothing cached means a 400, and the cached blobs are used when present. + let (headers, body) = envelope_request(Some("false"), signed.to_ssz()); + let response = post_envelope_waiting( + &store, + &p2p, + &PayloadCache::default(), + &headers, + &body, + wait, + ) + .await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let cache = PayloadCache::default(); + cache.insert( + produced.block.slot, + root, + CachedPayload { + envelope: produced.envelope.clone(), + blobs: vec![blob], + cell_proofs: proofs, + }, + ); + let response = post_envelope_waiting(&store, &p2p, &cache, &headers, &body, wait).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(rejected.envelopes.lock().unwrap().len(), 1); + } } diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs index 6de77cf45..eedba5fe2 100644 --- a/crates/net/rpc/src/beacon/proposal.rs +++ b/crates/net/rpc/src/beacon/proposal.rs @@ -111,8 +111,8 @@ async fn post_block( .get("eth-consensus-version") .and_then(|value| value.to_str().ok()) .and_then(ForkName::parse); - if fork != Some(ForkName::Fulu) { - return ApiError::BadRequest("Eth-Consensus-Version must be fulu").into_response(); + if !matches!(fork, Some(ForkName::Fulu | ForkName::Gloas)) { + return ApiError::BadRequest("Eth-Consensus-Version must be fulu or gloas").into_response(); } let is_ssz = headers .get(header::CONTENT_TYPE) @@ -125,6 +125,9 @@ async fn post_block( ) .into_response(); } + if fork == Some(ForkName::Gloas) { + return post_gloas_block(&store, &p2p, &body).await; + } let Ok(contents) = FuluSignedBlockContents::from_ssz_bytes(&body) else { return ApiError::BadRequest("the body is not fulu SignedBlockContents").into_response(); }; @@ -170,6 +173,88 @@ async fn post_block( StatusCode::OK.into_response() } +/// The gloas half of `publishBlockV2`: a bare SSZ `SignedBeaconBlock`, since a +/// gloas block carries no payload and no blobs of its own (the envelope and +/// the data columns follow through `publishExecutionPayloadEnvelope`). +/// +/// Checked as the fulu one is, before it goes anywhere: the parent is held, +/// the slot is after the parent's and scheduled at gloas, the proposer is the +/// slot's, and the proposer's signature verifies against the parent's state +/// advanced to the slot. The advance and the signature are CPU-bound and run +/// off the runtime. +async fn post_gloas_block(store: &Store, p2p: &RpcToP2PRef, body: &[u8]) -> Response { + let Ok(signed) = containers::gloas::SignedBeaconBlock::from_ssz_bytes(body) else { + return ApiError::BadRequest("the body is not a gloas SignedBeaconBlock").into_response(); + }; + let block = containers::SignedBeaconBlock::Gloas(signed); + let config = store.config(); + if let Err(err) = require_gloas_slot( + &config, + block.slot(), + "blocks are accepted for gloas slots only", + ) { + return err.into_response(); + } + let parent_state = match store.get_state(&block.parent_root()) { + Ok(Some(state)) => state, + Ok(None) => return ApiError::BadRequest("the block's parent is not held").into_response(), + Err(_) => return ApiError::Internal("store read failed").into_response(), + }; + if block.slot() <= parent_state.slot() { + return ApiError::BadRequest("the block is not after its parent").into_response(); + } + let verdict = { + let block = block.clone(); + tokio::task::spawn_blocking(move || { + let state = advance_to_slot(&parent_state, block.slot(), &config) + .map_err(|_| ApiError::Internal("advancing the parent state failed"))?; + let expected = + ethlambda_state_transition::beacon::helpers::accessors::get_beacon_proposer_index( + &state, + ) + .map_err(|_| ApiError::Internal("no proposer for the slot"))?; + if block.proposer_index() != expected { + return Err(ApiError::BadRequest( + "the block's proposer is not the slot's", + )); + } + if !verify_block_signature(&state, &block) { + return Err(ApiError::BadRequest("invalid block signature")); + } + Ok(()) + }) + .await + }; + match verdict { + Ok(Ok(())) => {} + Ok(Err(err)) => return err.into_response(), + Err(_) => return ApiError::Internal("verifying the block failed").into_response(), + } + if p2p.publish_beacon_block(block).is_err() { + return ApiError::Internal("the network actor is not running").into_response(); + } + StatusCode::OK.into_response() +} + +/// Refuses a slot the schedule does not place at gloas. +pub(crate) fn require_gloas_slot( + config: &Config, + slot: Slot, + message: &'static str, +) -> Result<(), ApiError> { + match config.fork_at_epoch(compute_epoch_at_slot(slot)) { + ForkName::Gloas => Ok(()), + ForkName::Phase0 + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Fulu + | ForkName::Lean => Err(ApiError::BadRequest(message)), + } +} + #[derive(Debug, Deserialize)] struct ProduceQuery { randao_reveal: BlsSignature, @@ -409,7 +494,7 @@ async fn build_payload( } /// A wei amount as the decimal string the Beacon API's value fields carry. -fn decimal(value: ðlambda_types::beacon::primitives::Uint256) -> String { +pub(crate) fn decimal(value: ðlambda_types::beacon::primitives::Uint256) -> String { let hex = uint256(value); u128::from_str_radix(hex.trim_start_matches("0x"), 16) .map(|value| value.to_string()) @@ -490,4 +575,56 @@ mod tests { assert_eq!(status, StatusCode::BAD_REQUEST); assert_eq!(json["message"], "blocks are accepted for fulu slots only"); } + + #[tokio::test] + async fn a_signed_gloas_block_is_gossiped_and_a_forged_one_is_refused() { + use ethlambda_state_transition::beacon::gloas_block_production::test_support::{ + config, parent_state, produce, sign_block, state_to_build_on, + }; + let parent = parent_state(); + let state = state_to_build_on(); + let produced = produce(&state, true, Vec::new()).unwrap(); + let (mut store, _anchor) = beacon_store_with_config(parent.clone(), config()); + store + .insert_state(produced.block.parent_root, parent) + .unwrap(); + + let signed = containers::gloas::SignedBeaconBlock { + signature: sign_block(&state, &produced.block), + message: produced.block.clone(), + }; + let post = |store: Store, + network: Arc, + signed: &containers::gloas::SignedBeaconBlock| { + let network: RpcToP2PRef = network; + let app = routes().with_state(store).layer(Extension(network)); + let request = Request::post("/eth/v2/beacon/blocks") + .header("eth-consensus-version", "gloas") + .header(header::CONTENT_TYPE, crate::SSZ_CONTENT_TYPE) + .body(Body::from(signed.to_ssz())) + .unwrap(); + respond(app, request) + }; + + let network = Arc::new(RecordingNetwork::default()); + let (status, _) = post(store.clone(), network.clone(), &signed).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(network.blocks.lock().unwrap().len(), 1); + + // A signature from another validator, and a block whose parent is not held. + let rejected = Arc::new(RecordingNetwork::default()); + let mut forged = signed.clone(); + forged.signature = sign_block(&state, &{ + let mut other = produced.block.clone(); + other.proposer_index += 1; + other + }); + let (status, _) = post(store.clone(), rejected.clone(), &forged).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + let mut orphan = signed; + orphan.message.parent_root = H256::repeat_byte(5); + let (status, _) = post(store, rejected.clone(), &orphan).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(rejected.blocks.lock().unwrap().is_empty()); + } } diff --git a/crates/net/rpc/src/lib.rs b/crates/net/rpc/src/lib.rs index ca9c94bad..6c76c73b2 100644 --- a/crates/net/rpc/src/lib.rs +++ b/crates/net/rpc/src/lib.rs @@ -202,8 +202,10 @@ pub struct BeaconApiHandles { /// Filled by gossip and the payload attestation pool endpoint, read by /// block production and the pool's GET. pub payload_attestation_pool: SharedPayloadAttestationPool, - /// The columns this node samples, which `payload_attestation_data` checks a - /// block's blob availability against. + /// The columns this node custodies: what `payload_attestation_data` checks + /// a block's blob availability against, and what block production tells + /// the execution client it samples for when asking it to build a gloas + /// payload. pub custody_columns: CustodyColumns, /// The execution client block production builds payloads with; `None` /// makes it answer 503. From 868e013d0378dc079dda6bff103af03c6875e1dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:13:26 -0300 Subject: [PATCH 098/132] feat(beacon): gossip a self-built payload envelope with its data columns The Beacon API hands the node a signed envelope and the columns of its blobs, and gossipsub never delivers a node its own messages, so the publish path also has to give them to the chain actor. The envelope goes to the actor as it is, which is safe before its block is imported: the actor holds envelopes for blocks it has not imported or has stored but not imported. The sidecars go through the chain checks, which park one whose block is not imported yet. --- crates/net/p2p/src/gossipsub/handler.rs | 86 ++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 8 deletions(-) diff --git a/crates/net/p2p/src/gossipsub/handler.rs b/crates/net/p2p/src/gossipsub/handler.rs index ae7be1be4..1b664a476 100644 --- a/crates/net/p2p/src/gossipsub/handler.rs +++ b/crates/net/p2p/src/gossipsub/handler.rs @@ -792,18 +792,88 @@ pub async fn publish_beacon_block(server: &mut P2PServer, block: SignedBeaconBlo /// handed over by the Beacon API, on `execution_payload`, its data column /// sidecars on their subnets, and pass all of them to the chain actor. /// -/// STUB: owned by the block-production task (see the contract), which -/// replaces this body. +/// Like [`publish_beacon_block`], the only way this node takes its own +/// envelope in, since gossipsub never delivers a node its own messages. The +/// envelope goes to the actor as it is. It is safe to hand over before the +/// block it reveals has been imported, which is the usual case (the validator +/// client publishes the envelope as soon as the block is on its way): the +/// actor holds an envelope for a block it has not imported (`awaiting_block`), +/// or for one it has stored but not imported (`awaiting_import`), and applies +/// it when the block's post-state exists. +/// +/// The sidecars take the route every sidecar gossip did not accept takes, the +/// chain checks (`column_checks::check_and_forward`), which parks one whose +/// block is not imported yet and hands it back once it is. Their subnet +/// topics are published to whether or not this node custodies the subnet: +/// gossipsub sends to a topic it is not subscribed to through its fanout +/// peers. pub async fn publish_execution_payload_envelope( - _server: &mut P2PServer, + server: &mut P2PServer, envelope: ethlambda_types::beacon::containers::gloas::SignedExecutionPayloadEnvelope, - sidecars: Vec, + sidecars: Vec, ) { - error!( - block_root = %ShortRoot(&envelope.message.beacon_block_root.0), - sidecars = sidecars.len(), - "Publishing execution payload envelopes is not implemented; dropping it" + let slot = envelope.message.payload.slot_number; + let block_root = envelope.message.beacon_block_root; + let Some(beacon) = server.wire.beacon() else { + error!( + slot, + "An execution payload envelope reached a lean node; dropping it" + ); + return; + }; + let Some(digest) = beacon.publish_digest(slot) else { + warn!( + slot, + "No held fork digest covers this envelope's slot; not publishing" + ); + return; + }; + let topic = IdentTopic::new(beacon_topics::topic_name( + digest, + beacon_topics::EXECUTION_PAYLOAD, + )); + server + .swarm_handle + .publish(topic, compress_message(&envelope.to_ssz())); + info!( + slot, + builder_index = envelope.message.builder_index, + block_root = %ShortRoot(&block_root.0), + "Published execution payload envelope to gossipsub" ); + + let mut published_columns = 0usize; + for sidecar in &sidecars { + let DataColumnSidecar::Gloas(gloas_sidecar) = sidecar else { + warn!( + slot, + "Skipping a non-gloas sidecar published with a gloas envelope" + ); + continue; + }; + let subnet_id = gloas_sidecar.index + % ethlambda_types::beacon::constants::DATA_COLUMN_SIDECAR_SUBNET_COUNT; + let topic = IdentTopic::new(beacon_topics::data_column_topic_name(digest, subnet_id)); + server + .swarm_handle + .publish(topic, compress_message(&gloas_sidecar.to_ssz())); + published_columns += 1; + } + if published_columns > 0 { + info!( + slot, + block_root = %ShortRoot(&block_root.0), + columns = published_columns, + "Published data column sidecars to gossipsub" + ); + } + + if let Some(blockchain) = server.blockchain.clone() { + let _ = blockchain + .new_execution_payload_envelope(Box::new(envelope), BlockArrival::now()) + .inspect_err(|err| error!(%err, "Failed to hand the published envelope to the chain")); + } + crate::beacon::column_checks::check_and_forward(server, sidecars); } /// Gossip a payload attestation message a validator client signed, handed From 390c3bebf00453307254c084613b2e3042adbeac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:27:59 -0300 Subject: [PATCH 099/132] docs(beacon): document gloas validator duties The beacon node and validator client now serve and perform gloas duties, but the docs still said the node refuses a gloas epoch and never builds a payload. Document the new endpoints (produceBlockV4, the envelope endpoints, PTC duties, payload attestation data and pool, gloas on the existing pool and block endpoints), the departures from the specification that came with them (self-build only, target gas limit, ignored skip_randao_verification, the pre-gloas FULL rule for attestation data, arrival-time payload_present), the validator client's slashing-protection scope for envelopes and PTC votes, the four new validator metrics, and the engine calls used to build a gloas payload. --- CLAUDE.md | 35 ++++++- docs/beacon_engine.md | 14 ++- docs/cli.md | 28 +++++- docs/metrics.md | 15 ++- docs/rpc.md | 197 ++++++++++++++++++++++++++++++++++------ docs/spec_deviations.md | 96 ++++++++++++++++++++ 6 files changed, 345 insertions(+), 40 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 06357a87d..8bbb0375c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -768,8 +768,24 @@ transitions are in `ethlambda-types`, per the section above. Nothing above the parking of gloas column sidecars (they carry no signature). Fork-choice events are timed at arrival, not at the slot tick, and the head's payload status is kept with its root (`Store::head_payload_status`, recomputed after - a restart). The node stays a follower: the validator-client endpoints still - refuse a gloas epoch (`refuse_validator_duties_from_gloas`). + a restart). The node also serves gloas validator duties, self-build only (no + bids, no proposer preferences; see `docs/spec_deviations.md`). + `state_transition/src/beacon/gloas_block_production.rs` assembles the block and + its envelope (`gloas_payload_inputs`, `parse_gloas_execution_requests`, + `pack_gloas_attestations`, `pack_payload_attestations`, `assemble_gloas_block`, + `gloas_data_column_sidecars`); `rpc/src/beacon/gloas_proposal.rs` serves + `produceBlockV4` and the envelope endpoints (cache of the current and previous + slot's builds; the envelope POST waits up to 4 s for its block, then gossips + the envelope and all column sidecars through `RpcToP2P`, relying on gossipsub + fanout for subnets the node does not subscribe to) and `rpc/src/beacon/ptc.rs` + the PTC duties, payload attestation data and pool. One + `SharedPayloadAttestationPool` is shared by p2p (filled on accepted gossip) and + rpc. The store keeps an in-memory envelope-arrival record + (`insert_beacon_envelope_seen`, keyed by root, pruned with the el-hash cache, + not persisted) that `payload_present` reads against + `fork_choice::get_payload_due_ms`; `helpers::gloas::get_ptc_assignments` backs + the duties. `attestation_data.index` at a gloas slot comes from + `Store::head_payload_status`. With `--execution-endpoint`, an envelope that passes the consensus checks is judged by `engine_newPayloadV5` (INVALID with a null `latestValidHash` is a hash mismatch that refuses only that envelope; INVALID with one condemns the @@ -777,8 +793,21 @@ transitions are in `ethlambda-types`, per the section above. Nothing above retries once a slot, probing with the oldest), and a gloas head sends `engine_forkchoiceUpdatedV4` with the custody columns (head hash by the head node's FULL or EMPTY status, finalized and safe from the checkpoint blocks' - `bid.parent_block_hash`, safe being a fallback for fast confirmation). See + `bid.parent_block_hash`, safe being a fallback for fast confirmation). Building + a gloas payload for `produceBlockV4` uses `PayloadAttributesV4` (V3's fields + plus `slot_number` and `target_gas_limit`, the parent bid's gas limit) through + `forkchoice_updated_v4_with_attributes` and `get_payload_v6` + (`BuiltGloasPayload`, `ExecutionPayloadV4`). See [`docs/beacon_engine.md`](docs/beacon_engine.md). + The validator client (`crates/validator`) at a gloas slot runs: propose at the + boundary (block published bare, then the proposer-signed envelope with + `Eth-Blob-Data-Included: true`, both inside the proposal's attester-offset + budget; an envelope failure is logged and counted, not a failed proposal), + attest at `ATTESTATION_DUE_BPS_GLOAS`, aggregate at `AGGREGATE_DUE_BPS_GLOAS`, + then the PTC vote at `PAYLOAD_ATTESTATION_DUE_BPS`. `SlotClock` picks the + offsets by each slot's own fork, and `/eth/v1/config/spec` supplies them. It + still keeps no slashing-protection record; PTC votes have an in-memory + `(validator, slot)` dedup only, and envelopes are unguarded. `get_head_node` is one bottom-up walk for every fork, with the current slot's fork selecting the payload rules and the boost gate, so a follower on any network that schedules gloas (Sepolia does) pays work diff --git a/docs/beacon_engine.md b/docs/beacon_engine.md index f26847faf..97cd52eef 100644 --- a/docs/beacon_engine.md +++ b/docs/beacon_engine.md @@ -33,10 +33,16 @@ used from gloas on; the startup handshake warns when the execution client does n Three method families a full client would have are deliberately absent: -- **`engine_getPayload*` and `PayloadAttributesV3`.** This node is a follower and - never proposes, so it never asks an execution client to start building a block. - That also keeps `forkchoiceUpdated` outside the fork-scheduling rules attached - to `payloadAttributes.timestamp`: the second parameter is always `null`. +- **`engine_getPayload*` outside the gloas path.** The follower itself never + asks an execution client to start building a block, so its own + `forkchoiceUpdated` calls carry `null` payload attributes. Block production for + a validator client (`produceBlockV3` for fulu, `produceBlockV4` for gloas) does: + `forkchoiceUpdatedV3` with `PayloadAttributesV3` then `engine_getPayloadV5` + before gloas, and `forkchoiceUpdatedV4` with `PayloadAttributesV4` (V3's fields + plus `slotNumber` and `targetGasLimit`, the parent bid's gas limit) then + `engine_getPayloadV6` (`ExecutionPayloadV4`, with `blockAccessList` and + `slotNumber`) from gloas on. The handshake warns when a gloas network's + execution client does not advertise `engine_getPayloadV6`. - **`engine_getBlobs*`.** There is no blob-pool fetch path here; data columns come from peers over the network. - **`engine_getPayloadBodies*`.** Nothing consumes them. diff --git a/docs/cli.md b/docs/cli.md index a013c07a8..358d8a3bf 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -284,7 +284,8 @@ implementation's, so none of the [common flags](#common-flags) apply to it. ### What it does today -Attestations, block proposals and attestation aggregation. +Attestations, block proposals and attestation aggregation, and from gloas on +the self-built payload envelope and the payload timeliness committee vote. Each epoch it resolves its validators' indices, fetches their attester duties for this epoch and the next, fetches this epoch's proposer duties, subscribes @@ -327,17 +328,38 @@ computing its `hash_tree_root`, which only the typed container can give; a hand-written JSON mapping of an execution payload would be a large surface on which a single wrong field silently produces a signature over the wrong block. +From gloas on a proposal is two messages. The client asks for the block with +`include_payload=true` (`produceBlockV4`), publishes the block bare, then signs +the envelope with the proposer's own key under the builder domain and publishes +it with its blobs and cell proofs (`Eth-Blob-Data-Included: true`). Both go out as +SSZ. The envelope shares the proposal's budget (the attester offset, below), so +a proposal that overruns it is abandoned wherever it is. A failure to publish +the envelope is logged at error and counted, and does not fail the proposal. A +block that commits to a builder's bid is left alone. + +Each epoch it also fetches the payload timeliness committee duties for the +validators it holds, for this epoch and the next when they are gloas ones. A member sleeps until `PAYLOAD_ATTESTATION_DUE_BPS` of the +slot, asks the node for the +payload attestation data, signs it under the PTC domain and submits it. A `204` +(the node knows no block for the slot) means no vote. So a gloas slot runs in +this order: propose at the boundary, attest at `ATTESTATION_DUE_BPS_GLOAS`, +aggregate at `AGGREGATE_DUE_BPS_GLOAS`, then the committee vote. The attestation +and the committee vote are each bounded by the end of the slot. + Not implemented: the builder flow and blinded blocks (the client asks for an unblinded block and refuses a blinded one), sync-committee duties, voluntary exits, doppelganger protection and remote signing. ### Duty offsets come from the network -The slot length and the two duty offsets are read from the beacon node's +The slot length and the duty offsets are read from the beacon node's `/eth/v1/config/spec`, not divided out of a compiled-in constant. The specification states them as `SLOT_DURATION_MS` plus basis points of it, `ATTESTATION_DUE_BPS` and `AGGREGATE_DUE_BPS`, which on mainnet's 12-second slot -work out at 3999 ms and 8000 ms. +work out at 3999 ms and 8000 ms. Gloas moves both earlier +(`ATTESTATION_DUE_BPS_GLOAS`, `AGGREGATE_DUE_BPS_GLOAS`) and adds +`PAYLOAD_ATTESTATION_DUE_BPS` (and `PAYLOAD_DUE_BPS`, which the node uses), so +the clock picks the pair by the fork of each slot's own epoch. `SECONDS_PER_SLOT` no longer exists in the specification and is accepted only as a fallback, since deployed nodes still send it. A node sending both is required diff --git a/docs/metrics.md b/docs/metrics.md index 64644d7c1..cbdc72d6d 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -607,6 +607,10 @@ and absent is not the same as zero. | `ethlambda_validator_blocks_broadcast_not_imported_total` | Counter | Blocks a node broadcast but could not import into its own database, which is what a 202 means. A subset of `blocks_proposed_total`, not a failure: the block reached the network. Points at that node's execution layer | | `ethlambda_validator_blocks_refused_total` | Counter | Blocks deliberately not signed, because this process had already proposed that slot for that validator. Should normally read zero | | `ethlambda_validator_block_proposal_failures_total` | Counter | Proposal duties that did not end in a published block, including ones abandoned for overrunning | +| `ethlambda_validator_envelopes_published_total` | Counter | Execution payload envelopes a beacon node accepted for blocks this client proposed (gloas) | +| `ethlambda_validator_envelope_failures_total` | Counter | Proposed gloas blocks whose self-built envelope was not published. The block is out and the proposal counts as made, but the slot's payload is withheld, so this is the failure that costs the payload. Should read zero | +| `ethlambda_validator_payload_attestations_published_total` | Counter | Payload timeliness committee votes a beacon node accepted (gloas) | +| `ethlambda_validator_payload_attestation_failures_total` | Counter | Slots whose payload timeliness committee duty did not result in published votes | | `ethlambda_validator_aggregates_published_total` | Counter | Aggregates accepted by a beacon node. Bursty rather than steady: a validator is selected a few times a day, so hours at zero are normal for a small deployment | | `ethlambda_validator_aggregation_failures_total` | Counter | Aggregation duties that ended in no published aggregate, including ones abandoned for overrunning the slot | | `ethlambda_validator_fee_recipient_mismatches_total` | Counter | Blocks paying execution rewards to an address this client did not request. Should read zero forever; a non-zero value means every proposal is paying somewhere else | @@ -632,11 +636,20 @@ a failure. `blocks_proposed_total` counts blocks a node accepted; but that the node answering could not import, which is a beacon-node fault rather than a validator one and is why it is not folded into either. +Gloas adds two pairs. A gloas proposal ends in a block and then an envelope, +and the envelope's failure does not fail the proposal: `blocks_proposed_total` +counts the block, and `envelope_failures_total` the reveal that did not follow. +The committee vote has the same split as the attestation's: `payload_attestations_published_total` +counts votes accepted and `payload_attestation_failures_total` the slots whose +duty produced none. A slot where the node knows no block for the committee to +vote on is neither, since there was nothing to sign. + Two series should read zero for the life of a healthy deployment, and are the ones worth alerting on at any non-zero value rather than on a rate: `attestations_refused_total` and `blocks_refused_total` mean this client's own guards caught a duty they judged unsafe, and `fee_recipient_mismatches_total` -means blocks are being proposed that pay someone else. +means blocks are being proposed that pay someone else. `envelope_failures_total` +belongs with them on a gloas network. ## Troubleshooting diff --git a/docs/rpc.md b/docs/rpc.md index 212bd87e5..a1d3e76fb 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -219,7 +219,7 @@ curl -X POST http://127.0.0.1:5052/lean/v0/admin/aggregator \ ## Beacon API Server (`:5052`, on `ethlambda beacon`) The subset of the [Ethereum Beacon API](https://ethereum.github.io/beacon-APIs/) -that this follower can answer from its own store. It **replaces** the `/lean/v0` +that this node can answer from its own store. It **replaces** the `/lean/v0` surface rather than sitting beside it; a `/lean/v0` path on a beacon node is a `404`. @@ -240,13 +240,20 @@ surface rather than sitting beside it; a `/lean/v0` path on a beacon node is a | `GET`, `POST` | `/eth/v1/beacon/states/{state_id}/validators` | JSON | Registry entries by index or pubkey, with status | | `GET` | `/eth/v1/validator/duties/proposer/{epoch}` | JSON | Proposers for the head's epoch or the next | | `POST` | `/eth/v1/validator/duties/attester/{epoch}` | JSON | Committee assignments for the given indices | +| `POST` | `/eth/v1/validator/duties/ptc/{epoch}` | JSON | Payload timeliness committee seats for the given indices (gloas) | | `GET` | `/eth/v1/validator/attestation_data` | JSON | What to attest to at `slot` | +| `GET` | `/eth/v1/validator/payload_attestation_data` | JSON or SSZ | What a committee member signs for `slot` (gloas) | | `POST` | `/eth/v2/beacon/pool/attestations` | *(status only)* | Validate and gossip `SingleAttestation`s | +| `POST` | `/eth/v1/beacon/pool/payload_attestations` | *(status only)* | Validate, pool and gossip `PayloadAttestationMessage`s (gloas) | +| `GET` | `/eth/v1/beacon/pool/payload_attestations` | JSON | The payload attestation messages the pool holds (gloas) | | `POST` | `/eth/v1/validator/beacon_committee_subscriptions` | *(status only)* | Aggregators' entries join their committee's subnet | | `GET` | `/eth/v2/validator/aggregate_attestation` | JSON | The pooled votes for a data root and committee, aggregated | | `POST` | `/eth/v2/validator/aggregate_and_proofs` | *(status only)* | Validate and gossip `SignedAggregateAndProof`s | -| `GET` | `/eth/v3/validator/blocks/{slot}` | SSZ or JSON | An unsigned block built on the head (`produceBlockV3`) | -| `POST` | `/eth/v2/beacon/blocks` | *(status only)* | Gossip and import a signed block (`publishBlockV2`, SSZ) | +| `GET` | `/eth/v3/validator/blocks/{slot}` | SSZ or JSON | An unsigned fulu block built on the head (`produceBlockV3`) | +| `POST` | `/eth/v4/validator/blocks/{slot}` | SSZ or JSON | An unsigned self-built gloas block, with its envelope and blobs when asked (`produceBlockV4`) | +| `GET` | `/eth/v1/validator/execution_payload_envelopes/{slot}/{beacon_block_root}` | SSZ or JSON | The unsigned envelope `produceBlockV4` built (gloas) | +| `POST` | `/eth/v2/beacon/blocks` | *(status only)* | Gossip and import a signed fulu or gloas block (`publishBlockV2`, SSZ) | +| `POST` | `/eth/v1/beacon/execution_payload_envelopes` | *(status only)* | Gossip a signed envelope and its data columns (gloas, SSZ) | | `POST` | `/eth/v1/validator/prepare_beacon_proposer` | *(status only)* | Acknowledged, not acted on (see below) | ### Validator endpoints @@ -260,17 +267,23 @@ the chain actor writes, so no request waits on the actor. next; attester duties cover the head's previous, current and next epoch, which is as far as its shuffling is already fixed. Anything else is a `400`. `dependent_root` follows each endpoint's v1 definition. Attester duties walk - every committee of the epoch, a full shuffle per request on mainnet. An - epoch the node's schedule places at gloas is a `400` on both duty endpoints: - the head is a fulu state, and the node serves no validator duties from gloas - (it follows the fork but has no builder, payload-timeliness or gloas proposer - support). + every committee of the epoch, a full shuffle per request on mainnet. Gloas + epochs are served like fulu ones: gloas keeps fulu's `proposer_lookahead` and + `upgrade_to_gloas` carries it over, so a fulu head already answers proposer + duties for the first gloas epoch. - **`attestation_data`** follows phase0's `validator.md`: the head block, the epoch's boundary block as target, and as source the current justified checkpoint of the head state advanced to the slot's epoch (through fork choice's cached `checkpoint_state`, and only when the head is in an earlier - epoch). A slot before the head, or past the wall clock, is a `400`, as is a - slot in a gloas epoch: the answer would be a fulu-shaped vote under gloas. + epoch). A slot before the head, or past the wall clock, is a `400`. + `committee_index` is optional and ignored: from electra on every committee of + a slot attests to the same data. `data.index` is `0` before gloas. At a gloas + slot it is the payload-present flag: `0` when the head block is from the + requested slot itself (its payload cannot be revealed yet), otherwise `1` + exactly when fork choice holds the head block's payload as FULL + (`Store::head_payload_status`, or one fresh `get_head_node` walk when none is + recorded for this head). A pre-gloas head counts as FULL (see + [Spec Deviations](./spec_deviations.md#a-pre-gloas-head-counts-as-full-in-attestation_dataindex)). - **`pool/attestations`** checks each attestation against the electra `beacon_attestation_{subnet_id}` gossip conditions it can evaluate (clock window, `data.index == 0`, target epoch, the voted block known and the target @@ -279,9 +292,15 @@ the chain actor writes, so no request waits on the actor. back in an `IndexedErrorMessage` with its position; the valid ones in the same batch are still published. There is no seen-attestation cache. Each accepted attestation also goes into the node's **attestation pool**, since - gossip never delivers a node its own messages. `Eth-Consensus-Version: - gloas` is refused with a `400` here and on `aggregate_and_proofs`, as is a - gloas slot on the aggregate query below: the pool holds electra-shaped votes. + gossip never delivers a node its own messages. `Eth-Consensus-Version` must + name electra, fulu or gloas here and on `aggregate_and_proofs`, and must match + the fork of each item's slot: a `gloas` header on a pre-gloas slot, or an older + one on a gloas slot, fails that item (not the batch) with + `Eth-Consensus-Version does not match the attestation slot's fork`. At a + gloas slot `data.index` may be `0` or `1`, checked with gloas's payload-status + gossip rule (a vote in the voted block's own slot must be `0`; `1` needs the + node to have verified the payload, otherwise the item fails). A gloas + `SingleAttestation` is electra's own container. - **`beacon_committee_subscriptions`**: each aggregator's entry makes the node join its committee's attestation subnet until the end of that slot, so the committee's votes from other validators reach the pool too: every @@ -290,18 +309,21 @@ the chain actor writes, so no request waits on the actor. subnets are left once their slot has passed, and never appear in `attnets`. - **`aggregate_attestation`** answers from the pool: every vote held for the data root and committee, as electra's `Attestation` with the BLS aggregate of - their signatures. `404` when nothing is held. + their signatures; at a gloas slot the answer is `{version: "gloas", data}` + with gloas's `Attestation` (the same JSON shape). JSON only. `404` when + nothing is held. - **`aggregate_and_proofs`** checks each aggregate with the same `beacon_aggregate_and_proof` gossip conditions this node applies to its peers' aggregates (`gossip::aggregate`), signatures included, against a fresh seen-cache (a node never receives its own messages, so P2P's says nothing about them). What passes is gossiped on the topic and goes into the - pool. + pool. The `Eth-Consensus-Version` header picks the decoder: `gloas` takes + gloas's `SignedAggregateAndProof`, the others electra's. - **The attestation pool** holds, the best-covered per data root and committee: votes from `pool/attestations` and the aggregator subnets, aggregates from `aggregate_and_proofs`, and every electra gossip aggregate - P2P accepts, once all three of its signatures have verified. Gossip - aggregates are pooled on arrival, so a slot's aggregates are there when the + P2P accepts (gloas aggregates included, in gloas's container), once all three + of its signatures have verified. Gossip aggregates are pooled on arrival, so a slot's aggregates are there when the next slot's block is asked for. Entries more than an epoch old are dropped once a slot. - **`prepare_beacon_proposer`** records each validator's fee recipient, in @@ -320,18 +342,135 @@ the chain actor writes, so no request waits on the actor. `Eth-Execution-Payload-Blinded: false`; there is no builder flow. It is a **`503`** without a configured execution client, or when the payload carries blobs. A slot the schedule does not place at fulu (gloas included) is a - `400`, checked before the execution client. -- **`POST beacon/blocks`** takes SSZ `SignedBlockContents`, checks the block - is after the head and its proposer signature, then gossips it on - `beacon_block` and hands it to the chain actor to import. A slot the - schedule does not place at fulu (gloas included) is a `400`, checked before - the head state is advanced. - -**Blobs are not supported yet.** Publishing a blob-carrying block means -computing and gossiping its data column sidecars, which this node does not do, -and peers will not import a block they cannot sample. Such payloads are refused -at production (`503`, which a validator client fails over on) and such blocks -at publication (`400`). + `400`, checked before the execution client; `produceBlockV3` is the fulu + endpoint, and a gloas slot goes to `produceBlockV4`. +- **`POST beacon/blocks`** takes SSZ and the `Eth-Consensus-Version` header + (`fulu` or `gloas`, else `400`; a non-SSZ content type is a `415`). Fulu: a + `SignedBlockContents`; the node checks the block is after the head and its + proposer signature, then gossips it on `beacon_block` and hands it to the + chain actor to import. A slot the schedule does not place at fulu is a `400`, + checked before the head state is advanced. Gloas: a bare + `gloas::SignedBeaconBlock` (a gloas block carries no payload or blobs; they + follow in the envelope). The slot must be scheduled at gloas, the parent must + be held, the slot must be after the parent's, and the proposer and signature + are checked against the parent's state advanced to the slot (all `400` + otherwise), before the block is gossiped and handed to the chain actor. + +**Fulu blobs are not supported.** Publishing a blob-carrying fulu block means +computing and gossiping its data column sidecars, which this node does not do +for fulu, and peers will not import a block they cannot sample. Such payloads +are refused at production (`503`, which a validator client fails over on) and +such blocks at publication (`400`). Gloas blobs are supported: the envelope +publication below builds and gossips the columns. + +### Gloas block production + +A gloas proposer signs two things on their own, the block and the envelope that +reveals its payload, and this node serves both halves. It only builds for +itself: no builder bid is taken from gossip or a builder API, and no +`SignedProposerPreferences` is read (see +[Spec Deviations](./spec_deviations.md#self-build-only)). + +**`POST /eth/v4/validator/blocks/{slot}`** (`produceBlockV4`): + +| | | +|---|---| +| Query | `randao_reveal` (required), `include_payload` (required, `true` or `false`), `graffiti` (optional, 32-byte hex), `skip_randao_verification` (accepted, ignored) | +| Request headers | `Eth-Consensus-Version` is optional but must be `gloas` when present. `Accept: application/octet-stream` for SSZ, JSON otherwise | +| Body | A `BuilderConfig` (`min_bid`, `builder_boost_factor`, `builders`), JSON, or SSZ with `Content-Type: application/octet-stream`. It is decoded and otherwise ignored; a missing or undecodable one is a `400` | +| `200` SSZ | With `include_payload=true`, `BlockContents` (`block`, `execution_payload_envelope`, `kzg_proofs`, `blobs`); with `false`, the bare `gloas::BeaconBlock` | +| `200` JSON | `{version: "gloas", consensus_block_value, execution_payload_value, execution_payload_included, data}`, where `data` is the same container as the SSZ body | +| Response headers | `Eth-Consensus-Version: gloas`, `Eth-Execution-Payload-Included` (`true` or `false`), `Eth-Execution-Payload-Value` (wei, decimal), `Eth-Consensus-Block-Value` (always `0`: no builder comparison happens on this node) | +| `400` | A slot the schedule does not place at gloas, a missing or malformed query, a wrong `Eth-Consensus-Version`, a bad body, a slot not after the head block, or a `randao_reveal` that does not verify against the slot's proposer. The slot check precedes the execution-client check, so it is a `400` on any node | +| `503` | No execution client configured, the execution client did not start or return a build, or the node is building on a FULL parent whose envelope it does not hold | + +The node advances the head state to the slot, and decides which parent payload +to build on with `should_build_on_full` over the payload status fork choice +recorded for the head. It then asks its execution client to build +(`forkchoiceUpdatedV4` with `PayloadAttributesV4`, then `getPayloadV6`) with the +proposer's `prepare_beacon_proposer` fee recipient. The body packs the +attestation pool's best aggregates and the payload attestation pool's votes for +the parent block (an aggregate that does not verify against the advanced state +is dropped, since one bad operation fails the block), and the state root comes +from running the block through `process_block`. The bid is a zero-value +self-build bid read off the built payload. What was built is cached by `(slot, +block root)`, for the current and previous slot only. + +**`GET /eth/v1/validator/execution_payload_envelopes/{slot}/{beacon_block_root}`** +serves the cached unsigned envelope, for a client that asked for the block with +`include_payload=false`: `{version: "gloas", data}` as JSON, or the SSZ +`ExecutionPayloadEnvelope` on `Accept: application/octet-stream`, with +`Eth-Consensus-Version: gloas`. `404` when nothing is cached for the pair +(unknown, or older than the previous slot); `400` for an unparseable slot or +root. + +**`POST /eth/v1/beacon/execution_payload_envelopes`** takes the signed envelope, +SSZ only (`415` otherwise), with `Eth-Consensus-Version: gloas` and +`Eth-Blob-Data-Included` (both required, else `400`): + +- `true`: the body is `SignedExecutionPayloadEnvelopeContents` (the signed + envelope, `kzg_proofs`, `blobs`). +- `false`: the body is the bare signed envelope, and the blobs and proofs come + from the production cache. `400` when none is cached and the block's bid + commits to blobs. + +The envelope's block is waited for, polling every 50 ms for up to 4 seconds, +since a validator client publishes the envelope right after the block returns, +which is before the chain actor has imported it; a block still unknown after +that is a `400` (`unknown block`). The envelope must fulfill the block's bid +(builder index and block hash), the blobs must match the bid's commitments, the +signature must verify under `DOMAIN_BEACON_BUILDER` against the block's +post-state, and every data column built from the blobs must verify against the +commitments and cell proofs; any failure is a `400`. A success is `200` once the +envelope and all `NUMBER_OF_COLUMNS` gloas data column sidecars are handed to +P2P, which gossips them together. A node that does not subscribe to a column's +subnet publishes through gossipsub fanout. + +### Payload timeliness committee + +- **`POST /eth/v1/validator/duties/ptc/{epoch}`** takes a JSON array of quoted + validator indices (an empty array or a bad index is a `400`) and answers + `{dependent_root, execution_optimistic, data: [{pubkey, validator_index, + slot}]}`: one duty per validator, the first slot of the epoch whose committee + seats it. An index with no seat, or unknown, gets none. The epoch may be at + most one past the later of the head state's and the wall clock's epoch (else + `400`). An epoch before the gloas fork answers `200` with an empty `data`, so + a client can ask every epoch. A gloas head state answers from its + `ptc_window`; an epoch the head has not caught up to, or the first gloas epoch + under a fulu head, is answered from a copy of the head state advanced to the + epoch's first slot, crossing the gloas upgrade where it falls, on a blocking + thread. A gloas epoch more than one before the head state's is a `400`. + `dependent_root` is defined as for attester duties. There is no `503`. +- **`GET /eth/v1/validator/payload_attestation_data?slot=`** answers + `{version: "gloas", data: {beacon_block_root, slot, payload_present, + blob_data_available}}` as JSON, or the SSZ `PayloadAttestationData` on + `Accept: application/octet-stream`, with `Eth-Consensus-Version: gloas`. The + data is for the block of that very slot on the head's chain; a slot with no + such block (or whose block is not a gloas block) answers `204` with no body, + which a client reads as "cast no vote". A slot before the gloas fork is a + `400`, and a node that is syncing answers `503`. `payload_present` is whether + the block's envelope reached this node before the slot's start plus + `PAYLOAD_DUE_BPS` of it, judged by arrival time (see + [Spec Deviations](./spec_deviations.md#payload_present-is-judged-by-the-envelopes-arrival-time)). + `blob_data_available` is true when the bid commits to no blobs, the payload is + verified, or every column this node custodies for the block is stored. +- **`POST /eth/v1/beacon/pool/payload_attestations`** takes a JSON array of + `PayloadAttestationMessage`s; `Eth-Consensus-Version` is optional and must be + `gloas` when present. Each message passes the checks gossip applies to a + peer's (the current slot, a known block at the slot, committee membership, + signature) against a fresh seen-cache. A valid one is pooled and gossiped; one + already pooled is neither pooled nor gossiped again and still counts as + accepted. A rejected one comes back in an `IndexedErrorMessage` (`400`) as + `{outcome}: {reason}` with its position, and the rest still go out; a + head-state cache miss is a per-item `ignore: state_unavailable`. +- **`GET /eth/v1/beacon/pool/payload_attestations?slot=`** answers `{version: + "gloas", data}` with the messages the pool holds, those of `slot` only when + given. They are the individual messages, not the aggregates the specification + names: the pool keeps them unaggregated so block production can combine the + ones it needs. + +The payload attestation pool is shared with P2P, which fills it from accepted +gossip, so a block this node builds carries the votes its peers sent too. `tooling/kurtosis-validator/network_params_ethlambda_beacon.yaml` points the validator client at this node alone, so every block on that devnet is one this diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index fd194d45f..f3e00b67b 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -64,6 +64,17 @@ same keys, already signed. close the shapes reachable through ordinary operation: a backward wall-clock step, or a duty schedule replaced mid-epoch. Both are held in memory only, so a restart empties them and neither knows anything about any other process. +- **Gloas messages:** a gloas proposer also signs an execution payload + envelope, and a payload timeliness committee member signs a payload + attestation. Neither is slashable, so neither has a guard. An envelope is + published once per proposal, right after its block, and a failure to publish it + is logged at error and counted without failing the proposal. A PTC vote has an + in-memory dedup of `(validator, slot)` pairs recorded at signing time + (`PayloadAttestationService`), so a duty abandoned between signing and + submission is lost rather than re-signed. It is emptied by a restart and says + nothing about another process, so it is not slashing protection either: a + second, different vote for one slot is equivocation that peers penalize in + gossip, not a slashing. - **Why a proposer slashing is the worse of the two:** a double vote needs a second validator's attestation to be caught. Two signed block headers for one slot are the whole of the evidence on their own. @@ -82,6 +93,91 @@ same keys, already signed. runs, and treat a restart as an event that needs the same care a manual key move would. The client warns about this at startup on every run. +## Self-build only + +The gloas validator duties are served for a proposer that builds its own +payload, and for no one else. + +- **The specification:** a gloas proposer may take a builder's signed bid + (`SignedExecutionPayloadBid`, from gossip or a builder API) instead of building + its own payload, and publishes a `SignedProposerPreferences` so builders know + its fee recipient and gas limit target. `produceBlockV4` takes a + `BuilderConfig` (`min_bid`, `builder_boost_factor`, `builders`) to steer that + choice. +- **ethlambda:** the beacon node never takes a bid, from gossip or otherwise, and + never builds or reads a `SignedProposerPreferences`. `produceBlockV4` decodes + the `BuilderConfig` body (a missing or undecodable one is a `400`, as the + specification requires) and ignores it, logging at debug when it names + builders. Every block commits to a zero-value self-build bid + (`BUILDER_INDEX_SELF_BUILD`, the G2 point at infinity as signature), and + `Eth-Consensus-Block-Value` is always `0` since there is nothing to compare. + The validator client signs the envelope with the proposer's own key under + `DOMAIN_BEACON_BUILDER`, and leaves a block that commits to anyone else's bid + alone. +- **Why:** a scope decision for the first gloas duties: taking bids needs a bid + pool, builder payment handling and a builder-facing API, none of which this + node has. +- **Consequence:** a validator run through this node never earns a builder's + payment, and an execution client's own block value is what + `Eth-Execution-Payload-Value` reports. + +## `target_gas_limit` is the parent bid's gas limit + +- **The specification:** the payload is built toward the `target_gas_limit` of + the proposer's `SignedProposerPreferences`, and `bid.gas_limit` must be + compatible with it (`is_gas_limit_target_compatible`). +- **ethlambda:** with no preferences to read, `PayloadAttributesV4.targetGasLimit` + is the `gas_limit` of `latest_execution_payload_bid` of the state being built + on (`gloas_payload_inputs`), so the execution client holds the gas limit where + it is. +- **Equivalence:** the bid is built from the payload the execution client + returns, so `bid.gas_limit` is whatever that payload carries and is always + consistent with the block. A validator that wants the limit to move cannot say + so through this node: it follows the previous block's. + +## `skip_randao_verification` is ignored + +- **The specification:** `produceBlockV4`'s `skip_randao_verification` lets a + caller skip the node's check of `randao_reveal`; when it is set, the reveal + must be the point at infinity. +- **ethlambda:** the parameter is accepted and not honored. The block's state + root comes from running it through the state transition, which verifies the + reveal, so the endpoint checks the reveal against the slot's proposer up front + and answers `400` (`invalid randao_reveal`) for one that does not verify, + infinity included. +- **Consequence:** a caller cannot get a block without a real reveal. Every + caller this client knows of (including its own validator client) sends one. + +## A pre-gloas head counts as FULL in `attestation_data.index` + +- **The specification:** at a gloas slot `data.index` is `0` when the attested + block is from the attestation's own slot, and otherwise `0` for an `EMPTY` and + `1` for a `FULL` payload status in the validator's fork choice. It does not say + which a pre-gloas block is. +- **ethlambda:** at a gloas slot `data.index` is `0` when the head block is from + the requested slot, `1` when fork choice holds the head's payload as FULL + (`Store::head_payload_status`, or one `get_head_node` walk when none is + recorded), and `1` for a pre-gloas head, the same boundary rule + [fork choice applies](#the-fulu-to-gloas-fork-choice-boundary). So the first + gloas slots vote for the last fulu block's payload. +- **Why:** the vote should name the node fork choice itself walks to, and + that rule makes a pre-gloas block's one node FULL. + +## `payload_present` is judged by the envelope's arrival time + +- **The specification:** a PTC member sets `payload_present` when it has + received the block's envelope by `get_payload_due_ms()` into the slot. +- **ethlambda:** `GET payload_attestation_data` compares the time the envelope + reached the node (`Store::beacon_envelope_seen_ms`, recorded by the chain + actor from the arrival time of the envelope, not the time it finished + verifying; the earliest arrival wins) with the slot's start plus + `get_payload_due_ms()`. An envelope held for its block or columns is + therefore judged by when it came, as the rule intends. The record lives in + memory only, pruned with the execution block hash cache. +- **Consequence:** a restart forgets it. Only the slot in flight can be wrong + (a member asked after a restart answers `payload_present = false` for an + envelope that arrived before it), and the next slot's record starts fresh. + ## `/eth/v1/node/identity` reports no ENR The endpoint's `enr`, `p2p_addresses` and `discovery_addresses` are empty From 6513e085ca2f0faa3601fc1f2c2980e9b7eadd81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:25:24 -0300 Subject: [PATCH 100/132] test(beacon): port the validator-client tests to the gloas-aware client API The client grew a fork argument on attestation_data, AggregateKind, a fork on BlockRequest and ProducedBlock accessors. The router also layers the payload attestation pool and custody columns as the real server does, so the harness keeps matching what start_beacon_rpc_server builds. --- .../rpc/src/beacon/validator_client_tests.rs | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/crates/net/rpc/src/beacon/validator_client_tests.rs b/crates/net/rpc/src/beacon/validator_client_tests.rs index 16663d75c..3762452d7 100644 --- a/crates/net/rpc/src/beacon/validator_client_tests.rs +++ b/crates/net/rpc/src/beacon/validator_client_tests.rs @@ -18,6 +18,7 @@ use ethlambda_state_transition::beacon::helpers::{ fulu::initialize_proposer_lookahead, test_state::{sign_for, with_signing_validators_at}, }; +use ethlambda_state_transition::beacon::payload_attestation_pool::SharedPayloadAttestationPool; use ethlambda_types::{ beacon::{ constants::DOMAIN_BEACON_ATTESTER, @@ -32,7 +33,7 @@ use ethlambda_types::{ use ethlambda_validator::{ Error, beacon_node::{ - BeaconNodeApi, BlockRequest, + AggregateKind, BeaconNodeApi, BlockRequest, SignedAggregates, dto::{ AttestationDataOutDto, CommitteeSubscriptionDto, ProposerPreparationDto, SingleAttestationDto, encode_hex, @@ -90,6 +91,8 @@ async fn serve_with_engine( .layer(Extension(SyncStatusController::new(SyncStatus::Synced))) .layer(Extension(p2p)) .layer(Extension(SharedAttestationPool::default())) + .layer(Extension(SharedPayloadAttestationPool::default())) + .layer(Extension(crate::CustodyColumns(Vec::new()))) .layer(Extension(crate::beacon::validator::FeeRecipients::default())) .layer(Extension(engine)); @@ -157,7 +160,7 @@ async fn the_validator_client_can_attest_through_this_node() { .unwrap(); // The attestation data is checked by the client itself before it signs. - let data = client.attestation_data(slot).await.unwrap(); + let data = client.attestation_data(slot, ForkName::Fulu).await.unwrap(); let domain = get_domain(&state, DOMAIN_BEACON_ATTESTER, Some(data.target.epoch)); let signing_root = compute_signing_root(data.hash_tree_root(), domain); let data_dto = AttestationDataOutDto::from(&data); @@ -197,14 +200,17 @@ async fn the_validator_client_can_attest_through_this_node() { .iter() .filter(|attestation| attestation.committee_index == duty.committee_index) .count(); + let AggregateKind::Electra(attestation) = aggregate.attestation else { + panic!("a fulu slot's aggregate is electra's container"); + }; let bits = (0..duty.committee_length as usize) - .filter(|&i| aggregate.attestation.aggregation_bits.get(i).unwrap()) + .filter(|&i| attestation.aggregation_bits.get(i).unwrap()) .count(); assert_eq!(bits, voters); - let signed = signed_aggregate(&state, duty.validator_index, aggregate.attestation); + let signed = signed_aggregate(&state, duty.validator_index, attestation); client - .publish_aggregates(ForkName::Fulu, &[signed]) + .publish_aggregates(ForkName::Fulu, &SignedAggregates::Electra(vec![signed])) .await .unwrap(); assert_eq!(network.aggregates.lock().unwrap().len(), 1); @@ -250,6 +256,7 @@ async fn block_production_without_an_execution_client_is_retryable() { let (client, state, _) = serve().await; let request = BlockRequest { slot: state.slot() + 1, + fork: ForkName::Fulu, proposer_index: 0, randao_reveal: Default::default(), graffiti: Default::default(), @@ -352,18 +359,19 @@ async fn the_validator_client_can_propose_through_this_node() { let request = BlockRequest { slot, + fork: ForkName::Fulu, proposer_index: proposer, randao_reveal, graffiti: Default::default(), }; let produced = client.produce_block(&request).await.unwrap(); - assert_eq!(produced.block().slot, slot); - assert_eq!(produced.block().proposer_index, proposer); + assert_eq!(produced.slot(), slot); + assert_eq!(produced.proposer_index(), proposer); let block_domain = get_domain(&advanced, DOMAIN_BEACON_PROPOSER, Some(epoch)); let signature = sign_for( proposer as usize, - compute_signing_root(produced.block().hash_tree_root(), block_domain), + compute_signing_root(produced.block_root(), block_domain), ); let body = produced.into_signed_ssz(signature); client.publish_block(ForkName::Fulu, &body).await.unwrap(); From 5535deb2ed74d5a4c2b26a756b2a0f6e630320c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:29:30 -0300 Subject: [PATCH 101/132] refactor(beacon): expose the per-slot payload attestation aggregation pack_payload_attestations grouped and verified votes inline, tied to a state exactly one slot past the parent. The pool listing endpoint needs the same grouping for any slot the head state can read a committee for, so the grouping moves into aggregate_payload_attestations and packing calls it. Packing behavior is unchanged. --- .../src/beacon/gloas_block_production.rs | 39 ++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs b/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs index ccb4bd9e5..a9800cba9 100644 --- a/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs +++ b/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs @@ -206,7 +206,34 @@ pub fn pack_payload_attestations( if parent_slot.checked_add(1) != Some(state.slot()) { return Vec::new(); } - let Ok(ptc) = get_ptc(state, parent_slot, config) else { + let messages = messages.into_iter().filter(|message| { + message.data.beacon_block_root == parent_root && message.data.slot == parent_slot + }); + let mut packed = aggregate_payload_attestations(state, parent_slot, messages, config); + packed.sort_by_key(|(seats, _)| std::cmp::Reverse(*seats)); + packed + .into_iter() + .map(|(_, attestation)| attestation) + .take(preset::MAX_PAYLOAD_ATTESTATIONS as usize) + .collect() +} + +/// Aggregate the votes of `messages` cast at `slot` into one +/// [`PayloadAttestation`] per distinct `PayloadAttestationData`, each paired +/// with the number of committee seats it covers. +/// +/// The committee is `get_ptc(state, slot)`, so `state` must be within the +/// window that function reads (the slot's epoch, the one after it, or one +/// past `state`'s). Votes for another slot are skipped, and an aggregate that +/// does not verify against `state` is dropped. The order is the data roots', +/// not a ranking: callers that must cap the count sort by seats themselves. +pub fn aggregate_payload_attestations( + state: &BeaconState, + slot: Slot, + messages: impl IntoIterator, + config: &Config, +) -> Vec<(usize, PayloadAttestation)> { + let Ok(ptc) = get_ptc(state, slot, config) else { return Vec::new(); }; @@ -219,7 +246,7 @@ pub fn pack_payload_attestations( ), > = BTreeMap::new(); for message in messages { - if message.data.beacon_block_root != parent_root || message.data.slot != parent_slot { + if message.data.slot != slot { continue; } groups @@ -230,7 +257,7 @@ pub fn pack_payload_attestations( .or_insert(message.signature); } - let mut packed: Vec<(usize, PayloadAttestation)> = groups + groups .into_values() .filter_map(|(data, votes)| { let mut aggregation_bits = gloas::PayloadTimelinessCommitteeBits::default(); @@ -250,12 +277,6 @@ pub fn pack_payload_attestations( let indexed = get_indexed_payload_attestation(state, &attestation, config).ok()?; is_valid_indexed_payload_attestation(state, &indexed).then_some((seats, attestation)) }) - .collect(); - packed.sort_by_key(|(seats, _)| std::cmp::Reverse(*seats)); - packed - .into_iter() - .map(|(_, attestation)| attestation) - .take(preset::MAX_PAYLOAD_ATTESTATIONS as usize) .collect() } From ef556a889b4109a5c71f3508b2e75da217b5b9ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:29:36 -0300 Subject: [PATCH 102/132] fix(beacon): list the payload attestation pool as aggregates beacon-APIs getPoolPayloadAttestations returns Gloas.PayloadAttestation (a bitvector over the slot's committee plus one aggregated signature), not raw messages, so a consumer built to the spec could not parse our answer. The handler now aggregates the pooled votes per slot and data with the logic block production uses, against the head state. The optional slot filter is kept. --- crates/net/rpc/src/beacon/ptc.rs | 102 ++++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 15 deletions(-) diff --git a/crates/net/rpc/src/beacon/ptc.rs b/crates/net/rpc/src/beacon/ptc.rs index 67a912a3c..e827d3e51 100644 --- a/crates/net/rpc/src/beacon/ptc.rs +++ b/crates/net/rpc/src/beacon/ptc.rs @@ -6,9 +6,10 @@ //! signs for a slot. //! - `POST /eth/v1/beacon/pool/payload_attestations`: a member's signed vote, //! validated as gossip would, pooled for block production and gossiped. -//! - `GET /eth/v1/beacon/pool/payload_attestations`: what the pool holds. +//! - `GET /eth/v1/beacon/pool/payload_attestations`: what the pool holds, +//! aggregated per slot and data. -use std::collections::HashSet; +use std::collections::{BTreeMap, HashSet}; use std::sync::Arc; use axum::{ @@ -24,6 +25,7 @@ use ethlambda_blockchain::metrics::SyncStatus; use ethlambda_network_api::RpcToP2PRef; use ethlambda_state_transition::beacon::{ fork_choice::{get_current_slot, get_payload_due_ms}, + gloas_block_production::aggregate_payload_attestations, gossip::{ Outcome, payload_attestation::{SeenPayloadAttestations, cheap_checks, stateful_checks}, @@ -38,7 +40,7 @@ use ethlambda_types::{ beacon::{ containers::{ BeaconState, SignedBeaconBlock, - gloas::{PayloadAttestationData, PayloadAttestationMessage}, + gloas::{PayloadAttestation, PayloadAttestationData, PayloadAttestationMessage}, }, fork::ForkName, primitives::{BlsPubkey, Epoch, Slot, ValidatorIndex}, @@ -454,28 +456,61 @@ struct PoolQuery { } /// `GET /eth/v1/beacon/pool/payload_attestations?slot=`: the votes this node -/// holds, those of `slot` only when given. +/// holds as `PayloadAttestation`s, those of `slot` only when given. /// -/// The messages themselves, not the aggregates the specification's response -/// names: the pool keeps them unaggregated so block production can combine -/// the ones it needs, and a consumer wanting the aggregate can build it. +/// The pool keeps the unaggregated messages (block production combines the +/// ones it needs), but the specification's response is the aggregate: one per +/// distinct `PayloadAttestationData`, its bitvector over the slot's committee +/// and one aggregated signature. Aggregated here with the logic block +/// production uses, against the head state, which can read the committee of +/// the slot it is in, the one before and the next one. A slot outside that +/// window has no readable committee, so its votes are left out rather than +/// listed unaggregated; the pool only ever accepts the current slot's votes, +/// so in practice that is a vote the head has since left far behind. async fn get_pool_payload_attestations( Query(query): Query, + State(store): State, Extension(pool): Extension, ) -> Response { let held = pool .lock() .expect("payload attestation pool lock poisoned") .all(query.slot); + let aggregated = tokio::task::spawn_blocking(move || aggregate_pool(&store, held)).await; + let data = match aggregated { + Ok(Ok(data)) => data, + Ok(Err(err)) => return err.into_response(), + Err(_) => return ApiError::Internal("aggregating the pool failed").into_response(), + }; with_consensus_version( crate::json_response(serde_json::json!({ "version": ForkName::Gloas.as_str(), - "data": held, + "data": data, })), ForkName::Gloas, ) } +/// Group `held` by slot and aggregate each slot's votes against the head state. +fn aggregate_pool( + store: &Store, + held: Vec, +) -> Result, ApiError> { + let config = store.config(); + let (_, state) = head(store)?; + let mut by_slot: BTreeMap> = BTreeMap::new(); + for message in held { + by_slot.entry(message.data.slot).or_default().push(message); + } + Ok(by_slot + .into_iter() + .flat_map(|(slot, messages)| { + aggregate_payload_attestations(&state, slot, messages, &config) + }) + .map(|(_, attestation)| attestation) + .collect()) +} + #[cfg(test)] mod tests { use super::*; @@ -1050,17 +1085,29 @@ mod tests { } #[tokio::test] - async fn the_pool_is_listed_by_slot() { + async fn the_pool_is_listed_as_aggregates_by_slot() { let state = gloas_state(); let state_epoch = compute_epoch_at_slot(state.slot()); let member = window_index(state_epoch, state_epoch, 0); let (store, root) = store_with_head(state.clone(), gloas_config(), 0); let pool = SharedPayloadAttestationPool::default(); let message = vote(&state, root, member, true); - let mut older = message.clone(); - older.data.slot -= 1; - pool.lock().unwrap().insert(older); - pool.lock().unwrap().insert(message.clone()); + + // The previous slot's committee is its own window entry, which names + // the validator `slot % SLOTS_PER_EPOCH` here. + let mut earlier_state = state.clone(); + let BeaconState::Gloas(inner) = &mut earlier_state else { + unreachable!("built as gloas") + }; + inner.slot -= 1; + let earlier_member = earlier_state.slot() % preset::SLOTS_PER_EPOCH; + let older = vote(&earlier_state, root, earlier_member, true); + + // Not in the head slot's committee: nothing to aggregate it into. + let outsider = vote(&state, root, member + 1, true); + for held in [&older, &message, &outsider] { + pool.lock().unwrap().insert(held.clone()); + } let list = |uri: String| { let store = store.clone(); let pool = pool.clone(); @@ -1081,7 +1128,32 @@ mod tests { assert_eq!(all.status, StatusCode::OK); assert_eq!(all.headers["eth-consensus-version"], "gloas"); assert_eq!(all.json()["version"], "gloas"); - assert_eq!(all.json()["data"].as_array().unwrap().len(), 2); + let data = all.json()["data"].as_array().unwrap().clone(); + assert_eq!( + data.len(), + 2, + "one aggregate per slot, the outsider's dropped" + ); + // Oldest slot first, each an aggregate and not a message. + assert_eq!(data[0]["data"]["slot"], older.data.slot.to_string()); + assert_eq!(data[1]["data"]["slot"], message.data.slot.to_string()); + for aggregate in &data { + assert!(aggregate.get("validator_index").is_none()); + // The single seat-holder fills every position of the committee. + let bits = aggregate["aggregation_bits"].as_str().unwrap(); + assert_eq!(bits, format!("0x{}", "ff".repeat(preset::PTC_SIZE / 8))); + } + + // It is the aggregate block production would pack, which only keeps + // one that verifies against the state. + let expected = ethlambda_state_transition::beacon::gloas_block_production::aggregate_payload_attestations( + &state, + message.data.slot, + [message.clone()], + &gloas_config(), + ); + assert_eq!(expected.len(), 1); + assert_eq!(data[1], serde_json::to_value(&expected[0].1).unwrap()); let slot = message.data.slot; let one = list(format!( @@ -1089,6 +1161,6 @@ mod tests { )) .await; assert_eq!(one.json()["data"].as_array().unwrap().len(), 1); - assert_eq!(one.json()["data"][0]["validator_index"], member.to_string()); + assert_eq!(one.json()["data"][0]["data"]["slot"], slot.to_string()); } } From ddcd935abec9b8c8eba76969322e360ddcc109de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:35:43 -0300 Subject: [PATCH 103/132] test(beacon): drive the validator client end to end at a gloas slot Adds gloas counterparts to the validator-client tests, each running the real client (HttpBeaconNode, ProposalService, PayloadAttestationService) against the real router over a socket, with a signature check against the state's own domains: - attest and aggregate: no committee index in the request, the payload signal in the answer (0, then 1 once fork choice holds the head's payload as full), the gloas header on submit, and the gloas aggregate container. - payload timeliness committee: duties equal the state's committees, 204 for a slot with no block, and votes signed by the client are gossiped and pooled. - proposal: produceBlockV4 with the payload, a signed block, a signed builder-domain envelope matching the bid, with and without a real blob (all data columns go out with the envelope). The fake execution client now also answers forkchoiceUpdatedV4 and getPayloadV6. The test state needs its fork set to the schedule's gloas version, which the builders leave zeroed, for the client's signatures to verify against it. --- .../rpc/src/beacon/validator_client_tests.rs | 630 +++++++++++++++++- crates/net/rpc/src/lib.rs | 45 ++ 2 files changed, 649 insertions(+), 26 deletions(-) diff --git a/crates/net/rpc/src/beacon/validator_client_tests.rs b/crates/net/rpc/src/beacon/validator_client_tests.rs index 3762452d7..931960490 100644 --- a/crates/net/rpc/src/beacon/validator_client_tests.rs +++ b/crates/net/rpc/src/beacon/validator_client_tests.rs @@ -42,6 +42,8 @@ use ethlambda_validator::{ }, }; +use ethlambda_storage::Store; + use crate::test_utils::{RecordingNetwork, beacon_store_at}; const COUNT: usize = 64; @@ -77,7 +79,21 @@ async fn serve_with_engine( fulu.current_sync_committee = sync_committee.clone(); fulu.next_sync_committee = sync_committee; - let (mut store, _root) = beacon_store_at(state.clone()); + let (store, _root) = beacon_store_at(state.clone()); + let (client, network, _pool) = spawn_server(store, engine).await; + (client, state, network) +} + +/// Serve `store` through the router the real server builds, over a socket on an +/// ephemeral port, and connect this crate's client to it. +async fn spawn_server( + mut store: Store, + engine: Option, +) -> ( + HttpBeaconNode, + Arc, + SharedPayloadAttestationPool, +) { // The store's clock, which the aggregate conditions read the current epoch // from, as the chain actor's ticks would have set it. let now_ms = std::time::SystemTime::now() @@ -87,11 +103,12 @@ async fn serve_with_engine( store.set_time_ms(now_ms).unwrap(); let network = Arc::new(RecordingNetwork::default()); let p2p: RpcToP2PRef = network.clone(); + let payload_pool = SharedPayloadAttestationPool::default(); let router = crate::build_beacon_api_router(store, "ethlambda/test", "peer".into()) .layer(Extension(SyncStatusController::new(SyncStatus::Synced))) .layer(Extension(p2p)) .layer(Extension(SharedAttestationPool::default())) - .layer(Extension(SharedPayloadAttestationPool::default())) + .layer(Extension(payload_pool.clone())) .layer(Extension(crate::CustodyColumns(Vec::new()))) .layer(Extension(crate::beacon::validator::FeeRecipients::default())) .layer(Extension(engine)); @@ -101,7 +118,7 @@ async fn serve_with_engine( tokio::spawn(async move { axum::serve(listener, router).await }); let client = HttpBeaconNode::new(format!("http://{address}")).unwrap(); - (client, state, network) + (client, network, payload_pool) } /// One slot of an attester's work, in the order `ethlambda validator` does it. @@ -267,19 +284,63 @@ async fn block_production_without_an_execution_client_is_retryable() { } /// A stand-in execution client: `forkchoiceUpdated` with attributes answers a -/// payload id, and `getPayloadV5` a payload that extends the requested head -/// with exactly the requested attributes, which is all a real one's payload -/// has to agree with for the block to verify. +/// payload id, and `getPayload` a payload that extends the requested head with +/// exactly the requested attributes, which is all a real one's payload has to +/// agree with for the block to verify. Answers both generations: V3/V5 for +/// fulu and V4/V6 (`PayloadAttributesV4`, `ExecutionPayloadV4`) for gloas. async fn fake_execution_client() -> ethlambda_engine::EngineClient { + fake_execution_client_with_blob(None).await +} + +/// [`fake_execution_client`] whose gloas payloads carry `blob`, with its real +/// commitment and cell proofs, in the blobs bundle. +async fn fake_execution_client_with_blob(blob: Option>) -> ethlambda_engine::EngineClient { use axum::{Json, routing::post}; + use ethlambda_state_transition::beacon::kzg::{ + blob_to_kzg_commitment, compute_cells_and_kzg_proofs, + }; use std::sync::Mutex; + let bundle = match &blob { + None => serde_json::json!({ "commitments": [], "proofs": [], "blobs": [] }), + Some(blob) => { + let commitment = blob_to_kzg_commitment(blob).unwrap(); + let (_, proofs) = compute_cells_and_kzg_proofs(blob).unwrap(); + serde_json::json!({ + "commitments": [encode_hex(&commitment.0)], + "proofs": proofs.iter().map(|proof| encode_hex(&proof.0)).collect::>(), + "blobs": [encode_hex(blob)], + }) + } + }; let requested: Arc>> = Arc::default(); let handler = move |Json(request): Json| { let requested = requested.clone(); + let bundle = bundle.clone(); async move { + let payload = |attributes: &serde_json::Value, gas_limit: serde_json::Value| { + serde_json::json!({ + "parentHash": attributes["parentHash"], + "feeRecipient": attributes["suggestedFeeRecipient"], + "stateRoot": format!("0x{}", "00".repeat(32)), + "receiptsRoot": format!("0x{}", "00".repeat(32)), + "logsBloom": format!("0x{}", "00".repeat(256)), + "prevRandao": attributes["prevRandao"], + "blockNumber": "0x1", + "gasLimit": gas_limit, + "gasUsed": "0x0", + "timestamp": attributes["timestamp"], + "extraData": "0x", + "baseFeePerGas": "0x7", + "blockHash": format!("0x{}", "ee".repeat(32)), + "transactions": [], + "withdrawals": attributes["withdrawals"], + "blobGasUsed": "0x0", + "excessBlobGas": "0x0", + }) + }; let result = match request["method"].as_str() { - Some("engine_forkchoiceUpdatedV3") => { + Some("engine_forkchoiceUpdatedV3" | "engine_forkchoiceUpdatedV4") => { let mut attributes = request["params"][1].clone(); attributes["parentHash"] = request["params"][0]["headBlockHash"].clone(); *requested.lock().unwrap() = Some(attributes); @@ -291,31 +352,27 @@ async fn fake_execution_client() -> ethlambda_engine::EngineClient { Some("engine_getPayloadV5") => { let attributes = requested.lock().unwrap().clone().unwrap(); serde_json::json!({ - "executionPayload": { - "parentHash": attributes["parentHash"], - "feeRecipient": attributes["suggestedFeeRecipient"], - "stateRoot": format!("0x{}", "00".repeat(32)), - "receiptsRoot": format!("0x{}", "00".repeat(32)), - "logsBloom": format!("0x{}", "00".repeat(256)), - "prevRandao": attributes["prevRandao"], - "blockNumber": "0x1", - "gasLimit": "0x1c9c380", - "gasUsed": "0x0", - "timestamp": attributes["timestamp"], - "extraData": "0x", - "baseFeePerGas": "0x7", - "blockHash": format!("0x{}", "ee".repeat(32)), - "transactions": [], - "withdrawals": attributes["withdrawals"], - "blobGasUsed": "0x0", - "excessBlobGas": "0x0", - }, + "executionPayload": payload(&attributes, "0x1c9c380".into()), "blockValue": "0x2a", "blobsBundle": { "commitments": [], "proofs": [], "blobs": [] }, "shouldOverrideBuilder": false, "executionRequests": [], }) } + Some("engine_getPayloadV6") => { + let attributes = requested.lock().unwrap().clone().unwrap(); + let mut execution_payload = + payload(&attributes, attributes["targetGasLimit"].clone()); + execution_payload["blockAccessList"] = "0xc0".into(); + execution_payload["slotNumber"] = attributes["slotNumber"].clone(); + serde_json::json!({ + "executionPayload": execution_payload, + "blockValue": "0x2a", + "blobsBundle": bundle, + "shouldOverrideBuilder": false, + "executionRequests": [], + }) + } _ => serde_json::Value::Null, }; Json(serde_json::json!({ "jsonrpc": "2.0", "id": request["id"], "result": result })) @@ -377,3 +434,524 @@ async fn the_validator_client_can_propose_through_this_node() { client.publish_block(ForkName::Fulu, &body).await.unwrap(); assert_eq!(network.blocks.lock().unwrap().len(), 1); } + +// --------------------------------------------------------------------------- +// Gloas +// --------------------------------------------------------------------------- + +mod gloas { + use std::time::Duration; + + use ethlambda_state_transition::beacon::{ + block_production::advance_to_slot, + bls, + constants::{DOMAIN_BEACON_BUILDER, DOMAIN_BEACON_PROPOSER, DOMAIN_PTC_ATTESTER}, + fork_choice::PayloadStatus, + gloas_block_production::test_support::{config, parent_state, post_state}, + helpers::{ + accessors::get_beacon_proposer_index, + gloas::{compute_ptc, get_ptc}, + test_state::secret_key_for, + }, + }; + use ethlambda_types::beacon::{ + constants::BUILDER_INDEX_SELF_BUILD, + containers::{SignedAggregateAndProof, SignedBeaconBlock, gloas as gloas_containers}, + primitives::Bytes32, + }; + use ethlambda_validator::{ + beacon_node::dto::{ProposerDutyDto, PtcDutyDto}, + keys::ValidatorStore, + payload_attestation::PayloadAttestationService, + proposal::ProposalService, + signing::SigningContext, + }; + use tokio::sync::RwLock; + + use super::*; + use crate::test_utils::{beacon_store_with_head_block, gloas_beacon_block}; + + /// A gloas head state at slot 32 under [`config`] (gloas from epoch 0), + /// with the proposer lookahead and sync committee a real registry gives it + /// and every `ptc_window` entry computed, so the committees the node reads + /// for the previous, current and next epoch are the real ones. + fn head_state() -> BeaconState { + let mut state = parent_state(); + let epoch = compute_epoch_at_slot(state.slot()); + let first_slot = compute_start_slot_at_epoch(epoch - 1); + let cache = + ethlambda_state_transition::beacon::helpers::accessors::CommitteeCache::default(); + let window: Vec<_> = (first_slot..first_slot + preset::PTC_WINDOW_LENGTH as u64) + .map(|slot| compute_ptc(&state, slot, &cache).unwrap()) + .collect(); + let BeaconState::Gloas(inner) = &mut state else { + unreachable!("built as gloas") + }; + for (index, committee) in window.into_iter().enumerate() { + inner.ptc_window[index] = committee; + } + // The fork the schedule puts the state in, so a signature made from + // this client's schedule verifies against the state's own domain: the + // builders leave `fork` zeroed, which no real state has. + let schedule = config(); + inner.fork = ethlambda_types::beacon::containers::shared::Fork { + previous_version: schedule.fork_version(ForkName::Fulu), + current_version: schedule.fork_version(ForkName::Gloas), + epoch: schedule.gloas_fork_epoch, + }; + state + } + + struct Served { + client: HttpBeaconNode, + state: BeaconState, + network: Arc, + payload_pool: SharedPayloadAttestationPool, + store: Store, + head_root: ethlambda_types::primitives::H256, + context: Arc, + } + + /// The gloas head state served over HTTP, its head block a gloas block of + /// the state's slot, on a clock where `clock_slot` is running now. + async fn serve_gloas( + clock_slot: u64, + engine: Option, + ) -> Served { + let state = head_state(); + let block = gloas_beacon_block( + state.slot(), + Default::default(), + Default::default(), + Bytes32::repeat_byte(1), + ); + // The root a block built on this state names as its parent: the + // state's latest header, its state root filled in as the slot + // transition does. + let mut header = state.latest_block_header().clone(); + header.state_root = state.hash_tree_root(); + let head_root = header.hash_tree_root(); + let store = + beacon_store_with_head_block(state.clone(), config(), block, head_root, clock_slot); + let (client, network, payload_pool) = spawn_server(store.clone(), engine).await; + let genesis = client.genesis().await.expect("genesis"); + let spec = client + .spec() + .await + .expect("the client reads this node's spec"); + let context = Arc::new(SigningContext { + config: spec, + genesis_validators_root: genesis.genesis_validators_root, + }); + Served { + client, + state, + network, + payload_pool, + store, + head_root, + context, + } + } + + /// A key store holding the test registry's secret for each of `indices`. + fn keys_for(state: &BeaconState, indices: &[u64]) -> RwLock { + let mut keys = ValidatorStore::new(); + for &index in indices { + let pubkey = keys + .insert_secret("test", &secret_key_for(index as usize).to_bytes()) + .expect("the test secret is a valid key"); + assert_eq!( + pubkey, + state.validator(index).unwrap().pubkey, + "the test registry's key for validator {index}" + ); + } + RwLock::new(keys) + } + + /// One slot of a gloas attester's work: the data comes back without a + /// committee index and names the payload signal, the gloas header goes out + /// with the votes, and the aggregate is the gloas container. + #[tokio::test] + async fn the_validator_client_can_attest_and_aggregate_at_a_gloas_slot() { + let Served { + client, + state, + network, + store, + head_root, + .. + } = serve_gloas(32, None).await; + let slot = state.slot(); + let epoch = compute_epoch_at_slot(slot); + assert_eq!( + client.spec().await.unwrap().fork_at_epoch(epoch), + ForkName::Gloas + ); + + let indices: Vec = (0..COUNT as u64).collect(); + let attesters = client.attester_duties(epoch, &indices).await.unwrap(); + assert_eq!(attesters.duties.len(), COUNT); + + // The attested block is the one of the requested slot: the payload + // signal is 0. + let data = client + .attestation_data(slot, ForkName::Gloas) + .await + .unwrap(); + assert_eq!(data.slot, slot); + assert_eq!(data.index, 0, "a same-slot vote carries no payload signal"); + // A vote for the next slot attests to the head block, now a slot + // old: its signal is whether fork choice holds that block's payload + // as full, and the client passes it on unchanged because it is signed. + let later = client + .attestation_data(slot + 1, ForkName::Gloas) + .await + .unwrap(); + assert_eq!(later.beacon_block_root, head_root); + assert_eq!(later.index, 0, "no envelope was seen for the head block"); + store.set_head_payload_status(head_root, PayloadStatus::Full); + let later = client + .attestation_data(slot + 1, ForkName::Gloas) + .await + .unwrap(); + assert_eq!(later.index, 1); + let domain = get_domain(&state, DOMAIN_BEACON_ATTESTER, Some(data.target.epoch)); + let signing_root = compute_signing_root(data.hash_tree_root(), domain); + let data_dto = AttestationDataOutDto::from(&data); + let attestations: Vec = attesters + .duties + .iter() + .filter(|duty| duty.slot == slot) + .map(|duty| SingleAttestationDto { + committee_index: duty.committee_index, + attester_index: duty.validator_index, + data: data_dto.clone(), + signature: encode_hex(&sign_for(duty.validator_index as usize, signing_root).0), + }) + .collect(); + assert!(!attestations.is_empty(), "someone attests at the head slot"); + + let accepted = client + .submit_attestations(&attestations, "gloas") + .await + .unwrap(); + assert_eq!(accepted, attestations.len()); + { + let published = network.published.lock().unwrap(); + assert_eq!(published.len(), attestations.len()); + for (_, attestation) in published.iter() { + assert_eq!(attestation.data, data); + } + } + + // Aggregation, as the first committee's aggregator would. + let duty = attesters + .duties + .iter() + .find(|duty| duty.slot == slot) + .expect("someone attests at the head slot"); + let aggregate = client + .aggregate_attestation(slot, data.hash_tree_root(), duty.committee_index) + .await + .unwrap(); + assert_eq!(aggregate.fork, ForkName::Gloas); + let AggregateKind::Gloas(attestation) = aggregate.attestation else { + panic!("a gloas slot's aggregate is gloas's container"); + }; + let voters = attestations + .iter() + .filter(|attestation| attestation.committee_index == duty.committee_index) + .count(); + let bits = (0..duty.committee_length as usize) + .filter(|&i| attestation.aggregation_bits.get(i).unwrap_or(false)) + .count(); + assert_eq!(bits, voters); + + let signed = signed_gloas_aggregate(&state, duty.validator_index, attestation); + client + .publish_aggregates( + ForkName::Gloas, + &SignedAggregates::Gloas(vec![signed.clone()]), + ) + .await + .unwrap(); + let recorded = network.aggregates.lock().unwrap(); + assert_eq!(recorded.len(), 1); + assert_eq!(recorded[0], SignedAggregateAndProof::Gloas(signed)); + } + + /// [`super::signed_aggregate`] for gloas, whose `Attestation` roots + /// differently: the proof is signed over the gloas container. + fn signed_gloas_aggregate( + state: &BeaconState, + aggregator: u64, + aggregate: gloas_containers::Attestation, + ) -> gloas_containers::SignedAggregateAndProof { + use ethlambda_types::beacon::constants::{ + DOMAIN_AGGREGATE_AND_PROOF, DOMAIN_SELECTION_PROOF, + }; + let slot = aggregate.data.slot; + let epoch = compute_epoch_at_slot(slot); + let selection_domain = get_domain(state, DOMAIN_SELECTION_PROOF, Some(epoch)); + let selection_proof = sign_for( + aggregator as usize, + compute_signing_root(slot.hash_tree_root(), selection_domain), + ); + let message = gloas_containers::AggregateAndProof { + aggregator_index: aggregator, + aggregate, + selection_proof, + }; + let domain = get_domain(state, DOMAIN_AGGREGATE_AND_PROOF, Some(epoch)); + let signature = sign_for( + aggregator as usize, + compute_signing_root(message.hash_tree_root(), domain), + ); + gloas_containers::SignedAggregateAndProof { message, signature } + } + + /// The committee's duties are the state's, no vote exists for a slot with + /// no block, and the client's own votes (signed with its own key store and + /// fork schedule) are accepted, gossiped and pooled. + #[tokio::test] + async fn the_validator_client_can_serve_on_the_payload_timeliness_committee() { + let Served { + client, + state, + network, + payload_pool, + head_root, + context, + .. + } = serve_gloas(32, None).await; + let slot = state.slot(); + let epoch = compute_epoch_at_slot(slot); + let config = config(); + + // The duties are the first seat each validator holds in the epoch's + // committees. + let mut expected = std::collections::BTreeMap::new(); + for epoch_slot in compute_start_slot_at_epoch(epoch)..compute_start_slot_at_epoch(epoch + 1) + { + for member in get_ptc(&state, epoch_slot, &config).unwrap().iter() { + expected.entry(*member).or_insert(epoch_slot); + } + } + let indices: Vec = (0..COUNT as u64).collect(); + let duties = client.ptc_duties(epoch, &indices).await.unwrap(); + let served: std::collections::BTreeMap = duties + .duties + .iter() + .map(|duty| (duty.validator_index, duty.slot)) + .collect(); + assert_eq!(served, expected); + for duty in &duties.duties { + assert_eq!( + duty.pubkey, + encode_hex(&state.validator(duty.validator_index).unwrap().pubkey.0) + ); + } + // An epoch before gloas has no committee; this schedule has none. + let next = client.ptc_duties(epoch + 1, &indices).await.unwrap(); + assert!( + !next.duties.is_empty(), + "the next epoch's window is readable" + ); + + // No block of the next slot, so no vote to cast; the head's own slot + // has one, naming the head block. + assert!( + client + .payload_attestation_data(slot + 1) + .await + .unwrap() + .is_none() + ); + let data = client + .payload_attestation_data(slot) + .await + .unwrap() + .expect("the head block's slot has data"); + assert_eq!(data.beacon_block_root, head_root); + assert_eq!(data.slot, slot); + assert!(!data.payload_present, "no envelope was seen"); + assert!(data.blob_data_available, "the bid commits to no blobs"); + + // The real service: members of the slot's committee, signing with the + // client's own schedule. + let members: Vec = duties + .duties + .iter() + .filter(|duty| duty.slot == slot) + .map(|duty| duty.validator_index) + .take(3) + .collect(); + assert!( + !members.is_empty(), + "someone sits on the head slot's committee" + ); + let keys = keys_for(&state, &members); + let ptc_duties: Vec = duties + .duties + .iter() + .filter(|duty| members.contains(&duty.validator_index)) + .cloned() + .collect(); + let service = PayloadAttestationService::new(Arc::new(client), context); + let accepted = service.attest(slot, &ptc_duties, &keys).await.unwrap(); + assert_eq!(accepted, members.len()); + + // Recorded for gossip, pooled, and signed so that it verifies under the + // state's own DOMAIN_PTC_ATTESTER. + let recorded = network.payload_attestations.lock().unwrap().clone(); + assert_eq!(recorded.len(), members.len()); + let domain = get_domain(&state, DOMAIN_PTC_ATTESTER, Some(epoch)); + let signing_root = compute_signing_root(data.hash_tree_root(), domain); + for message in &recorded { + assert!(members.contains(&message.validator_index)); + assert_eq!(message.data, data); + let pubkey = state.validator(message.validator_index).unwrap().pubkey; + assert!(bls::verify(&pubkey, signing_root, &message.signature)); + } + let pooled = payload_pool.lock().unwrap().all(Some(slot)); + assert_eq!(pooled.len(), members.len()); + for message in &recorded { + assert!(pooled.contains(message)); + } + } + + /// Insert the first block the node publishes into `store`, with its + /// post-state, as the chain actor would on importing it: nothing in this + /// harness does, and the envelope endpoint waits for the block. + fn import_published_block( + mut store: Store, + network: Arc, + parent: BeaconState, + ) { + tokio::spawn(async move { + loop { + let published = network.blocks.lock().unwrap().first().cloned(); + if let Some(SignedBeaconBlock::Gloas(signed)) = published { + let root = signed.message.hash_tree_root(); + let post = post_state(&parent, &signed.message); + store + .insert_signed_block(root, SignedBeaconBlock::Gloas(signed)) + .unwrap(); + store.insert_state(root, post).unwrap(); + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }); + } + + /// A proposer's slot at gloas, through `ProposalService`: produceBlockV4 + /// with the payload, the block signed and published, the envelope signed + /// and published. What the network was handed must verify against the + /// state, with the proposer's key. + async fn propose_through_the_service(blob: Option>) { + let engine = fake_execution_client_with_blob(blob.clone()).await; + let Served { + client, + state, + network, + store, + context, + .. + } = serve_gloas(33, Some(engine)).await; + let slot = state.slot() + 1; + let epoch = compute_epoch_at_slot(slot); + let advanced = advance_to_slot(&state, slot, &config()).unwrap(); + let proposer = get_beacon_proposer_index(&advanced).unwrap(); + let keys = keys_for(&state, &[proposer]); + let pubkey = state.validator(proposer).unwrap().pubkey; + import_published_block(store, network.clone(), advanced.clone()); + + let service = + ProposalService::new(Arc::new(client), context, Bytes32::repeat_byte(0xab), None); + let duty = ProposerDutyDto { + pubkey: encode_hex(&pubkey.0), + validator_index: proposer, + slot, + }; + service + .propose(slot, &duty, &keys) + .await + .expect("the proposal goes through this node"); + + // The block: gloas, the proposer's, and signed under the state's + // proposer domain. + let blocks = network.blocks.lock().unwrap().clone(); + assert_eq!(blocks.len(), 1); + let SignedBeaconBlock::Gloas(signed_block) = &blocks[0] else { + panic!("a gloas slot publishes a gloas block"); + }; + let block = &signed_block.message; + assert_eq!(block.slot, slot); + assert_eq!(block.proposer_index, proposer); + let block_root = block.hash_tree_root(); + let block_domain = get_domain(&advanced, DOMAIN_BEACON_PROPOSER, Some(epoch)); + assert!(bls::verify( + &pubkey, + compute_signing_root(block_root, block_domain), + &signed_block.signature + )); + + // The envelope: for this block, a self-build, matching the bid it + // reveals, and signed with the proposer's key under the builder domain. + let envelopes = network.envelopes.lock().unwrap().clone(); + assert_eq!(envelopes.len(), 1, "the envelope reached the network"); + let (signed_envelope, sidecars) = &envelopes[0]; + let envelope = &signed_envelope.message; + let bid = &block.body.signed_execution_payload_bid.message; + assert_eq!(bid.builder_index, BUILDER_INDEX_SELF_BUILD); + assert_eq!(envelope.builder_index, BUILDER_INDEX_SELF_BUILD); + assert_eq!(envelope.beacon_block_root, block_root); + assert_eq!(envelope.payload.block_hash, bid.block_hash); + assert_eq!(envelope.payload.parent_hash, bid.parent_block_hash); + assert_eq!(envelope.payload.gas_limit, bid.gas_limit); + assert_eq!(envelope.payload.slot_number, slot); + assert_eq!( + envelope.execution_requests.hash_tree_root(), + bid.execution_requests_root + ); + let builder_domain = get_domain(&advanced, DOMAIN_BEACON_BUILDER, Some(epoch)); + assert!(bls::verify( + &pubkey, + compute_signing_root(envelope.hash_tree_root(), builder_domain), + &signed_envelope.signature + )); + + // With a blob: the bid commits to it and every column goes out with the + // envelope (`Eth-Blob-Data-Included: true` carried the blobs). + match blob { + None => { + assert!(bid.blob_kzg_commitments.is_empty()); + assert!(sidecars.is_empty()); + } + Some(blob) => { + let expected = + ethlambda_state_transition::beacon::kzg::blob_to_kzg_commitment(&blob).unwrap(); + assert_eq!(bid.blob_kzg_commitments.len(), 1); + assert_eq!(bid.blob_kzg_commitments[0], expected); + assert_eq!(sidecars.len(), preset::NUMBER_OF_COLUMNS); + } + } + } + + #[tokio::test] + async fn the_validator_client_can_propose_and_reveal_a_gloas_block() { + propose_through_the_service(None).await; + } + + #[tokio::test] + async fn the_validator_client_can_propose_a_gloas_block_with_a_blob() { + let mut blob = vec![0u8; preset::BYTES_PER_BLOB]; + for (i, element) in blob.chunks_mut(32).enumerate() { + element[31] = (i % 100) as u8; + } + propose_through_the_service(Some(blob)).await; + } +} diff --git a/crates/net/rpc/src/lib.rs b/crates/net/rpc/src/lib.rs index 6c76c73b2..e03ae7862 100644 --- a/crates/net/rpc/src/lib.rs +++ b/crates/net/rpc/src/lib.rs @@ -649,6 +649,51 @@ pub(crate) mod test_utils { (store, root) } + /// A beacon store whose head is `block` (a block at `state`'s slot) with + /// `state` as its post-state, on a clock whose slot `clock_slot` began a + /// second ago. + /// + /// For endpoints that read the head block itself (the payload attestation + /// data names it) or that only accept the current slot (gossip-checked + /// votes), which [`beacon_store_at`]'s phase0 anchor on mainnet's real + /// clock cannot serve. + /// + /// The block is stored under `root`, which the caller names because a test + /// state's `latest_block_header` need not commit to the block built to + /// stand in for it, and a block built on the state names that header's root + /// as its parent. + pub(crate) fn beacon_store_with_head_block( + state: BeaconState, + config: Config, + block: SignedBeaconBlock, + root: H256, + clock_slot: u64, + ) -> Store { + let slot = state.slot(); + let slot_secs = config.slot_duration_ms / 1000; + let now_secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("the clock is after the epoch") + .as_secs(); + let genesis = now_secs - clock_slot * slot_secs - 1; + let mut store = Store::init_beacon( + Arc::new(InMemoryBackend::default()), + genesis, + config, + root, + Checkpoint { root, slot }, + slot, + ); + store + .insert_signed_block(root, block) + .expect("insert head block"); + store.insert_state(root, state).expect("insert head state"); + store + .update_checkpoints(ForkCheckpoints::head_only(root)) + .expect("make the block the head"); + store + } + /// Build a beacon store anchored at `anchor_slot`, with a real child block /// at `anchor_slot + 1` whose import moves the head for real. /// From 8dc16405f1f4f018db1bc98dd5d624fd828303d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:36:21 -0300 Subject: [PATCH 104/132] docs(beacon): the payload attestation pool is listed as aggregates The listing now follows the specification's PayloadAttestation shape; the doc still described the raw messages it returned before. --- docs/rpc.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/rpc.md b/docs/rpc.md index a1d3e76fb..3c335723e 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -245,7 +245,7 @@ surface rather than sitting beside it; a `/lean/v0` path on a beacon node is a | `GET` | `/eth/v1/validator/payload_attestation_data` | JSON or SSZ | What a committee member signs for `slot` (gloas) | | `POST` | `/eth/v2/beacon/pool/attestations` | *(status only)* | Validate and gossip `SingleAttestation`s | | `POST` | `/eth/v1/beacon/pool/payload_attestations` | *(status only)* | Validate, pool and gossip `PayloadAttestationMessage`s (gloas) | -| `GET` | `/eth/v1/beacon/pool/payload_attestations` | JSON | The payload attestation messages the pool holds (gloas) | +| `GET` | `/eth/v1/beacon/pool/payload_attestations` | JSON | The pool's votes as aggregated `PayloadAttestation`s (gloas) | | `POST` | `/eth/v1/validator/beacon_committee_subscriptions` | *(status only)* | Aggregators' entries join their committee's subnet | | `GET` | `/eth/v2/validator/aggregate_attestation` | JSON | The pooled votes for a data root and committee, aggregated | | `POST` | `/eth/v2/validator/aggregate_and_proofs` | *(status only)* | Validate and gossip `SignedAggregateAndProof`s | @@ -464,10 +464,13 @@ subnet publishes through gossipsub fanout. `{outcome}: {reason}` with its position, and the rest still go out; a head-state cache miss is a per-item `ignore: state_unavailable`. - **`GET /eth/v1/beacon/pool/payload_attestations?slot=`** answers `{version: - "gloas", data}` with the messages the pool holds, those of `slot` only when - given. They are the individual messages, not the aggregates the specification - names: the pool keeps them unaggregated so block production can combine the - ones it needs. + "gloas", data}` with the pool's votes as the specification's + `PayloadAttestation`s, those of `slot` only when given: one aggregate per slot + and `PayloadAttestationData`, a bit per seat of that slot's committee + (`get_ptc` on the head state) and the seats' signatures aggregated, the way + block production packs them. The pool itself keeps the individual messages. + JSON only; a vote whose slot falls outside the head state's committee window + is left out. The payload attestation pool is shared with P2P, which fills it from accepted gossip, so a block this node builds carries the votes its peers sent too. From 55adf1b3df3dd6634104f6465ea856cc20b8e12b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 17:17:11 -0300 Subject: [PATCH 105/132] fix(rpc): accept blob-carrying blocks and envelopes over 2 MiB Axum caps request bodies at 2 MiB by default, so POST /eth/v2/beacon/blocks with a fulu SignedBlockContents of 18-21 blobs (131072 bytes each, plus 128 cell proofs of 48 bytes per blob) answered 413. The gloas envelope POST with Eth-Blob-Data-Included carries the whole execution payload plus blobs and would hit the same limit. Raise the limit to MAX_PUBLISH_BODY_BYTES (64 MiB) with a route-level DefaultBodyLimit on exactly those two POST routes, not router-wide. The preset's MAX_BLOB_COMMITMENTS_PER_BLOCK is no practical bound (~550 MB), so the cap is sized from a generous blob count plus a maximal payload and margin. --- crates/net/rpc/src/beacon/gloas_proposal.rs | 19 +++++++++++++++++-- crates/net/rpc/src/beacon/mod.rs | 12 ++++++++++++ crates/net/rpc/src/beacon/proposal.rs | 21 +++++++++++++++++++-- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/crates/net/rpc/src/beacon/gloas_proposal.rs b/crates/net/rpc/src/beacon/gloas_proposal.rs index 951587597..fb9218a1e 100644 --- a/crates/net/rpc/src/beacon/gloas_proposal.rs +++ b/crates/net/rpc/src/beacon/gloas_proposal.rs @@ -33,7 +33,7 @@ use std::{ use axum::{ Extension, Router, body::Bytes, - extract::{Path, Query, State, rejection::QueryRejection}, + extract::{DefaultBodyLimit, Path, Query, State, rejection::QueryRejection}, http::{HeaderMap, HeaderValue, StatusCode, header}, response::{IntoResponse, Response}, routing::{get, post}, @@ -168,7 +168,7 @@ pub(crate) fn routes() -> Router { ) .route( "/eth/v1/beacon/execution_payload_envelopes", - post(post_envelope), + post(post_envelope).layer(DefaultBodyLimit::max(super::MAX_PUBLISH_BODY_BYTES)), ) .layer(Extension(PayloadCache::default())) } @@ -938,6 +938,21 @@ mod tests { .unwrap() } + #[tokio::test] + async fn an_envelope_over_the_default_body_limit_is_not_a_413() { + let (store, _) = gloas_scheduled_store(); + let network: RpcToP2PRef = std::sync::Arc::new(RecordingNetwork::default()); + let app = routes().with_state(store).layer(Extension(network)); + let request = Request::post("/eth/v1/beacon/execution_payload_envelopes") + .header("eth-consensus-version", "gloas") + .header("eth-blob-data-included", "true") + .header(header::CONTENT_TYPE, crate::SSZ_CONTENT_TYPE) + .body(Body::from(vec![0u8; 3 * 1024 * 1024])) + .unwrap(); + let (status, _) = respond(app, request).await; + assert_ne!(status, StatusCode::PAYLOAD_TOO_LARGE); + } + const EMPTY_CONFIG: &str = r#"{"min_bid":"0","builder_boost_factor":"0","builders":[]}"#; #[tokio::test] diff --git a/crates/net/rpc/src/beacon/mod.rs b/crates/net/rpc/src/beacon/mod.rs index 2da230150..3b94fbb1c 100644 --- a/crates/net/rpc/src/beacon/mod.rs +++ b/crates/net/rpc/src/beacon/mod.rs @@ -30,6 +30,18 @@ pub(crate) mod validator; #[cfg(test)] mod validator_client_tests; +/// Request-body cap for the routes that accept blocks, envelopes and blobs. +/// +/// Axum's default is 2 MiB, which a fulu `SignedBlockContents` outgrows at +/// about 14 blobs (each blob is 131072 bytes, plus 128 cell proofs of 48 bytes, +/// 6144 bytes, per blob). The preset's `MAX_BLOB_COMMITMENTS_PER_BLOCK` (4096, +/// roughly 550 MB) is no practical bound, so this is sized from what the blob +/// schedule allows in practice instead: a generous 128 blobs and their cell +/// proofs come to about 17.5 MiB, a maximal execution payload (the gloas +/// envelope carries it whole) takes up to roughly 16 MiB more, and the rest is +/// margin for the block and its operations. Applied per route, never router-wide. +pub(crate) const MAX_PUBLISH_BODY_BYTES: usize = 64 * 1024 * 1024; + /// The wrapper every fork-versioned Beacon API payload travels in. /// /// The fork travels here rather than inside `data` because the containers are diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs index eedba5fe2..52be05c62 100644 --- a/crates/net/rpc/src/beacon/proposal.rs +++ b/crates/net/rpc/src/beacon/proposal.rs @@ -15,7 +15,7 @@ use axum::{ Extension, Router, body::Bytes, - extract::{Path, Query, State}, + extract::{DefaultBodyLimit, Path, Query, State}, http::{HeaderMap, HeaderValue, StatusCode, header}, response::{IntoResponse, Response}, routing::{get, post}, @@ -89,7 +89,10 @@ pub(crate) struct FuluSignedBlockContents { pub(crate) fn routes() -> Router { Router::new() .route("/eth/v3/validator/blocks/{slot}", get(get_block)) - .route("/eth/v2/beacon/blocks", post(post_block)) + .route( + "/eth/v2/beacon/blocks", + post(post_block).layer(DefaultBodyLimit::max(super::MAX_PUBLISH_BODY_BYTES)), + ) } /// `POST /eth/v2/beacon/blocks`, SSZ-encoded `SignedBlockContents`. @@ -576,6 +579,20 @@ mod tests { assert_eq!(json["message"], "blocks are accepted for fulu slots only"); } + #[tokio::test] + async fn a_block_over_the_default_body_limit_is_not_a_413() { + let (store, _) = gloas_scheduled_store(); + let network: RpcToP2PRef = Arc::new(RecordingNetwork::default()); + let app = routes().with_state(store).layer(Extension(network)); + let request = Request::post("/eth/v2/beacon/blocks") + .header("eth-consensus-version", "fulu") + .header(header::CONTENT_TYPE, crate::SSZ_CONTENT_TYPE) + .body(Body::from(vec![0u8; 3 * 1024 * 1024])) + .unwrap(); + let (status, _) = respond(app, request).await; + assert_ne!(status, StatusCode::PAYLOAD_TOO_LARGE); + } + #[tokio::test] async fn a_signed_gloas_block_is_gossiped_and_a_forged_one_is_refused() { use ethlambda_state_transition::beacon::gloas_block_production::test_support::{ From 32ceea0afe0b583b6ec3fcbfeeae6c4384d9ad39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 19:40:39 -0300 Subject: [PATCH 106/132] feat(rpc): serve the single-validator and fork_schedule endpoints Foreign validator clients refuse or stall on a node that 404s these: - GET /eth/v1/beacon/states/{state_id}/validators/{validator_id}: lighthouse resolves every key to an index this way; without it all its validators stay inactive and no proposal is made. Same entry shape, state-id handling and flags as the list form; an unknown validator is a 404. - GET /eth/v1/config/fork_schedule: nimbus reads it every epoch and marks a node it cannot decode as incompatible. Phase0 first and its own predecessor, then each scheduled fork, so the list is the linked list nimbus validates. A test pins every key nimbus's spec comparison reads. --- crates/net/rpc/src/beacon/config.rs | 200 +++++++++++++++++++++++++++- crates/net/rpc/src/beacon/states.rs | 81 +++++++++++ 2 files changed, 280 insertions(+), 1 deletion(-) diff --git a/crates/net/rpc/src/beacon/config.rs b/crates/net/rpc/src/beacon/config.rs index 10e2279f8..f3c06db42 100644 --- a/crates/net/rpc/src/beacon/config.rs +++ b/crates/net/rpc/src/beacon/config.rs @@ -33,7 +33,55 @@ use ethlambda_types::beacon::{config::Config, constants, preset, serde_helpers:: use serde_json::{Map, Value}; pub(crate) fn routes() -> Router { - Router::new().route("/eth/v1/config/spec", get(get_spec)) + Router::new() + .route("/eth/v1/config/spec", get(get_spec)) + .route("/eth/v1/config/fork_schedule", get(get_fork_schedule)) +} + +/// `GET /eth/v1/config/fork_schedule`: every fork of the node's `Config` +/// that is scheduled, oldest first. +/// +/// Validator clients compare this against their own schedule at startup (nimbus +/// refuses a node that cannot answer it). +async fn get_fork_schedule(State(store): State) -> Response { + crate::json_response(serde_json::json!({ "data": fork_schedule(store.config().as_ref()) })) +} + +/// The `Fork` objects (`previous_version`, `current_version`, `epoch`) for +/// phase0 and every later fork whose epoch is not `FAR_FUTURE_EPOCH`. +/// +/// Phase0 is its own predecessor, as in the spec's genesis `Fork`. An +/// unscheduled fork is skipped without breaking the chain: the next scheduled +/// fork's `previous_version` is the last *scheduled* one's version. +fn fork_schedule(config: &Config) -> Vec { + let forks = [ + (config.altair_fork_version, config.altair_fork_epoch), + (config.bellatrix_fork_version, config.bellatrix_fork_epoch), + (config.capella_fork_version, config.capella_fork_epoch), + (config.deneb_fork_version, config.deneb_fork_epoch), + (config.electra_fork_version, config.electra_fork_epoch), + (config.fulu_fork_version, config.fulu_fork_epoch), + (config.gloas_fork_version, config.gloas_fork_epoch), + ]; + let fork = |previous: [u8; 4], current: [u8; 4], epoch: u64| { + serde_json::json!({ + "previous_version": hex_string(previous), + "current_version": hex_string(current), + "epoch": epoch.to_string(), + }) + }; + + let genesis = config.genesis_fork_version; + let mut schedule = vec![fork(genesis, genesis, 0)]; + let mut previous = genesis; + for (version, epoch) in forks { + if epoch == constants::FAR_FUTURE_EPOCH { + continue; + } + schedule.push(fork(previous, version, epoch)); + previous = version; + } + schedule } async fn get_spec(State(store): State) -> Response { @@ -287,6 +335,96 @@ mod tests { ); } + /// What nimbus's validator client (`checkConfig` and + /// `getConsensusForkConfig`, v26.10.0) reads off the spec before it will + /// use a node: a missing or differing key marks the node incompatible. + #[tokio::test] + async fn the_spec_carries_every_key_nimbus_checks() { + let json = get_spec_json().await; + let data = &json["data"]; + + let checked = [ + ( + "MAX_VALIDATORS_PER_COMMITTEE", + preset::MAX_VALIDATORS_PER_COMMITTEE.to_string(), + ), + ("SLOTS_PER_EPOCH", preset::SLOTS_PER_EPOCH.to_string()), + ( + "EPOCHS_PER_ETH1_VOTING_PERIOD", + preset::EPOCHS_PER_ETH1_VOTING_PERIOD.to_string(), + ), + ( + "SLOTS_PER_HISTORICAL_ROOT", + preset::SLOTS_PER_HISTORICAL_ROOT.to_string(), + ), + ( + "EPOCHS_PER_HISTORICAL_VECTOR", + preset::EPOCHS_PER_HISTORICAL_VECTOR.to_string(), + ), + ( + "EPOCHS_PER_SLASHINGS_VECTOR", + preset::EPOCHS_PER_SLASHINGS_VECTOR.to_string(), + ), + ( + "HISTORICAL_ROOTS_LIMIT", + preset::HISTORICAL_ROOTS_LIMIT.to_string(), + ), + ( + "VALIDATOR_REGISTRY_LIMIT", + preset::VALIDATOR_REGISTRY_LIMIT.to_string(), + ), + ( + "MAX_PROPOSER_SLASHINGS", + preset::MAX_PROPOSER_SLASHINGS.to_string(), + ), + ( + "MAX_ATTESTER_SLASHINGS", + preset::MAX_ATTESTER_SLASHINGS.to_string(), + ), + ("MAX_ATTESTATIONS", preset::MAX_ATTESTATIONS.to_string()), + ("MAX_DEPOSITS", preset::MAX_DEPOSITS.to_string()), + ( + "MAX_VOLUNTARY_EXITS", + preset::MAX_VOLUNTARY_EXITS.to_string(), + ), + ]; + for (key, expected) in checked { + assert_eq!(data[key], expected, "{key}"); + } + for domain in [ + "DOMAIN_BEACON_PROPOSER", + "DOMAIN_BEACON_ATTESTER", + "DOMAIN_RANDAO", + "DOMAIN_DEPOSIT", + "DOMAIN_VOLUNTARY_EXIT", + "DOMAIN_SELECTION_PROOF", + "DOMAIN_AGGREGATE_AND_PROOF", + ] { + assert!(data[domain].as_str().unwrap().starts_with("0x"), "{domain}"); + } + // One of the two keys that fix the slot time (it compares the one + // present with its own and refuses a node that has neither, unless + // it too runs the default). + assert!(data["SECONDS_PER_SLOT"].is_string()); + // Each fork's version and epoch, altair's scheduled. + for fork in [ + "ALTAIR", + "BELLATRIX", + "CAPELLA", + "DENEB", + "ELECTRA", + "FULU", + "GLOAS", + ] { + assert!(data[format!("{fork}_FORK_VERSION")].is_string(), "{fork}"); + assert!(data[format!("{fork}_FORK_EPOCH")].is_string(), "{fork}"); + } + assert_ne!( + data["ALTAIR_FORK_EPOCH"], + constants::FAR_FUTURE_EPOCH.to_string() + ); + } + #[tokio::test] async fn the_spec_carries_gloas_preset_and_constant_keys() { let json = get_spec_json().await; @@ -320,6 +458,66 @@ mod tests { assert!(data["MIN_BUILDER_WITHDRAWABILITY_DELAY"].is_string()); } + #[tokio::test] + async fn the_fork_schedule_lists_scheduled_forks_with_quoted_epochs() { + let fixture = beacon_fixture(64); + let config = fixture.store.config(); + let app = routes().with_state(fixture.store); + let response = app + .oneshot( + Request::builder() + .uri("/eth/v1/config/fork_schedule") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = response.into_body().collect().await.unwrap().to_bytes(); + let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let data = json["data"].as_array().unwrap(); + + // Phase0 comes first and is its own predecessor. + assert_eq!(data[0]["epoch"], "0"); + assert_eq!(data[0]["previous_version"], data[0]["current_version"]); + assert_eq!( + data[0]["current_version"], + hex_string(config.genesis_fork_version) + ); + // Each fork's predecessor is the previous entry's version, and epochs + // never go down. + let epoch = |v: &serde_json::Value| v["epoch"].as_str().unwrap().parse::().unwrap(); + for pair in data.windows(2) { + assert_eq!(pair[1]["previous_version"], pair[0]["current_version"]); + assert!(epoch(&pair[1]) >= epoch(&pair[0])); + } + for fork in data { + assert_ne!(fork["epoch"], constants::FAR_FUTURE_EPOCH.to_string()); + } + } + + #[test] + fn an_unscheduled_fork_is_left_out_of_the_schedule() { + let mut config = Config::mainnet(); + config.gloas_fork_epoch = constants::FAR_FUTURE_EPOCH; + let schedule = fork_schedule(&config); + let last = schedule.last().unwrap(); + assert_eq!( + last["current_version"], + hex_string(config.fulu_fork_version) + ); + + config.fulu_fork_epoch = constants::FAR_FUTURE_EPOCH; + config.gloas_fork_epoch = 100; + let schedule = fork_schedule(&config); + let last = schedule.last().unwrap(); + // Gloas follows the last scheduled fork, electra. + assert_eq!( + last["previous_version"], + hex_string(config.electra_fork_version) + ); + } + #[test] fn no_key_is_reported_twice() { let config = Config::mainnet(); diff --git a/crates/net/rpc/src/beacon/states.rs b/crates/net/rpc/src/beacon/states.rs index 2f83b1d1b..5d307019a 100644 --- a/crates/net/rpc/src/beacon/states.rs +++ b/crates/net/rpc/src/beacon/states.rs @@ -45,6 +45,10 @@ pub(crate) fn routes() -> Router { "/eth/v1/beacon/states/{state_id}/validators", get(get_validators).post(post_validators), ) + .route( + "/eth/v1/beacon/states/{state_id}/validators/{validator_id}", + get(get_validator), + ) } /// Resolve a `state_id` to the block root its state is stored under. @@ -258,6 +262,48 @@ async fn get_validators( validators_response(&store, &state_id, request) } +/// `GET .../validators/{validator_id}`: one registry entry, by index or public +/// key, in the shape of a list entry. Lighthouse's validator client resolves +/// each of its keys to an index this way. An id naming no validator is a 404 +/// here, unlike the list form, which omits it. +async fn get_validator( + Path((state_id, validator_id)): Path<(String, String)>, + State(store): State, +) -> Response { + let id = match ValidatorId::parse(&validator_id) { + Ok(id) => id, + Err(err) => return err.into_response(), + }; + let (root, state) = match load(&store, &state_id) { + Ok(found) => found, + Err(err) => return err.into_response(), + }; + + let found = match id { + ValidatorId::Index(index) => state + .validator(index) + .ok() + .zip(state.balance(index).ok()) + .map(|found| (index, found)), + ValidatorId::Pubkey(pubkey) => state + .iter_validators() + .zip(state.iter_balances()) + .enumerate() + .find(|(_, (validator, _))| validator.pubkey == pubkey) + .map(|(index, found)| (index as ValidatorIndex, found)), + }; + let Some((index, (validator, balance))) = found else { + return ApiError::NotFound("validator not found").into_response(); + }; + + let status = ValidatorStatus::of(validator, balance, compute_epoch_at_slot(state.slot())); + crate::json_response(serde_json::json!({ + "execution_optimistic": crate::shared::optimistic::block_is_optimistic(&store, root), + "finalized": is_finalized(&store, state.slot()), + "data": ValidatorEntry { index, balance, status: status.name(), validator }, + })) +} + /// `POST .../validators`, the form a validator client uses: a long list of /// public keys does not fit in a query string. async fn post_validators( @@ -528,6 +574,41 @@ mod tests { assert_eq!(indices, ["1", "3", "4"]); } + async fn get_one(id: &str) -> axum::response::Response { + let (app, _) = app(); + let request = Request::get(format!("/eth/v1/beacon/states/head/validators/{id}")) + .body(Body::empty()) + .unwrap(); + app.oneshot(request).await.unwrap() + } + + /// What lighthouse's validator client sends for each of its keys. + #[tokio::test] + async fn one_validator_resolves_by_pubkey_or_index() { + let (_, state) = app(); + for id in [pubkey_hex(&state, 3), "3".to_owned()] { + let response = get_one(&id).await; + assert_eq!(response.status(), StatusCode::OK); + let json = body_json(response).await; + assert_eq!(json["execution_optimistic"], false); + assert!(json["finalized"].is_boolean()); + // One object, not a list, in the shape of a list entry. + assert_eq!(json["data"]["index"], "3"); + assert_eq!(json["data"]["status"], "active_ongoing"); + assert!(json["data"]["balance"].is_string()); + assert_eq!(json["data"]["validator"]["pubkey"], pubkey_hex(&state, 3)); + } + } + + #[tokio::test] + async fn an_unknown_single_validator_is_a_404_and_a_malformed_id_a_400() { + assert_eq!(get_one("999").await.status(), StatusCode::NOT_FOUND); + let unknown = format!("0x{}", "ab".repeat(48)); + assert_eq!(get_one(&unknown).await.status(), StatusCode::NOT_FOUND); + assert_eq!(get_one("0x1234").await.status(), StatusCode::BAD_REQUEST); + assert_eq!(get_one("nope").await.status(), StatusCode::BAD_REQUEST); + } + #[test] fn status_follows_the_lifecycle() { let epoch = 10; From 573b7bbcea32ac434eedb755b102b91bf99aba8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 19:40:43 -0300 Subject: [PATCH 107/132] fix(rpc): accept SSZ bodies on attestation, aggregate and payload-vote submissions Prysm posts aggregate_and_proofs and payload attestations as application/octet-stream and only retries as JSON on a 415; our JSON-only decoder answered 400 "invalid request body" and its aggregates were never published. Nimbus submits attestations as SSZ too. The three batch endpoints now take either encoding, picked by Content-Type (none still means JSON), through one decode_list helper. SSZ is the specification's List[...] of the same items. Any other content type is a 415, the answer prysm's fallback keys on, and a body that does not decode is a 400. --- crates/net/rpc/src/beacon/mod.rs | 33 ++++++ crates/net/rpc/src/beacon/pool.rs | 188 ++++++++++++++++++++++++++---- crates/net/rpc/src/beacon/ptc.rs | 47 +++++++- 3 files changed, 245 insertions(+), 23 deletions(-) diff --git a/crates/net/rpc/src/beacon/mod.rs b/crates/net/rpc/src/beacon/mod.rs index 3b94fbb1c..2a44821b0 100644 --- a/crates/net/rpc/src/beacon/mod.rs +++ b/crates/net/rpc/src/beacon/mod.rs @@ -72,6 +72,9 @@ pub(crate) enum ApiError { /// A request this node cannot answer right now, typically because its /// execution client did not (the Beacon API's 503). ServiceUnavailable(&'static str), + /// A request body in an encoding this endpoint does not take (415). A + /// client that posted SSZ and gets this one falls back to JSON. + UnsupportedMediaType(&'static str), } impl From for ApiError { @@ -90,6 +93,7 @@ impl IntoResponse for ApiError { ApiError::NotFound(m) => (StatusCode::NOT_FOUND, m), ApiError::Internal(m) => (StatusCode::INTERNAL_SERVER_ERROR, m), ApiError::ServiceUnavailable(m) => (StatusCode::SERVICE_UNAVAILABLE, m), + ApiError::UnsupportedMediaType(m) => (StatusCode::UNSUPPORTED_MEDIA_TYPE, m), }; let body = serde_json::json!({ "code": status.as_u16(), "message": message }); let mut response = crate::json_response(body); @@ -98,6 +102,35 @@ impl IntoResponse for ApiError { } } +/// Decode the array a batch-submission endpoint takes, as JSON or as the SSZ +/// `List[T, ...]` of the same elements, by the request's `Content-Type`. +/// +/// An absent `Content-Type` is read as JSON, which is what every client that +/// predates SSZ submission sends. Anything else is a 415 rather than a guess, +/// so a client that tries SSZ first (prysm) learns this node wants the other. +pub(crate) fn decode_list( + headers: &axum::http::HeaderMap, + body: &[u8], +) -> Result, ApiError> +where + T: serde::de::DeserializeOwned + libssz::SszDecode, +{ + let content_type = headers + .get(axum::http::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(|value| value.split(';').next().unwrap_or("").trim()); + match content_type { + None | Some("application/json") => { + serde_json::from_slice(body).map_err(|_| ApiError::BadRequest("invalid request body")) + } + Some(crate::SSZ_CONTENT_TYPE) => as libssz::SszDecode>::from_ssz_bytes(body) + .map_err(|_| ApiError::BadRequest("invalid request body")), + Some(_) => Err(ApiError::UnsupportedMediaType( + "Content-Type must be application/json or application/octet-stream", + )), + } +} + /// Every route this surface serves. /// /// Deliberately not a superset of [`crate::build_api_router`]: the `/lean/v0` diff --git a/crates/net/rpc/src/beacon/pool.rs b/crates/net/rpc/src/beacon/pool.rs index 88ad1bab5..03bd1429e 100644 --- a/crates/net/rpc/src/beacon/pool.rs +++ b/crates/net/rpc/src/beacon/pool.rs @@ -51,7 +51,7 @@ use ethlambda_types::{ use serde::{Deserialize, Serialize}; use tracing::{debug, warn}; -use crate::beacon::{ApiError, validator::head}; +use crate::beacon::{ApiError, decode_list, validator::head}; pub(crate) fn routes() -> Router { Router::new() @@ -96,8 +96,9 @@ async fn post_pool_attestations( Ok(fork) => fork, Err(err) => return err.into_response(), }; - let Ok(attestations) = serde_json::from_slice::>(&body) else { - return ApiError::BadRequest("invalid request body").into_response(); + let attestations = match decode_list::(&headers, &body) { + Ok(attestations) => attestations, + Err(err) => return err.into_response(), }; let (_head_root, state) = match head(&store) { Ok(found) => found, @@ -340,23 +341,25 @@ async fn post_aggregate_and_proofs( Err(err) => return err.into_response(), }; // Gloas's aggregate is its own container, so the header picks the decoder. - let aggregates = if header_fork == ForkName::Gloas { - serde_json::from_slice::>(&body).map(|signed| { - signed - .into_iter() - .map(SignedAggregateAndProof::Gloas) - .collect() - }) - } else { - serde_json::from_slice::>(&body).map(|signed| { - signed - .into_iter() - .map(SignedAggregateAndProof::Electra) - .collect() - }) - }; - let Ok(aggregates): Result, _> = aggregates else { - return ApiError::BadRequest("invalid request body").into_response(); + let aggregates: Result, ApiError> = + if header_fork == ForkName::Gloas { + decode_list::(&headers, &body).map(|signed| { + signed + .into_iter() + .map(SignedAggregateAndProof::Gloas) + .collect() + }) + } else { + decode_list::(&headers, &body).map(|signed| { + signed + .into_iter() + .map(SignedAggregateAndProof::Electra) + .collect() + }) + }; + let aggregates = match aggregates { + Ok(aggregates) => aggregates, + Err(err) => return err.into_response(), }; let now_ms = std::time::SystemTime::now() @@ -1090,4 +1093,149 @@ mod tests { assert_eq!(response.status(), StatusCode::BAD_REQUEST); assert!(fixture.network.aggregates.lock().unwrap().is_empty()); } + + /// POST `body` with an explicit content type, as a client that does not + /// send JSON would. + async fn post_raw( + fixture: &Fixture, + uri: &str, + content_type: Option<&str>, + version: &str, + body: Vec, + ) -> (StatusCode, serde_json::Value) { + let network: RpcToP2PRef = fixture.network.clone(); + let app = routes() + .with_state(fixture.store.clone()) + .layer(Extension(network)) + .layer(Extension(fixture.pool.clone())); + let mut request = Request::post(uri).header("eth-consensus-version", version); + if let Some(content_type) = content_type { + request = request.header("content-type", content_type); + } + let response = app + .oneshot(request.body(Body::from(body)).unwrap()) + .await + .unwrap(); + let status = response.status(); + let body = response.into_body().collect().await.unwrap().to_bytes(); + ( + status, + serde_json::from_slice(&body).unwrap_or(serde_json::Value::Null), + ) + } + + /// Nimbus submits its attestations this way. + #[tokio::test] + async fn attestations_are_accepted_as_ssz() { + use libssz::SszEncode as _; + let fixture = fixture(); + let attestation = attestation(&fixture, 0, 0); + let (status, json) = post_raw( + &fixture, + "/eth/v2/beacon/pool/attestations", + Some("application/octet-stream"), + fixture.version, + vec![attestation.clone()].to_ssz(), + ) + .await; + assert_eq!(status, StatusCode::OK, "{json}"); + let published = fixture.network.published.lock().unwrap(); + assert_eq!(published.len(), 1); + assert_eq!(published[0].1, attestation); + } + + #[tokio::test] + async fn an_unsupported_content_type_is_a_415_and_malformed_ssz_a_400() { + let fixture = fixture(); + let uri = "/eth/v2/beacon/pool/attestations"; + let (status, json) = + post_raw(&fixture, uri, Some("text/plain"), fixture.version, vec![]).await; + assert_eq!(status, StatusCode::UNSUPPORTED_MEDIA_TYPE); + assert_eq!(json["code"], 415); + + let (status, _) = post_raw( + &fixture, + uri, + Some("application/octet-stream"), + fixture.version, + vec![1, 2, 3], + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + + // No content type at all is read as JSON, as it always was. + let (status, _) = post_raw(&fixture, uri, None, fixture.version, b"[]".to_vec()).await; + assert_eq!(status, StatusCode::OK); + } + + /// Prysm publishes its aggregates as SSZ and falls back to JSON only on a 415. + #[tokio::test] + async fn aggregates_are_accepted_as_ssz() { + use libssz::SszEncode as _; + let fixture = fixture(); + let slot = fixture.state.slot(); + let committee = get_beacon_committee(&fixture.state, slot, 0).unwrap(); + let votes: Vec = (0..committee.len()) + .map(|position| attestation(&fixture, 0, position)) + .collect(); + submit(&fixture, &votes).await; + let aggregate = fixture + .pool + .lock() + .unwrap() + .aggregate(votes[0].data.hash_tree_root(), slot, 0) + .unwrap(); + // Two entries, so the list's offset table is exercised. + let first = signed_aggregate(&fixture, committee[0], aggregate.clone()); + let second = signed_aggregate(&fixture, committee[1], aggregate); + + let (status, json) = post_raw( + &fixture, + "/eth/v2/validator/aggregate_and_proofs", + Some("application/octet-stream"), + "fulu", + vec![first.clone(), second.clone()].to_ssz(), + ) + .await; + assert_eq!(status, StatusCode::OK, "{json}"); + let published = fixture.network.aggregates.lock().unwrap(); + assert_eq!( + *published, + [ + SignedAggregateAndProof::Electra(first), + SignedAggregateAndProof::Electra(second) + ] + ); + } + + #[tokio::test] + async fn gloas_aggregates_are_accepted_as_ssz() { + use libssz::SszEncode as _; + let fixture = fixture_at(ForkName::Gloas, 0); + let slot = fixture.state.slot(); + let committee = get_beacon_committee(&fixture.state, slot, 0).unwrap(); + let votes: Vec = (0..committee.len()) + .map(|position| attestation(&fixture, 0, position)) + .collect(); + submit(&fixture, &votes).await; + let pooled = fixture + .pool + .lock() + .unwrap() + .aggregate(votes[0].data.hash_tree_root(), slot, 0) + .unwrap(); + let signed = signed_gloas_aggregate(&fixture, committee[0], &pooled); + + let (status, json) = post_raw( + &fixture, + "/eth/v2/validator/aggregate_and_proofs", + Some("application/octet-stream"), + "gloas", + vec![signed.clone()].to_ssz(), + ) + .await; + assert_eq!(status, StatusCode::OK, "{json}"); + let published = fixture.network.aggregates.lock().unwrap(); + assert_eq!(*published, [SignedAggregateAndProof::Gloas(signed)]); + } } diff --git a/crates/net/rpc/src/beacon/ptc.rs b/crates/net/rpc/src/beacon/ptc.rs index e827d3e51..3e57da6bd 100644 --- a/crates/net/rpc/src/beacon/ptc.rs +++ b/crates/net/rpc/src/beacon/ptc.rs @@ -54,7 +54,7 @@ use tracing::{debug, warn}; use crate::{ CustodyColumns, - beacon::{ApiError, validator::head}, + beacon::{ApiError, decode_list, validator::head}, shared::content::{Encoding, ssz_response, with_consensus_version}, }; @@ -377,8 +377,9 @@ async fn post_pool_payload_attestations( if let Err(err) = require_gloas_or_absent(&headers) { return err.into_response(); } - let Ok(messages) = serde_json::from_slice::>(&body) else { - return ApiError::BadRequest("invalid request body").into_response(); + let messages = match decode_list::(&headers, &body) { + Ok(messages) => messages, + Err(err) => return err.into_response(), }; let now_ms = std::time::SystemTime::now() @@ -1084,6 +1085,46 @@ mod tests { assert_eq!(reply.status, StatusCode::BAD_REQUEST); } + /// Lighthouse and prysm submit their payload votes as SSZ. + #[tokio::test] + async fn votes_are_accepted_as_ssz_and_other_content_types_are_a_415() { + use libssz::SszEncode as _; + let state = gloas_state(); + let state_epoch = compute_epoch_at_slot(state.slot()); + let member = window_index(state_epoch, state_epoch, 0); + let (store, root) = store_with_head(state.clone(), gloas_config(), 0); + let message = vote(&state, root, member, true); + + let pool = SharedPayloadAttestationPool::default(); + let network = Arc::new(RecordingNetwork::default()); + let ssz_request = Request::post("/eth/v1/beacon/pool/payload_attestations") + .header("content-type", "application/octet-stream") + .header("eth-consensus-version", "gloas") + .body(Body::from(vec![message.clone()].to_ssz())) + .unwrap(); + let reply = send( + store.clone(), + pool.clone(), + network.clone(), + Vec::new(), + Default::default(), + ssz_request, + ) + .await; + assert_eq!(reply.status, StatusCode::OK); + assert_eq!(*network.payload_attestations.lock().unwrap(), vec![message]); + + let reply = request( + store, + Request::post("/eth/v1/beacon/pool/payload_attestations") + .header("content-type", "text/plain") + .body(Body::from("[]")) + .unwrap(), + ) + .await; + assert_eq!(reply.status, StatusCode::UNSUPPORTED_MEDIA_TYPE); + } + #[tokio::test] async fn the_pool_is_listed_as_aggregates_by_slot() { let state = gloas_state(); From e893d3b69b2b53ba9988f5d6d40affca61ddc57f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 19:40:46 -0300 Subject: [PATCH 108/132] docs(rpc): document the endpoints and SSZ request bodies added for foreign validator clients --- docs/rpc.md | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/docs/rpc.md b/docs/rpc.md index 3c335723e..fd679a613 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -233,22 +233,24 @@ surface rather than sitting beside it; a `/lean/v0` path on a beacon node is a | `GET` | `/eth/v1/beacon/states/{state_id}/finality_checkpoints` | JSON | That state's three checkpoints | | `GET` | `/eth/v1/beacon/genesis` | JSON | Genesis time, validators root, fork version | | `GET` | `/eth/v1/config/spec` | JSON | The store's `Config`, plus `PRESET_BASE`, `CONFIG_NAME`, the preset and the constants (see below) | +| `GET` | `/eth/v1/config/fork_schedule` | JSON | Every scheduled fork as `{previous_version, current_version, epoch}` | | `GET` | `/eth/v1/node/syncing` | JSON | Head slot, sync distance, optimistic flag | | `GET` | `/eth/v1/node/health` | *(status only)* | `200` caught up, `206` syncing | | `GET` | `/eth/v1/node/version` | JSON | Client version string | | `GET` | `/eth/v1/node/identity` | JSON | Peer ID and metadata only (see below) | | `GET`, `POST` | `/eth/v1/beacon/states/{state_id}/validators` | JSON | Registry entries by index or pubkey, with status | +| `GET` | `/eth/v1/beacon/states/{state_id}/validators/{validator_id}` | JSON | One registry entry by index or `0x` pubkey; `404` when unknown | | `GET` | `/eth/v1/validator/duties/proposer/{epoch}` | JSON | Proposers for the head's epoch or the next | | `POST` | `/eth/v1/validator/duties/attester/{epoch}` | JSON | Committee assignments for the given indices | | `POST` | `/eth/v1/validator/duties/ptc/{epoch}` | JSON | Payload timeliness committee seats for the given indices (gloas) | | `GET` | `/eth/v1/validator/attestation_data` | JSON | What to attest to at `slot` | | `GET` | `/eth/v1/validator/payload_attestation_data` | JSON or SSZ | What a committee member signs for `slot` (gloas) | -| `POST` | `/eth/v2/beacon/pool/attestations` | *(status only)* | Validate and gossip `SingleAttestation`s | -| `POST` | `/eth/v1/beacon/pool/payload_attestations` | *(status only)* | Validate, pool and gossip `PayloadAttestationMessage`s (gloas) | +| `POST` | `/eth/v2/beacon/pool/attestations` | *(status only)* | Validate and gossip `SingleAttestation`s (JSON or SSZ body) | +| `POST` | `/eth/v1/beacon/pool/payload_attestations` | *(status only)* | Validate, pool and gossip `PayloadAttestationMessage`s (gloas; JSON or SSZ body) | | `GET` | `/eth/v1/beacon/pool/payload_attestations` | JSON | The pool's votes as aggregated `PayloadAttestation`s (gloas) | | `POST` | `/eth/v1/validator/beacon_committee_subscriptions` | *(status only)* | Aggregators' entries join their committee's subnet | | `GET` | `/eth/v2/validator/aggregate_attestation` | JSON | The pooled votes for a data root and committee, aggregated | -| `POST` | `/eth/v2/validator/aggregate_and_proofs` | *(status only)* | Validate and gossip `SignedAggregateAndProof`s | +| `POST` | `/eth/v2/validator/aggregate_and_proofs` | *(status only)* | Validate and gossip `SignedAggregateAndProof`s (JSON or SSZ body) | | `GET` | `/eth/v3/validator/blocks/{slot}` | SSZ or JSON | An unsigned fulu block built on the head (`produceBlockV3`) | | `POST` | `/eth/v4/validator/blocks/{slot}` | SSZ or JSON | An unsigned self-built gloas block, with its envelope and blobs when asked (`produceBlockV4`) | | `GET` | `/eth/v1/validator/execution_payload_envelopes/{slot}/{beacon_block_root}` | SSZ or JSON | The unsigned envelope `produceBlockV4` built (gloas) | @@ -284,6 +286,12 @@ the chain actor writes, so no request waits on the actor. (`Store::head_payload_status`, or one fresh `get_head_node` walk when none is recorded for this head). A pre-gloas head counts as FULL (see [Spec Deviations](./spec_deviations.md#a-pre-gloas-head-counts-as-full-in-attestation_dataindex)). +- **`states/{state_id}/validators/{validator_id}`** is one entry of the list + endpoint, in the same shape and with the same `state_id` handling, + `execution_optimistic` and `finalized`. The id is an index or a `0x` pubkey; + one naming no validator is a `404` (the list form omits it), a malformed one + a `400`. Lighthouse's validator client resolves each key to its index this + way, and stays inactive without it. - **`pool/attestations`** checks each attestation against the electra `beacon_attestation_{subnet_id}` gossip conditions it can evaluate (clock window, `data.index == 0`, target epoch, the voted block known and the target @@ -319,6 +327,16 @@ the chain actor writes, so no request waits on the actor. nothing about them). What passes is gossiped on the topic and goes into the pool. The `Eth-Consensus-Version` header picks the decoder: `gloas` takes gloas's `SignedAggregateAndProof`, the others electra's. +- **Request body encodings.** `pool/attestations`, `pool/payload_attestations` + and `aggregate_and_proofs` take either JSON (`application/json`, also the + reading of a request with no `Content-Type`) or the SSZ `List[...]` of the + same items (`application/octet-stream`): the offset-table encoding every + variable-size list uses, which is what prysm sends for aggregates and payload + votes, with a JSON retry only on a `415`, and what nimbus can send for + attestations. Any other content type is a `415`, and a body that does not + decode in the type's encoding a `400`. A block, an envelope and the other + validator-client submissions keep their own rules (SSZ only on `blocks` and + the envelope; JSON on the rest). - **The attestation pool** holds, the best-covered per data root and committee: votes from `pool/attestations` and the aggregator subnets, aggregates from `aggregate_and_proofs`, and every electra gossip aggregate @@ -536,6 +554,20 @@ Pre-gloas responses are unchanged. One helper implements both rules (`shared/optimistic.rs`) and every response that carries the flag goes through it. +### `GET /eth/v1/config/fork_schedule` + +The `Fork` objects of the `Config`'s schedule, oldest first, as +`{previous_version, current_version, epoch}` with the epoch quoted. Phase0 is +first, at epoch `0` and its own predecessor; every later fork whose epoch is not +`FAR_FUTURE_EPOCH` follows, and an unscheduled fork is skipped without breaking +the chain, so each entry's `previous_version` is the one before it in the list. +Nimbus's validator client reads this every epoch, requires that linked-list +shape, and marks a node it cannot decode as incompatible. Nimbus also compares +`/eth/v1/config/spec` against its own constants before use: the 13 preset +values, the 7 domain types, `SECONDS_PER_SLOT` (or `SLOT_DURATION_MS`, which +this node does not report) and every fork's `*_FORK_VERSION` and `*_FORK_EPOCH`, +with altair scheduled. All are served (a unit test pins the list). + ### `GET /eth/v1/config/spec` One flat object holding the network's configuration, the compiled preset, and From c6a1a915837e099da0c7b51bedb8e0965ee9969c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 19:58:04 -0300 Subject: [PATCH 109/132] feat(types): deserialize the block, envelope and request containers from Beacon API JSON The Beacon API's block and envelope request bodies are JSON as well as SSZ, and several validator clients post only JSON (teku's envelopes, nimbus's blocks, lodestar's everything). Every container a fulu or gloas signed block or execution payload envelope holds now derives Deserialize, mirroring each Serialize adapter: seq, quoted_u64_seq, ssz_hex and ssz_hex_seq gain the inverse (collected through the collection's TryFrom>, so a list bound is refused rather than truncated), and U256, KzgCommitment and KzgProof read the way they are written. Round-trip tests cover a populated fulu block, gloas block and gloas envelope. --- .../types/src/beacon/containers/altair.rs | 51 ++- .../types/src/beacon/containers/bellatrix.rs | 44 +- .../types/src/beacon/containers/capella.rs | 86 ++-- .../types/src/beacon/containers/deneb.rs | 70 +-- .../types/src/beacon/containers/electra.rs | 163 +++++-- .../types/src/beacon/containers/fulu.rs | 62 +-- .../types/src/beacon/containers/gloas.rs | 274 +++++++++--- .../types/src/beacon/containers/json_tests.rs | 405 ++++++++++++++++++ .../common/types/src/beacon/containers/mod.rs | 3 + .../types/src/beacon/containers/phase0.rs | 38 +- .../types/src/beacon/containers/shared.rs | 96 ++++- crates/common/types/src/beacon/primitives.rs | 19 +- .../common/types/src/beacon/serde_helpers.rs | 59 +++ 13 files changed, 1102 insertions(+), 268 deletions(-) create mode 100644 crates/common/types/src/beacon/containers/json_tests.rs diff --git a/crates/common/types/src/beacon/containers/altair.rs b/crates/common/types/src/beacon/containers/altair.rs index d968c1133..11da84ce8 100644 --- a/crates/common/types/src/beacon/containers/altair.rs +++ b/crates/common/types/src/beacon/containers/altair.rs @@ -70,10 +70,19 @@ pub type SyncSubcommitteeBits = /// needs one regardless of how many attestations or other operations it /// includes. #[derive( - Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] pub struct SyncAggregate { - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub sync_committee_bits: SyncCommitteeBits, /// The aggregate of every signature from a member set in /// `sync_committee_bits`, over the previous slot's block root. @@ -85,7 +94,7 @@ pub struct SyncAggregate { /// participates. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] pub struct SyncCommittee { - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub pubkeys: SyncCommitteePubkeys, /// The aggregate of every key in `pubkeys`, so `process_sync_aggregate` does /// not have to re-aggregate from scratch when the whole committee signs. @@ -109,15 +118,15 @@ pub struct BeaconBlockBody { pub eth1_data: Eth1Data, /// Arbitrary proposer-chosen bytes, which consensus never reads. pub graffiti: Bytes32, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub proposer_slashings: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub attester_slashings: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub attestations: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub deposits: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub voluntary_exits: SszList, /// The aggregated sync committee signature over the previous slot's block /// root, plus which members contributed. This is what lets a light client @@ -180,16 +189,16 @@ pub struct BeaconState { /// slot advances, since a block cannot commit to the root of the state /// containing it. pub latest_block_header: BeaconBlockHeader, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub block_roots: BlockRoots, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub state_roots: StateRoots, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub historical_roots: HistoricalRoots, // -- Eth1 -- pub eth1_data: Eth1Data, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub eth1_data_votes: Eth1DataVotes, /// How many deposits from the contract have been processed, which is where /// the next one will be read from. @@ -197,17 +206,17 @@ pub struct BeaconState { pub eth1_deposit_index: u64, // -- Registry -- - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub validators: Validators, - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub balances: Balances, // -- Randomness -- - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub randao_mixes: RandaoMixes, // -- Slashings -- - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub slashings: Slashings, // -- Participation -- @@ -217,15 +226,15 @@ pub struct BeaconState { /// it is processed instead of deferring to the epoch boundary, so there is /// no longer a backlog of whole attestations to keep around, only a flag /// per validator per epoch. - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub previous_epoch_participation: EpochParticipation, /// Flags for the current epoch, which become `previous_epoch_participation` /// at the next epoch boundary. - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub current_epoch_participation: EpochParticipation, // -- Finality -- - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub justification_bits: JustificationBits, pub previous_justified_checkpoint: Checkpoint, pub current_justified_checkpoint: Checkpoint, @@ -237,7 +246,7 @@ pub struct BeaconState { /// non-finalizing epoch and falls otherwise, which is what lets the /// inactivity leak single out validators who are actually offline rather /// than penalizing everyone during a stall. - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub inactivity_scores: InactivityScores, // -- Sync committees -- @@ -289,7 +298,7 @@ pub struct SyncCommitteeContribution { pub beacon_block_root: Root, #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub subcommittee_index: u64, - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub aggregation_bits: SyncSubcommitteeBits, /// The aggregate signature of every member set in `aggregation_bits`, over /// `beacon_block_root`. diff --git a/crates/common/types/src/beacon/containers/bellatrix.rs b/crates/common/types/src/beacon/containers/bellatrix.rs index 80993a8b7..daafe8ca6 100644 --- a/crates/common/types/src/beacon/containers/bellatrix.rs +++ b/crates/common/types/src/beacon/containers/bellatrix.rs @@ -91,7 +91,7 @@ pub struct ExecutionPayload { /// own `state_root`. pub state_root: Bytes32, pub receipts_root: Bytes32, - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub logs_bloom: LogsBloom, /// The randomness the beacon chain exposes to the EVM for this block; /// `difficulty` in the yellow paper before the merge repurposed the @@ -105,7 +105,7 @@ pub struct ExecutionPayload { pub gas_used: u64, #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub timestamp: u64, - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub extra_data: ExtraData, /// This block's EIP-1559 base fee, a `uint256` because the execution /// layer's fee market is not bounded to fit a `uint64`. @@ -113,7 +113,7 @@ pub struct ExecutionPayload { /// This payload's own hash, which the next payload's `parent_hash` must /// equal. pub block_hash: ExecutionBlockHash, - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex_seq")] pub transactions: Transactions, } @@ -127,7 +127,7 @@ pub struct ExecutionPayloadHeader { pub fee_recipient: ExecutionAddress, pub state_root: Bytes32, pub receipts_root: Bytes32, - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub logs_bloom: LogsBloom, pub prev_randao: Bytes32, #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] @@ -138,7 +138,7 @@ pub struct ExecutionPayloadHeader { pub gas_used: u64, #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub timestamp: u64, - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub extra_data: ExtraData, pub base_fee_per_gas: Uint256, /// The hash of the execution block this header summarizes. @@ -177,15 +177,15 @@ pub struct BeaconBlockBody { pub eth1_data: Eth1Data, /// Arbitrary proposer-chosen bytes, which consensus never reads. pub graffiti: Bytes32, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub proposer_slashings: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub attester_slashings: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub attestations: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub deposits: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub voluntary_exits: SszList, /// The aggregated sync committee signature over the previous slot's block /// root, plus which members contributed. @@ -292,16 +292,16 @@ pub struct BeaconState { /// slot advances, since a block cannot commit to the root of the state /// containing it. pub latest_block_header: BeaconBlockHeader, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub block_roots: BlockRoots, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub state_roots: StateRoots, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub historical_roots: HistoricalRoots, // -- Eth1 -- pub eth1_data: Eth1Data, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub eth1_data_votes: Eth1DataVotes, /// How many deposits from the contract have been processed, which is where /// the next one will be read from. @@ -309,31 +309,31 @@ pub struct BeaconState { pub eth1_deposit_index: u64, // -- Registry -- - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub validators: Validators, - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub balances: Balances, // -- Randomness -- - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub randao_mixes: RandaoMixes, // -- Slashings -- - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub slashings: Slashings, // -- Participation -- /// Per-validator participation flags for the previous epoch, positionally /// parallel to `validators`. - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub previous_epoch_participation: EpochParticipation, /// Flags for the current epoch, which become `previous_epoch_participation` /// at the next epoch boundary. - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub current_epoch_participation: EpochParticipation, // -- Finality -- - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub justification_bits: JustificationBits, pub previous_justified_checkpoint: Checkpoint, pub current_justified_checkpoint: Checkpoint, @@ -341,7 +341,7 @@ pub struct BeaconState { // -- Inactivity -- /// Per-validator inactivity score, positionally parallel to `validators`. - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub inactivity_scores: InactivityScores, // -- Sync committees -- diff --git a/crates/common/types/src/beacon/containers/capella.rs b/crates/common/types/src/beacon/containers/capella.rs index 0ec2e397d..c536b1be6 100644 --- a/crates/common/types/src/beacon/containers/capella.rs +++ b/crates/common/types/src/beacon/containers/capella.rs @@ -70,7 +70,17 @@ pub type Withdrawals = SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub attester_slashings: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub attestations: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub deposits: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub voluntary_exits: SszList, /// The aggregated sync committee signature over the previous slot's block /// root, plus which members contributed. @@ -234,7 +264,7 @@ pub struct BeaconBlockBody { /// holds here, so it never affects this same slot's withdrawal sweep: that /// sweep already ran, against whatever credentials were in effect before /// this block. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub bls_to_execution_changes: SszList, } @@ -336,18 +366,18 @@ pub struct BeaconState { /// slot advances, since a block cannot commit to the root of the state /// containing it. pub latest_block_header: BeaconBlockHeader, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub block_roots: BlockRoots, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub state_roots: StateRoots, /// Frozen as of this fork: no longer appended to. See /// [`Self::historical_summaries`]. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub historical_roots: HistoricalRoots, // -- Eth1 -- pub eth1_data: Eth1Data, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub eth1_data_votes: Eth1DataVotes, /// How many deposits from the contract have been processed, which is where /// the next one will be read from. @@ -355,31 +385,31 @@ pub struct BeaconState { pub eth1_deposit_index: u64, // -- Registry -- - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub validators: Validators, - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub balances: Balances, // -- Randomness -- - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub randao_mixes: RandaoMixes, // -- Slashings -- - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub slashings: Slashings, // -- Participation -- /// Per-validator participation flags for the previous epoch, positionally /// parallel to `validators`. - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub previous_epoch_participation: EpochParticipation, /// Flags for the current epoch, which become `previous_epoch_participation` /// at the next epoch boundary. - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub current_epoch_participation: EpochParticipation, // -- Finality -- - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub justification_bits: JustificationBits, pub previous_justified_checkpoint: Checkpoint, pub current_justified_checkpoint: Checkpoint, @@ -387,7 +417,7 @@ pub struct BeaconState { // -- Inactivity -- /// Per-validator inactivity score, positionally parallel to `validators`. - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub inactivity_scores: InactivityScores, // -- Sync committees -- @@ -418,7 +448,7 @@ pub struct BeaconState { /// root period, replacing the growth of `historical_roots` as of this /// fork. See [`super::shared::HistoricalSummary`] for why the two forms /// are `hash_tree_root`-compatible. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub historical_summaries: HistoricalSummaries, } diff --git a/crates/common/types/src/beacon/containers/deneb.rs b/crates/common/types/src/beacon/containers/deneb.rs index 3092844f8..195dbeb20 100644 --- a/crates/common/types/src/beacon/containers/deneb.rs +++ b/crates/common/types/src/beacon/containers/deneb.rs @@ -86,13 +86,23 @@ pub type VersionedHashes = SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub attester_slashings: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub attestations: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub deposits: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub voluntary_exits: SszList, /// The aggregated sync committee signature over the previous slot's block /// root, plus which members contributed. @@ -201,14 +211,14 @@ pub struct BeaconBlockBody { /// each [`SignedBLSToExecutionChange`] wraps a /// [`super::capella::BLSToExecutionChange`] with a signature proving its /// holder controls the credential being changed. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub bls_to_execution_changes: SszList, /// One KZG commitment per blob this block's proposer chose to include. /// Never the blobs themselves: those are propagated separately as /// [`BlobSidecar`]s, which is what keeps a block's own size independent /// of how much blob data it references. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub blob_kzg_commitments: KzgCommitments, } @@ -307,16 +317,16 @@ pub struct BeaconState { /// slot advances, since a block cannot commit to the root of the state /// containing it. pub latest_block_header: BeaconBlockHeader, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub block_roots: BlockRoots, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub state_roots: StateRoots, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub historical_roots: HistoricalRoots, // -- Eth1 -- pub eth1_data: Eth1Data, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub eth1_data_votes: Eth1DataVotes, /// How many deposits from the contract have been processed, which is /// where the next one will be read from. @@ -324,31 +334,31 @@ pub struct BeaconState { pub eth1_deposit_index: u64, // -- Registry -- - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub validators: Validators, - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub balances: Balances, // -- Randomness -- - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub randao_mixes: RandaoMixes, // -- Slashings -- - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub slashings: Slashings, // -- Participation -- /// Per-validator participation flags for the previous epoch, positionally /// parallel to `validators`. - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub previous_epoch_participation: EpochParticipation, /// Flags for the current epoch, which become /// `previous_epoch_participation` at the next epoch boundary. - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub current_epoch_participation: EpochParticipation, // -- Finality -- - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub justification_bits: JustificationBits, pub previous_justified_checkpoint: Checkpoint, pub current_justified_checkpoint: Checkpoint, @@ -356,7 +366,7 @@ pub struct BeaconState { // -- Inactivity -- /// Per-validator inactivity score, positionally parallel to `validators`. - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub inactivity_scores: InactivityScores, // -- Sync committees -- @@ -384,7 +394,7 @@ pub struct BeaconState { // -- History -- /// Capella's replacement for whole [`super::shared::HistoricalBatch`] /// roots: one summary per historical window, appended the same way. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub historical_summaries: HistoricalSummaries, } @@ -430,7 +440,7 @@ pub struct BlobSidecar { /// is also its leaf index for `kzg_commitment_inclusion_proof`. #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub index: BlobIndex, - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub blob: Blob, pub kzg_commitment: KzgCommitment, /// A KZG proof that `blob` evaluates to `kzg_commitment`, checkable @@ -439,7 +449,7 @@ pub struct BlobSidecar { /// The header of the block this blob belongs to, signed by its proposer, /// which is what `kzg_commitment_inclusion_proof` terminates at. pub signed_block_header: SignedBeaconBlockHeader, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub kzg_commitment_inclusion_proof: BlobKzgCommitmentInclusionProof, } diff --git a/crates/common/types/src/beacon/containers/electra.rs b/crates/common/types/src/beacon/containers/electra.rs index 4610957ca..20ed5e7c3 100644 --- a/crates/common/types/src/beacon/containers/electra.rs +++ b/crates/common/types/src/beacon/containers/electra.rs @@ -161,11 +161,21 @@ pub struct Attestation { /// the state transition converts an [`Attestation`] into this form before /// checking it. `attesting_indices` now spans every committee an /// [`Attestation`] covers, following the same widening as [`AggregationBits`]. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] pub struct IndexedAttestation { /// The attesters, which the specification requires to be sorted and /// unique. - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub attesting_indices: AttestingIndices, pub data: AttestationData, pub signature: BlsSignature, @@ -179,7 +189,17 @@ pub struct IndexedAttestation { /// `MAX_ATTESTER_SLASHINGS_ELECTRA` rather than phase0's larger /// `MAX_ATTESTER_SLASHINGS`: evidence spanning a whole slot is proportionally /// more expensive to include. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] pub struct AttesterSlashing { pub attestation_1: IndexedAttestation, pub attestation_2: IndexedAttestation, @@ -232,7 +252,17 @@ pub struct SingleAttestation { /// unlike a contract-log deposit, a request arrives already ordered by the /// execution layer, so there is no merkle proof to check, only a position to /// record. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] pub struct DepositRequest { pub pubkey: BlsPubkey, pub withdrawal_credentials: Bytes32, @@ -250,7 +280,17 @@ pub struct DepositRequest { /// credentials before honoring the request. An `amount` of /// `FULL_EXIT_REQUEST_AMOUNT` signals a full exit rather than a partial /// withdrawal of that amount. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] pub struct WithdrawalRequest { pub source_address: ExecutionAddress, pub validator_pubkey: BlsPubkey, @@ -262,7 +302,17 @@ pub struct WithdrawalRequest { /// to merge `source_pubkey`'s balance into `target_pubkey`'s and exit the /// source, which is how a validator raises its effective balance past /// `MAX_EFFECTIVE_BALANCE` without a fresh deposit. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] pub struct ConsolidationRequest { pub source_address: ExecutionAddress, pub source_pubkey: BlsPubkey, @@ -272,14 +322,23 @@ pub struct ConsolidationRequest { /// The execution payload's envelope for every execution-layer-triggered /// request in this block, grouped by kind. #[derive( - Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] pub struct ExecutionRequests { - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub deposits: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub withdrawals: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub consolidations: SszList, } @@ -294,7 +353,7 @@ pub struct ExecutionRequests { #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] pub struct NewPayloadRequest { pub execution_payload: ExecutionPayload, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub versioned_hashes: VersionedHashes, pub parent_beacon_block_root: Root, pub execution_requests: ExecutionRequests, @@ -367,7 +426,17 @@ pub struct PendingConsolidation { /// Unchanged from deneb except for three things: `attester_slashings` and /// `attestations` are now bounded (and, for attestations, shaped) /// differently, and `execution_requests` is appended at the end. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] pub struct BeaconBlockBody { /// The proposer's contribution to the chain's randomness, which is a /// signature over the current epoch and so cannot be chosen freely. @@ -376,24 +445,24 @@ pub struct BeaconBlockBody { pub eth1_data: Eth1Data, /// Arbitrary proposer-chosen bytes, which consensus never reads. pub graffiti: Bytes32, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub proposer_slashings: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub attester_slashings: AttesterSlashings, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub attestations: Attestations, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub deposits: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub voluntary_exits: SszList, /// The aggregated sync committee signature over the previous slot's block /// root, plus which members contributed. pub sync_aggregate: SyncAggregate, pub execution_payload: ExecutionPayload, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub bls_to_execution_changes: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub blob_kzg_commitments: KzgCommitments, /// The execution-layer-triggered deposit, withdrawal, and consolidation /// requests carried by this block's payload. The one field electra adds @@ -449,7 +518,17 @@ impl BeaconBlockBody { } /// A block. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] pub struct BeaconBlock { #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub slot: Slot, @@ -462,7 +541,17 @@ pub struct BeaconBlock { pub body: BeaconBlockBody, } -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] pub struct SignedBeaconBlock { pub message: BeaconBlock, pub signature: BlsSignature, @@ -508,16 +597,16 @@ pub struct BeaconState { /// slot advances, since a block cannot commit to the root of the state /// containing it. pub latest_block_header: BeaconBlockHeader, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub block_roots: BlockRoots, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub state_roots: StateRoots, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub historical_roots: HistoricalRoots, // -- Eth1 -- pub eth1_data: Eth1Data, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub eth1_data_votes: Eth1DataVotes, /// How many deposits from the contract have been processed, which is /// where the next one will be read from. Superseded for new deposits once @@ -527,34 +616,34 @@ pub struct BeaconState { pub eth1_deposit_index: u64, // -- Registry -- - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub validators: Validators, - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub balances: Balances, // -- Randomness -- - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub randao_mixes: RandaoMixes, // -- Slashings -- - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub slashings: Slashings, // -- Participation -- - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub previous_epoch_participation: EpochParticipation, - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub current_epoch_participation: EpochParticipation, // -- Finality -- - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub justification_bits: JustificationBits, pub previous_justified_checkpoint: Checkpoint, pub current_justified_checkpoint: Checkpoint, pub finalized_checkpoint: Checkpoint, // -- Inactivity -- - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub inactivity_scores: InactivityScores, // -- Sync committees -- @@ -577,7 +666,7 @@ pub struct BeaconState { pub next_withdrawal_validator_index: ValidatorIndex, // -- Deep history -- - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub historical_summaries: HistoricalSummaries, // -- Deposit requests (EIP-6110) -- @@ -614,15 +703,15 @@ pub struct BeaconState { // -- Pending queues (EIP-7251) -- /// Deposits known but not yet credited to the validator registry, /// drained a bounded amount per epoch by `process_pending_deposits`. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub pending_deposits: PendingDeposits, /// Partial withdrawals known but not yet paid out, drained a bounded /// amount per slot by `get_expected_withdrawals`. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub pending_partial_withdrawals: PendingPartialWithdrawals, /// Consolidations known but not yet applied, drained a bounded amount per /// epoch by `process_pending_consolidations`. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub pending_consolidations: PendingConsolidations, } diff --git a/crates/common/types/src/beacon/containers/fulu.rs b/crates/common/types/src/beacon/containers/fulu.rs index 83167aca2..a5f9d7b40 100644 --- a/crates/common/types/src/beacon/containers/fulu.rs +++ b/crates/common/types/src/beacon/containers/fulu.rs @@ -124,17 +124,17 @@ pub struct DataColumnSidecar { /// (which requires each element to already implement `Serialize`) or /// [`ssz_hex`](crate::beacon::serde_helpers::ssz_hex) (which would hex-encode /// the whole list as one string instead of one string per cell). - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex_seq")] pub column: DataColumn, /// The block's full `blob_kzg_commitments`, repeated in every one of that /// block's sidecars rather than fetched separately, which is what lets a /// sidecar be checked on its own. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub kzg_commitments: KzgCommitments, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub kzg_proofs: KzgProofs, pub signed_block_header: SignedBeaconBlockHeader, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub kzg_commitments_inclusion_proof: KzgCommitmentsInclusionProof, } @@ -153,7 +153,7 @@ pub struct MatrixEntry { /// what [`ssz_hex`](crate::beacon::serde_helpers::ssz_hex) exists for, /// unlike [`DataColumnSidecar::column`], which holds a whole list of /// cells. - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub cell: Cell, pub kzg_proof: KzgProof, #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] @@ -169,7 +169,7 @@ pub struct MatrixEntry { )] pub struct DataColumnsByRootIdentifier { pub block_root: Root, - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub columns: ColumnIndices, } @@ -208,13 +208,13 @@ pub type OptionalPartialDataColumnHeader = SszList; /// from the gossipsub topic's subnet. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] pub struct PartialDataColumnSidecar { - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub cells_present_bitmap: CellsBitList, - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex_seq")] pub partial_column: DataColumn, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub kzg_proofs: KzgProofs, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub header: OptionalPartialDataColumnHeader, } @@ -226,9 +226,9 @@ pub struct PartialDataColumnSidecar { /// already encodes. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] pub struct PartialDataColumnPartsMetadata { - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub available: CellsBitList, - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub requests: CellsBitList, } @@ -243,10 +243,10 @@ pub struct PartialDataColumnPartsMetadata { /// [`DataColumnSidecar::kzg_commitments_inclusion_proof`] uses. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] pub struct PartialDataColumnHeader { - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub kzg_commitments: KzgCommitments, pub signed_block_header: SignedBeaconBlockHeader, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub kzg_commitments_inclusion_proof: KzgCommitmentsInclusionProof, } @@ -290,20 +290,20 @@ pub struct BeaconState { /// slot advances, since a block cannot commit to the root of the state /// containing it. pub latest_block_header: BeaconBlockHeader, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub block_roots: BlockRoots, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub state_roots: StateRoots, /// Frozen since capella: further history is committed to by /// `historical_summaries` instead. Kept rather than removed so a fulu /// state's shape stays hash-tree-root-compatible with every root /// committed while this field was still growing. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub historical_roots: HistoricalRoots, // -- Eth1 -- pub eth1_data: Eth1Data, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub eth1_data_votes: Eth1DataVotes, /// How many deposits from the contract have been processed, which is /// where the next one will be read from. @@ -311,32 +311,32 @@ pub struct BeaconState { pub eth1_deposit_index: u64, // -- Registry -- - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub validators: Validators, - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub balances: Balances, // -- Randomness -- - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub randao_mixes: RandaoMixes, // -- Slashings -- - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub slashings: Slashings, // -- Participation -- /// Per-validator participation flags for the previous epoch, positionally /// parallel to `validators`. See altair for why this replaces phase0's /// accumulated attestations. - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub previous_epoch_participation: EpochParticipation, /// Flags for the current epoch, which become `previous_epoch_participation` /// at the next epoch boundary. - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub current_epoch_participation: EpochParticipation, // -- Finality -- - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub justification_bits: JustificationBits, pub previous_justified_checkpoint: Checkpoint, pub current_justified_checkpoint: Checkpoint, @@ -344,7 +344,7 @@ pub struct BeaconState { // -- Inactivity -- /// Per-validator inactivity score, positionally parallel to `validators`. - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub inactivity_scores: InactivityScores, // -- Sync committees -- @@ -373,7 +373,7 @@ pub struct BeaconState { // -- History (capella) -- /// The commitment to history from capella onward, appended to instead of /// `historical_roots` once that field was frozen. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub historical_summaries: HistoricalSummaries, // -- Deposits, exits, and consolidations (electra) -- @@ -404,13 +404,13 @@ pub struct BeaconState { pub earliest_consolidation_epoch: Epoch, /// Deposits credited on the execution layer but not yet applied to the /// registry, drained a bounded number at a time each epoch. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub pending_deposits: PendingDeposits, /// Partial withdrawals requested but not yet paid out. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub pending_partial_withdrawals: PendingPartialWithdrawals, /// Validator consolidations requested but not yet applied. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub pending_consolidations: PendingConsolidations, // -- Proposer lookahead (fulu) -- @@ -419,7 +419,7 @@ pub struct BeaconState { /// epoch boundary by `process_proposer_lookahead` so that /// `get_beacon_proposer_index` becomes a lookup into this vector rather /// than a shuffle computed on demand. - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub proposer_lookahead: ProposerLookahead, } diff --git a/crates/common/types/src/beacon/containers/gloas.rs b/crates/common/types/src/beacon/containers/gloas.rs index 51d236b7a..09838fa0f 100644 --- a/crates/common/types/src/beacon/containers/gloas.rs +++ b/crates/common/types/src/beacon/containers/gloas.rs @@ -232,7 +232,16 @@ pub struct BuilderPendingWithdrawal { /// An execution-layer-triggered builder deposit (EIP-8282), the builder-side /// counterpart of [`super::electra::DepositRequest`]. #[derive( - Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] pub struct BuilderDepositRequest { pub pubkey: BlsPubkey, @@ -245,7 +254,16 @@ pub struct BuilderDepositRequest { /// An execution-layer-triggered builder exit (EIP-8282), the builder-side /// counterpart of [`super::electra::WithdrawalRequest`]'s full-exit case. #[derive( - Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] pub struct BuilderExitRequest { pub source_address: ExecutionAddress, @@ -280,10 +298,20 @@ pub struct PayloadAttestationData { } /// An aggregate payload timeliness attestation, as included in a block. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] #[ssz(progressive_container)] pub struct PayloadAttestation { - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub aggregation_bits: PayloadTimelinessCommitteeBits, pub data: PayloadAttestationData, pub signature: BlsSignature, @@ -311,10 +339,20 @@ pub struct PayloadAttestationMessage { /// A [`PayloadAttestation`] with its attesters named rather than bit-encoded, /// the payload-attestation counterpart of [`IndexedAttestation`]. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] #[ssz(progressive_container)] pub struct IndexedPayloadAttestation { - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub attesting_indices: PayloadTimelinessCommitteeIndices, pub data: PayloadAttestationData, pub signature: BlsSignature, @@ -328,7 +366,16 @@ pub struct IndexedPayloadAttestation { /// proposer chooses among, and signs over, in place of embedding a payload in /// the block directly. #[derive( - Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] #[ssz(progressive_container)] pub struct ExecutionPayloadBid { @@ -349,7 +396,7 @@ pub struct ExecutionPayloadBid { /// the payload is never revealed or attested to as available. #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub execution_payment: Gwei, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub blob_kzg_commitments: BlobKzgCommitments, /// The root of the [`ExecutionRequests`] this payload's envelope will /// carry, committed to here so the bid is binding on the requests too, @@ -358,7 +405,16 @@ pub struct ExecutionPayloadBid { } #[derive( - Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] pub struct SignedExecutionPayloadBid { pub message: ExecutionPayloadBid, @@ -372,19 +428,28 @@ pub struct SignedExecutionPayloadBid { /// and `builder_exits` (EIP-8282), the execution-layer path [`Builder`]s are /// onboarded and retired through from the fork onward. #[derive( - Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] #[ssz(progressive_container)] pub struct ExecutionRequests { - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub deposits: DepositRequests, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub withdrawals: WithdrawalRequests, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub consolidations: ConsolidationRequests, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub builder_deposits: BuilderDepositRequests, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub builder_exits: BuilderExitRequests, } @@ -397,14 +462,24 @@ pub struct ExecutionRequests { /// (EIP-7928) and `slot_number` (EIP-8061) are appended. Not /// `#[derive(Default)]`: `logs_bloom` is an [`SszVector`], which, like every /// earlier fork's payload, has no meaningful empty value. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] #[ssz(progressive_container)] pub struct ExecutionPayload { pub parent_hash: ExecutionBlockHash, pub fee_recipient: ExecutionAddress, pub state_root: Bytes32, pub receipts_root: Bytes32, - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub logs_bloom: LogsBloom, pub prev_randao: Bytes32, #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] @@ -415,13 +490,13 @@ pub struct ExecutionPayload { pub gas_used: u64, #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub timestamp: u64, - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub extra_data: ExtraData, pub base_fee_per_gas: Uint256, pub block_hash: ExecutionBlockHash, - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex_seq")] pub transactions: Transactions, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub withdrawals: Withdrawals, #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub blob_gas_used: u64, @@ -429,7 +504,7 @@ pub struct ExecutionPayload { pub excess_blob_gas: u64, /// The serialized block access list this payload's execution produced /// (EIP-7928). - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub block_access_list: BlockAccessList, /// The slot this payload was built for (EIP-8061's gas limit schedule /// reads it against the schedule rather than the enclosing block's own @@ -441,7 +516,17 @@ pub struct ExecutionPayload { /// A builder-signed reveal of the payload it bid for: the payload itself, /// the execution-layer-triggered requests it carries, and enough to bind it /// to the block whose [`ExecutionPayloadBid`] it fulfills. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] #[ssz(progressive_container)] pub struct ExecutionPayloadEnvelope { pub payload: ExecutionPayload, @@ -452,7 +537,17 @@ pub struct ExecutionPayloadEnvelope { pub parent_beacon_block_root: Root, } -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] pub struct SignedExecutionPayloadEnvelope { pub message: ExecutionPayloadEnvelope, pub signature: BlsSignature, @@ -469,7 +564,7 @@ pub struct SignedExecutionPayloadEnvelope { #[ssz(progressive_container)] pub struct NewPayloadRequest { pub execution_payload: ExecutionPayload, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub versioned_hashes: VersionedHashes, pub parent_beacon_block_root: Root, pub execution_requests: ExecutionRequests, @@ -555,10 +650,20 @@ impl TryFrom<&Attestation> for super::electra::Attestation { /// An attestation with its attesters named rather than bit-encoded, the same /// role electra's plays, with `attesting_indices` now the unbounded /// [`AttestingIndices`]. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] #[ssz(progressive_container)] pub struct IndexedAttestation { - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub attesting_indices: AttestingIndices, pub data: AttestationData, pub signature: BlsSignature, @@ -567,7 +672,17 @@ pub struct IndexedAttestation { /// Evidence that a set of validators made two conflicting attestations. /// Unchanged in shape from electra; what changed is [`IndexedAttestation`] /// itself. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] pub struct AttesterSlashing { pub attestation_1: IndexedAttestation, pub attestation_2: IndexedAttestation, @@ -627,7 +742,16 @@ pub struct SignedAggregateAndProof { /// separate fetch). Every other field is electra's, now progressive /// (EIP-7688). #[derive( - Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] #[ssz(progressive_container)] pub struct BeaconBlockBody { @@ -638,20 +762,20 @@ pub struct BeaconBlockBody { pub eth1_data: Eth1Data, /// Arbitrary proposer-chosen bytes, which consensus never reads. pub graffiti: Bytes32, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub proposer_slashings: ProposerSlashings, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub attester_slashings: AttesterSlashings, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub attestations: Attestations, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub deposits: Deposits, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub voluntary_exits: VoluntaryExits, /// The aggregated sync committee signature over the previous slot's /// block root, plus which members contributed. pub sync_aggregate: SyncAggregate, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub bls_to_execution_changes: BlsToExecutionChanges, /// The proposer's binding commitment to the builder's bid for this /// slot's payload (EIP-7732); the payload itself is revealed afterward @@ -659,7 +783,7 @@ pub struct BeaconBlockBody { pub signed_execution_payload_bid: SignedExecutionPayloadBid, /// The payload timeliness committee's votes on the *previous* slot's /// payload. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub payload_attestations: PayloadAttestations, /// The previous slot's execution-layer-triggered requests, carried /// forward so `process_payload_attestation` and friends do not need a @@ -684,7 +808,16 @@ impl BeaconBlockBody { /// A block. #[derive( - Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] pub struct BeaconBlock { #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] @@ -699,7 +832,16 @@ pub struct BeaconBlock { } #[derive( - Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] pub struct SignedBeaconBlock { pub message: BeaconBlock, @@ -747,18 +889,18 @@ pub struct BeaconState { /// slot advances, since a block cannot commit to the root of the state /// containing it. pub latest_block_header: BeaconBlockHeader, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub block_roots: BlockRoots, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub state_roots: StateRoots, /// Frozen since capella: further history is committed to by /// `historical_summaries` instead. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub historical_roots: HistoricalRoots, // -- Eth1 -- pub eth1_data: Eth1Data, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub eth1_data_votes: Eth1DataVotes, /// How many deposits from the contract have been processed, which is /// where the next one will be read from. @@ -768,34 +910,34 @@ pub struct BeaconState { // -- Registry (EIP-7688) -- /// The validator registry, tree-backed and progressively merkleized. See /// [`ProgressiveValidators`]. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub validators: ProgressiveValidators, - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub balances: ProgressiveBalances, // -- Randomness -- - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub randao_mixes: RandaoMixes, // -- Slashings -- - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub slashings: Slashings, // -- Participation (EIP-7688) -- - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub previous_epoch_participation: EpochParticipation, - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub current_epoch_participation: EpochParticipation, // -- Finality -- - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub justification_bits: JustificationBits, pub previous_justified_checkpoint: Checkpoint, pub current_justified_checkpoint: Checkpoint, pub finalized_checkpoint: Checkpoint, // -- Inactivity (EIP-7688) -- - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub inactivity_scores: InactivityScores, // -- Sync committees -- @@ -815,7 +957,7 @@ pub struct BeaconState { pub next_withdrawal_validator_index: ValidatorIndex, // -- History (capella) -- - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub historical_summaries: HistoricalSummaries, // -- Deposits, exits, and consolidations (electra) -- @@ -831,20 +973,20 @@ pub struct BeaconState { pub consolidation_balance_to_consume: Gwei, #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub earliest_consolidation_epoch: Epoch, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub pending_deposits: PendingDeposits, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub pending_partial_withdrawals: PendingPartialWithdrawals, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub pending_consolidations: PendingConsolidations, // -- Proposer lookahead (fulu) -- - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub proposer_lookahead: ProposerLookahead, // -- Builders (EIP-7732) -- /// The builder registry. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub builders: Builders, /// Where the builder withdrawal sweep last stopped, the builder-side /// counterpart of `next_withdrawal_validator_index`. @@ -852,13 +994,13 @@ pub struct BeaconState { pub next_withdrawal_builder_index: BuilderIndex, /// Bits tracking, for each of the last `SLOTS_PER_HISTORICAL_ROOT` slots, /// whether that slot's payload was made available. - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub execution_payload_availability: ExecutionPayloadAvailability, /// Builder payments owed for the previous and current epoch, settled by /// `process_builder_pending_payments`. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub builder_pending_payments: BuilderPendingPayments, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub builder_pending_withdrawals: BuilderPendingWithdrawals, /// The bid the payload currently expected for this slot, if any, was /// bought under. @@ -866,7 +1008,7 @@ pub struct BeaconState { /// The withdrawals `get_expected_withdrawals` computed for the payload /// this slot expects, cached here so a late-revealed payload's /// `process_execution_payload` does not have to recompute the sweep. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub payload_expected_withdrawals: Withdrawals, /// The cached window of payload timeliness committees for the previous, /// current, and lookahead epochs, refreshed each epoch by @@ -896,9 +1038,9 @@ pub struct BeaconState { pub struct DataColumnSidecar { #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub index: ColumnIndex, - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex_seq")] pub column: DataColumn, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub kzg_proofs: KzgProofs, #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub slot: Slot, @@ -931,11 +1073,11 @@ pub struct DataColumnSidecar { Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, )] pub struct PartialDataColumnSidecar { - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub cells_present_bitmap: CellsBitList, - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex_seq")] pub partial_column: DataColumn, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub kzg_proofs: KzgProofs, } @@ -946,9 +1088,9 @@ pub struct PartialDataColumnSidecar { Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, )] pub struct PartialDataColumnPartsMetadata { - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub available: CellsBitList, - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub requests: CellsBitList, } diff --git a/crates/common/types/src/beacon/containers/json_tests.rs b/crates/common/types/src/beacon/containers/json_tests.rs new file mode 100644 index 000000000..62c6ca3e5 --- /dev/null +++ b/crates/common/types/src/beacon/containers/json_tests.rs @@ -0,0 +1,405 @@ +//! JSON round trips of the containers a validator client submits, which the +//! Beacon API's request bodies carry: serialize, deserialize, compare. Each +//! value has every list non-empty and every integer non-zero, so an adapter +//! that dropped or misread a field shows up as a difference. + +use super::{altair, capella, electra, gloas, shared}; +use crate::beacon::primitives::{BlsPubkey, BlsSignature, KzgCommitment, Uint256}; + +fn round_trip(value: &T) +where + T: serde::Serialize + serde::de::DeserializeOwned + PartialEq + std::fmt::Debug, +{ + let json = serde_json::to_string(value).unwrap(); + let back: T = serde_json::from_str(&json).unwrap(); + assert_eq!(&back, value, "{json}"); +} + +fn signature(byte: u8) -> BlsSignature { + let mut signature = BlsSignature::default(); + signature.0[0] = byte; + signature.0[95] = byte; + signature +} + +fn pubkey(byte: u8) -> BlsPubkey { + let mut pubkey = BlsPubkey::default(); + pubkey.0[0] = byte; + pubkey +} + +fn header(slot: u64) -> shared::SignedBeaconBlockHeader { + shared::SignedBeaconBlockHeader { + message: shared::BeaconBlockHeader { + slot, + proposer_index: 3, + parent_root: [1; 32].into(), + state_root: [2; 32].into(), + body_root: [3; 32].into(), + }, + signature: signature(9), + } +} + +fn data() -> shared::AttestationData { + shared::AttestationData { + slot: 12, + index: 0, + beacon_block_root: [4; 32].into(), + source: shared::Checkpoint { + epoch: 1, + root: [5; 32].into(), + }, + target: shared::Checkpoint { + epoch: 2, + root: [6; 32].into(), + }, + } +} + +fn deposit() -> shared::Deposit { + shared::Deposit { + proof: vec![[7u8; 32].into(); crate::beacon::constants::DEPOSIT_CONTRACT_TREE_DEPTH + 1] + .try_into() + .unwrap(), + data: shared::DepositData { + pubkey: pubkey(1), + withdrawal_credentials: [8; 32].into(), + amount: 32_000_000_000, + signature: signature(2), + }, + } +} + +fn exit() -> shared::SignedVoluntaryExit { + shared::SignedVoluntaryExit { + message: shared::VoluntaryExit { + epoch: 5, + validator_index: 6, + }, + signature: signature(3), + } +} + +fn bls_change() -> capella::SignedBLSToExecutionChange { + capella::SignedBLSToExecutionChange { + message: capella::BLSToExecutionChange { + validator_index: 4, + from_bls_pubkey: pubkey(4), + to_execution_address: [0xaa; 20].into(), + }, + signature: signature(4), + } +} + +fn sync_aggregate() -> altair::SyncAggregate { + let mut aggregate = altair::SyncAggregate::default(); + aggregate.sync_committee_bits.set(3, true).unwrap(); + aggregate.sync_committee_signature = signature(5); + aggregate +} + +fn electra_attestation() -> electra::Attestation { + let mut aggregation_bits = electra::AggregationBits::with_length(5).unwrap(); + aggregation_bits.set(1, true).unwrap(); + aggregation_bits.set(4, true).unwrap(); + let mut committee_bits = electra::CommitteeBits::default(); + committee_bits.set(2, true).unwrap(); + electra::Attestation { + aggregation_bits, + data: data(), + signature: signature(6), + committee_bits, + } +} + +fn electra_indexed() -> electra::IndexedAttestation { + electra::IndexedAttestation { + attesting_indices: vec![3, 9, 27].try_into().unwrap(), + data: data(), + signature: signature(7), + } +} + +fn electra_payload() -> super::deneb::ExecutionPayload { + use super::{bellatrix, deneb}; + deneb::ExecutionPayload { + parent_hash: [1; 32].into(), + fee_recipient: [2; 20].into(), + state_root: [3; 32].into(), + receipts_root: [4; 32].into(), + logs_bloom: bellatrix::LogsBloom::try_from(vec![ + 0xab; + crate::beacon::preset::BYTES_PER_LOGS_BLOOM + ]) + .unwrap(), + prev_randao: [5; 32].into(), + block_number: 10, + gas_limit: 30_000_000, + gas_used: 21_000, + timestamp: 1_700_000_000, + extra_data: vec![0xde, 0xad].try_into().unwrap(), + base_fee_per_gas: Uint256::from(7_000_000_000u64), + block_hash: [6; 32].into(), + transactions: vec![ + bellatrix::Transaction::try_from(vec![1, 2, 3]).unwrap(), + bellatrix::Transaction::try_from(vec![4]).unwrap(), + ] + .try_into() + .unwrap(), + withdrawals: vec![capella::Withdrawal { + index: 1, + validator_index: 2, + address: [3; 20].into(), + amount: 4, + }] + .try_into() + .unwrap(), + blob_gas_used: 131_072, + excess_blob_gas: 262_144, + } +} + +fn electra_requests() -> electra::ExecutionRequests { + electra::ExecutionRequests { + deposits: vec![electra::DepositRequest { + pubkey: pubkey(5), + withdrawal_credentials: [1; 32].into(), + amount: 32_000_000_000, + signature: signature(8), + index: 11, + }] + .try_into() + .unwrap(), + withdrawals: vec![electra::WithdrawalRequest { + source_address: [2; 20].into(), + validator_pubkey: pubkey(6), + amount: 5, + }] + .try_into() + .unwrap(), + consolidations: vec![electra::ConsolidationRequest { + source_address: [3; 20].into(), + source_pubkey: pubkey(7), + target_pubkey: pubkey(8), + }] + .try_into() + .unwrap(), + } +} + +fn electra_block() -> electra::SignedBeaconBlock { + let mut body = electra::BeaconBlockBody::empty(); + body.randao_reveal = signature(1); + body.graffiti = [9; 32].into(); + body.proposer_slashings = vec![shared::ProposerSlashing { + signed_header_1: header(7), + signed_header_2: header(7), + }] + .try_into() + .unwrap(); + body.attester_slashings = vec![electra::AttesterSlashing { + attestation_1: electra_indexed(), + attestation_2: electra_indexed(), + }] + .try_into() + .unwrap(); + body.attestations = vec![electra_attestation()].try_into().unwrap(); + body.deposits = vec![deposit()].try_into().unwrap(); + body.voluntary_exits = vec![exit()].try_into().unwrap(); + body.sync_aggregate = sync_aggregate(); + body.execution_payload = electra_payload(); + body.bls_to_execution_changes = vec![bls_change()].try_into().unwrap(); + body.blob_kzg_commitments = vec![KzgCommitment([5; 48])].try_into().unwrap(); + body.execution_requests = electra_requests(); + electra::SignedBeaconBlock { + message: electra::BeaconBlock { + slot: 33, + proposer_index: 4, + parent_root: [1; 32].into(), + state_root: [2; 32].into(), + body, + }, + signature: signature(9), + } +} + +fn gloas_attestation() -> gloas::Attestation { + gloas::Attestation::from(&electra_attestation()) +} + +fn gloas_payload() -> gloas::ExecutionPayload { + let payload = electra_payload(); + gloas::ExecutionPayload { + parent_hash: payload.parent_hash, + fee_recipient: payload.fee_recipient, + state_root: payload.state_root, + receipts_root: payload.receipts_root, + logs_bloom: payload.logs_bloom, + prev_randao: payload.prev_randao, + block_number: payload.block_number, + gas_limit: payload.gas_limit, + gas_used: payload.gas_used, + timestamp: payload.timestamp, + extra_data: payload.extra_data, + base_fee_per_gas: payload.base_fee_per_gas, + block_hash: payload.block_hash, + transactions: payload + .transactions + .iter() + .map(|transaction| gloas::Transaction::from(transaction.to_vec())) + .collect::>() + .into(), + withdrawals: payload + .withdrawals + .iter() + .cloned() + .collect::>() + .into(), + blob_gas_used: payload.blob_gas_used, + excess_blob_gas: payload.excess_blob_gas, + block_access_list: vec![0xca, 0xfe].into(), + slot_number: 33, + } +} + +fn gloas_requests() -> gloas::ExecutionRequests { + let requests = electra_requests(); + gloas::ExecutionRequests { + deposits: requests.deposits.iter().cloned().collect::>().into(), + withdrawals: requests + .withdrawals + .iter() + .cloned() + .collect::>() + .into(), + consolidations: requests + .consolidations + .iter() + .cloned() + .collect::>() + .into(), + builder_deposits: vec![gloas::BuilderDepositRequest { + pubkey: pubkey(9), + withdrawal_credentials: [4; 32].into(), + amount: 1, + signature: signature(1), + }] + .into(), + builder_exits: vec![gloas::BuilderExitRequest { + source_address: [5; 20].into(), + pubkey: pubkey(10), + }] + .into(), + } +} + +fn payload_attestation() -> gloas::PayloadAttestation { + let mut aggregation_bits = gloas::PayloadTimelinessCommitteeBits::default(); + aggregation_bits.set(1, true).unwrap(); + gloas::PayloadAttestation { + aggregation_bits, + data: gloas::PayloadAttestationData { + beacon_block_root: [6; 32].into(), + slot: 32, + payload_present: true, + blob_data_available: true, + }, + signature: signature(2), + } +} + +fn gloas_block() -> gloas::SignedBeaconBlock { + let electra = electra_block().message; + let mut body = gloas::BeaconBlockBody::empty(); + body.randao_reveal = electra.body.randao_reveal; + body.graffiti = electra.body.graffiti; + body.proposer_slashings = electra + .body + .proposer_slashings + .iter() + .cloned() + .collect::>() + .into(); + body.attestations = vec![gloas_attestation()].into(); + body.deposits = electra + .body + .deposits + .iter() + .cloned() + .collect::>() + .into(); + body.voluntary_exits = electra + .body + .voluntary_exits + .iter() + .cloned() + .collect::>() + .into(); + body.sync_aggregate = sync_aggregate(); + body.signed_execution_payload_bid = gloas::SignedExecutionPayloadBid { + message: gloas::ExecutionPayloadBid { + parent_block_hash: [1; 32].into(), + parent_block_root: [2; 32].into(), + block_hash: [3; 32].into(), + prev_randao: [4; 32].into(), + fee_recipient: [5; 20].into(), + gas_limit: 30_000_000, + builder_index: 7, + slot: 33, + value: 8, + execution_payment: 9, + execution_requests_root: [7; 32].into(), + blob_kzg_commitments: vec![KzgCommitment([6; 48])].try_into().unwrap(), + }, + signature: signature(3), + }; + body.payload_attestations = vec![payload_attestation()].try_into().unwrap(); + body.parent_execution_requests = gloas_requests(); + gloas::SignedBeaconBlock { + message: gloas::BeaconBlock { + slot: 33, + proposer_index: 4, + parent_root: [1; 32].into(), + state_root: [2; 32].into(), + body, + }, + signature: signature(9), + } +} + +#[test] +fn a_fulu_block_round_trips_through_json() { + round_trip(&electra_block()); +} + +#[test] +fn a_gloas_block_round_trips_through_json() { + round_trip(&gloas_block()); +} + +#[test] +fn a_gloas_envelope_round_trips_through_json() { + round_trip(&gloas::SignedExecutionPayloadEnvelope { + message: gloas::ExecutionPayloadEnvelope { + payload: gloas_payload(), + execution_requests: gloas_requests(), + builder_index: 7, + beacon_block_root: [1; 32].into(), + parent_beacon_block_root: [2; 32].into(), + }, + signature: signature(4), + }); +} + +#[test] +fn an_oversized_list_is_refused_rather_than_truncated() { + let mut json = serde_json::to_value(electra_block()).unwrap(); + let commitment = json["message"]["body"]["blob_kzg_commitments"][0].clone(); + json["message"]["body"]["blob_kzg_commitments"] = serde_json::Value::Array(vec![ + commitment; + crate::beacon::preset::MAX_BLOB_COMMITMENTS_PER_BLOCK + + 1 + ]); + assert!(serde_json::from_value::(json).is_err()); +} diff --git a/crates/common/types/src/beacon/containers/mod.rs b/crates/common/types/src/beacon/containers/mod.rs index 1e8b09e1c..367da3c75 100644 --- a/crates/common/types/src/beacon/containers/mod.rs +++ b/crates/common/types/src/beacon/containers/mod.rs @@ -1630,6 +1630,9 @@ impl DataColumnSidecar { } } +#[cfg(test)] +mod json_tests; + #[cfg(test)] mod tests { use super::*; diff --git a/crates/common/types/src/beacon/containers/phase0.rs b/crates/common/types/src/beacon/containers/phase0.rs index 631dbd8e3..d937795a4 100644 --- a/crates/common/types/src/beacon/containers/phase0.rs +++ b/crates/common/types/src/beacon/containers/phase0.rs @@ -42,7 +42,7 @@ pub type PendingAttestations = SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub attester_slashings: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub attestations: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub deposits: SszList, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub voluntary_exits: SszList, } @@ -167,16 +167,16 @@ pub struct BeaconState { /// advances, since a block cannot commit to the root of the state containing /// it. pub latest_block_header: BeaconBlockHeader, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub block_roots: BlockRoots, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub state_roots: StateRoots, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub historical_roots: HistoricalRoots, // -- Eth1 -- pub eth1_data: Eth1Data, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub eth1_data_votes: Eth1DataVotes, /// How many deposits from the contract have been processed, which is where /// the next one will be read from. @@ -184,31 +184,31 @@ pub struct BeaconState { pub eth1_deposit_index: u64, // -- Registry -- - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub validators: Validators, - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub balances: Balances, // -- Randomness -- - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub randao_mixes: RandaoMixes, // -- Slashings -- - #[serde(serialize_with = "crate::beacon::serde_helpers::quoted_u64_seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::quoted_u64_seq")] pub slashings: Slashings, // -- Attestations -- /// Attestations for the previous epoch, replayed at the epoch boundary. Only /// phase0 has these; altair replaces them with participation flags. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub previous_epoch_attestations: PendingAttestations, /// Attestations for the current epoch, which become /// `previous_epoch_attestations` at the next boundary. - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub current_epoch_attestations: PendingAttestations, // -- Finality -- - #[serde(serialize_with = "crate::beacon::serde_helpers::ssz_hex::serialize")] + #[serde(with = "crate::beacon::serde_helpers::ssz_hex")] pub justification_bits: JustificationBits, pub previous_justified_checkpoint: Checkpoint, pub current_justified_checkpoint: Checkpoint, diff --git a/crates/common/types/src/beacon/containers/shared.rs b/crates/common/types/src/beacon/containers/shared.rs index c7d857f1c..b2df90691 100644 --- a/crates/common/types/src/beacon/containers/shared.rs +++ b/crates/common/types/src/beacon/containers/shared.rs @@ -240,7 +240,16 @@ pub struct AttestationData { /// The execution chain's deposit state, as voted on by proposers. #[derive( - Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] pub struct Eth1Data { /// The deposit contract's merkle root at this point. @@ -272,9 +281,9 @@ pub struct Eth1Block { /// [`HistoricalRoots`] once the window is full. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] pub struct HistoricalBatch { - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub block_roots: BlockRoots, - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub state_roots: StateRoots, } @@ -324,7 +333,16 @@ pub struct DepositMessage { /// A deposit as recorded by the deposit contract. #[derive( - Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] pub struct DepositData { pub pubkey: BlsPubkey, @@ -339,9 +357,19 @@ pub struct DepositData { } /// A deposit together with its merkle proof against the deposit contract root. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, +)] pub struct Deposit { - #[serde(serialize_with = "crate::beacon::serde_helpers::seq::serialize")] + #[serde(with = "crate::beacon::serde_helpers::seq")] pub proof: DepositProof, pub data: DepositData, } @@ -353,7 +381,16 @@ pub struct Deposit { /// A block without its body, which is what the state retains and what proposer /// slashings compare. #[derive( - Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] pub struct BeaconBlockHeader { #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] @@ -371,7 +408,16 @@ pub struct BeaconBlockHeader { } #[derive( - Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] pub struct SignedBeaconBlockHeader { pub message: BeaconBlockHeader, @@ -380,7 +426,16 @@ pub struct SignedBeaconBlockHeader { /// Evidence that a proposer signed two different blocks for the same slot. #[derive( - Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] pub struct ProposerSlashing { pub signed_header_1: SignedBeaconBlockHeader, @@ -389,7 +444,17 @@ pub struct ProposerSlashing { /// A validator's request to stop validating. #[derive( - Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Copy, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] pub struct VoluntaryExit { /// The earliest epoch the exit may be processed at. @@ -400,7 +465,16 @@ pub struct VoluntaryExit { } #[derive( - Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] pub struct SignedVoluntaryExit { pub message: VoluntaryExit, diff --git a/crates/common/types/src/beacon/primitives.rs b/crates/common/types/src/beacon/primitives.rs index 43c2589b6..b19309a8f 100644 --- a/crates/common/types/src/beacon/primitives.rs +++ b/crates/common/types/src/beacon/primitives.rs @@ -337,6 +337,18 @@ impl serde::Serialize for U256 { } } +impl<'de> serde::Deserialize<'de> for U256 { + /// The inverse of [`serde::Serialize`] above: a decimal string, quoted or + /// bare like every integer [`super::serde_helpers::quoted_or_bare`] reads. + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let text = ::deserialize(deserializer)?; + U256::from_dec_str(text.trim()).map_err(serde::de::Error::custom) + } +} + /// Why a decimal string was not a [`U256`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] pub enum ParseU256Error { @@ -490,8 +502,9 @@ impl_hex_serialize!(H160, BlsPubkey, BlsSignature, KzgCommitment, KzgProof); /// The `Deserialize` counterpart of [`impl_hex_serialize`], for the types the /// Beacon API accepts in a request body: a validator's public key (as a -/// validator id), a signature (inside a submitted attestation) and an execution -/// address (a proposer's fee recipient). Hex with or +/// validator id), a signature (inside a submitted attestation), an execution +/// address (a proposer's fee recipient) and a block's KZG commitments and +/// proofs. Hex with or /// without the `0x` prefix, of exactly the type's width. macro_rules! impl_hex_deserialize { ($($ty:ty),* $(,)?) => { @@ -508,7 +521,7 @@ macro_rules! impl_hex_deserialize { }; } -impl_hex_deserialize!(BlsPubkey, BlsSignature, H160); +impl_hex_deserialize!(BlsPubkey, BlsSignature, H160, KzgCommitment, KzgProof); #[cfg(test)] mod tests { diff --git a/crates/common/types/src/beacon/serde_helpers.rs b/crates/common/types/src/beacon/serde_helpers.rs index 976b28901..a20db64af 100644 --- a/crates/common/types/src/beacon/serde_helpers.rs +++ b/crates/common/types/src/beacon/serde_helpers.rs @@ -179,6 +179,25 @@ pub mod quoted_u64_seq { } seq.end() } + + /// The inverse: an array of quoted (or bare) integers, which is how + /// [`quoted_or_bare`](super::quoted_or_bare) reads one. + pub fn deserialize<'de, D, C, T>(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + C: TryFrom>, + C::Error: std::fmt::Debug, + T: std::str::FromStr, + T::Err: std::fmt::Display, + { + let texts = as serde::Deserialize>::deserialize(deserializer)?; + let values = texts + .iter() + .map(|text| text.trim().parse().map_err(serde::de::Error::custom)) + .collect::, _>>()?; + C::try_from(values) + .map_err(|err| serde::de::Error::custom(format!("invalid sequence: {err:?}"))) + } } /// A sequence of sequences of integers, each innermost value written quoted. @@ -351,6 +370,31 @@ pub mod ssz_hex_seq { } seq.end() } + + /// The inverse: an array of hex strings, each decoded through the + /// element's `SszDecode`, so a byte list's bound is checked as on the wire. + pub fn deserialize<'de, D, C, T>(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + C: TryFrom>, + C::Error: std::fmt::Debug, + T: libssz::SszDecode, + { + let texts = as serde::Deserialize>::deserialize(deserializer)?; + let values = texts + .iter() + .map(|text| { + let digits = text.trim(); + let bytes = hex::decode(digits.strip_prefix("0x").unwrap_or(digits)) + .map_err(serde::de::Error::custom)?; + T::from_ssz_bytes(&bytes).map_err(|err| { + serde::de::Error::custom(format!("invalid SSZ encoding: {err:?}")) + }) + }) + .collect::, _>>()?; + C::try_from(values) + .map_err(|err| serde::de::Error::custom(format!("invalid sequence: {err:?}"))) + } } /// A sequence of values that serialize themselves, from a foreign collection. @@ -375,6 +419,21 @@ pub mod seq { } seq.end() } + + /// The inverse: a JSON array of elements, collected into the foreign + /// collection through its `TryFrom>`, which is where a length bound + /// is checked (a progressive list's conversion cannot fail). + pub fn deserialize<'de, D, C, T>(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + C: TryFrom>, + C::Error: std::fmt::Debug, + T: serde::Deserialize<'de>, + { + let values = as serde::Deserialize>::deserialize(deserializer)?; + C::try_from(values) + .map_err(|err| serde::de::Error::custom(format!("invalid sequence: {err:?}"))) + } } #[cfg(test)] From 73d2ead3b26884d96b57557571947d39a11794d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 19:58:07 -0300 Subject: [PATCH 110/132] fix(rpc): accept JSON bodies on block and envelope publication Teku's validator client posts the gloas execution payload envelope as JSON and got a 415, so its blocks never had their payload published; nimbus posts blocks as JSON and lodestar posts both as JSON by default. POST /eth/v2/beacon/blocks (fulu SignedBlockContents, gloas SignedBeaconBlock) and POST /eth/v1/beacon/execution_payload_envelopes (SignedExecutionPayloadEnvelope or, with Eth-Blob-Data-Included: true, the Contents form) now take either encoding by Content-Type, through the BodyEncoding that decode_list already used. The decoded value goes through the same validation as the SSZ one. Any other content type is still a 415, and a body that is not the container a 400. The envelope test that asserted JSON was refused now asserts what is still true: an unsupported type is a 415 and SSZ bytes labelled JSON are a 400. --- crates/net/rpc/src/beacon/gloas_proposal.rs | 205 ++++++++++++++++++-- crates/net/rpc/src/beacon/mod.rs | 61 ++++-- crates/net/rpc/src/beacon/proposal.rs | 123 +++++++++--- 3 files changed, 329 insertions(+), 60 deletions(-) diff --git a/crates/net/rpc/src/beacon/gloas_proposal.rs b/crates/net/rpc/src/beacon/gloas_proposal.rs index fb9218a1e..12a3a507d 100644 --- a/crates/net/rpc/src/beacon/gloas_proposal.rs +++ b/crates/net/rpc/src/beacon/gloas_proposal.rs @@ -84,13 +84,13 @@ use ethlambda_types::{ }, primitives::H256, }; -use libssz::{SszDecode as _, SszEncode as _}; +use libssz::SszEncode as _; use libssz_derive::{SszDecode, SszEncode}; use serde::Deserialize; use tracing::{debug, info, warn}; use crate::beacon::{ - ApiError, + ApiError, BodyEncoding, proposal::{Blobs, CellKzgProofs, decimal, require_gloas_slot}, validator::{FeeRecipients, head}, }; @@ -140,22 +140,24 @@ impl PayloadCache { /// Gloas's `BlockContents`, the Beacon API's envelope for a self-built block /// with its payload: not a consensus container, so it lives with the API. -#[derive(Debug, Clone, PartialEq, SszEncode, SszDecode, serde::Serialize)] +#[derive(Debug, Clone, PartialEq, SszEncode, SszDecode, serde::Serialize, serde::Deserialize)] pub(crate) struct GloasBlockContents { pub(crate) block: BeaconBlock, pub(crate) execution_payload_envelope: ExecutionPayloadEnvelope, - #[serde(serialize_with = "ethlambda_types::beacon::serde_helpers::seq::serialize")] + #[serde(with = "ethlambda_types::beacon::serde_helpers::seq")] pub(crate) kzg_proofs: CellKzgProofs, - #[serde(serialize_with = "ethlambda_types::beacon::serde_helpers::ssz_hex_seq::serialize")] + #[serde(with = "ethlambda_types::beacon::serde_helpers::ssz_hex_seq")] pub(crate) blobs: Blobs, } /// Gloas's `SignedExecutionPayloadEnvelopeContents`, what the stateless form of /// `publishExecutionPayloadEnvelope` receives. -#[derive(Debug, Clone, PartialEq, SszEncode, SszDecode)] +#[derive(Debug, Clone, PartialEq, SszEncode, SszDecode, serde::Serialize, serde::Deserialize)] pub(crate) struct GloasSignedEnvelopeContents { pub(crate) signed_execution_payload_envelope: SignedExecutionPayloadEnvelope, + #[serde(with = "ethlambda_types::beacon::serde_helpers::seq")] pub(crate) kzg_proofs: CellKzgProofs, + #[serde(with = "ethlambda_types::beacon::serde_helpers::ssz_hex_seq")] pub(crate) blobs: Blobs, } @@ -756,13 +758,10 @@ async fn post_envelope_waiting( if consensus_version(headers) != Some(ForkName::Gloas) { return ApiError::BadRequest("Eth-Consensus-Version must be gloas").into_response(); } - if !is_ssz(headers) { - return ( - StatusCode::UNSUPPORTED_MEDIA_TYPE, - "envelopes are accepted as application/octet-stream only", - ) - .into_response(); - } + let encoding = match BodyEncoding::from_headers(headers) { + Ok(encoding) => encoding, + Err(err) => return err.into_response(), + }; let blob_data_included = match headers .get("eth-blob-data-included") .and_then(|value| value.to_str().ok()) @@ -775,8 +774,8 @@ async fn post_envelope_waiting( } }; let (signed, supplied) = if blob_data_included { - match GloasSignedEnvelopeContents::from_ssz_bytes(body) { - Ok(contents) => ( + match encoding.decode::(body) { + Some(contents) => ( contents.signed_execution_payload_envelope, Some(( contents @@ -787,7 +786,7 @@ async fn post_envelope_waiting( contents.kzg_proofs.to_vec(), )), ), - Err(_) => { + None => { return ApiError::BadRequest( "the body is not a gloas SignedExecutionPayloadEnvelopeContents", ) @@ -795,9 +794,9 @@ async fn post_envelope_waiting( } } } else { - match SignedExecutionPayloadEnvelope::from_ssz_bytes(body) { - Ok(signed) => (signed, None), - Err(_) => { + match encoding.decode::(body) { + Some(signed) => (signed, None), + None => { return ApiError::BadRequest( "the body is not a gloas SignedExecutionPayloadEnvelope", ) @@ -1093,6 +1092,82 @@ mod tests { (headers, body) } + /// The same as [`envelope_request`] with a JSON body, which is what teku's + /// validator client posts. + fn json_envelope_request(included: Option<&str>, body: Vec) -> (HeaderMap, Vec) { + let (mut headers, body) = envelope_request(included, body); + headers.insert( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ); + (headers, body) + } + + #[tokio::test] + async fn an_envelope_is_accepted_as_json_exactly_as_it_is_as_ssz() { + let state = state_to_build_on(); + let produced = produced_without_blobs(&state); + let signature = sign_block(&state, &produced.block); + let (store, _root, post) = store_with_block(&state, &produced, signature); + let signed = signed_envelope(&post, &produced); + let wait = Duration::from_millis(100); + + let mut published = Vec::new(); + for json in [false, true] { + let network = Arc::new(RecordingNetwork::default()); + let p2p: RpcToP2PRef = network.clone(); + let (headers, body) = if json { + json_envelope_request(Some("false"), serde_json::to_vec(&signed).unwrap()) + } else { + envelope_request(Some("false"), signed.to_ssz()) + }; + let response = post_envelope_waiting( + &store, + &p2p, + &PayloadCache::default(), + &headers, + &body, + wait, + ) + .await; + assert_eq!(response.status(), StatusCode::OK, "json: {json}"); + published.push(network.envelopes.lock().unwrap().clone()); + } + assert_eq!(published[0].len(), 1); + assert_eq!(published[0], published[1]); + + // The same refusals: a forged signature, and a body that is not an envelope. + let network = Arc::new(RecordingNetwork::default()); + let p2p: RpcToP2PRef = network.clone(); + let mut forged = signed.clone(); + forged.signature = + sign_envelope(&post, produced.block.proposer_index + 1, &produced.envelope); + let (headers, body) = + json_envelope_request(Some("false"), serde_json::to_vec(&forged).unwrap()); + let response = post_envelope_waiting( + &store, + &p2p, + &PayloadCache::default(), + &headers, + &body, + wait, + ) + .await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let (headers, body) = json_envelope_request(Some("false"), b"{\"message\": 1}".to_vec()); + let response = post_envelope_waiting( + &store, + &p2p, + &PayloadCache::default(), + &headers, + &body, + wait, + ) + .await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert!(network.envelopes.lock().unwrap().is_empty()); + } + /// A produced block with no blobs, so the envelope needs no columns. fn produced_without_blobs(state: &BeaconState) -> GloasProduced { use ethlambda_state_transition::beacon::gloas_block_production::{ @@ -1242,15 +1317,21 @@ mod tests { let response = post_envelope_waiting(&store, &p2p, &cache, &headers, &body, wait).await; assert_eq!(response.status(), StatusCode::BAD_REQUEST); - // JSON is not accepted. + // Any other content type is a 415 (JSON is accepted, see + // `an_envelope_is_accepted_as_json_exactly_as_it_is_as_ssz`), and SSZ + // bytes labelled JSON are not a JSON envelope. let signed = signed_envelope(&post, &produced); let (mut headers, body) = envelope_request(Some("false"), signed.to_ssz()); + headers.insert(header::CONTENT_TYPE, HeaderValue::from_static("text/plain")); + let response = post_envelope_waiting(&store, &p2p, &cache, &headers, &body, wait).await; + assert_eq!(response.status(), StatusCode::UNSUPPORTED_MEDIA_TYPE); + let (mut headers, body) = envelope_request(Some("false"), signed.to_ssz()); headers.insert( header::CONTENT_TYPE, HeaderValue::from_static("application/json"), ); let response = post_envelope_waiting(&store, &p2p, &cache, &headers, &body, wait).await; - assert_eq!(response.status(), StatusCode::UNSUPPORTED_MEDIA_TYPE); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); // The blob-data header is required. let (headers, body) = envelope_request(None, signed.to_ssz()); @@ -1366,6 +1447,88 @@ mod tests { } } + #[tokio::test] + async fn json_envelope_contents_carry_the_blobs_and_are_checked_like_ssz() { + let WithBlob { + state, + produced, + blob, + proofs, + } = with_blob(); + let signature = sign_block(&state, &produced.block); + let (store, _root, post) = store_with_block(&state, &produced, signature); + let signed = signed_envelope(&post, &produced); + let wait = Duration::from_millis(100); + let contents = |blob: &[u8]| GloasSignedEnvelopeContents { + signed_execution_payload_envelope: signed.clone(), + kzg_proofs: CellKzgProofs::try_from(proofs.clone()).unwrap(), + blobs: blobs_list(vec![blob.to_vec()]).unwrap(), + }; + let send = |network: Arc, contents: GloasSignedEnvelopeContents| { + let store = store.clone(); + async move { + let p2p: RpcToP2PRef = network; + let (headers, body) = + json_envelope_request(Some("true"), serde_json::to_vec(&contents).unwrap()); + post_envelope_waiting( + &store, + &p2p, + &PayloadCache::default(), + &headers, + &body, + wait, + ) + .await + } + }; + + let network = Arc::new(RecordingNetwork::default()); + let response = send(network.clone(), contents(&blob)).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + network.envelopes.lock().unwrap()[0].1.len(), + ethlambda_types::beacon::preset::NUMBER_OF_COLUMNS + ); + + let mut tampered = blob.clone(); + tampered[63] ^= 1; + let rejected = Arc::new(RecordingNetwork::default()); + let response = send(rejected.clone(), contents(&tampered)).await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert!(rejected.envelopes.lock().unwrap().is_empty()); + } + + #[test] + fn the_block_and_envelope_contents_round_trip_through_json() { + let WithBlob { + state, + produced, + blob, + proofs, + } = with_blob(); + let signature = sign_block(&state, &produced.block); + let (_store, _root, post) = store_with_block(&state, &produced, signature); + let signed = signed_envelope(&post, &produced); + let contents = GloasSignedEnvelopeContents { + signed_execution_payload_envelope: signed, + kzg_proofs: CellKzgProofs::try_from(proofs.clone()).unwrap(), + blobs: blobs_list(vec![blob.clone()]).unwrap(), + }; + let back: GloasSignedEnvelopeContents = + serde_json::from_slice(&serde_json::to_vec(&contents).unwrap()).unwrap(); + assert_eq!(back, contents); + + let block_contents = GloasBlockContents { + block: produced.block.clone(), + execution_payload_envelope: produced.envelope.clone(), + kzg_proofs: CellKzgProofs::try_from(proofs).unwrap(), + blobs: blobs_list(vec![blob]).unwrap(), + }; + let back: GloasBlockContents = + serde_json::from_slice(&serde_json::to_vec(&block_contents).unwrap()).unwrap(); + assert_eq!(back, block_contents); + } + #[tokio::test] async fn an_envelope_with_blobs_gossips_every_column_and_a_tampered_blob_is_refused() { let WithBlob { diff --git a/crates/net/rpc/src/beacon/mod.rs b/crates/net/rpc/src/beacon/mod.rs index 2a44821b0..728a20876 100644 --- a/crates/net/rpc/src/beacon/mod.rs +++ b/crates/net/rpc/src/beacon/mod.rs @@ -102,12 +102,48 @@ impl IntoResponse for ApiError { } } +/// How a request body is encoded, from its `Content-Type`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum BodyEncoding { + Json, + Ssz, +} + +impl BodyEncoding { + /// An absent `Content-Type` is read as JSON, which is what every client + /// that predates SSZ submission sends. Anything else is a 415 rather than a + /// guess, so a client that tries SSZ first (prysm) learns this node wants + /// the other. + pub(crate) fn from_headers(headers: &axum::http::HeaderMap) -> Result { + let content_type = headers + .get(axum::http::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(|value| value.split(';').next().unwrap_or("").trim()); + match content_type { + None | Some("application/json") => Ok(Self::Json), + Some(crate::SSZ_CONTENT_TYPE) => Ok(Self::Ssz), + Some(_) => Err(ApiError::UnsupportedMediaType( + "Content-Type must be application/json or application/octet-stream", + )), + } + } + + /// Decode one container in this encoding, or `None` when the body is not + /// one. JSON and SSZ go through the same type, so what follows a decode + /// cannot depend on which one the client chose. + pub(crate) fn decode(self, body: &[u8]) -> Option + where + T: serde::de::DeserializeOwned + libssz::SszDecode, + { + match self { + Self::Json => serde_json::from_slice(body).ok(), + Self::Ssz => T::from_ssz_bytes(body).ok(), + } + } +} + /// Decode the array a batch-submission endpoint takes, as JSON or as the SSZ /// `List[T, ...]` of the same elements, by the request's `Content-Type`. -/// -/// An absent `Content-Type` is read as JSON, which is what every client that -/// predates SSZ submission sends. Anything else is a 415 rather than a guess, -/// so a client that tries SSZ first (prysm) learns this node wants the other. pub(crate) fn decode_list( headers: &axum::http::HeaderMap, body: &[u8], @@ -115,19 +151,12 @@ pub(crate) fn decode_list( where T: serde::de::DeserializeOwned + libssz::SszDecode, { - let content_type = headers - .get(axum::http::header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .map(|value| value.split(';').next().unwrap_or("").trim()); - match content_type { - None | Some("application/json") => { - serde_json::from_slice(body).map_err(|_| ApiError::BadRequest("invalid request body")) + let invalid = || ApiError::BadRequest("invalid request body"); + match BodyEncoding::from_headers(headers)? { + BodyEncoding::Json => serde_json::from_slice(body).map_err(|_| invalid()), + BodyEncoding::Ssz => { + as libssz::SszDecode>::from_ssz_bytes(body).map_err(|_| invalid()) } - Some(crate::SSZ_CONTENT_TYPE) => as libssz::SszDecode>::from_ssz_bytes(body) - .map_err(|_| ApiError::BadRequest("invalid request body")), - Some(_) => Err(ApiError::UnsupportedMediaType( - "Content-Type must be application/json or application/octet-stream", - )), } } diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs index 52be05c62..475c00052 100644 --- a/crates/net/rpc/src/beacon/proposal.rs +++ b/crates/net/rpc/src/beacon/proposal.rs @@ -50,13 +50,13 @@ use ethlambda_types::{ }, primitives::H256, }; -use libssz::{SszDecode as _, SszEncode as _}; +use libssz::SszEncode as _; use libssz_derive::{SszDecode, SszEncode}; use libssz_types::SszList; use serde::Deserialize; use tracing::{info, warn}; -use crate::beacon::{ApiError, validator::FeeRecipients, validator::head}; +use crate::beacon::{ApiError, BodyEncoding, validator::FeeRecipients, validator::head}; use crate::shared::content::{Encoding, ssz_response, with_consensus_version}; /// One KZG proof per cell of every blob, fulu's `kzg_proofs` bound. @@ -69,20 +69,22 @@ pub(crate) type Blobs = SszList Router { ) } -/// `POST /eth/v2/beacon/blocks`, SSZ-encoded `SignedBlockContents`. +/// `POST /eth/v2/beacon/blocks`, `SignedBlockContents` (fulu) or a bare +/// `SignedBeaconBlock` (gloas), as SSZ or JSON by `Content-Type`. /// /// Checked before it goes anywhere: the fork is fulu, it carries no blobs /// (whose data columns this node cannot publish yet, see the module docs), it @@ -117,21 +120,14 @@ async fn post_block( if !matches!(fork, Some(ForkName::Fulu | ForkName::Gloas)) { return ApiError::BadRequest("Eth-Consensus-Version must be fulu or gloas").into_response(); } - let is_ssz = headers - .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.starts_with(crate::SSZ_CONTENT_TYPE)); - if !is_ssz { - return ( - StatusCode::UNSUPPORTED_MEDIA_TYPE, - "blocks are accepted as application/octet-stream only", - ) - .into_response(); - } + let encoding = match BodyEncoding::from_headers(&headers) { + Ok(encoding) => encoding, + Err(err) => return err.into_response(), + }; if fork == Some(ForkName::Gloas) { - return post_gloas_block(&store, &p2p, &body).await; + return post_gloas_block(&store, &p2p, encoding, &body).await; } - let Ok(contents) = FuluSignedBlockContents::from_ssz_bytes(&body) else { + let Some(contents) = encoding.decode::(&body) else { return ApiError::BadRequest("the body is not fulu SignedBlockContents").into_response(); }; if !contents.blobs.is_empty() @@ -176,7 +172,7 @@ async fn post_block( StatusCode::OK.into_response() } -/// The gloas half of `publishBlockV2`: a bare SSZ `SignedBeaconBlock`, since a +/// The gloas half of `publishBlockV2`: a bare `SignedBeaconBlock`, since a /// gloas block carries no payload and no blobs of its own (the envelope and /// the data columns follow through `publishExecutionPayloadEnvelope`). /// @@ -185,8 +181,13 @@ async fn post_block( /// slot's, and the proposer's signature verifies against the parent's state /// advanced to the slot. The advance and the signature are CPU-bound and run /// off the runtime. -async fn post_gloas_block(store: &Store, p2p: &RpcToP2PRef, body: &[u8]) -> Response { - let Ok(signed) = containers::gloas::SignedBeaconBlock::from_ssz_bytes(body) else { +async fn post_gloas_block( + store: &Store, + p2p: &RpcToP2PRef, + encoding: BodyEncoding, + body: &[u8], +) -> Response { + let Some(signed) = encoding.decode::(body) else { return ApiError::BadRequest("the body is not a gloas SignedBeaconBlock").into_response(); }; let block = containers::SignedBeaconBlock::Gloas(signed); @@ -579,6 +580,61 @@ mod tests { assert_eq!(json["message"], "blocks are accepted for fulu slots only"); } + /// A fulu body in either encoding reaches the same check and gets the same + /// answer, here the refusal of a slot the schedule does not place at fulu. + #[tokio::test] + async fn fulu_block_contents_are_accepted_as_json_exactly_as_ssz() { + let (store, _) = gloas_scheduled_store(); + let contents = FuluSignedBlockContents { + signed_block: containers::electra::SignedBeaconBlock { + message: containers::electra::BeaconBlock { + slot: 64, + proposer_index: 0, + parent_root: H256::ZERO, + state_root: H256::ZERO, + body: containers::electra::BeaconBlockBody::empty(), + }, + signature: Default::default(), + }, + kzg_proofs: Default::default(), + blobs: Default::default(), + }; + let back: FuluSignedBlockContents = + serde_json::from_slice(&serde_json::to_vec(&contents).unwrap()).unwrap(); + assert_eq!(back, contents); + + let mut answers = Vec::new(); + for (content_type, body) in [ + (crate::SSZ_CONTENT_TYPE, contents.to_ssz()), + ("application/json", serde_json::to_vec(&contents).unwrap()), + ] { + let network: RpcToP2PRef = Arc::new(RecordingNetwork::default()); + let app = routes().with_state(store.clone()).layer(Extension(network)); + let request = Request::post("/eth/v2/beacon/blocks") + .header("eth-consensus-version", "fulu") + .header(header::CONTENT_TYPE, content_type) + .body(Body::from(body)) + .unwrap(); + answers.push(respond(app, request).await); + } + assert_eq!(answers[0].0, StatusCode::BAD_REQUEST); + assert_eq!(answers[0], answers[1]); + } + + #[tokio::test] + async fn a_block_content_type_other_than_json_or_ssz_is_a_415() { + let (store, _) = gloas_scheduled_store(); + let network: RpcToP2PRef = Arc::new(RecordingNetwork::default()); + let app = routes().with_state(store).layer(Extension(network)); + let request = Request::post("/eth/v2/beacon/blocks") + .header("eth-consensus-version", "fulu") + .header(header::CONTENT_TYPE, "text/plain") + .body(Body::from("{}")) + .unwrap(); + let (status, _) = respond(app, request).await; + assert_eq!(status, StatusCode::UNSUPPORTED_MEDIA_TYPE); + } + #[tokio::test] async fn a_block_over_the_default_body_limit_is_not_a_413() { let (store, _) = gloas_scheduled_store(); @@ -628,6 +684,27 @@ mod tests { assert_eq!(status, StatusCode::OK); assert_eq!(network.blocks.lock().unwrap().len(), 1); + // The same block as JSON is accepted and published identically, and a + // forged one refused. + let post_json = |network: Arc, + signed: &containers::gloas::SignedBeaconBlock| { + let network: RpcToP2PRef = network; + let app = routes().with_state(store.clone()).layer(Extension(network)); + let request = Request::post("/eth/v2/beacon/blocks") + .header("eth-consensus-version", "gloas") + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_vec(signed).unwrap())) + .unwrap(); + respond(app, request) + }; + let via_json = Arc::new(RecordingNetwork::default()); + let (status, _) = post_json(via_json.clone(), &signed).await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + *via_json.blocks.lock().unwrap(), + *network.blocks.lock().unwrap() + ); + // A signature from another validator, and a block whose parent is not held. let rejected = Arc::new(RecordingNetwork::default()); let mut forged = signed.clone(); From dcc325a4e0267defc0f769dff60e93a39c4e2e3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 19:58:12 -0300 Subject: [PATCH 111/132] docs(rpc): block and envelope publication take JSON or SSZ --- docs/rpc.md | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/rpc.md b/docs/rpc.md index fd679a613..17fd4b807 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -254,8 +254,8 @@ surface rather than sitting beside it; a `/lean/v0` path on a beacon node is a | `GET` | `/eth/v3/validator/blocks/{slot}` | SSZ or JSON | An unsigned fulu block built on the head (`produceBlockV3`) | | `POST` | `/eth/v4/validator/blocks/{slot}` | SSZ or JSON | An unsigned self-built gloas block, with its envelope and blobs when asked (`produceBlockV4`) | | `GET` | `/eth/v1/validator/execution_payload_envelopes/{slot}/{beacon_block_root}` | SSZ or JSON | The unsigned envelope `produceBlockV4` built (gloas) | -| `POST` | `/eth/v2/beacon/blocks` | *(status only)* | Gossip and import a signed fulu or gloas block (`publishBlockV2`, SSZ) | -| `POST` | `/eth/v1/beacon/execution_payload_envelopes` | *(status only)* | Gossip a signed envelope and its data columns (gloas, SSZ) | +| `POST` | `/eth/v2/beacon/blocks` | *(status only)* | Gossip and import a signed fulu or gloas block (`publishBlockV2`; JSON or SSZ body) | +| `POST` | `/eth/v1/beacon/execution_payload_envelopes` | *(status only)* | Gossip a signed envelope and its data columns (gloas; JSON or SSZ body) | | `POST` | `/eth/v1/validator/prepare_beacon_proposer` | *(status only)* | Acknowledged, not acted on (see below) | ### Validator endpoints @@ -334,9 +334,9 @@ the chain actor writes, so no request waits on the actor. variable-size list uses, which is what prysm sends for aggregates and payload votes, with a JSON retry only on a `415`, and what nimbus can send for attestations. Any other content type is a `415`, and a body that does not - decode in the type's encoding a `400`. A block, an envelope and the other - validator-client submissions keep their own rules (SSZ only on `blocks` and - the envelope; JSON on the rest). + decode in the type's encoding a `400`. `POST beacon/blocks` and + `POST beacon/execution_payload_envelopes` choose the same way (below); the + other validator-client submissions are JSON. - **The attestation pool** holds, the best-covered per data root and committee: votes from `pool/attestations` and the aggregator subnets, aggregates from `aggregate_and_proofs`, and every electra gossip aggregate @@ -362,8 +362,11 @@ the chain actor writes, so no request waits on the actor. blobs. A slot the schedule does not place at fulu (gloas included) is a `400`, checked before the execution client; `produceBlockV3` is the fulu endpoint, and a gloas slot goes to `produceBlockV4`. -- **`POST beacon/blocks`** takes SSZ and the `Eth-Consensus-Version` header - (`fulu` or `gloas`, else `400`; a non-SSZ content type is a `415`). Fulu: a +- **`POST beacon/blocks`** takes JSON or SSZ, by `Content-Type` as above, and + the `Eth-Consensus-Version` header (`fulu` or `gloas`, else `400`; any other + content type is a `415`). The JSON is the specification's: lodestar's and + nimbus's validator clients post it, teku's falls back to it, and lighthouse + and prysm post SSZ. Both reach the same checks after decoding. Fulu: a `SignedBlockContents`; the node checks the block is after the head and its proposer signature, then gossips it on `beacon_block` and hands it to the chain actor to import. A slot the schedule does not place at fulu is a `400`, @@ -423,8 +426,11 @@ serves the cached unsigned envelope, for a client that asked for the block with root. **`POST /eth/v1/beacon/execution_payload_envelopes`** takes the signed envelope, -SSZ only (`415` otherwise), with `Eth-Consensus-Version: gloas` and -`Eth-Blob-Data-Included` (both required, else `400`): +as JSON or SSZ by `Content-Type` (any other is a `415`; teku's validator client +posts JSON, nimbus posts the bare envelope as JSON and the contents as SSZ, +lighthouse posts SSZ, prysm posts SSZ and retries as JSON on a `415`), with +`Eth-Consensus-Version: gloas` and `Eth-Blob-Data-Included` (both required, else +`400`): - `true`: the body is `SignedExecutionPayloadEnvelopeContents` (the signed envelope, `kzg_proofs`, `blobs`). From a9ceb4c778bb262db98f6b71fdfe2aa1672587ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:40:21 -0300 Subject: [PATCH 112/132] fix(types): quote Builder integers and accept it, and proposer preferences, from JSON `Builder` serialized `version`, `balance`, `deposit_epoch` and `withdrawable_epoch` as bare numbers, which the Beacon API wants quoted (this also affected the gloas state's JSON). Give it the same `quoted_or_bare` adapters the other containers use, and `Deserialize`, along with the proposer preferences containers a builder-market endpoint takes in a request body. Round-trip tests cover the bid, the preferences and the builder, and the example bid and preferences events from beacon-APIs' event stream parse. --- .../types/src/beacon/containers/gloas.rs | 37 ++++++++- .../types/src/beacon/containers/json_tests.rs | 81 +++++++++++++++++++ 2 files changed, 115 insertions(+), 3 deletions(-) diff --git a/crates/common/types/src/beacon/containers/gloas.rs b/crates/common/types/src/beacon/containers/gloas.rs index 09838fa0f..8284480a6 100644 --- a/crates/common/types/src/beacon/containers/gloas.rs +++ b/crates/common/types/src/beacon/containers/gloas.rs @@ -180,23 +180,36 @@ pub type CellsBitList = ProgressiveBitlist; /// A registered builder's record in the builder registry (EIP-7732), the /// builder-side counterpart of [`super::shared::Validator`]. #[derive( - Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] pub struct Builder { pub pubkey: BlsPubkey, /// Which shape this record is in. Only [`crate::beacon::constants::PAYLOAD_BUILDER_VERSION`] /// exists today; the field exists so a future format change has /// somewhere to record it. + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub version: u8, pub execution_address: ExecutionAddress, + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub balance: Gwei, /// The epoch this builder's deposit was placed, which /// `is_active_builder` compares against the finalized checkpoint before /// treating the builder as eligible. + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub deposit_epoch: Epoch, /// `FAR_FUTURE_EPOCH` until this builder initiates an exit, the same /// sentinel convention [`super::shared::Validator::withdrawable_epoch`] /// uses. + #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")] pub withdrawable_epoch: Epoch, } @@ -1115,7 +1128,16 @@ pub struct PartialDataColumnGroupID { /// A proposer's broadcast, ahead of its slot, of the fee recipient and gas /// limit it wants a builder's bid to target (EIP-7732 p2p-interface.md). #[derive( - Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] pub struct ProposerPreferences { /// The root of the beacon state the proposer duty this message announces @@ -1132,7 +1154,16 @@ pub struct ProposerPreferences { } #[derive( - Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot, + Debug, + Clone, + Default, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + SszEncode, + SszDecode, + HashTreeRoot, )] pub struct SignedProposerPreferences { pub message: ProposerPreferences, diff --git a/crates/common/types/src/beacon/containers/json_tests.rs b/crates/common/types/src/beacon/containers/json_tests.rs index 62c6ca3e5..5506897b8 100644 --- a/crates/common/types/src/beacon/containers/json_tests.rs +++ b/crates/common/types/src/beacon/containers/json_tests.rs @@ -403,3 +403,84 @@ fn an_oversized_list_is_refused_rather_than_truncated() { ]); assert!(serde_json::from_value::(json).is_err()); } + +fn bid() -> gloas::SignedExecutionPayloadBid { + gloas::SignedExecutionPayloadBid { + message: gloas::ExecutionPayloadBid { + parent_block_hash: [1; 32].into(), + parent_block_root: [2; 32].into(), + block_hash: [3; 32].into(), + prev_randao: [4; 32].into(), + fee_recipient: [5; 20].into(), + gas_limit: 30_000_000, + builder_index: 7, + slot: 33, + value: 8, + execution_payment: 9, + execution_requests_root: [7; 32].into(), + blob_kzg_commitments: vec![KzgCommitment([6; 48])].try_into().unwrap(), + }, + signature: signature(3), + } +} + +#[test] +fn a_bid_round_trips_through_json() { + round_trip(&bid()); +} + +#[test] +fn proposer_preferences_round_trip_through_json() { + round_trip(&gloas::SignedProposerPreferences { + message: gloas::ProposerPreferences { + dependent_root: [1; 32].into(), + proposal_slot: 32, + validator_index: 123, + fee_recipient: [5; 20].into(), + target_gas_limit: 60_000_000, + }, + signature: signature(4), + }); +} + +#[test] +fn a_builder_round_trips_and_quotes_its_integers() { + let builder = gloas::Builder { + pubkey: pubkey(1), + version: 3, + execution_address: [5; 20].into(), + balance: 32_000_000_000, + deposit_epoch: 4, + withdrawable_epoch: u64::MAX, + }; + round_trip(&builder); + let json = serde_json::to_value(&builder).unwrap(); + for field in ["version", "balance", "deposit_epoch", "withdrawable_epoch"] { + assert!(json[field].is_string(), "{field} must be quoted: {json}"); + } +} + +/// The example `execution_payload_bid` and `proposer_preferences` events in +/// beacon-APIs' event stream, byte for byte. +#[test] +fn the_beacon_apis_example_messages_parse() { + let bid = r#"{"message": {"parent_block_hash": "0x9a2fefd2fdb57f74993c7780ea5b9030d2897b615b89f808011ca5aebed54eaf", "parent_block_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2", "block_hash": "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", "prev_randao": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2", "fee_recipient": "0x0000000000000000000000000000000000000000", "gas_limit": "30000000", "builder_index": "42", "slot": "10", "value": "1000000000", "execution_payment": "0", "blob_kzg_commitments": ["0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2"], "execution_requests_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"}, "signature": "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"}"#; + let parsed: gloas::SignedExecutionPayloadBid = serde_json::from_str(bid).unwrap(); + assert_eq!(parsed.message.builder_index, 42); + assert_eq!(parsed.message.slot, 10); + assert_eq!(parsed.message.value, 1_000_000_000); + assert_eq!(parsed.message.blob_kzg_commitments.len(), 1); + + let preferences = r#"{"message": {"dependent_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2", "proposal_slot": "32", "validator_index": "123", "fee_recipient": "0x0000000000000000000000000000000000000000", "target_gas_limit": "60000000"}, "signature": "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"}"#; + let parsed: gloas::SignedProposerPreferences = serde_json::from_str(preferences).unwrap(); + assert_eq!(parsed.message.proposal_slot, 32); + assert_eq!(parsed.message.validator_index, 123); + assert_eq!(parsed.message.target_gas_limit, 60_000_000); +} + +#[test] +fn signed_proposer_preferences_are_a_fixed_172_bytes() { + use libssz::SszEncode as _; + let bytes = gloas::SignedProposerPreferences::default().to_ssz(); + assert_eq!(bytes.len(), 172); +} From 1e9b6e476ea439348561b98ab3e2f84858898270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:40:28 -0300 Subject: [PATCH 113/132] feat(beacon): stub the gloas builder market so its three halves can land in parallel Adds the shared `BuilderMarket`, the `execution_payload_bid` and `proposer_preferences` gossip rule modules, the p2p triage, verdict and publish plumbing, the `RpcToP2P` publish methods, and the Beacon API route modules, all with the final signatures and placeholder bodies: every rule answers `Ignore(NoConsumer)`, the market holds nothing, and no route is registered yet. Splitting the wiring from the logic lets the rules and market, the p2p handling and the Beacon API be filled in on separate branches without touching each other's files. New reason variants, `Validated` arms and `RpcToP2P` methods sit beside the envelope ones, and everything else is a new file, so a parallel sync committee branch merges cleanly. --- bin/ethlambda/src/main.rs | 6 + .../src/beacon/builder_market.rs | 130 ++++++++++++++++++ .../src/beacon/gloas_block_production.rs | 36 +++++ .../beacon/gossip/execution_payload_bid.rs | 92 +++++++++++++ .../state_transition/src/beacon/gossip/mod.rs | 62 +++++++++ .../src/beacon/gossip/proposer_preferences.rs | 79 +++++++++++ .../state_transition/src/beacon/mod.rs | 1 + crates/net/api/src/lib.rs | 18 ++- crates/net/p2p/src/beacon/builder_market.rs | 57 ++++++++ crates/net/p2p/src/beacon/mod.rs | 1 + crates/net/p2p/src/beacon/topics.rs | 4 + crates/net/p2p/src/beacon/verdict.rs | 35 ++++- crates/net/p2p/src/gossipsub/handler.rs | 4 + crates/net/p2p/src/gossipsub/mod.rs | 2 + crates/net/p2p/src/lib.rs | 42 +++++- crates/net/p2p/src/req_resp/handlers.rs | 4 + crates/net/rpc/src/beacon/bid_selection.rs | 45 ++++++ crates/net/rpc/src/beacon/bids.rs | 13 ++ crates/net/rpc/src/beacon/builder_config.rs | 53 +++++++ crates/net/rpc/src/beacon/builders.rs | 12 ++ crates/net/rpc/src/beacon/mod.rs | 8 ++ .../rpc/src/beacon/proposer_preferences.rs | 13 ++ crates/net/rpc/src/lib.rs | 29 +++- 23 files changed, 739 insertions(+), 7 deletions(-) create mode 100644 crates/blockchain/state_transition/src/beacon/builder_market.rs create mode 100644 crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs create mode 100644 crates/blockchain/state_transition/src/beacon/gossip/proposer_preferences.rs create mode 100644 crates/net/p2p/src/beacon/builder_market.rs create mode 100644 crates/net/rpc/src/beacon/bid_selection.rs create mode 100644 crates/net/rpc/src/beacon/bids.rs create mode 100644 crates/net/rpc/src/beacon/builder_config.rs create mode 100644 crates/net/rpc/src/beacon/builders.rs create mode 100644 crates/net/rpc/src/beacon/proposer_preferences.rs diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index c86b692ad..475c22325 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -722,6 +722,10 @@ async fn run_node(options: Options) -> eyre::Result<()> { // by block production and `GET .../pool/payload_attestations`. let payload_attestation_pool = ethlambda_state_transition::beacon::payload_attestation_pool::SharedPayloadAttestationPool::default(); + // Bids, proposer preferences and known payloads: filled by gossip and the + // Beacon API's bid and preferences endpoints, read by block production. + let builder_market = + ethlambda_state_transition::beacon::builder_market::SharedBuilderMarket::default(); let p2p = P2P::spawn( built, setup.store.clone(), @@ -729,6 +733,7 @@ async fn run_node(options: Options) -> eyre::Result<()> { discovery, attestation_pool.clone(), payload_attestation_pool.clone(), + builder_market.clone(), ) .await .wrap_err("failed to start discv5 discovery")?; @@ -770,6 +775,7 @@ async fn run_node(options: Options) -> eyre::Result<()> { p2p: rpc_p2p, attestation_pool: attestation_pool.clone(), payload_attestation_pool: payload_attestation_pool.clone(), + builder_market: builder_market.clone(), custody_columns: rpc_custody_columns, engine: rpc_engine, }, diff --git a/crates/blockchain/state_transition/src/beacon/builder_market.rs b/crates/blockchain/state_transition/src/beacon/builder_market.rs new file mode 100644 index 000000000..a39c79736 --- /dev/null +++ b/crates/blockchain/state_transition/src/beacon/builder_market.rs @@ -0,0 +1,130 @@ +//! The shared state behind the gloas builder market: bids seen on +//! `execution_payload_bid` (or posted to the Beacon API) and pooled for block +//! production, the proposer preferences those bids are judged against, and the +//! execution payloads gossip has revealed. +//! +//! One [`SharedBuilderMarket`] exists per node. p2p validates against it and +//! the Beacon API reads it, like [`super::payload_attestation_pool`]. Gossip's +//! stateful checks run on blocking threads, so none of this can be owned by the +//! chain actor. +//! +//! The method signatures are the contract between the gossip rules, p2p and the +//! Beacon API; the bodies are placeholders until they are filled. + +use std::{ + num::NonZeroUsize, + sync::{Arc, Mutex}, +}; + +use lru::LruCache; + +use super::containers::gloas; +use super::gossip::IgnoreReason; +use super::primitives::{BlsPubkey, ExecutionAddress, ExecutionBlockHash, Root, Slot}; + +/// Exactly one per node. +pub type SharedBuilderMarket = Arc; + +/// Bids pooled per `(slot, parent hash, parent root)`, top values kept. +pub const MAX_BIDS_PER_PARENT: usize = 16; +/// A full slot refuses new keys: `record_bid` answers `false`. +pub const MAX_SEEN_BID_KEYS_PER_SLOT: usize = 4096; +/// Over the cap, the lowest `proposal_slot` is dropped first. +pub const MAX_PREFERENCES: usize = 1024; +/// Known payloads, evicted least recently used first, by block hash. +pub const KNOWN_PAYLOADS_CAPACITY: NonZeroUsize = NonZeroUsize::new(256).unwrap(); + +/// What gossip learned about an execution payload from its envelope. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct KnownPayload { + pub gas_limit: u64, + /// The block whose envelope revealed it. + pub beacon_block_root: Root, + /// `(pubkey, source_address)` of every builder exit request it carries. + pub builder_exits: Vec<(BlsPubkey, ExecutionAddress)>, +} + +#[derive(Debug)] +pub struct BuilderMarket { + // Filled by Agent A: the bid pool and the preferences cache join this. + #[allow(dead_code)] + payloads: Mutex>, +} + +impl Default for BuilderMarket { + fn default() -> Self { + Self { + payloads: Mutex::new(LruCache::new(KNOWN_PAYLOADS_CAPACITY)), + } + } +} + +#[allow(dead_code, unused_variables)] // filled by Agent A +impl BuilderMarket { + // Bids: seen.execution_payload_bids + seen.best_execution_payload_bid + pool. + + /// The spec's two seen rules, in order: `(slot, parent_hash, parent_root, + /// builder)` recorded -> `AlreadySeen`; value <= best for `(slot, + /// parent_hash, parent_root)` -> `NotHighestBid`. + pub fn check_bid_seen(&self, bid: &gloas::ExecutionPayloadBid) -> Result<(), IgnoreReason> { + Ok(()) + } + + /// Re-runs [`Self::check_bid_seen`] under the lock. If it passes, records + /// both seen keys and pools the bid. `false` = not recorded (race, or the + /// per-slot key cap). Prunes slots below `bid.slot - 1`. + pub fn record_bid(&self, signed: gloas::SignedExecutionPayloadBid) -> bool { + false + } + + /// The identical message and signature is pooled (API idempotency). + pub fn contains_bid(&self, signed: &gloas::SignedExecutionPayloadBid) -> bool { + false + } + + /// Pooled bids for the key: value descending, then builder index ascending. + pub fn bids_for( + &self, + slot: Slot, + parent_block_root: Root, + parent_block_hash: ExecutionBlockHash, + ) -> Vec { + Vec::new() + } + + pub fn has_bids_for_slot(&self, slot: Slot) -> bool { + false + } + + pub fn prune_bids_before(&self, slot: Slot) {} + + // Proposer preferences: seen.proposer_preferences. + + pub fn preferences( + &self, + proposal_slot: Slot, + dependent_root: Root, + ) -> Option { + None + } + + /// The first valid preferences per key win. `false` if one is held. Prunes + /// `proposal_slot < current_slot`. + pub fn record_preferences( + &self, + signed: gloas::SignedProposerPreferences, + current_slot: Slot, + ) -> bool { + false + } + + pub fn prune_preferences_before(&self, slot: Slot) {} + + // Known payloads: seen.execution_payloads. + + pub fn record_execution_payload(&self, envelope: &gloas::ExecutionPayloadEnvelope) {} + + pub fn known_payload(&self, block_hash: ExecutionBlockHash) -> Option { + None + } +} diff --git a/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs b/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs index a9800cba9..6b3af3aa2 100644 --- a/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs +++ b/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs @@ -347,6 +347,42 @@ pub struct GloasBlockInputs { pub execution_requests: ExecutionRequests, } +/// What a gloas block body carries when it commits to another builder's bid +/// rather than to a payload this node built. +#[derive(Debug, Clone)] +pub struct GloasBidBlockInputs { + pub randao_reveal: BlsSignature, + pub graffiti: Bytes32, + pub attestations: Vec, + pub payload_attestations: Vec, + pub parent_execution_requests: ExecutionRequests, + pub signed_bid: SignedExecutionPayloadBid, +} + +/// The unsigned block for `state.slot()` committing to `inputs.signed_bid`. +/// `process_block` on a copy fills `state_root` and enforces +/// `process_execution_payload_bid` (active, cover, signature, slot, parent +/// hash and root, randao). No envelope: the builder reveals it. +#[allow(dead_code)] // filled by Agent A +pub fn assemble_gloas_block_on_bid( + _state: &BeaconState, + _inputs: GloasBidBlockInputs, + _config: &Config, +) -> Result { + Err(Error::SpecAssert("unimplemented")) +} + +/// Cheap pre-filter on the state advanced to the slot: whether the bid could +/// be packed into a block on `state`. The signature is not checked here. +#[allow(dead_code)] // filled by Agent A +pub fn bid_is_includable( + _state: &BeaconState, + _signed_bid: &SignedExecutionPayloadBid, + _parent_requests: &ExecutionRequests, +) -> bool { + false +} + /// A produced block and the unsigned envelope that reveals its payload. #[derive(Debug, Clone)] pub struct GloasProduced { diff --git a/crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs b/crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs new file mode 100644 index 000000000..9cda084b6 --- /dev/null +++ b/crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs @@ -0,0 +1,92 @@ +//! Gossip validation for the gloas `execution_payload_bid` topic: a builder's +//! `SignedExecutionPayloadBid`, the commitment a proposer may choose in place +//! of building its own payload. +//! +//! Split like [`super::envelope`]: [`cheap_checks`] reads only the message, the +//! market's seen state and the clock, so the p2p actor runs it inline; +//! [`stateful_checks`] reads cached states and verifies the signature, so it +//! runs on a blocking thread. Never queues: every "MAY be queued" is IGNORE. +//! +//! Stubs until filled: every rule answers `Ignore(NoConsumer)`. + +use std::sync::Arc; + +use super::{IgnoreReason, Outcome}; +use crate::beacon::builder_market::BuilderMarket; +use crate::beacon::config::Config; +use crate::beacon::containers::{BeaconState, gloas}; +use crate::beacon::fork_choice::Store; +use crate::beacon::primitives::{Epoch, Root, Slot}; + +/// Gloas p2p preset: the largest decompressed `SignedExecutionPayloadBid`. +pub const MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE: usize = 196_932; + +/// The spec's `is_gas_limit_target_compatible`. +/// `max_diff = (parent / 1024).saturating_sub(1)`, `min = parent - max_diff`, +/// `max = parent.saturating_add(max_diff)`. +#[allow(dead_code, unused_variables)] // filled by Agent A +pub fn is_gas_limit_target_compatible( + parent_gas_limit: u64, + gas_limit: u64, + target_gas_limit: u64, +) -> bool { + false +} + +/// `is_current_slot(slot) || slot.checked_sub(1).is_some_and(is_current_slot)`. +#[allow(dead_code, unused_variables)] // filled by Agent A +pub(crate) fn is_current_or_next_slot(config: &Config, slot: Slot, now_ms: u64) -> bool { + false +} + +#[allow(unused_variables)] // filled by Agent A +pub fn cheap_checks( + market: &BuilderMarket, + store: &Store, + signed: &gloas::SignedExecutionPayloadBid, + now_ms: u64, +) -> Result<(), Outcome> { + Err(Outcome::Ignore(IgnoreReason::NoConsumer)) +} + +#[allow(unused_variables)] // filled by Agent A +pub fn stateful_checks( + store: &Store, + market: &BuilderMarket, + signed: &gloas::SignedExecutionPayloadBid, +) -> Outcome { + Outcome::Ignore(IgnoreReason::NoConsumer) +} + +/// Both halves. The caller records the bid on `Accept`. +pub fn validate( + market: &BuilderMarket, + store: &Store, + signed: &gloas::SignedExecutionPayloadBid, + now_ms: u64, +) -> Outcome { + if let Err(outcome) = cheap_checks(market, store, signed, now_ms) { + return outcome; + } + stateful_checks(store, market, signed) +} + +/// The spec's `is_bid_compatible_with_head`, against the recorded head. +#[allow(dead_code, unused_variables)] // filled by Agent A +pub fn is_bid_compatible_with_head( + store: &Store, + bid: &gloas::ExecutionPayloadBid, +) -> Result { + Err(Outcome::Ignore(IgnoreReason::NoConsumer)) +} + +/// Cached-only: a `CheckpointState{epoch, root}` hit; else the cached +/// `BlockState(root)` advanced to the epoch start and cached. `None` = miss. +#[allow(dead_code, unused_variables)] // filled by Agent A +pub(crate) fn cached_checkpoint_state( + store: &Store, + epoch: Epoch, + root: Root, +) -> Option> { + None +} diff --git a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs index 187f612b7..748df0ce8 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs @@ -11,7 +11,9 @@ pub mod attestation; pub mod block; pub mod column; pub mod envelope; +pub mod execution_payload_bid; pub mod payload_attestation; +pub mod proposer_preferences; #[cfg(test)] pub(crate) mod test_support; @@ -138,6 +140,32 @@ pub enum IgnoreReason { FinalizedNotAncestor, /// An ancestor lies outside what the state's `block_roots` can answer. AncestryUnknown, + /// A gloas bid or proposer preferences names a slot before the fork. + PreGloasSlot, + /// A bid's slot is neither the current nor the next slot. + NotCurrentOrNextSlot, + /// A bid's value does not beat the best already seen for its slot and parent. + NotHighestBid, + /// A bid or preferences name a slot beyond the proposer lookahead. + BeyondLookahead, + /// No proposer preferences are known for a bid's slot and dependent root. + PreferencesUnseen, + /// A bid's fee recipient is not the one in the proposer's preferences. + FeeRecipientMismatch, + /// A bid's parent block hash is not a known execution payload. + ParentPayloadUnknown, + /// A bid's gas limit cannot reach the preferences' target from the parent's. + GasLimitIncompatible, + /// A bid is not compatible with this node's head branch. + NotOnHeadBranch, + /// A bid's builder cannot cover its value. + BuilderCannotCover, + /// A bid's builder exits in the parent payload. + BuilderMayExit, + /// Proposer preferences arrived after their proposal slot began. + SlotStarted, + /// Proposer preferences name a dependent root no chain here can have. + ImpossibleDependentRoot, /// A gloas block builds on its parent's full payload branch, but the /// parent's envelope has not been seen and verified (the specification /// lets it be queued until it is). @@ -174,6 +202,19 @@ impl IgnoreReason { Self::StateUnavailable => "state_unavailable", Self::FinalizedNotAncestor => "finalized_not_ancestor", Self::AncestryUnknown => "ancestry_unknown", + Self::PreGloasSlot => "pre_gloas_slot", + Self::NotCurrentOrNextSlot => "not_current_or_next_slot", + Self::NotHighestBid => "not_highest_bid", + Self::BeyondLookahead => "beyond_lookahead", + Self::PreferencesUnseen => "preferences_unseen", + Self::FeeRecipientMismatch => "fee_recipient_mismatch", + Self::ParentPayloadUnknown => "parent_payload_unknown", + Self::GasLimitIncompatible => "gas_limit_incompatible", + Self::NotOnHeadBranch => "not_on_head_branch", + Self::BuilderCannotCover => "builder_cannot_cover", + Self::BuilderMayExit => "builder_may_exit", + Self::SlotStarted => "slot_started", + Self::ImpossibleDependentRoot => "impossible_dependent_root", Self::ParentPayloadUnverified => "parent_payload_unverified", Self::PayloadEnvelopeUnseen => "payload_envelope_unseen", Self::PayloadOptimistic => "payload_optimistic", @@ -231,6 +272,20 @@ pub enum RejectReason { AggregateSignature, /// The target is not the voted block's ancestor at the target epoch. TargetNotAncestor, + /// A bid promises a payment outside the bid value. + ExecutionPaymentNonZero, + /// A bid's block hash is its parent's. + BlockHashEqualsParent, + /// A bid's `prev_randao` is not the parent state's current mix. + PrevRandao, + /// A bid's builder index is not in the registry. + UnknownBuilder, + /// A bid's builder is not a payload builder. + NotPayloadBuilder, + /// A bid's builder is not active. + InactiveBuilder, + /// Preferences' dependent block is later than the shuffling's dependent slot. + DependentRootTooLate, /// A gloas block body (or its parent execution requests) carries more of /// an operation than its limit, or any deposit. OperationLimit, @@ -291,6 +346,13 @@ impl RejectReason { Self::AggregatorSignature => "aggregator_signature", Self::AggregateSignature => "aggregate_signature", Self::TargetNotAncestor => "target_not_ancestor", + Self::ExecutionPaymentNonZero => "execution_payment_nonzero", + Self::BlockHashEqualsParent => "block_hash_equals_parent", + Self::PrevRandao => "prev_randao", + Self::UnknownBuilder => "unknown_builder", + Self::NotPayloadBuilder => "not_payload_builder", + Self::InactiveBuilder => "inactive_builder", + Self::DependentRootTooLate => "dependent_root_too_late", Self::OperationLimit => "operation_limit", Self::BidParentMismatch => "bid_parent_mismatch", Self::BidNotOnParentHead => "bid_not_on_parent_head", diff --git a/crates/blockchain/state_transition/src/beacon/gossip/proposer_preferences.rs b/crates/blockchain/state_transition/src/beacon/gossip/proposer_preferences.rs new file mode 100644 index 000000000..50c73650e --- /dev/null +++ b/crates/blockchain/state_transition/src/beacon/gossip/proposer_preferences.rs @@ -0,0 +1,79 @@ +//! Gossip validation for the gloas `proposer_preferences` topic: a proposer's +//! `SignedProposerPreferences` (fee recipient and gas target) for a slot, which +//! bids on that slot are judged against. +//! +//! Split like [`super::envelope`]: [`cheap_checks`] inline in the p2p actor, +//! [`stateful_checks`] on a blocking thread, reading cached states only. +//! +//! Stubs until filled: every rule answers `Ignore(NoConsumer)`. + +use super::{IgnoreReason, Outcome}; +use crate::beacon::builder_market::BuilderMarket; +use crate::beacon::config::Config; +use crate::beacon::containers::{BeaconState, gloas}; +use crate::beacon::fork_choice::Store; +use crate::beacon::primitives::{Domain, Epoch, Root, Slot}; + +#[allow(unused_variables)] // filled by Agent A +pub fn cheap_checks( + market: &BuilderMarket, + store: &Store, + signed: &gloas::SignedProposerPreferences, + now_ms: u64, +) -> Result<(), Outcome> { + Err(Outcome::Ignore(IgnoreReason::NoConsumer)) +} + +#[allow(unused_variables)] // filled by Agent A +pub fn stateful_checks(store: &Store, signed: &gloas::SignedProposerPreferences) -> Outcome { + Outcome::Ignore(IgnoreReason::NoConsumer) +} + +/// Both halves. The caller records the preferences on `Accept`. +pub fn validate( + market: &BuilderMarket, + store: &Store, + signed: &gloas::SignedProposerPreferences, + now_ms: u64, +) -> Outcome { + if let Err(outcome) = cheap_checks(market, store, signed, now_ms) { + return outcome; + } + stateful_checks(store, signed) +} + +/// The spec's `is_valid_dependent_root`: `root == store.head()`, or some block +/// in the index has `parent_root == root` and `slot > dependent_slot`. +#[allow(dead_code, unused_variables)] // filled by Agent A +pub fn is_valid_dependent_root(store: &Store, root: Root, dependent_slot: Slot) -> bool { + false +} + +/// `ancestor_at(state, state_block_root, compute_shuffling_dependent_slot( +/// epoch(proposal_slot)))`. Used by `produceBlockV4` and the validator client. +#[allow(dead_code, unused_variables)] // filled by Agent A +pub fn dependent_root_at( + state: &BeaconState, + state_block_root: Root, + proposal_slot: Slot, +) -> Option { + None +} + +/// The distinct candidate signing domains, tried in order: `get_domain( +/// lookahead_state, DOMAIN_PROPOSER_PREFERENCES, Some(P))`; the schedule's fork +/// version at `P - MIN_SEED_LOOKAHEAD` (saturating); the schedule's at `P`. +#[allow(dead_code, unused_variables)] // filled by Agent A +pub fn proposer_preferences_domains( + lookahead_state: &BeaconState, + config: &Config, + proposal_epoch: Epoch, +) -> Vec { + Vec::new() +} + +/// `now > slot_start + 500 ms`. +#[allow(dead_code, unused_variables)] // filled by Agent A +pub(crate) fn is_past_slot(config: &Config, slot: Slot, now_ms: u64) -> bool { + false +} diff --git a/crates/blockchain/state_transition/src/beacon/mod.rs b/crates/blockchain/state_transition/src/beacon/mod.rs index 1a4471fa6..6a5457c81 100644 --- a/crates/blockchain/state_transition/src/beacon/mod.rs +++ b/crates/blockchain/state_transition/src/beacon/mod.rs @@ -72,6 +72,7 @@ pub mod aggregate; pub mod attestation_pool; pub mod block_production; pub mod bls; +pub mod builder_market; pub mod das; pub mod fork_choice; pub mod genesis; diff --git a/crates/net/api/src/lib.rs b/crates/net/api/src/lib.rs index 7c3c3305d..2d961591d 100644 --- a/crates/net/api/src/lib.rs +++ b/crates/net/api/src/lib.rs @@ -5,7 +5,10 @@ use ethlambda_types::{ beacon::containers::{ DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock, electra::SingleAttestation, - gloas::{PayloadAttestationMessage, SignedExecutionPayloadEnvelope}, + gloas::{ + PayloadAttestationMessage, SignedExecutionPayloadBid, SignedExecutionPayloadEnvelope, + SignedProposerPreferences, + }, }, beacon::primitives::ValidatorIndex, block::SignedBlock, @@ -366,6 +369,19 @@ pub trait RpcToP2P: Send + Sync { envelope: Box, sidecars: Vec, ) -> Result<(), ActorError>; + /// Gossip a signed execution payload bid on `execution_payload_bid`. The + /// caller ran the gossip rules and recorded it in the shared + /// `BuilderMarket` (seen keys and pool), so the p2p actor only publishes. + fn publish_execution_payload_bid( + &self, + bid: SignedExecutionPayloadBid, + ) -> Result<(), ActorError>; + /// Gossip signed proposer preferences on `proposer_preferences`, under the + /// proposal slot's digest. Validated and cached by the caller. + fn publish_proposer_preferences( + &self, + preferences: SignedProposerPreferences, + ) -> Result<(), ActorError>; /// Gossip a payload timeliness committee member's vote on /// `payload_attestation_message` and hand it to the chain actor. Checked by /// the caller as above. diff --git a/crates/net/p2p/src/beacon/builder_market.rs b/crates/net/p2p/src/beacon/builder_market.rs new file mode 100644 index 000000000..44e78bdc6 --- /dev/null +++ b/crates/net/p2p/src/beacon/builder_market.rs @@ -0,0 +1,57 @@ +//! Gloas builder market gossip: the `execution_payload_bid` and +//! `proposer_preferences` topics. +//! +//! Stubs until filled. Neither message type ever reaches the chain actor: the +//! rules live in `ethlambda_state_transition::beacon::gossip::{ +//! execution_payload_bid, proposer_preferences}`, and what they accept is +//! recorded in the node's shared `BuilderMarket` by the verdict. + +use ethlambda_state_transition::beacon::gossip::{IgnoreReason, Outcome}; +use ethlambda_types::beacon::containers::gloas::{ + SignedExecutionPayloadBid, SignedProposerPreferences, +}; +use ethlambda_types::beacon::primitives::Slot; +use ethlambda_types::time::unix_now_ms; + +use crate::P2PServer; +use crate::beacon::verdict::Dispatch; + +/// Size cap -> `Reject(Malformed)`; decode -> `Reject(Decode)`; +/// `inc_beacon_gossip(KIND, "decoded")`; +/// `gossip::execution_payload_bid::cheap_checks`; then +/// `Validate(Validated::ExecutionPayloadBid { .. })`. +#[allow(dead_code, unused_variables)] // filled by Agent B +pub(crate) fn triage_execution_payload_bid(server: &P2PServer, payload: &[u8]) -> Dispatch { + Dispatch::Report(Outcome::Ignore(IgnoreReason::NoConsumer)) +} + +/// As [`triage_execution_payload_bid`], for `proposer_preferences`. +#[allow(dead_code, unused_variables)] // filled by Agent B +pub(crate) fn triage_proposer_preferences(server: &P2PServer, payload: &[u8]) -> Dispatch { + Dispatch::Report(Outcome::Ignore(IgnoreReason::NoConsumer)) +} + +/// Publish on `publish_digest(bid.slot)` / `execution_payload_bid`. +/// Precondition: the caller already recorded it in the market. +#[allow(dead_code, unused_variables)] // filled by Agent B +pub(crate) fn publish_execution_payload_bid( + server: &mut P2PServer, + bid: SignedExecutionPayloadBid, +) { +} + +/// Publish on `publish_digest(proposal_slot)`: during the epoch before gloas +/// this is the gloas digest, which is held. +#[allow(dead_code, unused_variables)] // filled by Agent B +pub(crate) fn publish_proposer_preferences( + server: &mut P2PServer, + preferences: SignedProposerPreferences, +) { +} + +/// The wall-clock slot, from the store's config. +pub(crate) fn wall_slot(server: &P2PServer) -> Slot { + let config = server.store.config(); + let genesis_ms = config.genesis_time_ms(); + unix_now_ms().saturating_sub(genesis_ms) / config.slot_duration_ms.max(1) +} diff --git a/crates/net/p2p/src/beacon/mod.rs b/crates/net/p2p/src/beacon/mod.rs index 8ebc3721e..03bd60fa8 100644 --- a/crates/net/p2p/src/beacon/mod.rs +++ b/crates/net/p2p/src/beacon/mod.rs @@ -13,6 +13,7 @@ //! written once and handed the two things the chains disagree about: how wide //! the `` field is, and how a chunk body becomes a block. +pub mod builder_market; pub mod column_checks; pub mod decode; pub mod encoding; diff --git a/crates/net/p2p/src/beacon/topics.rs b/crates/net/p2p/src/beacon/topics.rs index ff4e6da60..7605d3d9d 100644 --- a/crates/net/p2p/src/beacon/topics.rs +++ b/crates/net/p2p/src/beacon/topics.rs @@ -51,6 +51,10 @@ pub const SYNC_COMMITTEE_CONTRIBUTION_AND_PROOF: &str = "sync_committee_contribu pub const EXECUTION_PAYLOAD: &str = "execution_payload"; /// Topic kind for gloas payload timeliness committee votes. pub const PAYLOAD_ATTESTATION_MESSAGE: &str = "payload_attestation_message"; +/// Topic kind for gloas builder bids. +pub const EXECUTION_PAYLOAD_BID: &str = "execution_payload_bid"; +/// Topic kind for gloas proposer preferences. +pub const PROPOSER_PREFERENCES: &str = "proposer_preferences"; /// The topic kinds gloas adds to [`SUBSCRIBED_TOPIC_KINDS`], subscribed from /// the gloas digest on and never under an earlier one. diff --git a/crates/net/p2p/src/beacon/verdict.rs b/crates/net/p2p/src/beacon/verdict.rs index 5d820eed1..62591872c 100644 --- a/crates/net/p2p/src/beacon/verdict.rs +++ b/crates/net/p2p/src/beacon/verdict.rs @@ -15,12 +15,14 @@ use std::panic::{AssertUnwindSafe, catch_unwind}; use std::time::Instant; use ethlambda_network_api::{AggregateArrival, BlockArrival, BlockSource}; +use ethlambda_state_transition::beacon::builder_market::SharedBuilderMarket; use ethlambda_state_transition::beacon::gossip::{self, IgnoreReason, Outcome}; use ethlambda_state_transition::beacon::helpers::accessors::CommitteeCacheExt as _; use ethlambda_storage::{CacheKey, Store}; use ethlambda_types::beacon::containers::electra::{self, SingleAttestation}; use ethlambda_types::beacon::containers::gloas::{ - PayloadAttestationMessage, SignedExecutionPayloadEnvelope, + PayloadAttestationMessage, SignedExecutionPayloadBid, SignedExecutionPayloadEnvelope, + SignedProposerPreferences, }; use ethlambda_types::beacon::containers::{ DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock, @@ -32,7 +34,7 @@ use spawned_concurrency::message::Message; use spawned_concurrency::tasks::{Context, Handler}; use tracing::{error, warn}; -use crate::beacon::column_checks; +use crate::beacon::{builder_market, column_checks}; use crate::{P2PServer, metrics}; /// Which gossip message a verdict is for. @@ -78,6 +80,16 @@ pub(crate) enum Validated { /// A gloas `execution_payload`. Boxed for the reason `Block` is: it /// carries a whole execution payload. Envelope(Box), + /// A gloas `execution_payload_bid`. Carries the market because + /// `stateful_checks` receives only the store. + #[allow(dead_code)] // constructed by Agent B's triage + ExecutionPayloadBid { + bid: Box, + market: SharedBuilderMarket, + }, + /// A gloas `proposer_preferences`. + #[allow(dead_code)] // constructed by Agent B's triage + ProposerPreferences(Box), /// A gloas `payload_attestation_message`. PayloadAttestation(PayloadAttestationMessage), } @@ -115,6 +127,12 @@ impl Validated { subnet_id, } => gossip::attestation::stateful_checks(store, attestation, *subnet_id), Self::Envelope(envelope) => gossip::envelope::stateful_checks(store, envelope), + Self::ExecutionPayloadBid { bid, market } => { + gossip::execution_payload_bid::stateful_checks(store, market, bid) + } + Self::ProposerPreferences(preferences) => { + gossip::proposer_preferences::stateful_checks(store, preferences) + } Self::PayloadAttestation(message) => { gossip::payload_attestation::stateful_checks(store, message) } @@ -147,6 +165,12 @@ impl Validated { envelope.message.beacon_block_root, envelope.message.builder_index, ), + Self::ExecutionPayloadBid { bid, .. } => { + server.builder_market.record_bid((**bid).clone()) + } + Self::ProposerPreferences(preferences) => server + .builder_market + .record_preferences((**preferences).clone(), builder_market::wall_slot(server)), Self::PayloadAttestation(message) => server .seen_payload_attestations .record(message.data.slot, message.validator_index), @@ -267,6 +291,10 @@ impl Validated { Self::Aggregate { .. } | Self::Attestation { .. } | Self::Envelope(_) + // The SSE `execution_payload_bid` and `proposer_preferences` events + // hook in here once the events endpoint lands. + | Self::ExecutionPayloadBid { .. } + | Self::ProposerPreferences(_) | Self::PayloadAttestation(_) => {} } } @@ -445,6 +473,9 @@ fn permits_for<'a>( Validated::Aggregate { .. } | Validated::Attestation { .. } | Validated::PayloadAttestation(_) => &server.attestation_validation_permits, + Validated::ExecutionPayloadBid { .. } | Validated::ProposerPreferences(_) => { + &server.builder_validation_permits + } } } diff --git a/crates/net/p2p/src/gossipsub/handler.rs b/crates/net/p2p/src/gossipsub/handler.rs index 1b664a476..ac1a0c478 100644 --- a/crates/net/p2p/src/gossipsub/handler.rs +++ b/crates/net/p2p/src/gossipsub/handler.rs @@ -253,6 +253,10 @@ fn handle_beacon_gossip( triage_envelope(server, payload) } else if kind == beacon_topics::PAYLOAD_ATTESTATION_MESSAGE { triage_payload_attestation(server, payload) + } else if kind == beacon_topics::EXECUTION_PAYLOAD_BID { + crate::beacon::builder_market::triage_execution_payload_bid(server, payload) + } else if kind == beacon_topics::PROPOSER_PREFERENCES { + crate::beacon::builder_market::triage_proposer_preferences(server, payload) } else { triage_other(wire, kind, payload) }; diff --git a/crates/net/p2p/src/gossipsub/mod.rs b/crates/net/p2p/src/gossipsub/mod.rs index 1701c5f3a..8c5580162 100644 --- a/crates/net/p2p/src/gossipsub/mod.rs +++ b/crates/net/p2p/src/gossipsub/mod.rs @@ -2,6 +2,8 @@ mod encoding; mod handler; mod messages; +#[allow(unused_imports)] // used by Agent B's publish functions +pub(crate) use encoding::compress_message; pub use encoding::decompress_message; pub use handler::{ handle_gossip_message, join_aggregator_subnets, leave_expired_aggregator_subnets, diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index 655b5a927..aa44d88c1 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -45,8 +45,8 @@ use ethlambda_network_api::{ }, rpc_to_p2p::{ PublishBeaconAggregate, PublishBeaconAttestation, PublishBeaconBlock, - PublishExecutionPayloadEnvelope, PublishPayloadAttestationMessage, - SubscribeAttestationSubnets, + PublishExecutionPayloadBid, PublishExecutionPayloadEnvelope, + PublishPayloadAttestationMessage, PublishProposerPreferences, SubscribeAttestationSubnets, }, }; use ethlambda_state_transition::beacon::aggregate::MAX_AGGREGATES_PER_SLOT; @@ -56,7 +56,8 @@ use ethlambda_state_transition::beacon::gossip::{ payload_attestation::SeenPayloadAttestations, }; use ethlambda_state_transition::beacon::{ - attestation_pool::SharedAttestationPool, payload_attestation_pool::SharedPayloadAttestationPool, + attestation_pool::SharedAttestationPool, builder_market::SharedBuilderMarket, + payload_attestation_pool::SharedPayloadAttestationPool, }; use ethlambda_storage::{Chain, Store}; use ethlambda_types::beacon::preset::{MAX_VALIDATORS_PER_COMMITTEE, SLOTS_PER_EPOCH}; @@ -244,6 +245,11 @@ const COLUMN_CHECK_PERMITS: usize = 16; /// has data from a follower. const ATTESTATION_VALIDATION_PERMITS: usize = 128; +/// How many `execution_payload_bid` and `proposer_preferences` stateful checks +/// may run at once. A pool of its own, so a burst of bids (each costs a BLS +/// verification) cannot starve blocks, columns or attestations of permits. +const BUILDER_VALIDATION_PERMITS: usize = 32; + /// Capacity of the first-valid-block cache, keyed by `(slot, proposer)`. /// How often to leave aggregator subnets whose slot has passed. One slot's /// worth: a subnet outlives its need by at most this, which costs a little @@ -1098,6 +1104,7 @@ impl P2P { discovery: Option, attestation_pool: SharedAttestationPool, payload_attestation_pool: SharedPayloadAttestationPool, + builder_market: SharedBuilderMarket, ) -> Result { let discovery = match discovery { Some(config) => Some(spawn_discovery(config).await?), @@ -1161,6 +1168,10 @@ impl P2P { )), attestation_pool, payload_attestation_pool, + builder_market, + builder_validation_permits: Arc::new(tokio::sync::Semaphore::new( + BUILDER_VALIDATION_PERMITS, + )), aggregator_subnets: HashMap::new(), }; let discovery_enabled = server.discovery.is_some(); @@ -1302,6 +1313,15 @@ pub struct P2PServer { /// lean never touches it. pub(crate) payload_attestation_pool: SharedPayloadAttestationPool, + /// Bids, proposer preferences and known payloads, shared with the Beacon + /// API (which posts bids and preferences, and builds blocks from the pool). + /// Gossip's stateful checks read it from blocking threads, so it cannot + /// live in the chain actor; lean never touches it. + pub(crate) builder_market: SharedBuilderMarket, + /// Permits for `execution_payload_bid` and `proposer_preferences` stateful + /// checks, see [`BUILDER_VALIDATION_PERMITS`]. + pub(crate) builder_validation_permits: Arc, + /// The attestation subnets joined for a validator client's aggregators, /// each with the last slot it is needed for. Short-lived by design: never /// advertised in `attnets`, and left once the slot has passed. The @@ -1612,6 +1632,18 @@ impl Handler for P2PServer { } } +impl Handler for P2PServer { + async fn handle(&mut self, msg: PublishExecutionPayloadBid, _ctx: &Context) { + beacon::builder_market::publish_execution_payload_bid(self, msg.bid); + } +} + +impl Handler for P2PServer { + async fn handle(&mut self, msg: PublishProposerPreferences, _ctx: &Context) { + beacon::builder_market::publish_proposer_preferences(self, msg.preferences); + } +} + impl Handler for P2PServer { async fn handle(&mut self, msg: PublishPayloadAttestationMessage, _ctx: &Context) { gossipsub::publish_payload_attestation_message(self, msg.message).await; @@ -2790,6 +2822,10 @@ pub(crate) mod test_support { )), attestation_pool: Default::default(), payload_attestation_pool: Default::default(), + builder_market: Default::default(), + builder_validation_permits: std::sync::Arc::new(tokio::sync::Semaphore::new( + crate::BUILDER_VALIDATION_PERMITS, + )), aggregator_subnets: HashMap::new(), } } diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs index 4db3f424b..c08ce605c 100644 --- a/crates/net/p2p/src/req_resp/handlers.rs +++ b/crates/net/p2p/src/req_resp/handlers.rs @@ -2811,6 +2811,10 @@ pub(crate) mod tests { )), attestation_pool: Default::default(), payload_attestation_pool: Default::default(), + builder_market: Default::default(), + builder_validation_permits: std::sync::Arc::new(tokio::sync::Semaphore::new( + crate::BUILDER_VALIDATION_PERMITS, + )), aggregator_subnets: HashMap::new(), } } diff --git a/crates/net/rpc/src/beacon/bid_selection.rs b/crates/net/rpc/src/beacon/bid_selection.rs new file mode 100644 index 000000000..0fd438380 --- /dev/null +++ b/crates/net/rpc/src/beacon/bid_selection.rs @@ -0,0 +1,45 @@ +//! Choosing between this node's own build and a pooled builder bid for +//! `produceBlockV4`. Pure: no store, no clock. +//! +//! Stubs until filled. + +use ethlambda_types::beacon::containers::gloas::{ExecutionPayloadBid, SignedExecutionPayloadBid}; +use ethlambda_types::beacon::primitives::Uint256; + +/// The local build, as the choice sees it. +#[allow(dead_code)] // filled by Agent C +pub(crate) struct LocalCandidate { + pub(crate) value_wei: u128, + pub(crate) should_override_builder: bool, +} + +// The enum is short-lived (one per block production), so boxing the bid buys +// nothing. +#[allow(dead_code, clippy::large_enum_variant)] // filled by Agent C +pub(crate) enum PayloadChoice { + Local, + Bid(SignedExecutionPayloadBid), +} + +/// `value.saturating_add(execution_payment)`; a p2p bid's payment is zero. +#[allow(dead_code, unused_variables)] // filled by Agent C +pub(crate) fn bid_total_gwei(bid: &ExecutionPayloadBid) -> u64 { + 0 +} + +/// Saturating conversion of a wei amount. +#[allow(dead_code, unused_variables)] // filled by Agent C +pub(crate) fn wei_u128(value: &Uint256) -> u128 { + 0 +} + +/// `None` when there is neither a local build nor a bid at or above `min_bid`. +#[allow(dead_code, unused_variables)] // filled by Agent C +pub(crate) fn choose_payload( + local: Option<&LocalCandidate>, + bids_desc: &[SignedExecutionPayloadBid], + min_bid: u64, + builder_boost_factor: u64, +) -> Option { + None +} diff --git a/crates/net/rpc/src/beacon/bids.rs b/crates/net/rpc/src/beacon/bids.rs new file mode 100644 index 000000000..21d3ecacd --- /dev/null +++ b/crates/net/rpc/src/beacon/bids.rs @@ -0,0 +1,13 @@ +//! `POST /eth/v1/beacon/execution_payload_bids`: a builder's bid handed to this +//! node, validated with the gossip rules, pooled in the shared +//! `BuilderMarket` and gossiped on `execution_payload_bid`. +//! +//! Stub: no routes until filled. + +use axum::Router; +use ethlambda_storage::Store; + +#[allow(dead_code)] // filled by Agent C +pub(crate) fn routes() -> Router { + Router::new() +} diff --git a/crates/net/rpc/src/beacon/builder_config.rs b/crates/net/rpc/src/beacon/builder_config.rs new file mode 100644 index 000000000..5408aa847 --- /dev/null +++ b/crates/net/rpc/src/beacon/builder_config.rs @@ -0,0 +1,53 @@ +//! The `BuilderConfig` a validator client sends with `produceBlockV4`. +//! +//! Stubs until filled. Field types below are placeholders; the SSZ and JSON +//! forms are added with the real containers. + +use axum::http::HeaderMap; + +use crate::beacon::ApiError; + +#[allow(dead_code)] // filled by Agent C +pub(crate) const MAX_BUILDER_ENTRIES: usize = 64; +#[allow(dead_code)] // filled by Agent C +pub(crate) const MAX_BUILDER_URL_SIZE: usize = 2048; +#[allow(dead_code)] // filled by Agent C +pub(crate) const MAX_BUILDER_PUBKEYS: usize = 64; +#[allow(dead_code)] // filled by Agent C +pub(crate) const MAX_BUILDER_AUTH_DATA_SIZE: usize = 4096; + +#[derive(Debug, Clone, Default)] +#[allow(dead_code)] // filled by Agent C +pub(crate) struct BuilderEntry { + pub(crate) url: String, + pub(crate) auth_data: Vec, + pub(crate) auth_slot: u64, + pub(crate) min_bid: u64, + pub(crate) builder_boost_factor: u64, +} + +#[derive(Debug, Clone, Default)] +#[allow(dead_code)] // filled by Agent C +pub(crate) struct BuilderConfig { + pub(crate) min_bid: u64, + pub(crate) builder_boost_factor: u64, + pub(crate) builders: Vec, +} + +#[allow(dead_code)] // filled by Agent C +impl BuilderEntry { + /// A non-empty url, non-empty `auth.data` and `auth.message.slot == slot`. + /// An unusable entry never fails the request. + pub(crate) fn is_usable_for(&self, _slot: u64) -> bool { + false + } +} + +/// A missing or undecodable body is a 400. +#[allow(dead_code)] // filled by Agent C +pub(crate) fn decode_builder_config( + _headers: &HeaderMap, + _body: &[u8], +) -> Result { + Err(ApiError::BadRequest("the body is not a BuilderConfig")) +} diff --git a/crates/net/rpc/src/beacon/builders.rs b/crates/net/rpc/src/beacon/builders.rs new file mode 100644 index 000000000..59995e4e0 --- /dev/null +++ b/crates/net/rpc/src/beacon/builders.rs @@ -0,0 +1,12 @@ +//! `POST /eth/v1/beacon/states/{state_id}/builders`: the builder registry of a +//! gloas state, filtered by id and status. +//! +//! Stub: no routes until filled. + +use axum::Router; +use ethlambda_storage::Store; + +#[allow(dead_code)] // filled by Agent C +pub(crate) fn routes() -> Router { + Router::new() +} diff --git a/crates/net/rpc/src/beacon/mod.rs b/crates/net/rpc/src/beacon/mod.rs index 728a20876..922a1867f 100644 --- a/crates/net/rpc/src/beacon/mod.rs +++ b/crates/net/rpc/src/beacon/mod.rs @@ -15,7 +15,11 @@ use serde::Serialize; use crate::shared::block_id::IdError; +pub(crate) mod bid_selection; +pub(crate) mod bids; pub(crate) mod blocks; +pub(crate) mod builder_config; +pub(crate) mod builders; pub(crate) mod config; pub(crate) mod envelopes; pub(crate) mod genesis; @@ -24,6 +28,7 @@ pub(crate) mod headers; pub(crate) mod node; pub(crate) mod pool; pub(crate) mod proposal; +pub(crate) mod proposer_preferences; pub(crate) mod ptc; pub(crate) mod states; pub(crate) mod validator; @@ -180,6 +185,9 @@ pub(crate) fn routes(version: &'static str, peer_id: String) -> Router { .merge(proposal::routes()) .merge(gloas_proposal::routes()) .merge(ptc::routes()) + .merge(bids::routes()) + .merge(proposer_preferences::routes()) + .merge(builders::routes()) } #[cfg(test)] diff --git a/crates/net/rpc/src/beacon/proposer_preferences.rs b/crates/net/rpc/src/beacon/proposer_preferences.rs new file mode 100644 index 000000000..6dd900087 --- /dev/null +++ b/crates/net/rpc/src/beacon/proposer_preferences.rs @@ -0,0 +1,13 @@ +//! `POST /eth/v1/validator/proposer_preferences`: signed proposer preferences +//! from a validator client, validated with the gossip rules, cached in the +//! shared `BuilderMarket` and gossiped on `proposer_preferences`. +//! +//! Stub: no routes until filled. + +use axum::Router; +use ethlambda_storage::Store; + +#[allow(dead_code)] // filled by Agent C +pub(crate) fn routes() -> Router { + Router::new() +} diff --git a/crates/net/rpc/src/lib.rs b/crates/net/rpc/src/lib.rs index e03ae7862..7844f941b 100644 --- a/crates/net/rpc/src/lib.rs +++ b/crates/net/rpc/src/lib.rs @@ -4,7 +4,8 @@ use axum::{Extension, Router}; use ethlambda_blockchain::{EventBus, SyncStatusController}; use ethlambda_network_api::RpcToP2PRef; use ethlambda_state_transition::beacon::{ - attestation_pool::SharedAttestationPool, payload_attestation_pool::SharedPayloadAttestationPool, + attestation_pool::SharedAttestationPool, builder_market::SharedBuilderMarket, + payload_attestation_pool::SharedPayloadAttestationPool, }; use ethlambda_storage::Store; use ethlambda_types::aggregator::AggregatorController; @@ -202,6 +203,9 @@ pub struct BeaconApiHandles { /// Filled by gossip and the payload attestation pool endpoint, read by /// block production and the pool's GET. pub payload_attestation_pool: SharedPayloadAttestationPool, + /// Bids, proposer preferences and known payloads: filled by the bid and + /// preferences endpoints and by gossip, read by block production. + pub builder_market: SharedBuilderMarket, /// The columns this node custodies: what `payload_attestation_data` checks /// a block's blob availability against, and what block production tells /// the execution client it samples for when asking it to build a gloas @@ -231,6 +235,7 @@ pub async fn start_beacon_rpc_server( .layer(Extension(handles.p2p)) .layer(Extension(handles.attestation_pool)) .layer(Extension(handles.payload_attestation_pool)) + .layer(Extension(handles.builder_market)) .layer(Extension(handles.custody_columns)) .layer(Extension(beacon::validator::FeeRecipients::default())) .layer(Extension(handles.engine)); @@ -443,6 +448,12 @@ pub(crate) mod test_utils { pub(crate) payload_attestations: std::sync::Mutex< Vec, >, + pub(crate) bids: std::sync::Mutex< + Vec, + >, + pub(crate) proposer_preferences: std::sync::Mutex< + Vec, + >, } impl ethlambda_network_api::RpcToP2P for RecordingNetwork { @@ -493,6 +504,22 @@ pub(crate) mod test_utils { Ok(()) } + fn publish_execution_payload_bid( + &self, + bid: ethlambda_types::beacon::containers::gloas::SignedExecutionPayloadBid, + ) -> Result<(), spawned_concurrency::error::ActorError> { + self.bids.lock().unwrap().push(bid); + Ok(()) + } + + fn publish_proposer_preferences( + &self, + preferences: ethlambda_types::beacon::containers::gloas::SignedProposerPreferences, + ) -> Result<(), spawned_concurrency::error::ActorError> { + self.proposer_preferences.lock().unwrap().push(preferences); + Ok(()) + } + fn publish_payload_attestation_message( &self, message: ethlambda_types::beacon::containers::gloas::PayloadAttestationMessage, From f9a92da21d12e7acbc411710d3a9a1fda740ee84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:48:17 -0300 Subject: [PATCH 114/132] feat(state-transition): implement the gloas builder market state Bids, proposer preferences and revealed execution payloads are judged and consumed from three places (gossip rules on blocking threads, the Beacon API and block production), so they live in one shared object rather than in the chain actor. The market keeps the spec's seen sets and best-bid bar apart from the pooled bids: the pool is truncated to the top values per parent, but the bar a new bid must strictly beat is not. Known payloads are a bounded LRU, and every collection is bounded so spam cannot grow memory. Also adds the test-utils fixtures (builder registry state, bid and preference signing, envelopes) the gossip rules, p2p and rpc tests build on. --- .../src/beacon/builder_market.rs | 529 +++++++++++++++++- 1 file changed, 511 insertions(+), 18 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/builder_market.rs b/crates/blockchain/state_transition/src/beacon/builder_market.rs index a39c79736..6eda031bd 100644 --- a/crates/blockchain/state_transition/src/beacon/builder_market.rs +++ b/crates/blockchain/state_transition/src/beacon/builder_market.rs @@ -7,13 +7,11 @@ //! the Beacon API reads it, like [`super::payload_attestation_pool`]. Gossip's //! stateful checks run on blocking threads, so none of this can be owned by the //! chain actor. -//! -//! The method signatures are the contract between the gossip rules, p2p and the -//! Beacon API; the bodies are placeholders until they are filled. use std::{ + collections::{BTreeMap, BTreeSet}, num::NonZeroUsize, - sync::{Arc, Mutex}, + sync::{Arc, Mutex, MutexGuard}, }; use lru::LruCache; @@ -44,22 +42,78 @@ pub struct KnownPayload { pub builder_exits: Vec<(BlsPubkey, ExecutionAddress)>, } +/// Bids for one `(parent_block_hash, parent_block_root)` of a slot. +#[derive(Debug, Default)] +struct ParentBids { + /// The highest value recorded, which a later bid must strictly beat. Kept + /// apart from `bids` because the pool is truncated and the bar is not. + best_value: Option, + /// Value descending, then builder index ascending. + bids: Vec, +} + +type ParentKey = (ExecutionBlockHash, Root); + +#[derive(Debug, Default)] +struct SlotBids { + /// The spec's `seen.execution_payload_bids`: one bid per builder per + /// `(slot, parent_hash, parent_root)`. + seen: BTreeSet<(ParentKey, u64)>, + parents: BTreeMap, +} + +#[derive(Debug, Default)] +struct BidPool { + slots: BTreeMap, +} + +impl BidPool { + fn check(&self, bid: &gloas::ExecutionPayloadBid) -> Result<(), IgnoreReason> { + let Some(slot) = self.slots.get(&bid.slot) else { + return Ok(()); + }; + let parent = (bid.parent_block_hash, bid.parent_block_root); + if slot.seen.contains(&(parent, bid.builder_index)) { + return Err(IgnoreReason::AlreadySeen); + } + if let Some(best) = slot.parents.get(&parent).and_then(|p| p.best_value) + && bid.value <= best + { + return Err(IgnoreReason::NotHighestBid); + } + Ok(()) + } +} + +/// Preferences by `(proposal_slot, dependent_root)`: the first valid one wins. +type PreferencesCache = BTreeMap<(Slot, Root), gloas::SignedProposerPreferences>; + #[derive(Debug)] pub struct BuilderMarket { - // Filled by Agent A: the bid pool and the preferences cache join this. - #[allow(dead_code)] + bids: Mutex, + preferences: Mutex, payloads: Mutex>, } impl Default for BuilderMarket { fn default() -> Self { Self { + bids: Mutex::default(), + preferences: Mutex::default(), payloads: Mutex::new(LruCache::new(KNOWN_PAYLOADS_CAPACITY)), } } } -#[allow(dead_code, unused_variables)] // filled by Agent A +/// A poisoned lock means a panic elsewhere interrupted an update, but every +/// mutation here is a single insert or remove, so the data is still consistent +/// and gossip should keep working. +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + impl BuilderMarket { // Bids: seen.execution_payload_bids + seen.best_execution_payload_bid + pool. @@ -67,19 +121,54 @@ impl BuilderMarket { /// builder)` recorded -> `AlreadySeen`; value <= best for `(slot, /// parent_hash, parent_root)` -> `NotHighestBid`. pub fn check_bid_seen(&self, bid: &gloas::ExecutionPayloadBid) -> Result<(), IgnoreReason> { - Ok(()) + lock(&self.bids).check(bid) } /// Re-runs [`Self::check_bid_seen`] under the lock. If it passes, records /// both seen keys and pools the bid. `false` = not recorded (race, or the /// per-slot key cap). Prunes slots below `bid.slot - 1`. pub fn record_bid(&self, signed: gloas::SignedExecutionPayloadBid) -> bool { - false + let mut pool = lock(&self.bids); + if pool.check(&signed.message).is_err() { + return false; + } + let bid_slot = signed.message.slot; + let slot = pool.slots.entry(bid_slot).or_default(); + if slot.seen.len() >= MAX_SEEN_BID_KEYS_PER_SLOT { + return false; + } + let parent = ( + signed.message.parent_block_hash, + signed.message.parent_block_root, + ); + slot.seen.insert((parent, signed.message.builder_index)); + let entry = slot.parents.entry(parent).or_default(); + entry.best_value = Some(signed.message.value); + entry.bids.push(signed); + entry.bids.sort_by(|a, b| { + b.message + .value + .cmp(&a.message.value) + .then(a.message.builder_index.cmp(&b.message.builder_index)) + }); + entry.bids.truncate(MAX_BIDS_PER_PARENT); + let keep_from = bid_slot.saturating_sub(1); + pool.slots = pool.slots.split_off(&keep_from); + true } /// The identical message and signature is pooled (API idempotency). pub fn contains_bid(&self, signed: &gloas::SignedExecutionPayloadBid) -> bool { - false + let pool = lock(&self.bids); + pool.slots + .get(&signed.message.slot) + .and_then(|slot| { + slot.parents.get(&( + signed.message.parent_block_hash, + signed.message.parent_block_root, + )) + }) + .is_some_and(|parent| parent.bids.contains(signed)) } /// Pooled bids for the key: value descending, then builder index ascending. @@ -89,14 +178,25 @@ impl BuilderMarket { parent_block_root: Root, parent_block_hash: ExecutionBlockHash, ) -> Vec { - Vec::new() + lock(&self.bids) + .slots + .get(&slot) + .and_then(|s| s.parents.get(&(parent_block_hash, parent_block_root))) + .map(|p| p.bids.clone()) + .unwrap_or_default() } pub fn has_bids_for_slot(&self, slot: Slot) -> bool { - false + lock(&self.bids) + .slots + .get(&slot) + .is_some_and(|s| s.parents.values().any(|p| !p.bids.is_empty())) } - pub fn prune_bids_before(&self, slot: Slot) {} + pub fn prune_bids_before(&self, slot: Slot) { + let mut pool = lock(&self.bids); + pool.slots = pool.slots.split_off(&slot); + } // Proposer preferences: seen.proposer_preferences. @@ -105,7 +205,9 @@ impl BuilderMarket { proposal_slot: Slot, dependent_root: Root, ) -> Option { - None + lock(&self.preferences) + .get(&(proposal_slot, dependent_root)) + .cloned() } /// The first valid preferences per key win. `false` if one is held. Prunes @@ -115,16 +217,407 @@ impl BuilderMarket { signed: gloas::SignedProposerPreferences, current_slot: Slot, ) -> bool { - false + let mut cache = lock(&self.preferences); + let key = (signed.message.proposal_slot, signed.message.dependent_root); + if cache.contains_key(&key) { + return false; + } + *cache = cache.split_off(&(current_slot, Root::ZERO)); + cache.insert(key, signed); + while cache.len() > MAX_PREFERENCES { + cache.pop_first(); + } + cache.contains_key(&key) } - pub fn prune_preferences_before(&self, slot: Slot) {} + pub fn prune_preferences_before(&self, slot: Slot) { + let mut cache = lock(&self.preferences); + *cache = cache.split_off(&(slot, Root::ZERO)); + } // Known payloads: seen.execution_payloads. - pub fn record_execution_payload(&self, envelope: &gloas::ExecutionPayloadEnvelope) {} + pub fn record_execution_payload(&self, envelope: &gloas::ExecutionPayloadEnvelope) { + let builder_exits = envelope + .execution_requests + .builder_exits + .iter() + .map(|exit| (exit.pubkey, exit.source_address)) + .collect(); + let known = KnownPayload { + gas_limit: envelope.payload.gas_limit, + beacon_block_root: envelope.beacon_block_root, + builder_exits, + }; + lock(&self.payloads).put(envelope.payload.block_hash, known); + } pub fn known_payload(&self, block_hash: ExecutionBlockHash) -> Option { - None + lock(&self.payloads).get(&block_hash).cloned() + } +} + +/// Builder-market fixtures shared by this module's tests, the gossip rules' +/// and (`test-utils` feature) the Beacon API's and p2p's: a gloas state with a +/// registered builder, and signatures that state verifies. +#[cfg(any(test, feature = "test-utils"))] +pub mod test_support { + use ethlambda_types::beacon::containers::bellatrix::{ExtraData, LogsBloom}; + use ethlambda_types::beacon::primitives::{BlsSignature, Bytes32, Uint256}; + + use super::*; + use crate::beacon::containers::BeaconState; + use crate::beacon::gloas_block_production::test_support::parent_state; + use crate::beacon::helpers::accessors::get_domain; + use crate::beacon::helpers::misc::{compute_epoch_at_slot, compute_signing_root}; + use crate::beacon::helpers::test_state::{secret_key_for, sign_for}; + use crate::beacon::primitives::HashTreeRoot as _; + use crate::beacon::{constants, preset}; + + /// The secret key behind builder `index`'s registered pubkey. Offset from + /// the validators' keys so the two never collide. + pub fn builder_secret(index: usize) -> blst::min_pk::SecretKey { + secret_key_for(1000 + index) + } + + /// `gloas_block_production::test_support::parent_state` with builders + /// `0..=builder_index` registered (each funded with `balance`, deposited at + /// `deposit_epoch`), and the finalized checkpoint one epoch past + /// `deposit_epoch` so they are active. + pub fn gloas_state_with_builder( + builder_index: u64, + balance: u64, + deposit_epoch: u64, + ) -> BeaconState { + let mut state = parent_state(); + let BeaconState::Gloas(inner) = &mut state else { + unreachable!("built as gloas") + }; + for index in 0..=builder_index { + let builder = gloas::Builder { + pubkey: BlsPubkey(builder_secret(index as usize).sk_to_pk().to_bytes()), + version: constants::PAYLOAD_BUILDER_VERSION, + execution_address: ExecutionAddress::repeat_byte(index as u8 + 1), + balance, + deposit_epoch, + withdrawable_epoch: constants::FAR_FUTURE_EPOCH, + }; + inner.builders.push(builder); + } + inner.finalized_checkpoint.epoch = deposit_epoch + 1; + state + } + + /// `bid` signed by builder `builder_index` under `state`'s builder domain. + pub fn sign_bid( + state: &BeaconState, + bid: gloas::ExecutionPayloadBid, + builder_index: u64, + ) -> gloas::SignedExecutionPayloadBid { + let domain = get_domain(state, constants::DOMAIN_BEACON_BUILDER, None); + let root = compute_signing_root(bid.hash_tree_root(), domain); + let signature = builder_secret(builder_index as usize).sign( + root.as_slice(), + crate::beacon::bls::DST, + &[], + ); + gloas::SignedExecutionPayloadBid { + message: bid, + signature: BlsSignature(signature.to_bytes()), + } + } + + /// `prefs` signed by its own `validator_index` under `state`'s preferences + /// domain at the proposal slot's epoch (the spec's). + pub fn sign_preferences( + state: &BeaconState, + prefs: gloas::ProposerPreferences, + ) -> gloas::SignedProposerPreferences { + let epoch = compute_epoch_at_slot(prefs.proposal_slot); + let domain = get_domain(state, constants::DOMAIN_PROPOSER_PREFERENCES, Some(epoch)); + let root = compute_signing_root(prefs.hash_tree_root(), domain); + let signature = sign_for(prefs.validator_index as usize, root); + gloas::SignedProposerPreferences { + message: prefs, + signature, + } + } + + /// An envelope revealing payload `block_hash` with `gas_limit`, for the + /// block `beacon_block_root`, carrying one exit request per `exits` entry. + pub fn envelope_with_gas_limit( + block_hash: ExecutionBlockHash, + gas_limit: u64, + beacon_block_root: Root, + exits: Vec<(BlsPubkey, ExecutionAddress)>, + ) -> gloas::ExecutionPayloadEnvelope { + let payload = gloas::ExecutionPayload { + parent_hash: ExecutionBlockHash::ZERO, + fee_recipient: Default::default(), + state_root: Bytes32::repeat_byte(1), + receipts_root: Bytes32::repeat_byte(2), + logs_bloom: LogsBloom::try_from(vec![0u8; preset::BYTES_PER_LOGS_BLOOM]).unwrap(), + prev_randao: Default::default(), + block_number: 7, + gas_limit, + gas_used: 0, + timestamp: 0, + extra_data: ExtraData::default(), + base_fee_per_gas: Uint256::from_u128(7), + block_hash, + transactions: Default::default(), + withdrawals: Default::default(), + blob_gas_used: 0, + excess_blob_gas: 0, + block_access_list: Default::default(), + slot_number: 0, + }; + let builder_exits: Vec<_> = exits + .into_iter() + .map(|(pubkey, source_address)| gloas::BuilderExitRequest { + source_address, + pubkey, + }) + .collect(); + let execution_requests = gloas::ExecutionRequests { + builder_exits: builder_exits.into(), + ..Default::default() + }; + gloas::ExecutionPayloadEnvelope { + payload, + execution_requests, + builder_index: 0, + beacon_block_root, + parent_beacon_block_root: Root::ZERO, + } + } +} + +#[cfg(test)] +mod tests { + use super::test_support::envelope_with_gas_limit; + use super::*; + + fn hash(byte: u8) -> ExecutionBlockHash { + ExecutionBlockHash::repeat_byte(byte) + } + + fn bid(slot: Slot, builder: u64, value: u64) -> gloas::SignedExecutionPayloadBid { + gloas::SignedExecutionPayloadBid { + message: gloas::ExecutionPayloadBid { + slot, + builder_index: builder, + value, + parent_block_hash: hash(1), + parent_block_root: Root::repeat_byte(2), + block_hash: hash(3), + ..Default::default() + }, + signature: Default::default(), + } + } + + fn prefs(slot: Slot, dependent: u8, validator: u64) -> gloas::SignedProposerPreferences { + gloas::SignedProposerPreferences { + message: gloas::ProposerPreferences { + dependent_root: Root::repeat_byte(dependent), + proposal_slot: slot, + validator_index: validator, + ..Default::default() + }, + signature: Default::default(), + } + } + + #[test] + fn a_builder_bids_once_per_parent() { + let market = BuilderMarket::default(); + assert!(market.record_bid(bid(10, 1, 5))); + assert_eq!( + market.check_bid_seen(&bid(10, 1, 9).message), + Err(IgnoreReason::AlreadySeen) + ); + assert!(!market.record_bid(bid(10, 1, 9))); + // Another parent hash is another key. + let mut other = bid(10, 1, 9); + other.message.parent_block_hash = hash(9); + assert!(market.record_bid(other)); + } + + #[test] + fn a_bid_must_strictly_beat_the_best() { + let market = BuilderMarket::default(); + assert!(market.record_bid(bid(10, 1, 5))); + assert_eq!( + market.check_bid_seen(&bid(10, 2, 5).message), + Err(IgnoreReason::NotHighestBid) + ); + assert_eq!( + market.check_bid_seen(&bid(10, 2, 4).message), + Err(IgnoreReason::NotHighestBid) + ); + assert_eq!(market.check_bid_seen(&bid(10, 2, 6).message), Ok(())); + assert!(market.record_bid(bid(10, 2, 6))); + // A lower bid never reached the pool, the bar stays at the best. + assert!(!market.record_bid(bid(10, 3, 6))); + } + + #[test] + fn a_stale_check_fails_the_record() { + let market = BuilderMarket::default(); + let racing = bid(10, 2, 7); + assert_eq!(market.check_bid_seen(&racing.message), Ok(())); + assert!(market.record_bid(bid(10, 1, 8))); + assert!(!market.record_bid(racing)); + } + + #[test] + fn a_full_slot_refuses_new_keys() { + let market = BuilderMarket::default(); + for builder in 0..MAX_SEEN_BID_KEYS_PER_SLOT as u64 { + assert!(market.record_bid(bid(10, builder, builder + 1))); + } + let next = MAX_SEEN_BID_KEYS_PER_SLOT as u64; + assert!(!market.record_bid(bid(10, next, next + 1))); + // Other slots are unaffected. + assert!(market.record_bid(bid(11, next, 1))); + } + + #[test] + fn bids_for_orders_by_value_and_keeps_the_top() { + let market = BuilderMarket::default(); + let total = MAX_BIDS_PER_PARENT as u64 + 4; + for builder in 0..total { + assert!(market.record_bid(bid(10, builder, builder + 1))); + } + let pooled = market.bids_for(10, Root::repeat_byte(2), hash(1)); + assert_eq!(pooled.len(), MAX_BIDS_PER_PARENT); + let values: Vec = pooled.iter().map(|b| b.message.value).collect(); + let expected: Vec = (5..=total).rev().collect(); + assert_eq!(values, expected); + // The truncated bar still holds: the best value is the latest. + assert_eq!( + market.check_bid_seen(&bid(10, 99, total).message), + Err(IgnoreReason::NotHighestBid) + ); + assert!(market.bids_for(10, Root::ZERO, hash(1)).is_empty()); + } + + #[test] + fn contains_bid_matches_the_exact_message() { + let market = BuilderMarket::default(); + let signed = bid(10, 1, 5); + assert!(!market.contains_bid(&signed)); + assert!(market.record_bid(signed.clone())); + assert!(market.contains_bid(&signed)); + let mut other = signed.clone(); + other.signature.0[0] = 1; + assert!(!market.contains_bid(&other)); + assert!(market.has_bids_for_slot(10)); + assert!(!market.has_bids_for_slot(11)); + } + + #[test] + fn bids_prune_below_the_previous_slot() { + let market = BuilderMarket::default(); + assert!(market.record_bid(bid(10, 1, 5))); + assert!(market.record_bid(bid(11, 1, 5))); + assert!(market.has_bids_for_slot(10)); + assert!(market.record_bid(bid(12, 1, 5))); + assert!(!market.has_bids_for_slot(10)); + assert!(market.has_bids_for_slot(11)); + market.prune_bids_before(12); + assert!(!market.has_bids_for_slot(11)); + assert!(market.has_bids_for_slot(12)); + } + + #[test] + fn the_first_preferences_win() { + let market = BuilderMarket::default(); + let first = prefs(40, 1, 3); + assert!(market.record_preferences(first.clone(), 38)); + assert!(!market.record_preferences(prefs(40, 1, 4), 38)); + assert_eq!(market.preferences(40, Root::repeat_byte(1)), Some(first)); + // Another dependent root is another key. + assert!(market.record_preferences(prefs(40, 2, 3), 38)); + assert_eq!(market.preferences(41, Root::repeat_byte(1)), None); + } + + #[test] + fn preferences_prune_by_proposal_slot() { + let market = BuilderMarket::default(); + assert!(market.record_preferences(prefs(40, 1, 3), 38)); + assert!(market.record_preferences(prefs(41, 1, 3), 41)); + assert!(market.preferences(40, Root::repeat_byte(1)).is_none()); + assert!(market.preferences(41, Root::repeat_byte(1)).is_some()); + market.prune_preferences_before(42); + assert!(market.preferences(41, Root::repeat_byte(1)).is_none()); + } + + #[test] + fn preferences_over_the_cap_evict_the_lowest_slot() { + let market = BuilderMarket::default(); + for slot in 0..MAX_PREFERENCES as u64 { + assert!(market.record_preferences(prefs(100 + slot, 1, 3), 0)); + } + let high = 100 + MAX_PREFERENCES as u64; + assert!(market.record_preferences(prefs(high, 1, 3), 0)); + assert!(market.preferences(100, Root::repeat_byte(1)).is_none()); + assert!(market.preferences(101, Root::repeat_byte(1)).is_some()); + // A message below everything held is itself the one dropped. + assert!(!market.record_preferences(prefs(50, 1, 3), 0)); + } + + #[test] + fn a_revealed_payload_is_known_with_its_exits() { + let market = BuilderMarket::default(); + let pubkey = BlsPubkey([7; 48]); + let source = ExecutionAddress::repeat_byte(9); + let envelope = envelope_with_gas_limit( + hash(5), + 36_000_000, + Root::repeat_byte(4), + vec![(pubkey, source)], + ); + assert!(market.known_payload(hash(5)).is_none()); + market.record_execution_payload(&envelope); + assert_eq!( + market.known_payload(hash(5)), + Some(KnownPayload { + gas_limit: 36_000_000, + beacon_block_root: Root::repeat_byte(4), + builder_exits: vec![(pubkey, source)], + }) + ); + } + + #[test] + fn known_payloads_are_a_bounded_lru() { + let market = BuilderMarket::default(); + let capacity = KNOWN_PAYLOADS_CAPACITY.get(); + for index in 0..capacity { + let mut block_hash = ExecutionBlockHash::ZERO; + block_hash.0[..8].copy_from_slice(&(index as u64).to_le_bytes()); + market.record_execution_payload(&envelope_with_gas_limit( + block_hash, + 1, + Root::ZERO, + vec![], + )); + } + let mut first = ExecutionBlockHash::ZERO; + first.0[..8].copy_from_slice(&0u64.to_le_bytes()); + // Touch the oldest so the second becomes the least recently used. + assert!(market.known_payload(first).is_some()); + market.record_execution_payload(&envelope_with_gas_limit( + hash(0xff), + 1, + Root::ZERO, + vec![], + )); + let mut second = ExecutionBlockHash::ZERO; + second.0[..8].copy_from_slice(&1u64.to_le_bytes()); + assert!(market.known_payload(first).is_some()); + assert!(market.known_payload(second).is_none()); } } From 551dab5b6a982d199dea3a0c0c13ec67880e0726 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:03:46 -0300 Subject: [PATCH 115/132] feat(state-transition): validate gloas bids and preferences, assemble blocks on a bid Gives the builder market its rules: the spec's execution_payload_bid and proposer_preferences gossip validation, split into cheap and stateful halves like the other topics, plus block assembly on a builder's bid. The rules read cached states only and never queue, so a verdict gossipsub waits on is never stalled by a replay. Two shortcuts keep that honest: the parent's own state stands in for the advanced one within its epoch (gloas's process_slot touches none of the fields the later rules read), and a proposer's lookahead comes from the head state when it shares the dependent root, since a dependent block about an epoch old is usually out of the state cache. Preference signatures verify under the fork versions on both sides of the gloas boundary, because clients disagree there. assemble_gloas_block_on_bid and bid_is_includable let produceBlockV4 commit a block to another builder's bid; the pre-filter keeps a bid that would fail process_block from being tried. Both gossip vector handlers now run (44 and 26 mainnet cases pass). The runner records a delivered envelope the way the spec generators do, without re-verifying it, and applies the fixture's finalized-checkpoint override to the stored states, which is how those generators activate builders. docs/spec_deviations.md lists what departs from the spec. --- .../src/beacon/gloas_block_production.rs | 284 ++++- .../beacon/gossip/execution_payload_bid.rs | 1094 ++++++++++++++++- .../src/beacon/gossip/proposer_preferences.rs | 594 ++++++++- .../src/beacon/gossip/test_support.rs | 159 +++ .../tests/beacon_spec/gossip.rs | 135 +- docs/spec_deviations.md | 173 ++- 6 files changed, 2311 insertions(+), 128 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs b/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs index 6b3af3aa2..1a0fa8edc 100644 --- a/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs +++ b/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs @@ -29,8 +29,8 @@ use super::helpers::accessors::{ CommitteeCache, get_beacon_proposer_index, get_current_epoch, get_randao_mix, }; use super::helpers::gloas::{ - get_indexed_payload_attestation, get_ptc, gloas_state_ref, is_attestation_same_slot, - is_valid_indexed_payload_attestation, + can_builder_cover_bid, get_indexed_payload_attestation, get_ptc, gloas_state_ref, + is_active_builder, is_attestation_same_slot, is_valid_indexed_payload_attestation, }; use super::stf; use ethlambda_types::beacon::{ @@ -359,28 +359,88 @@ pub struct GloasBidBlockInputs { pub signed_bid: SignedExecutionPayloadBid, } -/// The unsigned block for `state.slot()` committing to `inputs.signed_bid`. -/// `process_block` on a copy fills `state_root` and enforces -/// `process_execution_payload_bid` (active, cover, signature, slot, parent -/// hash and root, randao). No envelope: the builder reveals it. -#[allow(dead_code)] // filled by Agent A +/// The unsigned block for `state.slot()` committing to `inputs.signed_bid`, a +/// builder's bid rather than a payload this node built. +/// +/// The body is [`assemble_gloas_block`]'s (empty sync aggregate, the state's own +/// `eth1_data`), with the bid in place of the self-build one. `process_block` on +/// a copy fills `state_root` and enforces `process_execution_payload_bid`: +/// the builder is active, covers the bid and signed it, and the bid's slot, +/// parent hash and root and randao are the state's. There is no envelope: the +/// builder reveals the payload itself. pub fn assemble_gloas_block_on_bid( - _state: &BeaconState, - _inputs: GloasBidBlockInputs, - _config: &Config, + state: &BeaconState, + inputs: GloasBidBlockInputs, + config: &Config, ) -> Result { - Err(Error::SpecAssert("unimplemented")) + verify( + matches!(state, BeaconState::Gloas(_)), + "gloas block production runs on a gloas state", + )?; + let body = BeaconBlockBody { + randao_reveal: inputs.randao_reveal, + eth1_data: state.eth1_data().clone(), + graffiti: inputs.graffiti, + attestations: inputs.attestations.into(), + sync_aggregate: empty_sync_aggregate(), + signed_execution_payload_bid: inputs.signed_bid, + payload_attestations: inputs.payload_attestations.into(), + parent_execution_requests: inputs.parent_execution_requests, + ..BeaconBlockBody::empty() + }; + let mut block = BeaconBlock { + slot: state.slot(), + proposer_index: get_beacon_proposer_index(state)?, + parent_root: state.latest_block_header().hash_tree_root(), + state_root: Root::ZERO, + body, + }; + let mut post = state.clone(); + stf::gloas::process_block(&mut post, &block, config, &CommitteeCache::default())?; + block.state_root = post.hash_tree_root(); + Ok(block) } /// Cheap pre-filter on the state advanced to the slot: whether the bid could -/// be packed into a block on `state`. The signature is not checked here. -#[allow(dead_code)] // filled by Agent A +/// be packed into a block on `state`, so the producer skips a bid whose block +/// would fail rather than learning it from a full `process_block`. +/// +/// Checks the bid is for `state`'s slot, parent root, previous randao and +/// parent payload (full: the parent bid's block hash; empty: the state's +/// `latest_block_hash`), that the builder exists, is a payload builder, is +/// active and covers the value, that the commitment count is within the preset +/// bound, and that the parent payload does not exit the builder when the bid +/// builds on it (`process_parent_execution_payload` runs before the bid, so a +/// builder that exits there is no longer active). The signature is not checked +/// here; `process_block` does. pub fn bid_is_includable( - _state: &BeaconState, - _signed_bid: &SignedExecutionPayloadBid, - _parent_requests: &ExecutionRequests, + state: &BeaconState, + signed_bid: &SignedExecutionPayloadBid, + parent_requests: &ExecutionRequests, ) -> bool { - false + let BeaconState::Gloas(inner) = state else { + return false; + }; + let bid = &signed_bid.message; + let parent_is_full = bid.parent_block_hash == inner.latest_execution_payload_bid.block_hash; + let builds_on_known_parent = parent_is_full || bid.parent_block_hash == inner.latest_block_hash; + let Some(builder) = inner.builders.get(bid.builder_index as usize) else { + return false; + }; + let exited_by_parent = parent_is_full + && parent_requests.builder_exits.iter().any(|exit| { + exit.pubkey == builder.pubkey && exit.source_address == builder.execution_address + }); + bid.slot == state.slot() + && bid.parent_block_root == state.latest_block_header().hash_tree_root() + && bid.prev_randao == get_randao_mix(state, get_current_epoch(state)) + && builds_on_known_parent + && bid.block_hash != bid.parent_block_hash + && builder.version == constants::PAYLOAD_BUILDER_VERSION + && bid.blob_kzg_commitments.len() <= preset::MAX_BLOB_COMMITMENTS_PER_BLOCK + && !exited_by_parent + && is_active_builder(inner, bid.builder_index).unwrap_or(false) + && can_builder_cover_bid(inner, bid.builder_index, bid.value).unwrap_or(false) } /// A produced block and the unsigned envelope that reveals its payload. @@ -1113,3 +1173,193 @@ mod gloas_block_production_tests { assert!(parse_gloas_execution_requests(&[vec![0x7f, 0]]).is_err()); } } + +#[cfg(test)] +mod bid_assembly_tests { + use super::test_support::*; + use super::*; + use crate::beacon::ForkName; + use crate::beacon::block_production::advance_to_slot; + use crate::beacon::builder_market::test_support::{ + builder_secret, gloas_state_with_builder, sign_bid, + }; + use ethlambda_types::beacon::primitives::{BlsPubkey, ExecutionAddress}; + + /// A state at slot 33 with a funded, active builder 0, and a bid on its + /// full parent that a block can carry. + fn scene() -> (BeaconState, SignedExecutionPayloadBid) { + let parent = gloas_state_with_builder(0, 100_000_000_000, 0); + let state = advance_to_slot(&parent, parent.slot() + 1, &config()).unwrap(); + let bid = bid_for(&state, |_| {}); + (state, bid) + } + + /// A bid on `state`'s full parent payload, signed by builder 0 after `edit`. + fn bid_for( + state: &BeaconState, + edit: impl FnOnce(&mut ExecutionPayloadBid), + ) -> SignedExecutionPayloadBid { + let mut bid = ExecutionPayloadBid { + parent_block_hash: ExecutionBlockHash::repeat_byte(PARENT_BLOCK_HASH), + parent_block_root: state.latest_block_header().hash_tree_root(), + block_hash: ExecutionBlockHash::repeat_byte(0x33), + prev_randao: get_randao_mix(state, get_current_epoch(state)), + gas_limit: 30_000_000, + builder_index: 0, + slot: state.slot(), + value: 7, + ..Default::default() + }; + edit(&mut bid); + sign_bid(state, bid, 0) + } + + fn inputs(state: &BeaconState, signed_bid: SignedExecutionPayloadBid) -> GloasBidBlockInputs { + GloasBidBlockInputs { + randao_reveal: randao_reveal(state), + graffiti: Bytes32::repeat_byte(7), + attestations: Vec::new(), + payload_attestations: Vec::new(), + parent_execution_requests: ExecutionRequests::default(), + signed_bid, + } + } + + #[test] + fn a_funded_signed_bid_becomes_a_block_that_carries_it() { + let (state, bid) = scene(); + assert!(bid_is_includable( + &state, + &bid, + &ExecutionRequests::default() + )); + let block = + assemble_gloas_block_on_bid(&state, inputs(&state, bid.clone()), &config()).unwrap(); + assert_eq!(block.body.signed_execution_payload_bid, bid); + assert_eq!(block.slot, state.slot()); + assert_ne!(block.state_root, Root::ZERO); + // The block goes through the state transition as the network would run it. + let post = post_state(&state, &block); + let BeaconState::Gloas(inner) = &post else { + unreachable!("gloas") + }; + assert_eq!(inner.latest_execution_payload_bid, bid.message); + assert_eq!(block.state_root, post.hash_tree_root()); + } + + #[test] + fn an_inactive_builder_cannot_have_a_block_built() { + // Deposited at the finalized epoch itself: not yet active. + let parent = gloas_state_with_builder(0, 100_000_000_000, 0); + let mut state = advance_to_slot(&parent, parent.slot() + 1, &config()).unwrap(); + let BeaconState::Gloas(inner) = &mut state else { + unreachable!("gloas") + }; + inner.builders[0].deposit_epoch = inner.finalized_checkpoint.epoch; + let bid = bid_for(&state, |_| {}); + assert!(!bid_is_includable( + &state, + &bid, + &ExecutionRequests::default() + )); + assert!(assemble_gloas_block_on_bid(&state, inputs(&state, bid), &config()).is_err()); + } + + #[test] + fn a_bid_on_the_wrong_parent_hash_is_refused() { + let (state, _) = scene(); + let bid = bid_for(&state, |bid| { + bid.parent_block_hash = ExecutionBlockHash::repeat_byte(0x77) + }); + assert!(!bid_is_includable( + &state, + &bid, + &ExecutionRequests::default() + )); + assert!(assemble_gloas_block_on_bid(&state, inputs(&state, bid), &config()).is_err()); + } + + #[test] + fn a_bid_with_the_wrong_randao_is_refused() { + let (state, _) = scene(); + let bid = bid_for(&state, |bid| bid.prev_randao = Bytes32::repeat_byte(9)); + assert!(!bid_is_includable( + &state, + &bid, + &ExecutionRequests::default() + )); + assert!(assemble_gloas_block_on_bid(&state, inputs(&state, bid), &config()).is_err()); + } + + #[test] + fn a_bid_with_a_bad_signature_is_built_into_nothing() { + let (state, mut bid) = scene(); + bid.signature.0[3] ^= 1; + // The cheap filter does not verify signatures, `process_block` does. + assert!(bid_is_includable( + &state, + &bid, + &ExecutionRequests::default() + )); + assert!(assemble_gloas_block_on_bid(&state, inputs(&state, bid), &config()).is_err()); + } + + #[test] + fn a_bid_for_another_slot_or_parent_root_is_not_includable() { + let (state, _) = scene(); + let requests = ExecutionRequests::default(); + let later = bid_for(&state, |bid| bid.slot += 1); + assert!(!bid_is_includable(&state, &later, &requests)); + let other_root = bid_for(&state, |bid| bid.parent_block_root = Root::repeat_byte(5)); + assert!(!bid_is_includable(&state, &other_root, &requests)); + let same_hash = bid_for(&state, |bid| bid.block_hash = bid.parent_block_hash); + assert!(!bid_is_includable(&state, &same_hash, &requests)); + } + + #[test] + fn an_unknown_or_overdrawn_builder_is_not_includable() { + let (state, _) = scene(); + let requests = ExecutionRequests::default(); + let unknown = bid_for(&state, |bid| bid.builder_index = 4); + assert!(!bid_is_includable(&state, &unknown, &requests)); + let overdrawn = bid_for(&state, |bid| bid.value = 200_000_000_000); + assert!(!bid_is_includable(&state, &overdrawn, &requests)); + } + + #[test] + fn a_builder_exited_by_the_parents_requests_is_not_includable() { + let (state, bid) = scene(); + let pubkey = BlsPubkey(builder_secret(0).sk_to_pk().to_bytes()); + let exit = |source_address| ExecutionRequests { + builder_exits: vec![gloas::BuilderExitRequest { + source_address, + pubkey, + }] + .into(), + ..Default::default() + }; + // The builder's execution address is `index + 1` repeated. + assert!(!bid_is_includable( + &state, + &bid, + &exit(ExecutionAddress::repeat_byte(1)) + )); + // A request from another address does not exit it. + assert!(bid_is_includable( + &state, + &bid, + &exit(ExecutionAddress::repeat_byte(9)) + )); + } + + #[test] + fn a_non_gloas_state_has_no_includable_bid() { + let (_, bid) = scene(); + let fulu = crate::beacon::helpers::test_state::with_validators_at(ForkName::Fulu, 8); + assert!(!bid_is_includable( + &fulu, + &bid, + &ExecutionRequests::default() + )); + } +} diff --git a/crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs b/crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs index 9cda084b6..59ad8c922 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs @@ -2,63 +2,454 @@ //! `SignedExecutionPayloadBid`, the commitment a proposer may choose in place //! of building its own payload. //! -//! Split like [`super::envelope`]: [`cheap_checks`] reads only the message, the -//! market's seen state and the clock, so the p2p actor runs it inline; -//! [`stateful_checks`] reads cached states and verifies the signature, so it -//! runs on a blocking thread. Never queues: every "MAY be queued" is IGNORE. +//! The rules are the specification's `validate_execution_payload_bid_gossip` +//! (`specs/gloas/p2p-interface.md`), split like [`super::envelope`]: +//! [`cheap_checks`] reads only the message, the market's seen state and the +//! clock, so the p2p actor runs it inline; [`stateful_checks`] reads cached +//! states and verifies the signature, so it runs on a blocking thread. The +//! caller records the bid in the [`BuilderMarket`] on `Accept`. //! -//! Stubs until filled: every rule answers `Ignore(NoConsumer)`. +//! Deliberate departures from the specification (`docs/spec_deviations.md`): +//! +//! - Never queues. Every "MAY be queued" is IGNORE, and a state that is not +//! cached is IGNORE rather than rebuilt from disk, so a verdict gossipsub +//! waits on is never stalled by a replay. +//! - `store.block_states[parent]` advanced with `process_slots` to `bid.slot` +//! becomes the parent's cached post-state when the bid is in the parent's +//! own epoch (gloas's `process_slot` touches none of the fields rules 17 to +//! 22 read), and the cached checkpoint state of the bid's epoch otherwise. +//! - `get_head(store)` is the head the chain actor recorded, with a fresh walk +//! only when none is recorded. +//! - `seen.execution_payloads` is the market's known payloads: envelopes gossip +//! accepted or this node published, plus a pre-gloas parent's own payload. use std::sync::Arc; -use super::{IgnoreReason, Outcome}; -use crate::beacon::builder_market::BuilderMarket; +use ethlambda_storage::CacheKey; + +use super::{ + IgnoreReason, Outcome, RejectReason, is_current_slot, is_gloas_slot, + proposer_preferences::dependent_root_at, +}; +use crate::beacon::builder_market::{BuilderMarket, KnownPayload}; use crate::beacon::config::Config; -use crate::beacon::containers::{BeaconState, gloas}; -use crate::beacon::fork_choice::Store; -use crate::beacon::primitives::{Epoch, Root, Slot}; +use crate::beacon::constants::PAYLOAD_BUILDER_VERSION; +use crate::beacon::containers::{BeaconState, SignedBeaconBlock, gloas}; +use crate::beacon::fork_choice::{self, ForkChoiceNode, PayloadStatus, Store}; +use crate::beacon::helpers::accessors::{get_current_epoch, get_randao_mix}; +use crate::beacon::helpers::gloas::{can_builder_cover_bid, is_active_builder}; +use crate::beacon::helpers::misc::{compute_epoch_at_slot, compute_start_slot_at_epoch}; +use crate::beacon::lean_boundary::lean_state_unreachable; +use crate::beacon::preset; +use crate::beacon::primitives::{Epoch, ExecutionBlockHash, Root, Slot}; +use crate::beacon::stf; +use crate::beacon::stf::gloas::verify_execution_payload_bid_signature; /// Gloas p2p preset: the largest decompressed `SignedExecutionPayloadBid`. pub const MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE: usize = 196_932; -/// The spec's `is_gas_limit_target_compatible`. -/// `max_diff = (parent / 1024).saturating_sub(1)`, `min = parent - max_diff`, -/// `max = parent.saturating_add(max_diff)`. -#[allow(dead_code, unused_variables)] // filled by Agent A +/// The spec's `is_gas_limit_target_compatible`: whether `gas_limit` is what +/// the EIP-1559 transition rule from `parent_gas_limit` allows when steering +/// towards `target_gas_limit`. pub fn is_gas_limit_target_compatible( parent_gas_limit: u64, gas_limit: u64, target_gas_limit: u64, ) -> bool { - false + let max_difference = (parent_gas_limit / 1024).saturating_sub(1); + let min_gas_limit = parent_gas_limit - max_difference; + let max_gas_limit = parent_gas_limit.saturating_add(max_difference); + if target_gas_limit < min_gas_limit { + return gas_limit == min_gas_limit; + } + if target_gas_limit > max_gas_limit { + return gas_limit == max_gas_limit; + } + gas_limit == target_gas_limit } -/// `is_current_slot(slot) || slot.checked_sub(1).is_some_and(is_current_slot)`. -#[allow(dead_code, unused_variables)] // filled by Agent A +/// The spec's `is_current_or_next_slot`. pub(crate) fn is_current_or_next_slot(config: &Config, slot: Slot, now_ms: u64) -> bool { - false + is_current_slot(config, slot, now_ms) + || slot + .checked_sub(1) + .is_some_and(|previous| is_current_slot(config, previous, now_ms)) } -#[allow(unused_variables)] // filled by Agent A +/// The rules that read only the message, the market's seen state and the +/// clock. pub fn cheap_checks( market: &BuilderMarket, store: &Store, signed: &gloas::SignedExecutionPayloadBid, now_ms: u64, ) -> Result<(), Outcome> { - Err(Outcome::Ignore(IgnoreReason::NoConsumer)) + let bid = &signed.message; + let config = store.config(); + // [IGNORE] The first bid for this slot, parent and builder, and [IGNORE] + // the highest value seen for the slot and parent. + market.check_bid_seen(bid).map_err(Outcome::Ignore)?; + // [IGNORE] The bid's slot is the current slot or the next slot. + if !is_current_or_next_slot(&config, bid.slot, now_ms) { + return Err(Outcome::Ignore(IgnoreReason::NotCurrentOrNextSlot)); + } + // [REJECT] The bid's execution payment is zero. + if bid.execution_payment != 0 { + return Err(Outcome::Reject(RejectReason::ExecutionPaymentNonZero)); + } + // [REJECT] The bid's block hash is not its parent block hash. + if bid.block_hash == bid.parent_block_hash { + return Err(Outcome::Reject(RejectReason::BlockHashEqualsParent)); + } + // [REJECT] The commitment count is within the epoch's blob limit. + let proposal_epoch = compute_epoch_at_slot(bid.slot); + if bid.blob_kzg_commitments.len() as u64 > config.max_blobs_per_block(proposal_epoch) { + return Err(Outcome::Reject(RejectReason::TooManyBlobs)); + } + // Ours: a slot before the fork has no bids. Placed after the rejects so + // a malformed message is still penalized, whichever fork it names. + if !is_gloas_slot(&config, bid.slot) { + return Err(Outcome::Ignore(IgnoreReason::PreGloasSlot)); + } + Ok(()) +} + +/// A pre-gloas state's execution payload header, as far as a bid needs it: +/// `(block_hash, gas_limit)`. `None` for a state with no payload header. +fn pre_gloas_payload(state: &BeaconState) -> Option<(ExecutionBlockHash, u64)> { + match state { + BeaconState::Bellatrix(s) => { + let header = &s.latest_execution_payload_header; + Some((header.block_hash, header.gas_limit)) + } + BeaconState::Capella(s) => { + let header = &s.latest_execution_payload_header; + Some((header.block_hash, header.gas_limit)) + } + BeaconState::Deneb(s) => { + let header = &s.latest_execution_payload_header; + Some((header.block_hash, header.gas_limit)) + } + BeaconState::Electra(s) => { + let header = &s.latest_execution_payload_header; + Some((header.block_hash, header.gas_limit)) + } + BeaconState::Fulu(s) => { + let header = &s.latest_execution_payload_header; + Some((header.block_hash, header.gas_limit)) + } + BeaconState::Phase0(_) | BeaconState::Altair(_) | BeaconState::Gloas(_) => None, + BeaconState::Lean(_) => lean_state_unreachable("execution_payload_bid::pre_gloas_payload"), + } +} + +/// The head's payload-relevant facts, off whichever source answers. +enum HeadView { + Gloas { + node: ForkChoiceNode, + parent_root: Root, + bid_parent_hash: ExecutionBlockHash, + bid_block_hash: ExecutionBlockHash, + }, + PreGloas { + root: Root, + block_hash: ExecutionBlockHash, + }, +} + +/// The head, from the chain actor's record (a fresh walk only when no status +/// is recorded), and its payload hashes from the cached post-state (the +/// decoded block when that is not cached). +fn head_view(store: &Store) -> Result { + let config = store.config(); + let internal = || Outcome::Ignore(IgnoreReason::Internal); + let node = match (store.head().ok(), store.head_payload_status()) { + (Some(root), Some(payload_status)) => ForkChoiceNode { + root, + payload_status, + }, + _ => fork_choice::get_head_node(store, &config).map_err(|_| internal())?, + }; + if let Some(state) = store.cached_state(CacheKey::BlockState(node.root)) { + return match &*state { + BeaconState::Gloas(inner) => Ok(HeadView::Gloas { + node, + parent_root: inner.latest_block_header.parent_root, + bid_parent_hash: inner.latest_execution_payload_bid.parent_block_hash, + bid_block_hash: inner.latest_execution_payload_bid.block_hash, + }), + other => match pre_gloas_payload(other) { + Some((block_hash, _)) => Ok(HeadView::PreGloas { + root: node.root, + block_hash, + }), + None => Err(Outcome::Ignore(IgnoreReason::NotOnHeadBranch)), + }, + }; + } + let block = store + .get_signed_block(&node.root) + .map_err(|_| internal())? + .ok_or(Outcome::Ignore(IgnoreReason::StateUnavailable))?; + match block { + SignedBeaconBlock::Gloas(block) => { + let bid = &block.message.body.signed_execution_payload_bid.message; + Ok(HeadView::Gloas { + node, + parent_root: block.message.parent_root, + bid_parent_hash: bid.parent_block_hash, + bid_block_hash: bid.block_hash, + }) + } + other => match other.execution_block_hash() { + Some(block_hash) => Ok(HeadView::PreGloas { + root: node.root, + block_hash, + }), + None => Err(Outcome::Ignore(IgnoreReason::NotOnHeadBranch)), + }, + } +} + +/// The spec's `is_bid_compatible_with_head`, against the recorded head. +/// +/// A pre-gloas head has no bid, so a bid is compatible when it builds on that +/// head and its payload (the boundary rule: pre-gloas parent payloads count as +/// full). +pub fn is_bid_compatible_with_head( + store: &Store, + bid: &gloas::ExecutionPayloadBid, +) -> Result { + match head_view(store)? { + HeadView::PreGloas { root, block_hash } => { + Ok(bid.parent_block_root == root && bid.parent_block_hash == block_hash) + } + HeadView::Gloas { + node, + parent_root, + bid_parent_hash, + bid_block_hash, + } => { + let builds_on_parent_block = bid.parent_block_root == parent_root; + let builds_on_parent_payload = bid.parent_block_hash == bid_parent_hash; + if builds_on_parent_block && builds_on_parent_payload { + return Ok(true); + } + if bid.parent_block_root != node.root { + return Ok(false); + } + let builds_on_head_payload = bid.parent_block_hash == bid_block_hash; + // The head's status can only be PENDING if a caller recorded a + // walk's intermediate node, which `get_head_node` never returns. + debug_assert_ne!(node.payload_status, PayloadStatus::Pending); + let build_on_full = fork_choice::should_build_on_full(store, node, bid.slot) + .map_err(|_| Outcome::Ignore(IgnoreReason::Internal))?; + Ok(if build_on_full { + builds_on_head_payload + } else { + builds_on_parent_payload + }) + } + } +} + +/// Cached-only: a `CheckpointState{epoch, root}` hit; else the cached +/// `BlockState(root)` advanced to the epoch start and cached. `None` = miss. +/// +/// The rebuild-from-disk `fork_choice::checkpoint_state` does on a miss would +/// stall a gossip verdict, so a state that is not cached is a miss here. +pub(crate) fn cached_checkpoint_state( + store: &Store, + epoch: Epoch, + root: Root, +) -> Option> { + let key = CacheKey::CheckpointState { epoch, root }; + if let Some(state) = store.cached_state(key) { + return Some(state); + } + let state = store.cached_state(CacheKey::BlockState(root))?; + let target_slot = compute_start_slot_at_epoch(epoch); + let state = if state.slot() < target_slot { + let mut advanced = (*state).clone(); + stf::process_slots(&mut advanced, target_slot, &store.config()).ok()?; + Arc::new(advanced) + } else { + state + }; + store.cache_state(key, state.clone()); + Some(state) } -#[allow(unused_variables)] // filled by Agent A +/// The exits a parent payload carried, for rule 21: the market's known payload +/// when it is the parent block's own, else the verified envelope in the store. +/// `None` when neither is available. +fn parent_payload_exits( + store: &Store, + market: &BuilderMarket, + bid: &gloas::ExecutionPayloadBid, +) -> Option< + Vec<( + crate::beacon::primitives::BlsPubkey, + crate::beacon::primitives::ExecutionAddress, + )>, +> { + if let Some(KnownPayload { + beacon_block_root, + builder_exits, + .. + }) = market.known_payload(bid.parent_block_hash) + && beacon_block_root == bid.parent_block_root + { + return Some(builder_exits); + } + let envelope = store + .get_execution_payload_envelope(&bid.parent_block_root) + .ok()??; + Some( + envelope + .message + .execution_requests + .builder_exits + .iter() + .map(|exit| (exit.pubkey, exit.source_address)) + .collect(), + ) +} + +/// The rules that need states, then the signature. Runs on a blocking thread. pub fn stateful_checks( store: &Store, market: &BuilderMarket, signed: &gloas::SignedExecutionPayloadBid, ) -> Outcome { - Outcome::Ignore(IgnoreReason::NoConsumer) + match stateful_rules(store, market, signed) { + Ok(()) => Outcome::Accept, + Err(outcome) => outcome, + } } -/// Both halves. The caller records the bid on `Accept`. +fn stateful_rules( + store: &Store, + market: &BuilderMarket, + signed: &gloas::SignedExecutionPayloadBid, +) -> Result<(), Outcome> { + let bid = &signed.message; + let ignore = |reason| Outcome::Ignore(reason); + let reject = |reason| Outcome::Reject(reason); + let proposal_epoch = compute_epoch_at_slot(bid.slot); + + // [IGNORE] The parent block is known (never queued). + if !store.has_block(&bid.parent_block_root) { + return Err(ignore(IgnoreReason::UnknownBlock)); + } + // [REJECT] The bid is for a higher slot than its parent. + let (parent_slot, _) = store + .block_entry(&bid.parent_block_root) + .ok_or(ignore(IgnoreReason::UnknownBlock))?; + if bid.slot <= parent_slot { + return Err(reject(RejectReason::NotAfterParent)); + } + // [IGNORE] The parent has been imported (its post-state is cached). + let parent_state = store + .cached_state(CacheKey::BlockState(bid.parent_block_root)) + .ok_or(ignore(IgnoreReason::StateUnavailable))?; + // [IGNORE] The bid's slot is within the parent's proposer lookahead. + if proposal_epoch > get_current_epoch(&parent_state) + preset::MIN_SEED_LOOKAHEAD { + return Err(ignore(IgnoreReason::BeyondLookahead)); + } + // [IGNORE] The matching proposer preferences have been seen. Rule 10 + // keeps the dependent slot inside the parent state's `block_roots`. + let dependent_root = dependent_root_at(&parent_state, bid.parent_block_root, bid.slot) + .ok_or(ignore(IgnoreReason::AncestryUnknown))?; + let preferences = market + .preferences(bid.slot, dependent_root) + .ok_or(ignore(IgnoreReason::PreferencesUnseen))? + .message; + // [IGNORE] The fee recipient matches the proposer's preference. + if bid.fee_recipient != preferences.fee_recipient { + return Err(ignore(IgnoreReason::FeeRecipientMismatch)); + } + // [IGNORE] The parent block hash is a known execution payload. Across the + // fork boundary, a pre-gloas parent's own payload counts. + let parent_gas_limit = match market.known_payload(bid.parent_block_hash) { + Some(known) => known.gas_limit, + None => match pre_gloas_payload(&parent_state) { + Some((block_hash, gas_limit)) if block_hash == bid.parent_block_hash => gas_limit, + _ => return Err(ignore(IgnoreReason::ParentPayloadUnknown)), + }, + }; + // [IGNORE] The gas limit is compatible with the proposer's target. + if !is_gas_limit_target_compatible( + parent_gas_limit, + bid.gas_limit, + preferences.target_gas_limit, + ) { + return Err(ignore(IgnoreReason::GasLimitIncompatible)); + } + // [IGNORE] The bid is compatible with the head branch. + if !is_bid_compatible_with_head(store, bid)? { + return Err(ignore(IgnoreReason::NotOnHeadBranch)); + } + // [REJECT] The previous randao is the parent state's. + if bid.prev_randao != get_randao_mix(&parent_state, get_current_epoch(&parent_state)) { + return Err(reject(RejectReason::PrevRandao)); + } + + // The parent state advanced to the bid's slot. Within the parent's own + // epoch the parent state answers identically, see the module docs. + let state = if proposal_epoch == get_current_epoch(&parent_state) { + parent_state.clone() + } else { + cached_checkpoint_state(store, proposal_epoch, bid.parent_block_root) + .ok_or(ignore(IgnoreReason::StateUnavailable))? + }; + let BeaconState::Gloas(inner) = &*state else { + // A bid in a gloas epoch is advanced into gloas by the epoch + // transition; anything else is a state this cannot judge. + return Err(ignore(IgnoreReason::StateUnavailable)); + }; + + // [REJECT] The builder index is valid, [REJECT] it is a payload builder + // and [REJECT] active. + let builder = inner + .builders + .get(bid.builder_index as usize) + .ok_or(reject(RejectReason::UnknownBuilder))?; + if builder.version != PAYLOAD_BUILDER_VERSION { + return Err(reject(RejectReason::NotPayloadBuilder)); + } + if !is_active_builder(inner, bid.builder_index).unwrap_or(false) { + return Err(reject(RejectReason::InactiveBuilder)); + } + // [IGNORE] The builder can cover the bid. + if !can_builder_cover_bid(inner, bid.builder_index, bid.value).unwrap_or(false) { + return Err(ignore(IgnoreReason::BuilderCannotCover)); + } + // [IGNORE] The parent's payload does not try to exit the builder. Only a + // gloas parent has an envelope to carry the request. + let parent_is_gloas = matches!(&*parent_state, BeaconState::Gloas(_)); + if parent_is_gloas && bid.parent_block_hash == inner.latest_execution_payload_bid.block_hash { + let exits = parent_payload_exits(store, market, bid) + .ok_or(ignore(IgnoreReason::ParentPayloadUnverified))?; + if exits.iter().any(|(pubkey, source)| { + *pubkey == builder.pubkey && *source == builder.execution_address + }) { + return Err(ignore(IgnoreReason::BuilderMayExit)); + } + } + // [REJECT] The signature is valid. An error (an index the state lacks) + // is a signature that cannot be. + if !matches!( + verify_execution_payload_bid_signature(&state, signed), + Ok(true) + ) { + return Err(reject(RejectReason::BadSignature)); + } + Ok(()) +} + +/// Both halves. The caller records the bid on `Accept`: the specification's +/// `validate_execution_payload_bid_gossip`. pub fn validate( market: &BuilderMarket, store: &Store, @@ -71,22 +462,645 @@ pub fn validate( stateful_checks(store, market, signed) } -/// The spec's `is_bid_compatible_with_head`, against the recorded head. -#[allow(dead_code, unused_variables)] // filled by Agent A -pub fn is_bid_compatible_with_head( - store: &Store, - bid: &gloas::ExecutionPayloadBid, -) -> Result { - Err(Outcome::Ignore(IgnoreReason::NoConsumer)) -} +#[cfg(test)] +mod tests { + use ethlambda_types::beacon::primitives::{ + BlsPubkey, Bytes32, ExecutionAddress, KzgCommitment, + }; -/// Cached-only: a `CheckpointState{epoch, root}` hit; else the cached -/// `BlockState(root)` advanced to the epoch start and cached. `None` = miss. -#[allow(dead_code, unused_variables)] // filled by Agent A -pub(crate) fn cached_checkpoint_state( - store: &Store, - epoch: Epoch, - root: Root, -) -> Option> { - None + use super::*; + use crate::beacon::builder_market::test_support::{ + builder_secret, envelope_with_gas_limit, sign_preferences, + }; + use crate::beacon::gossip::proposer_preferences::dependent_root_at; + use crate::beacon::gossip::test_support::builder_scene::*; + use crate::beacon::gossip::test_support::{slot_start_ms, store}; + use crate::beacon::helpers::accessors::get_current_epoch; + use crate::beacon::precheck::fixed_proposer; + + fn ignore(reason: IgnoreReason) -> Outcome { + Outcome::Ignore(reason) + } + + fn reject(reason: RejectReason) -> Outcome { + Outcome::Reject(reason) + } + + /// `stateful_checks` on the scene's market and store. + fn stateful(scene: &Scene, bid: &gloas::SignedExecutionPayloadBid) -> Outcome { + stateful_checks(&scene.store, &scene.market, bid) + } + + fn builder_pubkey() -> BlsPubkey { + BlsPubkey(builder_secret(0).sk_to_pk().to_bytes()) + } + + #[test] + fn the_scenes_bid_is_accepted() { + let scene = scene(); + let outcome = validate(&scene.market, &scene.store, &scene.bid, scene.now_ms()); + assert_eq!(outcome, Outcome::Accept); + } + + // ---- is_gas_limit_target_compatible ---- + + #[test] + fn a_gas_limit_may_step_towards_a_nearby_target() { + let parent = 60_000_000; + // max_difference = 60_000_000 / 1024 - 1 = 58_592. + assert!(is_gas_limit_target_compatible( + parent, + parent + 100, + parent + 100 + )); + assert!(is_gas_limit_target_compatible( + parent, + parent - 10, + parent - 10 + )); + assert!(is_gas_limit_target_compatible(parent, parent, parent)); + assert!(!is_gas_limit_target_compatible( + parent, + parent + 99, + parent + 100 + )); + } + + #[test] + fn a_gas_limit_is_pinned_to_the_step_limit_beyond_it() { + let parent = 60_000_000; + assert!(is_gas_limit_target_compatible( + parent, + parent + 58_592, + 100_000_000 + )); + assert!(!is_gas_limit_target_compatible( + parent, + parent + 58_593, + 100_000_000 + )); + assert!(!is_gas_limit_target_compatible( + parent, + parent + 58_591, + 100_000_000 + )); + assert!(is_gas_limit_target_compatible( + parent, + parent - 58_592, + 30_000_000 + )); + assert!(!is_gas_limit_target_compatible( + parent, + parent - 58_593, + 30_000_000 + )); + // The edge itself: a target exactly at the limit is no longer beyond it. + assert!(is_gas_limit_target_compatible( + parent, + parent + 58_592, + parent + 58_592 + )); + } + + #[test] + fn a_small_parent_cannot_move() { + // 1023 / 1024 = 0, so the allowed difference saturates to zero. + assert!(is_gas_limit_target_compatible(1023, 1023, 5_000)); + assert!(!is_gas_limit_target_compatible(1023, 1024, 5_000)); + assert!(is_gas_limit_target_compatible(0, 0, 100)); + } + + #[test] + fn the_step_limit_saturates_near_the_top_of_the_range() { + let parent = u64::MAX - 5; + assert!(is_gas_limit_target_compatible(parent, u64::MAX, u64::MAX)); + assert!(is_gas_limit_target_compatible(parent, parent, parent)); + } + + // ---- the clock ---- + + #[test] + fn a_bid_is_timely_from_just_before_the_previous_slot_until_just_after_its_own() { + let store = store(0); + let config = store.config(); + let slot = 34; + let earliest = slot_start_ms(&store, slot - 1) - 500; + let latest = slot_start_ms(&store, slot + 1) + 500; + assert!(is_current_or_next_slot(&config, slot, earliest)); + assert!(!is_current_or_next_slot(&config, slot, earliest - 1)); + assert!(is_current_or_next_slot(&config, slot, latest)); + assert!(!is_current_or_next_slot(&config, slot, latest + 1)); + // Slot zero has no previous slot. + assert!(is_current_or_next_slot( + &config, + 0, + slot_start_ms(&store, 0) + )); + } + + // ---- cheap rules ---- + + fn cheap(scene: &Scene, bid: &gloas::SignedExecutionPayloadBid) -> Result<(), Outcome> { + cheap_checks(&scene.market, &scene.store, bid, scene.now_ms()) + } + + #[test] + fn a_builders_second_bid_for_a_parent_is_already_seen() { + let scene = scene(); + assert!(scene.market.record_bid(scene.bid.clone())); + assert_eq!( + cheap(&scene, &scene.signed(|bid| bid.value = 9)), + Err(ignore(IgnoreReason::AlreadySeen)) + ); + } + + #[test] + fn a_bid_that_does_not_beat_the_best_is_not_highest() { + let scene = scene(); + let mut other = scene.bid.clone(); + other.message.builder_index = 1; + other.message.value = 5; + assert!(scene.market.record_bid(other)); + assert_eq!( + cheap(&scene, &scene.bid), + Err(ignore(IgnoreReason::NotHighestBid)) + ); + assert_eq!( + cheap(&scene, &scene.signed(|bid| bid.value = 5)), + Err(ignore(IgnoreReason::NotHighestBid)) + ); + } + + #[test] + fn a_bid_for_a_distant_slot_is_not_current_or_next() { + let scene = scene(); + let far = scene.signed(|bid| bid.slot = 40); + assert_eq!( + cheap(&scene, &far), + Err(ignore(IgnoreReason::NotCurrentOrNextSlot)) + ); + } + + #[test] + fn a_bid_with_an_execution_payment_is_rejected() { + let scene = scene(); + let paid = scene.signed(|bid| bid.execution_payment = 1); + assert_eq!( + cheap(&scene, &paid), + Err(reject(RejectReason::ExecutionPaymentNonZero)) + ); + } + + #[test] + fn a_bid_whose_block_hash_is_its_parents_is_rejected() { + let scene = scene(); + let same = scene.signed(|bid| bid.block_hash = bid.parent_block_hash); + assert_eq!( + cheap(&scene, &same), + Err(reject(RejectReason::BlockHashEqualsParent)) + ); + } + + #[test] + fn a_bid_with_too_many_commitments_is_rejected() { + let scene = scene(); + let limit = scene.store.config().max_blobs_per_block(1) as usize; + let at_limit = scene.signed(|bid| { + bid.blob_kzg_commitments = vec![KzgCommitment([0; 48]); limit].into(); + }); + assert_eq!(cheap(&scene, &at_limit), Ok(())); + let over = scene.signed(|bid| { + bid.blob_kzg_commitments = vec![KzgCommitment([0; 48]); limit + 1].into(); + }); + assert_eq!( + cheap(&scene, &over), + Err(reject(RejectReason::TooManyBlobs)) + ); + } + + #[test] + fn a_bid_before_the_fork_is_ignored() { + // A fulu-only store has no gloas slot at all. + let fulu = store(0); + let scene = scene(); + let now_ms = slot_start_ms(&fulu, BID_SLOT) + 100; + assert_eq!( + cheap_checks(&scene.market, &fulu, &scene.signed(|_| {}), now_ms), + Err(ignore(IgnoreReason::PreGloasSlot)) + ); + } + + // ---- stateful rules ---- + + #[test] + fn a_bid_on_an_unknown_parent_is_ignored() { + let scene = scene(); + let bid = scene.signed(|bid| bid.parent_block_root = Root::repeat_byte(9)); + assert_eq!(stateful(&scene, &bid), ignore(IgnoreReason::UnknownBlock)); + } + + #[test] + fn a_bid_not_after_its_parent_is_rejected() { + let scene = scene(); + let bid = scene.signed(|bid| bid.slot = 32); + assert_eq!(stateful(&scene, &bid), reject(RejectReason::NotAfterParent)); + } + + #[test] + fn a_parent_without_a_cached_state_is_ignored() { + let mut scene = scene(); + let stateless = Root::repeat_byte(0x60); + scene + .store + .insert_pending_block(stateless, block_at(10)) + .expect("insert the block"); + let bid = scene.signed(|bid| bid.parent_block_root = stateless); + assert_eq!( + stateful(&scene, &bid), + ignore(IgnoreReason::StateUnavailable) + ); + } + + #[test] + fn a_bid_beyond_the_parents_lookahead_is_ignored() { + let scene = scene(); + // The parent is in epoch 1, so epoch 3 is past its lookahead. + let bid = scene.signed(|bid| bid.slot = 96); + assert_eq!( + stateful(&scene, &bid), + ignore(IgnoreReason::BeyondLookahead) + ); + } + + #[test] + fn a_bid_without_preferences_is_ignored() { + let scene = scene(); + let empty = BuilderMarket::default(); + assert_eq!( + stateful_checks(&scene.store, &empty, &scene.bid), + ignore(IgnoreReason::PreferencesUnseen) + ); + } + + #[test] + fn a_bid_for_another_fee_recipient_is_ignored() { + let scene = scene(); + let bid = scene.signed(|bid| bid.fee_recipient = ExecutionAddress::repeat_byte(0x99)); + assert_eq!( + stateful(&scene, &bid), + ignore(IgnoreReason::FeeRecipientMismatch) + ); + } + + #[test] + fn a_bid_on_an_unknown_payload_is_ignored() { + let scene = scene(); + let bid = scene.signed(|bid| bid.parent_block_hash = ExecutionBlockHash::repeat_byte(0x77)); + assert_eq!( + stateful(&scene, &bid), + ignore(IgnoreReason::ParentPayloadUnknown) + ); + } + + #[test] + fn a_gas_limit_the_target_cannot_reach_is_ignored() { + let scene = scene(); + let bid = scene.signed(|bid| bid.gas_limit = PARENT_GAS_LIMIT + 1_000); + assert_eq!( + stateful(&scene, &bid), + ignore(IgnoreReason::GasLimitIncompatible) + ); + } + + #[test] + fn a_bid_off_the_head_branch_is_ignored() { + let scene = scene(); + // A known payload, but neither the head's nor its parent's. + let other = ExecutionBlockHash::repeat_byte(0x44); + scene + .market + .record_execution_payload(&envelope_with_gas_limit( + other, + PARENT_GAS_LIMIT, + PARENT, + vec![], + )); + let bid = scene.signed(|bid| bid.parent_block_hash = other); + assert_eq!( + stateful(&scene, &bid), + ignore(IgnoreReason::NotOnHeadBranch) + ); + } + + #[test] + fn a_wrong_previous_randao_is_rejected() { + let scene = scene(); + let bid = scene.signed(|bid| bid.prev_randao = Bytes32::repeat_byte(9)); + assert_eq!(stateful(&scene, &bid), reject(RejectReason::PrevRandao)); + } + + #[test] + fn an_unregistered_builder_is_rejected() { + let scene = scene(); + let bid = scene.signed(|bid| bid.builder_index = 5); + assert_eq!(stateful(&scene, &bid), reject(RejectReason::UnknownBuilder)); + } + + #[test] + fn a_builder_of_another_version_is_rejected() { + let scene = scene_with(|state| state.builders[0].version = 7); + assert_eq!( + stateful(&scene, &scene.bid), + reject(RejectReason::NotPayloadBuilder) + ); + } + + #[test] + fn an_inactive_builder_is_rejected() { + let exiting = scene_with(|state| state.builders[0].withdrawable_epoch = 5); + assert_eq!( + stateful(&exiting, &exiting.bid), + reject(RejectReason::InactiveBuilder) + ); + // Deposited at or after the finalized epoch. + let young = scene_with(|state| state.builders[0].deposit_epoch = 1); + assert_eq!( + stateful(&young, &young.bid), + reject(RejectReason::InactiveBuilder) + ); + } + + #[test] + fn a_builder_that_cannot_cover_the_bid_is_ignored() { + let scene = scene(); + let bid = scene.signed(|bid| bid.value = 200_000_000_000); + assert_eq!( + stateful(&scene, &bid), + ignore(IgnoreReason::BuilderCannotCover) + ); + } + + #[test] + fn a_builder_the_parent_payload_exits_is_ignored() { + let scene = scene(); + let exits = vec![(builder_pubkey(), ExecutionAddress::repeat_byte(1))]; + scene + .market + .record_execution_payload(&envelope_with_gas_limit( + parent_block_hash(), + PARENT_GAS_LIMIT, + PARENT, + exits, + )); + assert_eq!( + stateful(&scene, &scene.bid), + ignore(IgnoreReason::BuilderMayExit) + ); + } + + #[test] + fn an_exit_for_another_builder_does_not_matter() { + let scene = scene(); + let wrong_key = (BlsPubkey([3; 48]), ExecutionAddress::repeat_byte(1)); + let wrong_source = (builder_pubkey(), ExecutionAddress::repeat_byte(9)); + scene + .market + .record_execution_payload(&envelope_with_gas_limit( + parent_block_hash(), + PARENT_GAS_LIMIT, + PARENT, + vec![wrong_key, wrong_source], + )); + assert_eq!(stateful(&scene, &scene.bid), Outcome::Accept); + } + + #[test] + fn exits_are_read_from_the_stored_envelope_when_the_known_payload_is_another_blocks() { + let mut scene = scene(); + // The market knows the hash from some other block, so it cannot answer + // for the parent's own envelope. + scene + .market + .record_execution_payload(&envelope_with_gas_limit( + parent_block_hash(), + PARENT_GAS_LIMIT, + Root::repeat_byte(0x61), + vec![], + )); + assert_eq!( + stateful(&scene, &scene.bid), + ignore(IgnoreReason::ParentPayloadUnverified) + ); + + let exits = vec![(builder_pubkey(), ExecutionAddress::repeat_byte(1))]; + let envelope = gloas::SignedExecutionPayloadEnvelope { + message: envelope_with_gas_limit(parent_block_hash(), PARENT_GAS_LIMIT, PARENT, exits), + signature: Default::default(), + }; + scene.store.insert_verified_payload(32, &envelope); + assert_eq!( + stateful(&scene, &scene.bid), + ignore(IgnoreReason::BuilderMayExit) + ); + } + + #[test] + fn a_bad_signature_is_rejected() { + let scene = scene(); + let mut bid = scene.bid.clone(); + bid.signature.0[5] ^= 1; + assert_eq!(stateful(&scene, &bid), reject(RejectReason::BadSignature)); + // Signed by another builder's key. + let forged = test_support_sign_by(&scene, 1); + assert_eq!( + stateful(&scene, &forged), + reject(RejectReason::BadSignature) + ); + } + + fn test_support_sign_by(scene: &Scene, secret: u64) -> gloas::SignedExecutionPayloadBid { + crate::beacon::builder_market::test_support::sign_bid( + &scene.state, + scene.bid.message.clone(), + secret, + ) + } + + // ---- the parent's state stands in for the advanced one ---- + + #[test] + fn a_state_advanced_within_its_epoch_keeps_what_the_rules_read() { + let scene = scene(); + let config = scene.store.config(); + let mut advanced = scene.state.clone(); + stf::process_slots(&mut advanced, BID_SLOT, &config).expect("advance"); + assert_eq!(advanced.slot(), BID_SLOT); + let (BeaconState::Gloas(before), BeaconState::Gloas(after)) = (&scene.state, &advanced) + else { + unreachable!("gloas states") + }; + assert_eq!(before.builders, after.builders); + assert_eq!(before.finalized_checkpoint, after.finalized_checkpoint); + assert_eq!(before.fork, after.fork); + assert_eq!( + before.builder_pending_payments, + after.builder_pending_payments + ); + assert_eq!( + before.builder_pending_withdrawals, + after.builder_pending_withdrawals + ); + assert_eq!( + before.latest_execution_payload_bid, + after.latest_execution_payload_bid + ); + assert_eq!( + get_randao_mix(&scene.state, get_current_epoch(&scene.state)), + get_randao_mix(&advanced, get_current_epoch(&advanced)) + ); + } + + #[test] + fn a_bid_across_an_epoch_uses_and_caches_the_checkpoint_state() { + let scene = scene(); + let slot = 64; + // Preferences for the later epoch's proposer. + let dependent = dependent_root_at(&scene.state, PARENT, slot).expect("in the window"); + let proposer = fixed_proposer(&scene.state, slot).expect("in the lookahead window"); + let preferences = sign_preferences( + &scene.state, + gloas::ProposerPreferences { + dependent_root: dependent, + proposal_slot: slot, + validator_index: proposer, + fee_recipient: fee_recipient(), + target_gas_limit: PARENT_GAS_LIMIT, + }, + ); + assert!(scene.market.record_preferences(preferences, slot - 1)); + let bid = scene.signed(|bid| bid.slot = slot); + let key = CacheKey::CheckpointState { + epoch: 2, + root: PARENT, + }; + assert!(scene.store.cached_state(key).is_none()); + assert_eq!(stateful(&scene, &bid), Outcome::Accept); + let cached = scene.store.cached_state(key).expect("the advanced state"); + assert_eq!(get_current_epoch(&cached), 2); + // A second bid finds it. + assert_eq!(stateful(&scene, &bid), Outcome::Accept); + } + + /// A fulu parent one slot before gloas's first epoch: the rules reach the + /// advanced state (which the upgrade has made gloas, with no builders yet) + /// only if the parent's own payload counted as a known one. + fn fulu_parent_scene( + edit: impl FnOnce(&mut crate::beacon::containers::BeaconState), + ) -> (Store, BuilderMarket, gloas::SignedExecutionPayloadBid) { + use ethlambda_storage::ForkCheckpoints; + + use crate::beacon::fork::ForkName; + use crate::beacon::gossip::test_support::{GENESIS_TIME, builder_scene::block_at}; + use crate::beacon::helpers::test_state::with_signing_validators_at; + + let config = Config::mainnet() + .with_fork_epoch(ForkName::Fulu, 0) + .with_fork_epoch(ForkName::Gloas, 2); + let parent = Root::repeat_byte(0x70); + let header_hash = ExecutionBlockHash::repeat_byte(0x41); + let mut state = with_signing_validators_at(ForkName::Fulu, 64); + *state.slot_mut() = 62; + if let BeaconState::Fulu(fulu) = &mut state { + fulu.latest_execution_payload_header.block_hash = header_hash; + fulu.latest_execution_payload_header.gas_limit = 30_000_000; + } + edit(&mut state); + state.apply_pending_mutations(); + + let mut store = Store::init_beacon( + Arc::new(ethlambda_storage::backend::InMemoryBackend::new()), + GENESIS_TIME, + config, + parent, + ethlambda_types::checkpoint::Checkpoint { + root: parent, + slot: 62, + }, + 62, + ); + store + .insert_pending_block(parent, block_at(62)) + .expect("insert the parent"); + store + .insert_state(parent, state.clone()) + .expect("insert the parent state"); + store + .update_checkpoints(ForkCheckpoints::head_only(parent)) + .expect("move the head"); + // The status the chain actor records; a pre-gloas head has one node. + store.set_head_payload_status(parent, PayloadStatus::Empty); + + let slot = 64; + let market = BuilderMarket::default(); + let proposer = fixed_proposer(&state, slot).expect("in the window"); + let dependent_root = dependent_root_at(&state, parent, slot).expect("in the window"); + let preferences = sign_preferences( + &state, + gloas::ProposerPreferences { + dependent_root, + proposal_slot: slot, + validator_index: proposer, + fee_recipient: fee_recipient(), + target_gas_limit: 30_000_000, + }, + ); + assert!(market.record_preferences(preferences, slot - 1)); + let bid = gloas::SignedExecutionPayloadBid { + message: gloas::ExecutionPayloadBid { + parent_block_hash: header_hash, + parent_block_root: parent, + block_hash: ExecutionBlockHash::repeat_byte(0x33), + prev_randao: get_randao_mix(&state, get_current_epoch(&state)), + fee_recipient: fee_recipient(), + gas_limit: 30_000_000, + builder_index: 0, + slot, + value: 1, + ..Default::default() + }, + signature: Default::default(), + }; + (store, market, bid) + } + + #[test] + fn a_pre_gloas_parents_payload_counts_as_known_with_its_header_gas_limit() { + let (store, market, bid) = fulu_parent_scene(|_| {}); + // Past the payload, gas, head and randao rules, the advanced state is + // gloas's first and has no builders yet. + assert_eq!( + stateful_checks(&store, &market, &bid), + reject(RejectReason::UnknownBuilder) + ); + } + + #[test] + fn a_pre_gloas_parents_header_gas_limit_bounds_the_bid() { + let (store, market, mut bid) = fulu_parent_scene(|_| {}); + bid.message.gas_limit += 1_000; + assert_eq!( + stateful_checks(&store, &market, &bid), + ignore(IgnoreReason::GasLimitIncompatible) + ); + } + + #[test] + fn only_the_pre_gloas_parents_own_payload_hash_counts() { + let (store, market, mut bid) = fulu_parent_scene(|_| {}); + bid.message.parent_block_hash = ExecutionBlockHash::repeat_byte(0x42); + assert_eq!( + stateful_checks(&store, &market, &bid), + ignore(IgnoreReason::ParentPayloadUnknown) + ); + } } diff --git a/crates/blockchain/state_transition/src/beacon/gossip/proposer_preferences.rs b/crates/blockchain/state_transition/src/beacon/gossip/proposer_preferences.rs index 50c73650e..a3bb007a8 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/proposer_preferences.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/proposer_preferences.rs @@ -2,34 +2,175 @@ //! `SignedProposerPreferences` (fee recipient and gas target) for a slot, which //! bids on that slot are judged against. //! -//! Split like [`super::envelope`]: [`cheap_checks`] inline in the p2p actor, -//! [`stateful_checks`] on a blocking thread, reading cached states only. +//! The rules are the specification's `validate_proposer_preferences_gossip` +//! (`specs/gloas/p2p-interface.md`), split like [`super::envelope`]: +//! [`cheap_checks`] inline in the p2p actor, [`stateful_checks`] on a blocking +//! thread, reading cached states only. The caller records the preferences in +//! the [`BuilderMarket`] on `Accept`. //! -//! Stubs until filled: every rule answers `Ignore(NoConsumer)`. +//! Deliberate departures from the specification (`docs/spec_deviations.md`): +//! +//! - Never queues: an unseen dependent block is IGNORE, and a state that is +//! not cached is IGNORE, not rebuilt from disk. +//! - The lookahead comes from the cached head state when the head shares the +//! dependent root (the whole canonical case, and a dependent block about an +//! epoch old is usually evicted from the state cache), else from the cached +//! checkpoint state of the epoch before the proposal's. +//! - The signature verifies under any of three domains, since clients disagree +//! at the fork boundary: see [`proposer_preferences_domains`]. + +use ethlambda_storage::CacheKey; -use super::{IgnoreReason, Outcome}; +use super::execution_payload_bid::cached_checkpoint_state; +use super::{ + IgnoreReason, Outcome, RejectReason, ancestor_at, is_future_slot, is_gloas_slot, slot_start_ms, +}; +use crate::beacon::bls; use crate::beacon::builder_market::BuilderMarket; use crate::beacon::config::Config; +use crate::beacon::constants::{DOMAIN_PROPOSER_PREFERENCES, MAXIMUM_GOSSIP_CLOCK_DISPARITY}; use crate::beacon::containers::{BeaconState, gloas}; -use crate::beacon::fork_choice::Store; -use crate::beacon::primitives::{Domain, Epoch, Root, Slot}; +use crate::beacon::fork_choice::{ + Store, compute_shuffling_dependent_slot, compute_shuffling_lookahead_start_slot, +}; +use crate::beacon::helpers::accessors::get_domain; +use crate::beacon::helpers::misc::{compute_domain, compute_epoch_at_slot, compute_signing_root}; +use crate::beacon::precheck::fixed_proposer; +use crate::beacon::preset; +use crate::beacon::primitives::{Domain, Epoch, HashTreeRoot as _, Root, Slot}; -#[allow(unused_variables)] // filled by Agent A +/// The rules that read only the message, the market's seen state and the +/// clock. pub fn cheap_checks( market: &BuilderMarket, store: &Store, signed: &gloas::SignedProposerPreferences, now_ms: u64, ) -> Result<(), Outcome> { - Err(Outcome::Ignore(IgnoreReason::NoConsumer)) + let preferences = &signed.message; + let config = store.config(); + // [IGNORE] The first valid preferences for this dependent root and slot. + if market + .preferences(preferences.proposal_slot, preferences.dependent_root) + .is_some() + { + return Err(Outcome::Ignore(IgnoreReason::AlreadySeen)); + } + // [IGNORE] The proposal epoch is after the gloas upgrade. + if !is_gloas_slot(&config, preferences.proposal_slot) { + return Err(Outcome::Ignore(IgnoreReason::PreGloasSlot)); + } + // [IGNORE] The proposal slot has not started yet. + if is_past_slot(&config, preferences.proposal_slot, now_ms) { + return Err(Outcome::Ignore(IgnoreReason::SlotStarted)); + } + // [IGNORE] The proposer for the proposal slot is known. + let proposal_epoch = compute_epoch_at_slot(preferences.proposal_slot); + let lookahead_start_slot = compute_shuffling_lookahead_start_slot(proposal_epoch); + if is_future_slot(&config, lookahead_start_slot, now_ms) { + return Err(Outcome::Ignore(IgnoreReason::BeyondLookahead)); + } + Ok(()) } -#[allow(unused_variables)] // filled by Agent A +/// The rules that need the dependent block and a state, then the signature. +/// Runs on a blocking thread. pub fn stateful_checks(store: &Store, signed: &gloas::SignedProposerPreferences) -> Outcome { - Outcome::Ignore(IgnoreReason::NoConsumer) + match stateful_rules(store, signed) { + Ok(()) => Outcome::Accept, + Err(outcome) => outcome, + } } -/// Both halves. The caller records the preferences on `Accept`. +fn stateful_rules(store: &Store, signed: &gloas::SignedProposerPreferences) -> Result<(), Outcome> { + let preferences = &signed.message; + let config = store.config(); + let proposal_epoch = compute_epoch_at_slot(preferences.proposal_slot); + let dependent_slot = compute_shuffling_dependent_slot(proposal_epoch); + + // [IGNORE] The dependent block has been seen (never queued). + if !store.has_block(&preferences.dependent_root) { + return Err(Outcome::Ignore(IgnoreReason::UnknownBlock)); + } + // [IGNORE] The dependent block passes validation, i.e. has a post-state. + if !store + .has_state(&preferences.dependent_root) + .unwrap_or(false) + { + return Err(Outcome::Ignore(IgnoreReason::StateUnavailable)); + } + // [REJECT] The dependent block is not after the shuffling dependent slot. + let (block_slot, _) = store + .block_entry(&preferences.dependent_root) + .ok_or(Outcome::Ignore(IgnoreReason::UnknownBlock))?; + if block_slot > dependent_slot { + return Err(Outcome::Reject(RejectReason::DependentRootTooLate)); + } + // [IGNORE] The dependent block is a possible dependent block. + if !is_valid_dependent_root(store, preferences.dependent_root, dependent_slot) { + return Err(Outcome::Ignore(IgnoreReason::ImpossibleDependentRoot)); + } + + let state = lookahead_state(store, preferences, proposal_epoch) + .ok_or(Outcome::Ignore(IgnoreReason::StateUnavailable))?; + + // [REJECT] The validator is the proposer for the slot in the lookahead. + if fixed_proposer(&state, preferences.proposal_slot) != Some(preferences.validator_index) { + return Err(Outcome::Reject(RejectReason::WrongProposer)); + } + // [REJECT] The signature is valid, under any candidate domain. + let pubkey = state + .validator(preferences.validator_index) + .map_err(|_| Outcome::Reject(RejectReason::WrongProposer))? + .pubkey; + let message_root = preferences.hash_tree_root(); + let valid = proposer_preferences_domains(&state, &config, proposal_epoch) + .into_iter() + .any(|domain| { + bls::verify( + &pubkey, + compute_signing_root(message_root, domain), + &signed.signature, + ) + }); + if !valid { + return Err(Outcome::Reject(RejectReason::BadSignature)); + } + Ok(()) +} + +/// A cached state whose proposer lookahead answers for `preferences`: the +/// head's when the head shares the dependent root, else the dependent block's +/// state advanced to the epoch before the proposal's. +fn lookahead_state( + store: &Store, + preferences: &gloas::ProposerPreferences, + proposal_epoch: Epoch, +) -> Option> { + if let Ok(head_root) = store.head() + && let Some(head_state) = store.cached_state(CacheKey::BlockState(head_root)) + { + // A state in the epoch before the proposal's, or in the proposal's + // own, holds the proposal slot in its two-epoch window. + let state_epoch = compute_epoch_at_slot(head_state.slot()); + let covers = state_epoch == proposal_epoch + || state_epoch + preset::MIN_SEED_LOOKAHEAD == proposal_epoch; + if covers + && dependent_root_at(&head_state, head_root, preferences.proposal_slot) + == Some(preferences.dependent_root) + { + return Some(head_state); + } + } + cached_checkpoint_state( + store, + proposal_epoch.saturating_sub(preset::MIN_SEED_LOOKAHEAD), + preferences.dependent_root, + ) +} + +/// Both halves. The caller records the preferences on `Accept`: the +/// specification's `validate_proposer_preferences_gossip`. pub fn validate( market: &BuilderMarket, store: &Store, @@ -44,36 +185,443 @@ pub fn validate( /// The spec's `is_valid_dependent_root`: `root == store.head()`, or some block /// in the index has `parent_root == root` and `slot > dependent_slot`. -#[allow(dead_code, unused_variables)] // filled by Agent A pub fn is_valid_dependent_root(store: &Store, root: Root, dependent_slot: Slot) -> bool { - false + if store.head().is_ok_and(|head| head == root) { + return true; + } + store + .block_index() + .values() + .any(|&(slot, parent_root)| parent_root == root && slot > dependent_slot) } /// `ancestor_at(state, state_block_root, compute_shuffling_dependent_slot( -/// epoch(proposal_slot)))`. Used by `produceBlockV4` and the validator client. -#[allow(dead_code, unused_variables)] // filled by Agent A +/// epoch(proposal_slot)))`: the block that fixed the proposal slot's proposer +/// shuffling on the chain `state` is the post-state of. Used by `produceBlockV4` +/// and the validator client. pub fn dependent_root_at( state: &BeaconState, state_block_root: Root, proposal_slot: Slot, ) -> Option { - None + let dependent_slot = compute_shuffling_dependent_slot(compute_epoch_at_slot(proposal_slot)); + ancestor_at(state, state_block_root, dependent_slot) } /// The distinct candidate signing domains, tried in order: `get_domain( -/// lookahead_state, DOMAIN_PROPOSER_PREFERENCES, Some(P))`; the schedule's fork -/// version at `P - MIN_SEED_LOOKAHEAD` (saturating); the schedule's at `P`. -#[allow(dead_code, unused_variables)] // filled by Agent A +/// lookahead_state, DOMAIN_PROPOSER_PREFERENCES, Some(P))` (the specification's); +/// the schedule's fork version at `P - MIN_SEED_LOOKAHEAD` (saturating); and the +/// schedule's at `P`. +/// +/// Outside the first epoch of a fork the three coincide. Across a boundary +/// the specification's gives the version of the epoch before the proposal's, +/// while lighthouse signs with the proposal epoch's, so accepting both avoids +/// rejecting an honest client's messages around the gloas upgrade. pub fn proposer_preferences_domains( lookahead_state: &BeaconState, config: &Config, proposal_epoch: Epoch, ) -> Vec { - Vec::new() + let genesis_validators_root = lookahead_state.genesis_validators_root(); + let at = |epoch: Epoch| { + compute_domain( + DOMAIN_PROPOSER_PREFERENCES, + config.fork_version(config.fork_at_epoch(epoch)), + genesis_validators_root, + ) + }; + let domains = [ + get_domain( + lookahead_state, + DOMAIN_PROPOSER_PREFERENCES, + Some(proposal_epoch), + ), + at(proposal_epoch.saturating_sub(preset::MIN_SEED_LOOKAHEAD)), + at(proposal_epoch), + ]; + let mut distinct: Vec = Vec::with_capacity(domains.len()); + for domain in domains { + if !distinct.contains(&domain) { + distinct.push(domain); + } + } + distinct } -/// `now > slot_start + 500 ms`. -#[allow(dead_code, unused_variables)] // filled by Agent A +/// `now > slot_start + MAXIMUM_GOSSIP_CLOCK_DISPARITY`: the specification's +/// `is_past_slot`. pub(crate) fn is_past_slot(config: &Config, slot: Slot, now_ms: u64) -> bool { - false + now_ms > slot_start_ms(config, slot).saturating_add(MAXIMUM_GOSSIP_CLOCK_DISPARITY) +} + +#[cfg(test)] +mod tests { + use ethlambda_storage::ForkCheckpoints; + use ethlambda_types::beacon::containers::Fork; + + use super::*; + use crate::beacon::builder_market::test_support::sign_preferences; + use crate::beacon::fork::ForkName; + use crate::beacon::gossip::test_support::builder_scene::*; + use crate::beacon::gossip::test_support::{slot_start_ms, store}; + use crate::beacon::helpers::test_state::{secret_key_for, with_signing_validators_at}; + use crate::beacon::primitives::ValidatorIndex; + use crate::beacon::stf; + + /// The epoch-2 slot the scene's proposer preferences name, and the block + /// that fixed its proposer: the genesis-slot block, an ancestor of the + /// head (slot 32) whose `block_roots` entry covers slot 31. + const PROPOSAL_SLOT: Slot = 64; + const DEPENDENT: Root = Root::repeat_byte(0x31); + + fn ignore(reason: IgnoreReason) -> Outcome { + Outcome::Ignore(reason) + } + + fn reject(reason: RejectReason) -> Outcome { + Outcome::Reject(reason) + } + + /// The builder scene, with the parent's `block_roots` naming `DEPENDENT` at + /// slot 31, and `DEPENDENT` stored (block and state) as the head's parent. + fn dependent_scene() -> Scene { + let mut scene = scene_with(|state| state.block_roots[31] = DEPENDENT); + scene + .store + .insert_pending_block(DEPENDENT, block_at(0)) + .expect("insert the dependent block"); + // The head, as a live-chain child of the dependent block: the rule + // that a dependent block is possible reads the live chain. + scene + .store + .insert_signed_block(PARENT, block_with_parent(32, DEPENDENT)) + .expect("link the head to the dependent block"); + let mut dependent_state = scene.state.clone(); + *dependent_state.slot_mut() = 0; + scene + .store + .insert_state(DEPENDENT, dependent_state) + .expect("insert the dependent state"); + scene + } + + fn preferences_for( + scene: &Scene, + dependent_root: Root, + proposer: ValidatorIndex, + ) -> gloas::SignedProposerPreferences { + sign_preferences( + &scene.state, + gloas::ProposerPreferences { + dependent_root, + proposal_slot: PROPOSAL_SLOT, + validator_index: proposer, + fee_recipient: fee_recipient(), + target_gas_limit: 30_000_000, + }, + ) + } + + /// Valid preferences from the head's lookahead. + fn valid_preferences(scene: &Scene) -> gloas::SignedProposerPreferences { + let proposer = + crate::beacon::precheck::fixed_proposer(&scene.state, PROPOSAL_SLOT).expect("window"); + preferences_for(scene, DEPENDENT, proposer) + } + + fn now_ms(scene: &Scene) -> u64 { + scene.now_ms() + } + + #[test] + fn valid_preferences_are_accepted_from_the_head_state() { + let scene = dependent_scene(); + let preferences = valid_preferences(&scene); + let outcome = validate(&scene.market, &scene.store, &preferences, now_ms(&scene)); + assert_eq!(outcome, Outcome::Accept); + // The head's own lookahead answered: nothing was advanced and cached. + let key = CacheKey::CheckpointState { + epoch: 1, + root: DEPENDENT, + }; + assert!(scene.store.cached_state(key).is_none()); + } + + // ---- cheap rules ---- + + #[test] + fn a_second_message_for_a_key_is_already_seen() { + let scene = dependent_scene(); + let preferences = valid_preferences(&scene); + assert!(scene.market.record_preferences(preferences.clone(), 33)); + assert_eq!( + cheap_checks(&scene.market, &scene.store, &preferences, now_ms(&scene)), + Err(ignore(IgnoreReason::AlreadySeen)) + ); + } + + #[test] + fn preferences_before_the_fork_are_ignored() { + let scene = dependent_scene(); + let fulu = store(0); + let preferences = valid_preferences(&scene); + assert_eq!( + cheap_checks(&scene.market, &fulu, &preferences, now_ms(&scene)), + Err(ignore(IgnoreReason::PreGloasSlot)) + ); + } + + #[test] + fn preferences_are_late_once_the_slot_began_by_more_than_the_disparity() { + let scene = dependent_scene(); + let preferences = valid_preferences(&scene); + let started = slot_start_ms(&scene.store, PROPOSAL_SLOT); + let check = |now| cheap_checks(&scene.market, &scene.store, &preferences, now); + assert_eq!(check(started + 500), Ok(())); + assert_eq!(check(started + 501), Err(ignore(IgnoreReason::SlotStarted))); + } + + #[test] + fn preferences_are_early_before_the_lookahead_opens() { + let scene = dependent_scene(); + let mut preferences = valid_preferences(&scene); + // Epoch 3's lookahead opens with epoch 2, at slot 64. + preferences.message.proposal_slot = 96; + let opens = slot_start_ms(&scene.store, 64); + let check = |now| cheap_checks(&scene.market, &scene.store, &preferences, now); + assert_eq!(check(opens - 500), Ok(())); + assert_eq!( + check(opens - 501), + Err(ignore(IgnoreReason::BeyondLookahead)) + ); + } + + // ---- stateful rules ---- + + #[test] + fn preferences_naming_an_unseen_dependent_block_are_ignored() { + let scene = dependent_scene(); + let preferences = preferences_for(&scene, Root::repeat_byte(0x99), 0); + assert_eq!( + stateful_checks(&scene.store, &preferences), + ignore(IgnoreReason::UnknownBlock) + ); + } + + #[test] + fn a_dependent_block_without_a_state_is_ignored() { + let mut scene = dependent_scene(); + let stateless = Root::repeat_byte(0x62); + scene + .store + .insert_pending_block(stateless, block_at(20)) + .expect("insert the block"); + let preferences = preferences_for(&scene, stateless, 0); + assert_eq!( + stateful_checks(&scene.store, &preferences), + ignore(IgnoreReason::StateUnavailable) + ); + } + + #[test] + fn a_dependent_block_after_the_dependent_slot_is_rejected() { + let scene = dependent_scene(); + // The head itself is at slot 32, after slot 31. + let preferences = preferences_for(&scene, PARENT, 0); + assert_eq!( + stateful_checks(&scene.store, &preferences), + reject(RejectReason::DependentRootTooLate) + ); + } + + #[test] + fn a_dependent_block_no_chain_can_use_is_ignored() { + let mut scene = dependent_scene(); + let stray = Root::repeat_byte(0x63); + scene + .store + .insert_pending_block(stray, block_at(10)) + .expect("insert the block"); + scene + .store + .insert_state(stray, scene.state.clone()) + .expect("insert the state"); + let preferences = preferences_for(&scene, stray, 0); + assert_eq!( + stateful_checks(&scene.store, &preferences), + ignore(IgnoreReason::ImpossibleDependentRoot) + ); + assert!(!is_valid_dependent_root(&scene.store, stray, 31)); + assert!(is_valid_dependent_root(&scene.store, PARENT, 31)); + } + + #[test] + fn a_validator_that_is_not_the_proposer_is_rejected() { + let scene = dependent_scene(); + let proposer = + crate::beacon::precheck::fixed_proposer(&scene.state, PROPOSAL_SLOT).expect("window"); + let other = (proposer + 1) % 8; + let preferences = preferences_for(&scene, DEPENDENT, other); + assert_eq!( + stateful_checks(&scene.store, &preferences), + reject(RejectReason::WrongProposer) + ); + } + + #[test] + fn a_bad_signature_is_rejected() { + let scene = dependent_scene(); + let mut preferences = valid_preferences(&scene); + preferences.signature.0[5] ^= 1; + assert_eq!( + stateful_checks(&scene.store, &preferences), + reject(RejectReason::BadSignature) + ); + } + + #[test] + fn a_dependent_block_off_the_head_uses_the_cached_checkpoint_state() { + let mut scene = dependent_scene(); + // The dependent block is the head, whose own state is epoch 0, so the + // lookahead has to come from its state advanced to epoch 1. + scene + .store + .update_checkpoints(ForkCheckpoints::head_only(DEPENDENT)) + .expect("move the head"); + let mut advanced = scene + .store + .cached_state(CacheKey::BlockState(DEPENDENT)) + .expect("stored"); + let mut owned = (*advanced).clone(); + stf::process_slots(&mut owned, 32, &scene.store.config()).expect("advance"); + advanced = std::sync::Arc::new(owned); + let proposer = + crate::beacon::precheck::fixed_proposer(&advanced, PROPOSAL_SLOT).expect("window"); + let preferences = preferences_for(&scene, DEPENDENT, proposer); + let key = CacheKey::CheckpointState { + epoch: 1, + root: DEPENDENT, + }; + assert!(scene.store.cached_state(key).is_none()); + assert_eq!(stateful_checks(&scene.store, &preferences), Outcome::Accept); + assert!(scene.store.cached_state(key).is_some()); + } + + #[test] + fn a_dependent_state_evicted_from_the_cache_is_ignored_not_rebuilt() { + let mut scene = dependent_scene(); + scene + .store + .update_checkpoints(ForkCheckpoints::head_only(DEPENDENT)) + .expect("move the head"); + let preferences = valid_preferences(&scene); + // Fill the bounded state cache with other blocks' states until the + // dependent block's is pushed out. The state still exists (`has_state`), + // but reading it would mean a rebuild from disk. + for byte in 0..64u8 { + let mut root = Root::repeat_byte(0xA0); + root.0[1] = byte; + scene + .store + .insert_state(root, scene.state.clone()) + .expect("insert a filler state"); + } + assert!( + scene + .store + .cached_state(CacheKey::BlockState(DEPENDENT)) + .is_none() + ); + assert!(scene.store.has_state(&DEPENDENT).expect("has_state")); + assert_eq!( + stateful_checks(&scene.store, &preferences), + ignore(IgnoreReason::StateUnavailable) + ); + } + + // ---- dependent roots and domains ---- + + #[test] + fn the_dependent_root_is_the_ancestor_at_the_shuffling_dependent_slot() { + let scene = dependent_scene(); + assert_eq!( + dependent_root_at(&scene.state, PARENT, PROPOSAL_SLOT), + Some(DEPENDENT) + ); + // At genesis the slot saturates to the state's own block. + let mut genesis = scene.state.clone(); + *genesis.slot_mut() = 0; + assert_eq!(dependent_root_at(&genesis, PARENT, 5), Some(PARENT)); + } + + fn boundary_config() -> Config { + Config::mainnet() + .with_fork_epoch(ForkName::Fulu, 0) + .with_fork_epoch(ForkName::Gloas, 2) + } + + fn fulu_state_with_fork(config: &Config) -> BeaconState { + let mut state = with_signing_validators_at(ForkName::Fulu, 8); + *state.fork_mut() = Fork { + previous_version: config.fork_version(ForkName::Electra), + current_version: config.fork_version(ForkName::Fulu), + epoch: 0, + }; + state + } + + #[test] + fn across_the_fork_both_neighbouring_versions_are_candidates() { + let config = boundary_config(); + let state = fulu_state_with_fork(&config); + let root = state.genesis_validators_root(); + let at = + |fork| compute_domain(DOMAIN_PROPOSER_PREFERENCES, config.fork_version(fork), root); + // Proposal epoch 2 is gloas's first: the lookahead state's fork (fulu) + // and the epoch before the proposal's agree, the proposal epoch's is gloas. + let domains = proposer_preferences_domains(&state, &config, 2); + assert_eq!(domains, vec![at(ForkName::Fulu), at(ForkName::Gloas)]); + + // A signature under either verifies; one under neither does not. + let preferences = gloas::ProposerPreferences { + proposal_slot: 64, + ..Default::default() + }; + let message_root = preferences.hash_tree_root(); + let sign = |domain| { + let signature = secret_key_for(0).sign( + compute_signing_root(message_root, domain).as_slice(), + crate::beacon::bls::DST, + &[], + ); + crate::beacon::primitives::BlsSignature(signature.to_bytes()) + }; + let pubkey = state.validator(0).expect("validator 0").pubkey; + let accepted = |signature| { + domains.iter().any(|domain| { + bls::verify( + &pubkey, + compute_signing_root(message_root, *domain), + &signature, + ) + }) + }; + assert!(accepted(sign(at(ForkName::Fulu)))); + assert!(accepted(sign(at(ForkName::Gloas)))); + assert!(!accepted(sign(at(ForkName::Electra)))); + } + + #[test] + fn away_from_a_fork_there_is_one_candidate_domain() { + let config = boundary_config(); + let mut state = fulu_state_with_fork(&config); + *state.fork_mut() = Fork { + previous_version: config.fork_version(ForkName::Fulu), + current_version: config.fork_version(ForkName::Gloas), + epoch: 2, + }; + assert_eq!(proposer_preferences_domains(&state, &config, 5).len(), 1); + // And at epoch 0 the saturating subtraction does not wrap. + assert!(!proposer_preferences_domains(&state, &config, 0).is_empty()); + } } diff --git a/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs b/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs index 78527bc26..451320a6c 100644 --- a/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs +++ b/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs @@ -97,3 +97,162 @@ pub(crate) fn fulu_parent(proposer: ValidatorIndex) -> BeaconState { state.apply_pending_mutations(); state } + +/// The builder-market rules' shared scene: a gloas chain whose head `PARENT` +/// (slot 32, epoch 1) is a FULL block with a funded active builder 0, the +/// proposer preferences and parent payload gossip would have delivered, and a +/// signed bid for slot 34 that passes every rule. +pub(crate) mod builder_scene { + use ethlambda_storage::ForkCheckpoints; + use ethlambda_types::beacon::containers::{SignedBeaconBlock, electra, gloas}; + + use super::*; + use crate::beacon::builder_market::{BuilderMarket, test_support}; + use crate::beacon::fork_choice::PayloadStatus; + use crate::beacon::gloas_block_production::test_support as gloas_support; + use crate::beacon::gossip::proposer_preferences::dependent_root_at; + use crate::beacon::helpers::accessors::{get_current_epoch, get_randao_mix}; + use crate::beacon::primitives::{ExecutionAddress, ExecutionBlockHash}; + + pub(crate) const PARENT: Root = Root::repeat_byte(0x50); + pub(crate) const BID_SLOT: Slot = 34; + pub(crate) const PARENT_GAS_LIMIT: u64 = 30_000_000; + + pub(crate) fn fee_recipient() -> ExecutionAddress { + ExecutionAddress::repeat_byte(0x11) + } + + /// The hash of the FULL parent's payload, which the bid builds on. + pub(crate) fn parent_block_hash() -> ExecutionBlockHash { + ExecutionBlockHash::repeat_byte(gloas_support::PARENT_BLOCK_HASH) + } + + /// A fulu-shaped block at `slot`: the store only reads its slot and parent. + pub(crate) fn block_at(slot: Slot) -> SignedBeaconBlock { + block_with_parent(slot, Root::ZERO) + } + + pub(crate) fn block_with_parent(slot: Slot, parent_root: Root) -> SignedBeaconBlock { + SignedBeaconBlock::Fulu(electra::SignedBeaconBlock { + message: electra::BeaconBlock { + slot, + proposer_index: 0, + parent_root, + state_root: Root::ZERO, + body: electra::BeaconBlockBody::empty(), + }, + signature: Default::default(), + }) + } + + /// An empty store, gloas from genesis, anchored (and headed) at `PARENT`, + /// whose block the caller must store. + pub(crate) fn gloas_store() -> Store { + Store::init_beacon( + Arc::new(InMemoryBackend::new()), + GENESIS_TIME, + gloas_support::config(), + PARENT, + Checkpoint { + root: PARENT, + slot: 32, + }, + 32, + ) + } + + pub(crate) struct Scene { + pub store: Store, + pub market: BuilderMarket, + /// `PARENT`'s post-state. + pub state: BeaconState, + /// A signed bid that passes every rule at [`Scene::now_ms`]. + pub bid: gloas::SignedExecutionPayloadBid, + } + + impl Scene { + /// 100 ms into the bid's slot. + pub(crate) fn now_ms(&self) -> u64 { + slot_start_ms(&self.store, BID_SLOT) + 100 + } + + /// The scene's bid after `edit`, re-signed by its builder. + pub(crate) fn signed( + &self, + edit: impl FnOnce(&mut gloas::ExecutionPayloadBid), + ) -> gloas::SignedExecutionPayloadBid { + let mut bid = self.bid.message.clone(); + edit(&mut bid); + test_support::sign_bid(&self.state, bid, self.bid.message.builder_index) + } + } + + /// The scene with `edit` applied to the parent's state before it is stored, + /// and the prerequisites of a passing bid recorded in the market. + pub(crate) fn scene_with(edit: impl FnOnce(&mut gloas::BeaconState)) -> Scene { + let mut state = test_support::gloas_state_with_builder(0, 100_000_000_000, 0); + let BeaconState::Gloas(inner) = &mut state else { + unreachable!("built as gloas") + }; + edit(inner); + state.apply_pending_mutations(); + let mut store = gloas_store(); + store + .insert_pending_block(PARENT, block_at(state.slot())) + .expect("insert the parent"); + store + .insert_state(PARENT, state.clone()) + .expect("insert the parent state"); + store + .update_checkpoints(ForkCheckpoints::head_only(PARENT)) + .expect("move the head"); + store.set_head_payload_status(PARENT, PayloadStatus::Full); + + let dependent_root = dependent_root_at(&state, PARENT, BID_SLOT).expect("in the window"); + let proposer = crate::beacon::precheck::fixed_proposer(&state, BID_SLOT).expect("window"); + let market = BuilderMarket::default(); + let preferences = test_support::sign_preferences( + &state, + gloas::ProposerPreferences { + dependent_root, + proposal_slot: BID_SLOT, + validator_index: proposer, + fee_recipient: fee_recipient(), + target_gas_limit: PARENT_GAS_LIMIT, + }, + ); + assert!(market.record_preferences(preferences, BID_SLOT - 1)); + market.record_execution_payload(&test_support::envelope_with_gas_limit( + parent_block_hash(), + PARENT_GAS_LIMIT, + PARENT, + vec![], + )); + let bid = test_support::sign_bid( + &state, + gloas::ExecutionPayloadBid { + parent_block_hash: parent_block_hash(), + parent_block_root: PARENT, + block_hash: ExecutionBlockHash::repeat_byte(0x33), + prev_randao: get_randao_mix(&state, get_current_epoch(&state)), + fee_recipient: fee_recipient(), + gas_limit: PARENT_GAS_LIMIT, + builder_index: 0, + slot: BID_SLOT, + value: 1, + ..Default::default() + }, + 0, + ); + Scene { + store, + market, + state, + bid, + } + } + + pub(crate) fn scene() -> Scene { + scene_with(|_| {}) + } +} diff --git a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs index e98ec5ed2..1d3e32ca2 100644 --- a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs +++ b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs @@ -14,6 +14,7 @@ use std::num::NonZeroUsize; use std::sync::Arc; use ethlambda_state_transition::beacon::ForkName; +use ethlambda_state_transition::beacon::builder_market::BuilderMarket; use ethlambda_state_transition::beacon::config::Config; use ethlambda_state_transition::beacon::containers::{ BeaconState, Checkpoint, DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock, @@ -43,6 +44,8 @@ const HANDLERS: &[&str] = &[ "gossip_beacon_attestation", "gossip_execution_payload_envelope", "gossip_payload_attestation_message", + "gossip_execution_payload_bid", + "gossip_proposer_preferences", ]; /// The forks each of [`HANDLERS`] validates. A case from any other fork is @@ -61,9 +64,10 @@ fn validated_forks(handler: &str) -> &'static [ForkName] { "gossip_beacon_aggregate_and_proof" | "gossip_beacon_attestation" => { &[ForkName::Fulu, ForkName::Gloas] } - "gossip_execution_payload_envelope" | "gossip_payload_attestation_message" => { - &[ForkName::Gloas] - } + "gossip_execution_payload_envelope" + | "gossip_payload_attestation_message" + | "gossip_execution_payload_bid" + | "gossip_proposer_preferences" => &[ForkName::Gloas], other => panic!("{other} is not in HANDLERS, so it has no validated forks"), } } @@ -77,20 +81,14 @@ fn validated_forks(handler: &str) -> &'static [ForkName] { /// list, the same way [`super::UNMODELED_FORKS`] forces a decision on a new /// fork directory. /// -/// `gossip_execution_payload_bid` and `gossip_proposer_preferences` are -/// gloas's own topics (EIP-7732 ePBS): the builder's bid and a builder's -/// advertised preferences, respectively. (Its envelope and payload -/// attestation topics are in [`HANDLERS`].) None of them existed -/// until gloas's fixture directory started parsing (`ForkName::Gloas`), so -/// they land here rather than silently in `unknown` the first time this -/// runner sees them. Alphabetized with the rest rather than kept together. +/// Gloas's own topics (EIP-7732 ePBS) are all in [`HANDLERS`] now: the +/// envelope, the payload attestation, the builder's bid and the proposer's +/// preferences. const IGNORED_HANDLERS: &[&str] = &[ "gossip_attester_slashing", "gossip_blob_sidecar", "gossip_bls_to_execution_change", - "gossip_execution_payload_bid", "gossip_partial_data_column_sidecar", - "gossip_proposer_preferences", "gossip_proposer_slashing", "gossip_sync_committee_contribution_and_proof", "gossip_sync_committee_message", @@ -259,17 +257,28 @@ fn case_config(case: &Case, state: &BeaconState) -> Config { /// Delivers `entry`'s execution payload envelope, if it lists one, so /// `is_payload_verified` answers true for its block. +/// +/// `trusted` records the envelope the way the specification's own bid and +/// preferences generators do (`store.payloads[root] = envelope.message`, no +/// verification): their gas-limit cases deliberately give the head an envelope +/// whose payload gas limit differs from the block's bid, which +/// `verify_execution_payload_envelope` would refuse. fn deliver_payload( store: &mut Store, case: &Case, entry: &StoreBlock, config: &Config, + trusted: bool, ) -> Result<(), String> { let Some(name) = &entry.payload else { return Ok(()); }; let envelope = gloas::SignedExecutionPayloadEnvelope::from_ssz_bytes(&case.ssz_bytes(name)) .map_err(|err| format!("decoding {name}: {err:?}"))?; + if trusted { + fork_choice::accept_execution_payload_envelope(store, &envelope); + return Ok(()); + } // No sampled columns are named, so the empty retrieval reads as available, // as in the fork-choice runner's envelope step. fork_choice::on_execution_payload_envelope( @@ -282,6 +291,11 @@ fn deliver_payload( .map_err(|err| format!("delivering {name}: {err:?}")) } +/// The two topics whose cases share one store setup and one market. +fn is_builder_market_topic(topic: &str) -> bool { + matches!(topic, "execution_payload_bid" | "proposer_preferences") +} + /// The store the case describes: its anchor, then each listed block. fn build_store( case: &Case, @@ -297,6 +311,9 @@ fn build_store( let backend = Arc::new(ethlambda_storage::backend::InMemoryBackend::new()); let mut store = fork_choice::get_forkchoice_store(backend, state, anchor_block, config) .map_err(|err| format!("get_forkchoice_store: {err:?}"))?; + let builder_market = is_builder_market_topic(&meta.topic); + let trusted = builder_market; + let mut imported = Vec::new(); // The store's clock at the case's base time, so `on_block` accepts every // listed block; a block from a slot past that time advances it to that @@ -304,7 +321,7 @@ fn build_store( // slot a clock-disparity case sends its sidecar for. let mut clock_s = (config.genesis_time_ms() + meta.current_time_ms) / 1000; fork_choice::on_tick(&mut store, clock_s, config); - deliver_payload(&mut store, case, anchor, config)?; + deliver_payload(&mut store, case, anchor, config, trusted)?; for entry in rest { let block = decode_block(case, &entry.block)?; @@ -348,7 +365,8 @@ fn build_store( &CommitteeCache::default(), ) .map_err(|err| format!("importing {}: {err:?}", entry.block))?; - deliver_payload(&mut store, case, entry, config)?; + deliver_payload(&mut store, case, entry, config, trusted)?; + imported.push(root); if gloas && let Some(status) = entry.payload_status.as_deref() { let status = match status { "VALID" => PayloadStatusEnum::Valid, @@ -376,6 +394,27 @@ fn build_store( store .update_checkpoints(ForkCheckpoints::new(head, None, Some(checkpoint))) .map_err(|err| format!("overriding the finalized checkpoint: {err}"))?; + if builder_market { + // The bid generators activate their builders by finalizing epoch 1 + // in the store *and* in the head's post-state (a replayed chain of + // empty blocks never finalizes), and say so through this override. + for block_root in &imported { + let Some(state) = store + .get_state(block_root) + .map_err(|err| format!("reading a state: {err}"))? + else { + continue; + }; + let mut state = (*state).clone(); + *state.finalized_checkpoint_mut() = Checkpoint { + epoch: finalized.epoch, + root, + }; + store + .insert_state(*block_root, state) + .map_err(|err| format!("overriding a state's finalized checkpoint: {err}"))?; + } + } } Ok(store) @@ -406,7 +445,14 @@ fn run_case(case: &Case) -> Result<(), String> { let state = BeaconState::from_ssz(case.fork, &case.ssz_bytes("state")) .map_err(|err| format!("decoding state: {err:?}"))?; let config = case_config(case, &state); - let store = build_store(case, &meta, state, &config)?; + let mut store = build_store(case, &meta, state, &config)?; + if is_builder_market_topic(&meta.topic) { + // `on_block` records no head, and the bid and preference rules read + // the recorded one. + fork_choice::get_head(&mut store, &config) + .map_err(|err| format!("computing the head: {err:?}"))?; + } + let market = BuilderMarket::default(); let capacity = NonZeroUsize::new(SEEN_CAPACITY).expect("non-zero"); let mut seen_blocks = SeenBlocks::new(capacity); let mut seen_columns = SeenColumns::new(capacity); @@ -534,6 +580,19 @@ fn run_case(case: &Case) -> Result<(), String> { } outcome } + // The bid cases mix three message types under one topic (the + // preferences and envelope a bid depends on arrive in order, and + // share the case's seen state), so the message's own name says + // which rule judges it. + "execution_payload_bid" | "proposer_preferences" => run_builder_market_message( + case, + &store, + &market, + &mut seen_envelopes, + message, + now_ms, + &config, + )?, other => return Err(format!("topic {other} has no runner")), }; check(message, outcome).map_err(|err| format!("message {index}: {err}"))?; @@ -541,6 +600,52 @@ fn run_case(case: &Case) -> Result<(), String> { Ok(()) } +/// One message of a `execution_payload_bid` or `proposer_preferences` case. +fn run_builder_market_message( + case: &Case, + store: &Store, + market: &BuilderMarket, + seen_envelopes: &mut SeenEnvelopes, + message: &GossipMessage, + now_ms: u64, + config: &Config, +) -> Result { + let bytes = case.ssz_bytes(&message.message); + let decode_err = |err| format!("decoding {}: {err:?}", message.message); + if message.message.starts_with("proposer_preferences_") { + let preferences = + gloas::SignedProposerPreferences::from_ssz_bytes(&bytes).map_err(decode_err)?; + let outcome = rules::proposer_preferences::validate(market, store, &preferences, now_ms); + if outcome == Outcome::Accept { + let wall_slot = + now_ms.saturating_sub(config.genesis_time_ms()) / config.slot_duration_ms; + market.record_preferences(preferences, wall_slot); + } + Ok(outcome) + } else if message.message.starts_with("execution_payload_envelope_") { + let envelope = + gloas::SignedExecutionPayloadEnvelope::from_ssz_bytes(&bytes).map_err(decode_err)?; + let outcome = rules::envelope::validate(seen_envelopes, store, &envelope); + if outcome == Outcome::Accept { + seen_envelopes.record( + envelope.message.beacon_block_root, + envelope.message.builder_index, + ); + market.record_execution_payload(&envelope.message); + } + Ok(outcome) + } else if message.message.starts_with("execution_payload_bid_") { + let bid = gloas::SignedExecutionPayloadBid::from_ssz_bytes(&bytes).map_err(decode_err)?; + let outcome = rules::execution_payload_bid::validate(market, store, &bid, now_ms); + if outcome == Outcome::Accept { + market.record_bid(bid); + } + Ok(outcome) + } else { + Err(format!("{} is of no known message type", message.message)) + } +} + pub fn trials() -> Vec { let mut trials = Vec::new(); for handler in HANDLERS { diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index f3e00b67b..b5fb5f41d 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -93,47 +93,154 @@ same keys, already signed. runs, and treat a restart as an event that needs the same care a manual key move would. The client warns about this at startup on every run. -## Self-build only +## Builder bids: gossip and API only (no builder API) -The gloas validator duties are served for a proposer that builds its own -payload, and for no one else. +A gloas proposer served by this node can take a builder's bid from gossip or +from the Beacon API, and no other way. - **The specification:** a gloas proposer may take a builder's signed bid (`SignedExecutionPayloadBid`, from gossip or a builder API) instead of building - its own payload, and publishes a `SignedProposerPreferences` so builders know - its fee recipient and gas limit target. `produceBlockV4` takes a - `BuilderConfig` (`min_bid`, `builder_boost_factor`, `builders`) to steer that - choice. -- **ethlambda:** the beacon node never takes a bid, from gossip or otherwise, and - never builds or reads a `SignedProposerPreferences`. `produceBlockV4` decodes - the `BuilderConfig` body (a missing or undecodable one is a `400`, as the - specification requires) and ignores it, logging at debug when it names - builders. Every block commits to a zero-value self-build bid - (`BUILDER_INDEX_SELF_BUILD`, the G2 point at infinity as signature), and - `Eth-Consensus-Block-Value` is always `0` since there is nothing to compare. - The validator client signs the envelope with the proposer's own key under - `DOMAIN_BEACON_BUILDER`, and leaves a block that commits to anyone else's bid - alone. -- **Why:** a scope decision for the first gloas duties: taking bids needs a bid - pool, builder payment handling and a builder-facing API, none of which this - node has. -- **Consequence:** a validator run through this node never earns a builder's - payment, and an execution client's own block value is what - `Eth-Execution-Payload-Value` reports. - -## `target_gas_limit` is the parent bid's gas limit + its own payload. `produceBlockV4` takes a `BuilderConfig` (`min_bid`, + `builder_boost_factor`, `builders`) to steer that choice. +- **ethlambda:** bids seen on `execution_payload_bid` or posted to + `POST /eth/v1/beacon/execution_payload_bids` are pooled, and `produceBlockV4` + compares the best one with the local build under the top-level `min_bid` and + `builder_boost_factor`: the bid must reach `min_bid`, and wins when + `builder_boost_factor * bid_value > local_value / 10^7` (in Gwei against Wei, + so a factor of 100 is parity). The local build wins a tie, and + `shouldOverrideBuilder` from the execution client is honored. With no + execution client, or a failed local build, the best viable bid is taken. +- **What it does not do:** the `builders` entries are decoded and ignored, so + no builder is asked for a bid over HTTP and no `Eth-Builder-Url` is returned. + Gossip bids are not filtered by `builder_pubkeys` either, since the Beacon API + puts that list on each entry and it governs only that entry's own bid. + `Eth-Consensus-Block-Value` stays `0`, because the consensus reward is not + computed. +- **A bid win returns no envelope.** The block commits to the builder's bid, the + builder reveals the payload, and nothing is cached for + `GET .../execution_payload_envelopes`, so that endpoint answers `404` and + `Eth-Execution-Payload-Included` is `false`. The node never signs or + publishes an envelope for such a block, and the proposer-signed + self-build envelope is refused for it, since its `builder_index` differs from + the bid's. +- **Why:** gossip and API bids need only a pool and the gossip rules. A builder + API client is a separate crate with its own failure modes and timeouts, and + is left for a later phase. + +## Fee recipient and gas target come from proposer preferences, with fallbacks - **The specification:** the payload is built toward the `target_gas_limit` of - the proposer's `SignedProposerPreferences`, and `bid.gas_limit` must be - compatible with it (`is_gas_limit_target_compatible`). -- **ethlambda:** with no preferences to read, `PayloadAttributesV4.targetGasLimit` - is the `gas_limit` of `latest_execution_payload_bid` of the state being built - on (`gloas_payload_inputs`), so the execution client holds the gas limit where - it is. + the proposer's `SignedProposerPreferences`, which also names the fee + recipient builders must pay, and `bid.gas_limit` must be compatible with the + target (`is_gas_limit_target_compatible`). +- **ethlambda:** the self-build reads the signed preferences for + `(slot, dependent_root)` that name the proposer, from gossip or the Beacon API. + The fee recipient is theirs, else `prepare_beacon_proposer`'s, else zero with + a warning. The target gas limit is theirs, else the `gas_limit` of + `latest_execution_payload_bid` of the state being built on, so the execution + client holds the gas limit where it is. - **Equivalence:** the bid is built from the payload the execution client returns, so `bid.gas_limit` is whatever that payload carries and is always - consistent with the block. A validator that wants the limit to move cannot say - so through this node: it follows the previous block's. + consistent with the block. A proposer that wants the limit to move without + preferences cannot say so through this node: it follows the previous block's. + +## Bid and preference gossip never queues + +- **The specification:** several rules say the message "MAY be queued": an + unknown parent block, an unimported parent, an unseen dependent block. +- **ethlambda:** every one of them is IGNORE. A bid or preferences message that + names a block or state this node lacks is dropped, never parked, so a burst + of them holds no memory and nothing is replayed later. +- **Why:** both topics are only useful for the next slot or two, and a bid that + arrives after its parent was imported would be stale by the time a replay ran. + A proposer that missed a message builds locally. + +## Bid gossip judges against the recorded head and cached states + +- **The specification:** `validate_execution_payload_bid_gossip` reads + `get_head(store)`, `store.block_states[parent]` and the parent state advanced + with `process_slots` to the bid's slot. +- **ethlambda:** the head node is the one the chain actor recorded + (`Store::head` and `head_payload_status`), with a fresh `get_head_node` walk + only when no status is recorded. The dependent root is read from the parent + state's `block_roots`, which the lookahead rule keeps in range. The parent + state stands in for the advanced one when the bid is in the parent's own + epoch, since gloas's `process_slot` touches none of `builders`, + `finalized_checkpoint`, `builder_pending_*`, `fork` or + `latest_execution_payload_bid`, which are all the later rules read. Across an + epoch the cached `CheckpointState` of the bid's epoch is used and filled, the + same entry attestation target states use. A state that is not cached is + IGNORE and is never rebuilt from disk. +- **Why:** gossip verdicts are waited on by gossipsub, and the state cache holds + 32 states, so a spec-literal read of a parent about an epoch old would miss + often. The head record and the equivalence above give the same verdict + without a replay. + +## Known execution payloads are gossip-accepted or self-published envelopes only + +- **The specification:** `seen.execution_payloads` holds a payload for every + envelope accepted from gossip. +- **ethlambda:** the known payloads are the market's, a 256-entry LRU filled by + envelopes that passed gossip validation and by envelopes this node publishes + (gossip never echoes a node's own messages). They are not persisted, and an + envelope that was queued and verified later, or fetched by request and + response, does not count. After a restart bids on a pre-restart payload are + IGNORE until new envelopes arrive. +- **Exception:** a pre-gloas parent's own payload counts as known, with its + execution payload header's gas limit, see the fork boundary entry below. + +## Proposer preferences are judged off cached states only + +- **The specification:** the lookahead is read from + `store.block_states[dependent_root]` advanced to the epoch before the + proposal's. +- **ethlambda:** the cached head state is used when it shares the dependent root + and is in the epoch before the proposal's or the proposal's own (the whole + canonical case), else the cached `CheckpointState` of the epoch before the + proposal's, else IGNORE. Nothing is rebuilt from disk. +- **Why:** a dependent block about an epoch old is usually out of the 32-state + cache, so reading it would IGNORE most honest preferences. + +## The fulu-to-gloas boundary for bids and preferences + +- **The specification:** says nothing about a parent that is not a gloas block. +- **ethlambda:** a pre-gloas parent's payload counts as known, with its header's + `gas_limit`. A pre-gloas head's payload hashes come from its payload header, + and a bid is compatible with it when it builds on that head and its payload. + The builder-exit check is skipped for a pre-gloas parent, which carries no + envelope. +- **Preference signatures** are accepted under the lookahead state's own + `DOMAIN_PROPOSER_PREFERENCES` domain (the specification's), the fork version of + the epoch before the proposal's, or the proposal epoch's. They coincide outside + the first epoch of a fork. Lighthouse signs with the proposal epoch's version + and the specification gives the earlier one, so accepting both avoids + rejecting an honest client's messages around the gloas upgrade. +- **Consequence:** builders onboarded at the fork are inactive until their + deposit epoch is finalized, so gossip bids are rejected for the first epochs of + gloas and `produceBlockV4` self-builds. + +## Beacon API answers for bids and preferences + +- **An IGNORE verdict is a `400`,** the same as a REJECT, because the API has no + separate status for it. The message names the verdict and the reason, such as + `ignore: preferences_unseen`. +- **An identical resubmission is a `200`** and is not published again. +- **The prose and the rules disagree** on a mismatched fee recipient or gas + limit: the Beacon API text says the bid is rejected, while consensus-specs + IGNOREs it. ethlambda follows consensus-specs. +- **`Eth-Consensus-Version` may be absent** on the bid and preferences posts. If + present it must name a gloas-compatible fork. + +## No minimum bid increment or rate limit + +- **The specification:** a note says implementations SHOULD guard against + builders spamming bids with minimal increments, for example with a minimum + threshold or by forwarding only the best bid at intervals. +- **ethlambda:** neither is implemented. Spam is bounded by one bid per builder + per `(slot, parent_hash, parent_root)`, a strictly higher value than the best + seen, a funded active registered builder with a valid signature, a cap on keys + per slot and on bids pooled per parent, and a separate permit pool for + validating them. A configurable minimum increment is a follow-up. ## `skip_randao_verification` is ignored From 470062a59a1b62ba5530452a729cccdbcd5b2a5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:48:58 -0300 Subject: [PATCH 116/132] feat(p2p): gossip and publish gloas builder bids and proposer preferences Subscribe the execution_payload_bid and proposer_preferences topics under gloas digests only, decode them, triage them (size cap, decode, cheap rules) and hand the rest to the blocking pool on a permit pool of their own so a bid burst cannot starve blocks, columns or attestations. Neither type reaches the chain actor; accepted ones are recorded in the shared builder market. Envelopes accepted from gossip, and envelopes this node publishes itself, are recorded as known payloads, since bid validation reads them and gossip never echoes a node's own message. Bids and preferences are published on the digest of their own slot, so preferences for the first gloas epoch go out on the gloas digest during the epoch before the fork. --- crates/net/p2p/src/beacon/builder_market.rs | 389 +++++++++++++++++++- crates/net/p2p/src/beacon/decode.rs | 53 +++ crates/net/p2p/src/beacon/topics.rs | 24 +- crates/net/p2p/src/beacon/verdict.rs | 215 ++++++++++- crates/net/p2p/src/gossipsub/handler.rs | 5 + crates/net/p2p/src/gossipsub/mod.rs | 1 - docs/beacon_wire.md | 24 +- docs/metrics.md | 29 +- 8 files changed, 700 insertions(+), 40 deletions(-) diff --git a/crates/net/p2p/src/beacon/builder_market.rs b/crates/net/p2p/src/beacon/builder_market.rs index 44e78bdc6..f44ce4bbd 100644 --- a/crates/net/p2p/src/beacon/builder_market.rs +++ b/crates/net/p2p/src/beacon/builder_market.rs @@ -1,52 +1,183 @@ //! Gloas builder market gossip: the `execution_payload_bid` and //! `proposer_preferences` topics. //! -//! Stubs until filled. Neither message type ever reaches the chain actor: the +//! Neither message type ever reaches the chain actor: the //! rules live in `ethlambda_state_transition::beacon::gossip::{ //! execution_payload_bid, proposer_preferences}`, and what they accept is //! recorded in the node's shared `BuilderMarket` by the verdict. -use ethlambda_state_transition::beacon::gossip::{IgnoreReason, Outcome}; +use ethlambda_state_transition::beacon::gossip::{ + self, Outcome, RejectReason, execution_payload_bid::MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE, +}; use ethlambda_types::beacon::containers::gloas::{ SignedExecutionPayloadBid, SignedProposerPreferences, }; use ethlambda_types::beacon::primitives::Slot; use ethlambda_types::time::unix_now_ms; +use libp2p::gossipsub::IdentTopic; +use libssz::SszEncode; +use tracing::{debug, error, info, warn}; -use crate::P2PServer; -use crate::beacon::verdict::Dispatch; +use crate::beacon::verdict::{Dispatch, Validated}; +use crate::beacon::{decode as beacon_decode, topics as beacon_topics}; +use crate::gossipsub::compress_message; +use crate::{P2PServer, metrics}; -/// Size cap -> `Reject(Malformed)`; decode -> `Reject(Decode)`; -/// `inc_beacon_gossip(KIND, "decoded")`; -/// `gossip::execution_payload_bid::cheap_checks`; then -/// `Validate(Validated::ExecutionPayloadBid { .. })`. -#[allow(dead_code, unused_variables)] // filled by Agent B +/// Decode a gloas execution payload bid and run its cheap gossip checks. +/// +/// The size cap is the specification's decompressed bound, applied before any +/// decode so an oversized payload costs nothing. Same shape as the envelope's +/// triage, except the seen state lives in the shared +/// [`BuilderMarket`](ethlambda_state_transition::beacon::builder_market::BuilderMarket), +/// and the object carries the market on to its stateful checks. pub(crate) fn triage_execution_payload_bid(server: &P2PServer, payload: &[u8]) -> Dispatch { - Dispatch::Report(Outcome::Ignore(IgnoreReason::NoConsumer)) + const KIND: &str = beacon_topics::EXECUTION_PAYLOAD_BID; + if payload.len() > MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE { + metrics::inc_beacon_gossip(KIND, "decode_failed"); + debug!( + kind = KIND, + bytes = payload.len(), + "Beacon gossip payload over the size cap" + ); + return Dispatch::Report(Outcome::Reject(RejectReason::Malformed)); + } + let bid = match beacon_decode::decode_execution_payload_bid(payload) { + Ok(bid) => bid, + Err(err) => { + metrics::inc_beacon_gossip(KIND, "decode_failed"); + debug!(kind = KIND, %err, bytes = payload.len(), "Beacon gossip decode failed"); + return Dispatch::Report(Outcome::Reject(RejectReason::Decode)); + } + }; + metrics::inc_beacon_gossip(KIND, "decoded"); + debug!( + slot = bid.message.slot, + builder_index = bid.message.builder_index, + value = bid.message.value, + bytes = payload.len(), + "Beacon execution payload bid decoded" + ); + if let Err(outcome) = gossip::execution_payload_bid::cheap_checks( + &server.builder_market, + &server.store, + &bid, + unix_now_ms(), + ) { + return Dispatch::Report(outcome); + } + Dispatch::Validate(Validated::ExecutionPayloadBid { + bid: Box::new(bid), + market: server.builder_market.clone(), + }) } -/// As [`triage_execution_payload_bid`], for `proposer_preferences`. -#[allow(dead_code, unused_variables)] // filled by Agent B +/// As [`triage_execution_payload_bid`], for `proposer_preferences`. The +/// container is fixed-size, so decode is its own size check. pub(crate) fn triage_proposer_preferences(server: &P2PServer, payload: &[u8]) -> Dispatch { - Dispatch::Report(Outcome::Ignore(IgnoreReason::NoConsumer)) + const KIND: &str = beacon_topics::PROPOSER_PREFERENCES; + let preferences = match beacon_decode::decode_proposer_preferences(payload) { + Ok(preferences) => preferences, + Err(err) => { + metrics::inc_beacon_gossip(KIND, "decode_failed"); + debug!(kind = KIND, %err, bytes = payload.len(), "Beacon gossip decode failed"); + return Dispatch::Report(Outcome::Reject(RejectReason::Decode)); + } + }; + metrics::inc_beacon_gossip(KIND, "decoded"); + debug!( + proposal_slot = preferences.message.proposal_slot, + validator_index = preferences.message.validator_index, + bytes = payload.len(), + "Beacon proposer preferences decoded" + ); + if let Err(outcome) = gossip::proposer_preferences::cheap_checks( + &server.builder_market, + &server.store, + &preferences, + unix_now_ms(), + ) { + return Dispatch::Report(outcome); + } + Dispatch::Validate(Validated::ProposerPreferences(Box::new(preferences))) +} + +/// Where and what to publish for one of this module's messages: the topic of +/// `kind` under the digest `slot` names, and the compressed SSZ. `None` when +/// this node is not on the beacon wire or holds no digest covering `slot`. +/// +/// Split from the publish functions so the topic and digest choice can be +/// checked without a swarm to read the command back from. +fn publication( + server: &P2PServer, + kind: &'static str, + slot: Slot, + ssz: &[u8], +) -> Option<(IdentTopic, Vec)> { + let Some(beacon) = server.wire.beacon() else { + error!( + kind, + slot, "A builder market message reached a lean node; dropping it" + ); + return None; + }; + let Some(digest) = beacon.publish_digest(slot) else { + warn!( + kind, + slot, "No held fork digest covers this message's slot; not publishing" + ); + return None; + }; + let topic = IdentTopic::new(beacon_topics::topic_name(digest, kind)); + Some((topic, compress_message(ssz))) } /// Publish on `publish_digest(bid.slot)` / `execution_payload_bid`. -/// Precondition: the caller already recorded it in the market. -#[allow(dead_code, unused_variables)] // filled by Agent B +/// +/// Precondition: the caller already recorded it in the market, since +/// gossipsub never delivers a node its own messages. pub(crate) fn publish_execution_payload_bid( server: &mut P2PServer, bid: SignedExecutionPayloadBid, ) { + let slot = bid.message.slot; + let Some((topic, data)) = publication( + server, + beacon_topics::EXECUTION_PAYLOAD_BID, + slot, + &bid.to_ssz(), + ) else { + return; + }; + server.swarm_handle.publish(topic, data); + info!( + slot, + builder_index = bid.message.builder_index, + value = bid.message.value, + "Published execution payload bid to gossipsub" + ); } /// Publish on `publish_digest(proposal_slot)`: during the epoch before gloas /// this is the gloas digest, which is held. -#[allow(dead_code, unused_variables)] // filled by Agent B pub(crate) fn publish_proposer_preferences( server: &mut P2PServer, preferences: SignedProposerPreferences, ) { + let proposal_slot = preferences.message.proposal_slot; + let Some((topic, data)) = publication( + server, + beacon_topics::PROPOSER_PREFERENCES, + proposal_slot, + &preferences.to_ssz(), + ) else { + return; + }; + server.swarm_handle.publish(topic, data); + info!( + proposal_slot, + validator_index = preferences.message.validator_index, + "Published proposer preferences to gossipsub" + ); } /// The wall-clock slot, from the store's config. @@ -55,3 +186,229 @@ pub(crate) fn wall_slot(server: &P2PServer) -> Slot { let genesis_ms = config.genesis_time_ms(); unix_now_ms().saturating_sub(genesis_ms) / config.slot_duration_ms.max(1) } + +#[cfg(test)] +mod tests { + use ethlambda_state_transition::beacon::gossip::{IgnoreReason, Outcome, RejectReason}; + use ethlambda_types::beacon::config::Config; + use ethlambda_types::beacon::fork::ForkName; + use ethlambda_types::beacon::preset::SLOTS_PER_EPOCH; + use ethlambda_types::beacon::primitives::{Epoch, ExecutionBlockHash}; + use libssz::SszEncode; + + use super::*; + use crate::beacon::transition::apply; + use crate::test_support::unconnected_beacon_server; + + const FULU: Epoch = 1_000; + const GLOAS: Epoch = 5_000; + + /// Fulu, then gloas at [`GLOAS`], so the rollover window can be opened. + fn rollover_config() -> Config { + Config::mainnet() + .with_fork_epoch(ForkName::Altair, 1) + .with_fork_epoch(ForkName::Bellatrix, 2) + .with_fork_epoch(ForkName::Capella, 3) + .with_fork_epoch(ForkName::Deneb, 4) + .with_fork_epoch(ForkName::Electra, 5) + .with_fork_epoch(ForkName::Fulu, FULU) + .with_fork_epoch(ForkName::Gloas, GLOAS) + } + + fn gloas_from_genesis() -> Config { + Config::mainnet().with_fork_epoch(ForkName::Gloas, 0) + } + + /// A bid the cheap checks have nothing to say against at `slot`. + fn bid_at(slot: Slot) -> SignedExecutionPayloadBid { + let mut bid = SignedExecutionPayloadBid::default(); + bid.message.slot = slot; + bid.message.builder_index = 3; + bid.message.value = 10; + bid.message.block_hash = ExecutionBlockHash::repeat_byte(1); + bid + } + + fn preferences_at(proposal_slot: Slot) -> SignedProposerPreferences { + let mut preferences = SignedProposerPreferences::default(); + preferences.message.proposal_slot = proposal_slot; + preferences.message.validator_index = 5; + preferences + } + + // -- Triage, independent of the gossip rules -- + + #[tokio::test] + async fn an_oversized_bid_is_malformed_before_it_is_decoded() { + let server = unconnected_beacon_server(gloas_from_genesis(), 0).await; + let payload = vec![0xff; MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE + 1]; + assert!(matches!( + triage_execution_payload_bid(&server, &payload), + Dispatch::Report(Outcome::Reject(RejectReason::Malformed)) + )); + } + + #[tokio::test] + async fn garbage_on_either_builder_topic_is_undecodable() { + let server = unconnected_beacon_server(gloas_from_genesis(), 0).await; + assert!(matches!( + triage_execution_payload_bid(&server, &[0xff; 3]), + Dispatch::Report(Outcome::Reject(RejectReason::Decode)) + )); + assert!(matches!( + triage_proposer_preferences(&server, &[0xff; 3]), + Dispatch::Report(Outcome::Reject(RejectReason::Decode)) + )); + // Preferences are fixed-size, so one byte over is not a preference. + let mut bytes = preferences_at(1).to_ssz(); + bytes.push(0); + assert!(matches!( + triage_proposer_preferences(&server, &bytes), + Dispatch::Report(Outcome::Reject(RejectReason::Decode)) + )); + } + + // -- Triage, which needs the real cheap checks (Agent A) -- + + #[tokio::test] + async fn a_far_slot_bid_is_not_current_or_next() { + let server = unconnected_beacon_server(gloas_from_genesis(), 0).await; + let bid = bid_at(wall_slot(&server) + 1_000); + assert!(matches!( + triage_execution_payload_bid(&server, &bid.to_ssz()), + Dispatch::Report(Outcome::Ignore(IgnoreReason::NotCurrentOrNextSlot)) + )); + } + + #[tokio::test] + async fn a_bid_with_a_nonzero_payment_is_rejected() { + let server = unconnected_beacon_server(gloas_from_genesis(), 0).await; + let mut bid = bid_at(wall_slot(&server) + 1); + bid.message.execution_payment = 1; + assert!(matches!( + triage_execution_payload_bid(&server, &bid.to_ssz()), + Dispatch::Report(Outcome::Reject(RejectReason::ExecutionPaymentNonZero)) + )); + } + + /// A valid-shaped bid goes on to the stateful checks carrying the shared + /// market, and once the market holds its builder's key a second is ignored + /// before them. + #[tokio::test] + async fn a_valid_shaped_bid_is_validated_unless_its_builder_key_was_seen() { + let server = unconnected_beacon_server(gloas_from_genesis(), 0).await; + let bid = bid_at(wall_slot(&server) + 1); + let payload = bid.to_ssz(); + match triage_execution_payload_bid(&server, &payload) { + Dispatch::Validate(Validated::ExecutionPayloadBid { + bid: decoded, + market, + }) => { + assert_eq!(*decoded, bid); + assert!(std::sync::Arc::ptr_eq(&market, &server.builder_market)); + } + _ => panic!("expected the bid to go to its stateful checks"), + } + + assert!(server.builder_market.record_bid(bid)); + assert!(matches!( + triage_execution_payload_bid(&server, &payload), + Dispatch::Report(Outcome::Ignore(IgnoreReason::AlreadySeen)) + )); + } + + #[tokio::test] + async fn preferences_are_judged_on_the_clock() { + let server = unconnected_beacon_server(gloas_from_genesis(), 0).await; + let wall = wall_slot(&server); + let past = preferences_at(wall.saturating_sub(5)); + assert!(matches!( + triage_proposer_preferences(&server, &past.to_ssz()), + Dispatch::Report(Outcome::Ignore(IgnoreReason::SlotStarted)) + )); + let two_epochs_ahead = preferences_at((wall / SLOTS_PER_EPOCH + 2) * SLOTS_PER_EPOCH + 1); + assert!(matches!( + triage_proposer_preferences(&server, &two_epochs_ahead.to_ssz()), + Dispatch::Report(Outcome::Ignore(IgnoreReason::BeyondLookahead)) + )); + } + + #[tokio::test] + async fn preferences_for_a_cached_key_are_already_seen() { + let server = unconnected_beacon_server(gloas_from_genesis(), 0).await; + let wall = wall_slot(&server); + let preferences = preferences_at(wall + 2); + let payload = preferences.to_ssz(); + assert!(matches!( + triage_proposer_preferences(&server, &payload), + Dispatch::Validate(Validated::ProposerPreferences(decoded)) if *decoded == preferences + )); + assert!(server.builder_market.record_preferences(preferences, wall)); + assert!(matches!( + triage_proposer_preferences(&server, &payload), + Dispatch::Report(Outcome::Ignore(IgnoreReason::AlreadySeen)) + )); + } + + // -- Publishing -- + + fn topic_of(server: &P2PServer, kind: &'static str, slot: Slot) -> Option { + publication(server, kind, slot, b"payload").map(|(topic, _)| topic.to_string()) + } + + #[tokio::test] + async fn a_bid_is_published_on_its_slots_digest_and_compressed() { + let mut server = unconnected_beacon_server(gloas_from_genesis(), 0).await; + // The test server starts on a placeholder digest; this is the startup + // call that puts the schedule's own digest in. + apply(&mut server, 0); + let wire = server.wire.beacon().expect("a beacon wire"); + let slot = 7; + let expected = beacon_topics::topic_name( + wire.publish_digest(slot).expect("held"), + beacon_topics::EXECUTION_PAYLOAD_BID, + ); + let ssz = bid_at(slot).to_ssz(); + let (topic, data) = + publication(&server, beacon_topics::EXECUTION_PAYLOAD_BID, slot, &ssz).unwrap(); + assert_eq!(topic.to_string(), expected); + assert_eq!(data, compress_message(&ssz)); + assert!(expected.ends_with("/execution_payload_bid/ssz_snappy")); + } + + /// During the epoch before gloas, preferences for a gloas slot go out on + /// the gloas digest the node has already joined, and a slot nobody listens + /// to is not published at all. + #[tokio::test] + async fn preferences_for_a_next_fork_slot_use_the_next_forks_digest() { + let mut server = unconnected_beacon_server(rollover_config(), 0).await; + apply(&mut server, GLOAS - 1); + let wire = server.wire.beacon().expect("a beacon wire"); + let gloas_digest = wire.schedule.digest_at(GLOAS); + let fulu_digest = wire.schedule.digest_at(GLOAS - 1); + assert_ne!(gloas_digest, fulu_digest); + + let kind = beacon_topics::PROPOSER_PREFERENCES; + let first_gloas_slot = GLOAS * SLOTS_PER_EPOCH; + assert_eq!( + topic_of(&server, kind, first_gloas_slot), + Some(beacon_topics::topic_name(gloas_digest, kind)) + ); + assert_eq!( + topic_of(&server, kind, first_gloas_slot - 1), + Some(beacon_topics::topic_name(fulu_digest, kind)) + ); + // Long past: its digest is no longer held. + assert_eq!(topic_of(&server, kind, 0), None); + } + + /// Before the window opens the gloas digest is not held, so there is + /// nowhere to publish a bid for a gloas slot. + #[tokio::test] + async fn nothing_is_published_for_a_digest_that_is_not_held() { + let mut server = unconnected_beacon_server(rollover_config(), 0).await; + apply(&mut server, GLOAS - 3); + let kind = beacon_topics::EXECUTION_PAYLOAD_BID; + assert_eq!(topic_of(&server, kind, GLOAS * SLOTS_PER_EPOCH), None); + } +} diff --git a/crates/net/p2p/src/beacon/decode.rs b/crates/net/p2p/src/beacon/decode.rs index b4fac1c01..4547d52ac 100644 --- a/crates/net/p2p/src/beacon/decode.rs +++ b/crates/net/p2p/src/beacon/decode.rs @@ -24,6 +24,7 @@ //! | `sync_committee_contribution_and_proof` | No, altair onward | //! | `data_column_sidecar_{subnet_id}` | Yes, at gloas, by topic digest | //! | `execution_payload`, `payload_attestation_message` | No, gloas onward | +//! | `execution_payload_bid`, `proposer_preferences` | No, gloas onward | use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::containers::{ @@ -237,6 +238,22 @@ pub fn decode_payload_attestation_message( gloas::PayloadAttestationMessage::from_ssz_bytes(bytes).map_err(|_| DecodeError::Ssz) } +/// Decode an `execution_payload_bid` payload. Gloas on, like +/// [`decode_execution_payload_envelope`]. +pub fn decode_execution_payload_bid( + bytes: &[u8], +) -> Result { + gloas::SignedExecutionPayloadBid::from_ssz_bytes(bytes).map_err(|_| DecodeError::Ssz) +} + +/// Decode a `proposer_preferences` payload. Gloas on, like +/// [`decode_execution_payload_envelope`]. +pub fn decode_proposer_preferences( + bytes: &[u8], +) -> Result { + gloas::SignedProposerPreferences::from_ssz_bytes(bytes).map_err(|_| DecodeError::Ssz) +} + /// Decode a `beacon_aggregate_and_proof` payload, at the fork its slot names. pub fn decode_aggregate_and_proof( config: &Config, @@ -653,6 +670,42 @@ mod tests { } } + #[test] + fn an_execution_payload_bid_round_trips() { + let mut bid = gloas::SignedExecutionPayloadBid::default(); + bid.message.slot = slot_of(10); + bid.message.builder_index = 4; + bid.message.value = 99; + bid.message.parent_block_root = Root::repeat_byte(7); + let bytes = bid.to_ssz(); + assert_eq!(decode_execution_payload_bid(&bytes), Ok(bid)); + assert!(decode_execution_payload_bid(&bytes[..bytes.len() - 1]).is_err()); + assert_eq!( + decode_execution_payload_bid(&[0xff; 3]), + Err(DecodeError::Ssz) + ); + } + + #[test] + fn proposer_preferences_round_trip() { + let mut preferences = gloas::SignedProposerPreferences::default(); + preferences.message.proposal_slot = slot_of(10); + preferences.message.validator_index = 9; + preferences.message.dependent_root = Root::repeat_byte(2); + let bytes = preferences.to_ssz(); + assert_eq!(decode_proposer_preferences(&bytes), Ok(preferences)); + for length in 0..bytes.len() { + assert!(decode_proposer_preferences(&bytes[..length]).is_err()); + } + let mut longer = bytes; + longer.push(0); + assert!(decode_proposer_preferences(&longer).is_err()); + assert_eq!( + decode_proposer_preferences(&[0xff; 3]), + Err(DecodeError::Ssz) + ); + } + #[test] fn an_execution_payload_envelope_round_trips() { let envelope = crate::test_support::envelope(5, 9); diff --git a/crates/net/p2p/src/beacon/topics.rs b/crates/net/p2p/src/beacon/topics.rs index 7605d3d9d..2fe68e720 100644 --- a/crates/net/p2p/src/beacon/topics.rs +++ b/crates/net/p2p/src/beacon/topics.rs @@ -1,6 +1,8 @@ //! The gossipsub topics `ethlambda beacon` subscribes to. //! -//! Seven global topics, plus two families this node's own node id selects a +//! Seven global topics (four more from gloas: the execution payload envelope, +//! the payload attestation message, the builder bid and the proposer +//! preferences), plus two families this node's own node id selects a //! narrow slice of: the data column subnets it custodies, and the //! `SUBNETS_PER_NODE` attestation subnets it backbones. //! @@ -58,7 +60,12 @@ pub const PROPOSER_PREFERENCES: &str = "proposer_preferences"; /// The topic kinds gloas adds to [`SUBSCRIBED_TOPIC_KINDS`], subscribed from /// the gloas digest on and never under an earlier one. -pub const GLOAS_TOPIC_KINDS: [&str; 2] = [EXECUTION_PAYLOAD, PAYLOAD_ATTESTATION_MESSAGE]; +pub const GLOAS_TOPIC_KINDS: [&str; 4] = [ + EXECUTION_PAYLOAD, + PAYLOAD_ATTESTATION_MESSAGE, + EXECUTION_PAYLOAD_BID, + PROPOSER_PREFERENCES, +]; /// Every topic kind this node subscribes to at every fork, in the order they /// are subscribed. [`GLOAS_TOPIC_KINDS`] follow from gloas. @@ -348,7 +355,7 @@ mod tests { } #[test] - fn gloas_adds_the_envelope_and_payload_attestation_topics() { + fn gloas_adds_the_envelope_vote_bid_and_preferences_topics() { let fork_topics = |fork| { BeaconTopics::for_fork(fork, MAINNET, &[], &[]) .topics @@ -358,8 +365,17 @@ mod tests { }; let gloas = fork_topics(ForkName::Gloas); let fulu = fork_topics(ForkName::Fulu); - assert_eq!(gloas.len(), SUBSCRIBED_TOPIC_KINDS.len() + 2); + assert_eq!(gloas.len(), SUBSCRIBED_TOPIC_KINDS.len() + 4); assert_eq!(fulu.len(), SUBSCRIBED_TOPIC_KINDS.len()); + assert_eq!( + GLOAS_TOPIC_KINDS, + [ + "execution_payload", + "payload_attestation_message", + "execution_payload_bid", + "proposer_preferences", + ] + ); for kind in GLOAS_TOPIC_KINDS { let name = topic_name(MAINNET, kind); assert!(gloas.contains(&name), "gloas lacks {name}"); diff --git a/crates/net/p2p/src/beacon/verdict.rs b/crates/net/p2p/src/beacon/verdict.rs index 62591872c..ef263492a 100644 --- a/crates/net/p2p/src/beacon/verdict.rs +++ b/crates/net/p2p/src/beacon/verdict.rs @@ -82,13 +82,11 @@ pub(crate) enum Validated { Envelope(Box), /// A gloas `execution_payload_bid`. Carries the market because /// `stateful_checks` receives only the store. - #[allow(dead_code)] // constructed by Agent B's triage ExecutionPayloadBid { bid: Box, market: SharedBuilderMarket, }, /// A gloas `proposer_preferences`. - #[allow(dead_code)] // constructed by Agent B's triage ProposerPreferences(Box), /// A gloas `payload_attestation_message`. PayloadAttestation(PayloadAttestationMessage), @@ -161,10 +159,20 @@ impl Validated { }, Self::Aggregate { aggregate, .. } => server.seen_aggregates.record(aggregate), Self::Attestation { attestation, .. } => server.seen_attestations.record(attestation), - Self::Envelope(envelope) => server.seen_envelopes.record( - envelope.message.beacon_block_root, - envelope.message.builder_index, - ), + Self::Envelope(envelope) => { + let recorded = server.seen_envelopes.record( + envelope.message.beacon_block_root, + envelope.message.builder_index, + ); + if recorded { + // A bid's parent payload is known once its envelope passed + // gossip; the builder market judges bids against this. + server + .builder_market + .record_execution_payload(&envelope.message); + } + recorded + } Self::ExecutionPayloadBid { bid, .. } => { server.builder_market.record_bid((**bid).clone()) } @@ -1180,6 +1188,201 @@ mod tests { assert!(server.attestation_validation_permits.try_acquire().is_ok()); } + fn bid(slot: u64, builder_index: u64, value: u64) -> SignedExecutionPayloadBid { + let mut bid = SignedExecutionPayloadBid::default(); + bid.message.slot = slot; + bid.message.builder_index = builder_index; + bid.message.value = value; + bid + } + + fn preferences(proposal_slot: u64, validator: u64) -> SignedProposerPreferences { + let mut preferences = SignedProposerPreferences::default(); + preferences.message.proposal_slot = proposal_slot; + preferences.message.validator_index = validator; + preferences + } + + fn bid_object(server: &P2PServer, bid: SignedExecutionPayloadBid) -> Validated { + Validated::ExecutionPayloadBid { + bid: Box::new(bid), + market: server.builder_market.clone(), + } + } + + /// Needs the real market (Agent A): its stub never records. + #[tokio::test] + async fn the_first_accept_for_a_bid_key_stands_and_the_second_is_marked_seen() { + let mut server = unconnected_beacon_server(Config::mainnet(), 0).await; + let object = bid_object(&server, bid(5, 3, 10)); + + assert_eq!( + settle(&mut server, Outcome::Accept, &object), + Outcome::Accept + ); + assert_eq!( + settle(&mut server, Outcome::Accept, &object), + Outcome::Ignore(IgnoreReason::AlreadySeen) + ); + // Anything but an accept records nothing. + let other = bid_object(&server, bid(5, 4, 10)); + assert_eq!( + settle( + &mut server, + Outcome::Ignore(IgnoreReason::StateUnavailable), + &other + ), + Outcome::Ignore(IgnoreReason::StateUnavailable) + ); + assert_eq!( + settle(&mut server, Outcome::Accept, &other), + Outcome::Accept + ); + } + + /// An accepted bid is pooled in the market the API and `produceBlockV4` + /// read. Needs the real market (Agent A). + #[tokio::test] + async fn an_accepted_bid_is_pooled_in_the_shared_market() { + let mut server = unconnected_beacon_server(Config::mainnet(), 0).await; + let signed = bid(5, 3, 10); + let object = bid_object(&server, signed.clone()); + assert!(!server.builder_market.contains_bid(&signed)); + + assert_eq!( + settle(&mut server, Outcome::Accept, &object), + Outcome::Accept + ); + assert!(server.builder_market.contains_bid(&signed)); + assert_eq!( + server.builder_market.bids_for( + 5, + signed.message.parent_block_root, + signed.message.parent_block_hash + ), + vec![signed] + ); + } + + /// Needs the real market (Agent A). + #[tokio::test] + async fn the_first_accept_for_a_preferences_key_stands_and_the_second_is_marked_seen() { + let mut server = unconnected_beacon_server(Config::mainnet(), 0).await; + let object = Validated::ProposerPreferences(Box::new(preferences(40, 5))); + + assert_eq!( + settle(&mut server, Outcome::Accept, &object), + Outcome::Accept + ); + assert_eq!( + settle(&mut server, Outcome::Accept, &object), + Outcome::Ignore(IgnoreReason::AlreadySeen) + ); + assert!(server.builder_market.preferences(40, Root::ZERO).is_some()); + } + + /// Neither builder market type has a consumer on the chain actor, on any + /// outcome. + #[tokio::test] + async fn bids_and_preferences_never_reach_the_chain_actor() { + let mut server = unconnected_beacon_server(Config::mainnet(), 0).await; + let chain = Arc::new(RecordingChain(AtomicBool::new(false))); + server.blockchain = Some(chain.clone()); + + for outcome in [ + Outcome::Accept, + Outcome::Queue(QueueReason::BlockUnknown), + Outcome::Ignore(IgnoreReason::Overloaded), + ] { + bid_object(&server, bid(5, 3, 10)).forward(&server, Instant::now(), outcome); + Validated::ProposerPreferences(Box::new(preferences(40, 5))).forward( + &server, + Instant::now(), + outcome, + ); + } + + assert!(!chain.0.load(Ordering::SeqCst)); + } + + /// Bids and preferences draw from a pool of their own, so a burst of them + /// cannot starve blocks, columns or attestations, and the reverse. + #[tokio::test] + async fn the_builder_permit_pool_is_independent_of_the_others() { + let server = unconnected_beacon_server(Config::mainnet(), 0).await; + let bid = bid_object(&server, bid(5, 3, 10)); + let preferences = Validated::ProposerPreferences(Box::new(preferences(40, 5))); + for object in [&bid, &preferences] { + assert!(Arc::ptr_eq( + permits_for(&server, object), + &server.builder_validation_permits + )); + } + + let gossip_before = server.gossip_validation_permits.available_permits(); + let attestation_before = server.attestation_validation_permits.available_permits(); + let mut held = Vec::new(); + while let Ok(permit) = server + .builder_validation_permits + .clone() + .try_acquire_owned() + { + held.push(permit); + } + // Exhausted: a bid's stateful checks would answer `Ignore(Overloaded)`. + assert_eq!(server.builder_validation_permits.available_permits(), 0); + assert!( + permits_for(&server, &bid) + .clone() + .try_acquire_owned() + .is_err() + ); + assert_eq!( + server.gossip_validation_permits.available_permits(), + gossip_before + ); + assert_eq!( + server.attestation_validation_permits.available_permits(), + attestation_before + ); + } + + /// An accepted envelope is a known payload for bid validation. Needs the + /// real market (Agent A). + #[tokio::test] + async fn an_accepted_envelope_becomes_a_known_payload() { + let mut server = unconnected_beacon_server(Config::mainnet(), 0).await; + let signed = envelope(1, 3); + let hash = signed.message.payload.block_hash; + assert!(server.builder_market.known_payload(hash).is_none()); + let object = Validated::Envelope(Box::new(signed)); + + // A queued envelope has not been judged, so it is not known. + settle( + &mut server, + Outcome::Queue(QueueReason::BlockUnknown), + &object, + ); + assert!(server.builder_market.known_payload(hash).is_none()); + + settle(&mut server, Outcome::Accept, &object); + assert!(server.builder_market.known_payload(hash).is_some()); + } + + /// The node's own envelope is known too, since gossip never echoes it. + /// Needs the real market (Agent A). + #[tokio::test] + async fn a_published_envelope_becomes_a_known_payload() { + let mut server = unconnected_beacon_server(Config::mainnet(), 0).await; + let signed = envelope(2, 3); + let hash = signed.message.payload.block_hash; + assert!(server.builder_market.known_payload(hash).is_none()); + + crate::gossipsub::publish_execution_payload_envelope(&mut server, signed, Vec::new()).await; + + assert!(server.builder_market.known_payload(hash).is_some()); + } + #[test] fn only_accept_propagates_and_only_accept_or_queue_reaches_the_chain() { assert!(matches!( diff --git a/crates/net/p2p/src/gossipsub/handler.rs b/crates/net/p2p/src/gossipsub/handler.rs index ac1a0c478..49915f73e 100644 --- a/crates/net/p2p/src/gossipsub/handler.rs +++ b/crates/net/p2p/src/gossipsub/handler.rs @@ -818,6 +818,11 @@ pub async fn publish_execution_payload_envelope( ) { let slot = envelope.message.payload.slot_number; let block_root = envelope.message.beacon_block_root; + // Gossip never echoes a node's own message, so its own envelope is a + // known payload for bid validation only because it is recorded here. + server + .builder_market + .record_execution_payload(&envelope.message); let Some(beacon) = server.wire.beacon() else { error!( slot, diff --git a/crates/net/p2p/src/gossipsub/mod.rs b/crates/net/p2p/src/gossipsub/mod.rs index 8c5580162..1e2b402fb 100644 --- a/crates/net/p2p/src/gossipsub/mod.rs +++ b/crates/net/p2p/src/gossipsub/mod.rs @@ -2,7 +2,6 @@ mod encoding; mod handler; mod messages; -#[allow(unused_imports)] // used by Agent B's publish functions pub(crate) use encoding::compress_message; pub use encoding::decompress_message; pub use handler::{ diff --git a/docs/beacon_wire.md b/docs/beacon_wire.md index 83addec59..731777450 100644 --- a/docs/beacon_wire.md +++ b/docs/beacon_wire.md @@ -85,14 +85,28 @@ after a boundary read the stale digest and reject the record. Seven global topics, `/eth2/{digest}/{name}/ssz_snappy`, plus two subnet families this node's own node id selects a narrow slice of: the data column subnets it custodies and the attestation subnets it backbones, both described -below. A gloas digest adds two more, `execution_payload` and -`payload_attestation_message`, which `BeaconTopics::for_fork` subscribes under -gloas digests and not under earlier ones. An envelope is validated like a block +below. A gloas digest adds four more, `execution_payload`, +`payload_attestation_message`, `execution_payload_bid` and +`proposer_preferences`, which `BeaconTopics::for_fork` subscribes under gloas +digests and not under earlier ones (so one epoch before the fork, when the +digest is joined). An envelope is validated like a block (its stateful half on the blocking pool) and goes to the chain actor on `Accept` and on `Queue`, since the actor holds an envelope whose block is not imported yet. A payload attestation is validated on the attestation permit pool and goes to the actor on `Accept` only: a vote for a block not imported here is dropped, -since it is valid only within its own slot. +since it is valid only within its own slot. Bids and preferences are the +builder market: they never reach the chain actor. Their rules live in +`state_transition::beacon::gossip::{execution_payload_bid, proposer_preferences}`, +the cheap half runs in p2p's triage (`p2p/src/beacon/builder_market.rs`) and the +stateful half on a permit pool of their own (`builder_validation_permits`), so +a bid burst cannot starve blocks, columns or attestations. Whatever is accepted +goes into the node's one shared `BuilderMarket`, which the Beacon API and block +production read. Neither is ever queued: a message whose parent or dependent +block is unknown is ignored. A bid is capped at 196,932 decompressed bytes +(`Reject(Malformed)` above that). The node publishes both: a bid on the digest of +its slot, preferences on the digest of the proposal slot, which in the epoch +before gloas is the gloas digest already joined. Envelopes the node publishes +itself are recorded as known payloads, since gossip never echoes them. | Topic | Decoded as | | --- | --- | @@ -106,6 +120,8 @@ since it is valid only within its own slot. | `beacon_attestation_{subnet_id}` | `Attestation`, phase0 or electra's `SingleAttestation` (gloas keeps the latter) | | `execution_payload` | `SignedExecutionPayloadEnvelope`, gloas digests only | | `payload_attestation_message` | `PayloadAttestationMessage`, gloas digests only | +| `execution_payload_bid` | `SignedExecutionPayloadBid`, gloas digests only | +| `proposer_preferences` | `SignedProposerPreferences`, gloas digests only | `beacon_attestation_{0..63}` is no longer wholly unsubscribed. This node holds `SUBNETS_PER_NODE` (2 on mainnet) long-lived subscriptions from that family, diff --git a/docs/metrics.md b/docs/metrics.md index cbdc72d6d..6d6f7b9e1 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -363,15 +363,17 @@ own section. These are ethlambda-specific, not part of the leanMetrics spec. `kind` is the topic kind, with every `data_column_sidecar_{subnet}` sharing the label `data_column_sidecar` and every `beacon_attestation_{subnet_id}` sharing -`beacon_attestation`. The gloas topics `execution_payload` and -`payload_attestation_message` are labelled by their own name. `queue` means +`beacon_attestation`. The gloas topics `execution_payload`, +`payload_attestation_message`, `execution_payload_bid` and +`proposer_preferences` are labelled by their own name. `queue` means IGNORE to gossipsub while the chain actor still receives the object and parks it; of the gloas topics only `execution_payload` answers it, for an envelope whose block is not known yet (`block_unknown`), has no post-state yet (`block_not_ready`), or has one that is not in the cache the checks read (`state_not_cached`, as when the block was imported moments ago; the actor -verifies the signature itself, so the envelope is forwarded rather than dropped). The aggregate, attestation and -`payload_attestation_message` topics never answer `queue`, since the vote +verifies the signature itself, so the envelope is forwarded rather than dropped). The aggregate, attestation, +`payload_attestation_message`, `execution_payload_bid` and `proposer_preferences` +topics never answer `queue`, since the vote block's post-state is either cached or it is not (`IgnoreReason::UnknownBlock`/`StateUnavailable`), with nothing to hold the message for. **`verdict_expired_total` should stay at @@ -390,16 +392,25 @@ only), `not_aggregator` (aggregate only), `not_in_committee`, `aggregator_signature` (aggregate only), `aggregate_signature` (aggregate only), `target_not_ancestor`, `wrong_subnet` (attestation only), and gloas's `data_index_out_of_range`, `same_slot_payload_flag` and `payload_invalid` on the -reject side. `already_seen`, `overloaded` and `unsupported_fork` are shared with the +reject side. The builder market topics add, on the ignore side, +`pre_gloas_slot`, `not_current_or_next_slot`, `not_highest_bid`, +`beyond_lookahead`, `preferences_unseen`, `fee_recipient_mismatch`, +`parent_payload_unknown`, `gas_limit_incompatible`, `not_on_head_branch`, +`builder_cannot_cover`, `builder_may_exit`, `slot_started` and +`impossible_dependent_root`, and on the reject side `execution_payment_nonzero`, +`block_hash_equals_parent`, `prev_randao`, `unknown_builder`, +`not_payload_builder`, `inactive_builder` and `dependent_root_too_late`; a bid +over its size cap is `malformed`. `already_seen`, `overloaded` and `unsupported_fork` are shared with the other topics: `unsupported_fork` is an ignore reason for a message of a fork this build has no gossip rules for, so an honest peer past the fork epoch is not scored as a bad decoder. Gloas blocks, data columns, aggregates and attestations are validated, so they carry their own verdict reasons instead. -Two permit pools bound the blocking-thread half of validation: -`gossip_validation_permits` for blocks and columns, -`attestation_validation_permits` for aggregates and subnet attestations. They -are deliberately separate: a mainnet slot's worth of aggregates and backbone +Three permit pools bound the blocking-thread half of validation: +`gossip_validation_permits` for blocks, columns and envelopes, +`attestation_validation_permits` for aggregates, subnet attestations and +payload votes, and `builder_validation_permits` for bids and proposer +preferences. They are deliberately separate: a mainnet slot's worth of aggregates and backbone attestations arrives every slot, not only during a range sync, and sharing one pool would let that burst answer `Ignore(Overloaded)` for a block or a column instead. Neither pool has a metric of its own yet; a permit exhausted on From bc87986c48dd09bb0033d843cdf052cf8ac72659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:54:43 -0300 Subject: [PATCH 117/132] feat(engine): keep shouldOverrideBuilder from getPayloadV6 An engine that wants its own payload used whatever a builder pays says so with this flag; dropping it would let a bid comparison override that. --- crates/net/engine/src/building.rs | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/crates/net/engine/src/building.rs b/crates/net/engine/src/building.rs index b407c9326..6ff109248 100644 --- a/crates/net/engine/src/building.rs +++ b/crates/net/engine/src/building.rs @@ -130,6 +130,9 @@ pub struct BuiltGloasPayload { pub blobs_bundle: BlobsBundle, /// The EIP-7685 request list, each entry its type byte then its data. pub execution_requests: Vec>, + /// The engine's `shouldOverrideBuilder`: it wants this payload built + /// locally whatever a builder bid pays. Absent from an answer means false. + pub should_override_builder: bool, } /// `BlobsBundleV2`: the payload's blobs, their commitments, and every blob's @@ -159,6 +162,8 @@ pub(crate) struct GetPayloadV6Response { block_value: String, blobs_bundle: BlobsBundleJson, #[serde(default)] + should_override_builder: bool, + #[serde(default)] execution_requests: Vec, } @@ -307,6 +312,7 @@ impl TryFrom for BuiltGloasPayload { block_value: parse_uint256(&response.block_value)?, blobs_bundle: decode_blobs_bundle(response.blobs_bundle)?, execution_requests: decode_requests(&response.execution_requests)?, + should_override_builder: response.should_override_builder, }) } } @@ -527,6 +533,30 @@ mod tests { built.execution_requests, vec![vec![0x00, 0x11], vec![0x02, 0xff]] ); + assert!(!built.should_override_builder); + } + + #[test] + fn a_v6_answer_carries_should_override_builder_and_defaults_it_to_false() { + let mut json = v6_json(); + json["shouldOverrideBuilder"] = true.into(); + let response: GetPayloadV6Response = serde_json::from_value(json).unwrap(); + assert!( + BuiltGloasPayload::try_from(response) + .unwrap() + .should_override_builder + ); + + let mut json = v6_json(); + json.as_object_mut() + .unwrap() + .remove("shouldOverrideBuilder"); + let response: GetPayloadV6Response = serde_json::from_value(json).unwrap(); + assert!( + !BuiltGloasPayload::try_from(response) + .unwrap() + .should_override_builder + ); } #[test] From 86a3c86750833422bef5c4c8134b9403ed6057a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:54:43 -0300 Subject: [PATCH 118/132] feat(rpc): serve the builder market endpoints and build on bids Add POST execution_payload_bids, POST proposer_preferences and POST states/{id}/builders. Bids and preferences run through the gossip rules and land in the shared builder market that p2p also fills. produceBlockV4 now reads the real BuilderConfig, builds locally when it has an engine and takes the best pooled bid when the config's min_bid and boost factor say so (the local payload wins ties, and shouldOverrideBuilder keeps it). A bid-won block comes back bare and caches nothing, since the builder reveals the payload. A self-built payload takes its fee recipient and gas target from the proposer's signed preferences when the market holds them. --- crates/net/rpc/src/beacon/bid_selection.rs | 207 ++++++- crates/net/rpc/src/beacon/bids.rs | 149 ++++- crates/net/rpc/src/beacon/builder_config.rs | 270 +++++++++- crates/net/rpc/src/beacon/builders.rs | 332 +++++++++++- crates/net/rpc/src/beacon/gloas_proposal.rs | 507 +++++++++++++----- .../rpc/src/beacon/proposer_preferences.rs | 139 ++++- crates/net/rpc/src/beacon/states.rs | 5 +- .../rpc/src/beacon/validator_client_tests.rs | 3 + docs/rpc.md | 87 ++- 9 files changed, 1499 insertions(+), 200 deletions(-) diff --git a/crates/net/rpc/src/beacon/bid_selection.rs b/crates/net/rpc/src/beacon/bid_selection.rs index 0fd438380..82b890c56 100644 --- a/crates/net/rpc/src/beacon/bid_selection.rs +++ b/crates/net/rpc/src/beacon/bid_selection.rs @@ -1,45 +1,230 @@ //! Choosing between this node's own build and a pooled builder bid for //! `produceBlockV4`. Pure: no store, no clock. //! -//! Stubs until filled. +//! Units differ on the two sides: a bid's value is in Gwei and the execution +//! client's `blockValue` in Wei, and the specification weights the local value +//! by 100 against a bid weighted by `builder_boost_factor`. Everything that +//! compares the two goes through [`choose_payload`], so the conversion lives in +//! one place. use ethlambda_types::beacon::containers::gloas::{ExecutionPayloadBid, SignedExecutionPayloadBid}; use ethlambda_types::beacon::primitives::Uint256; +/// Dividing a wei amount by this yields the amount in Gwei times 100, the +/// local value's weight: `wei / 1e9 * 100 == wei / 1e7`. +const WEI_PER_WEIGHTED_GWEI: u128 = 10_000_000; + /// The local build, as the choice sees it. -#[allow(dead_code)] // filled by Agent C pub(crate) struct LocalCandidate { pub(crate) value_wei: u128, + /// The execution client's `shouldOverrideBuilder`. pub(crate) should_override_builder: bool, } // The enum is short-lived (one per block production), so boxing the bid buys // nothing. -#[allow(dead_code, clippy::large_enum_variant)] // filled by Agent C +#[allow(clippy::large_enum_variant)] +#[derive(Debug, PartialEq, Eq)] pub(crate) enum PayloadChoice { Local, Bid(SignedExecutionPayloadBid), } /// `value.saturating_add(execution_payment)`; a p2p bid's payment is zero. -#[allow(dead_code, unused_variables)] // filled by Agent C pub(crate) fn bid_total_gwei(bid: &ExecutionPayloadBid) -> u64 { - 0 + bid.value.saturating_add(bid.execution_payment) } -/// Saturating conversion of a wei amount. -#[allow(dead_code, unused_variables)] // filled by Agent C +/// Saturating conversion of a wei amount: the 32 little-endian bytes of a +/// `uint256` clamped to `u128::MAX`. pub(crate) fn wei_u128(value: &Uint256) -> u128 { - 0 + let (low, high) = value.0.split_at(16); + if high.iter().any(|byte| *byte != 0) { + return u128::MAX; + } + u128::from_le_bytes(low.try_into().expect("sixteen bytes")) } -/// `None` when there is neither a local build nor a bid at or above `min_bid`. -#[allow(dead_code, unused_variables)] // filled by Agent C +/// The bid to build on, or the local build. +/// +/// 1. The best bid is the first of `bids_desc` (value descending) whose total +/// is at least `min_bid`. +/// 2. With no local build, that bid, or `None` if there is none. +/// 3. A local build that asks to override builders wins. +/// 4. Otherwise the bid wins iff `builder_boost_factor * bid_gwei` exceeds the +/// local value in the same weighting: `factor * gwei * 1e9 > 100 * wei`, +/// which for integers is `factor * gwei > floor(wei / 1e7)`. The left side +/// is a product of two `u64`s, so it cannot overflow `u128`. +/// 5. The local build wins a tie, so a factor of `0` prefers it and `u64::MAX` +/// prefers the bid, each unless step 2 or 3 says otherwise. pub(crate) fn choose_payload( local: Option<&LocalCandidate>, bids_desc: &[SignedExecutionPayloadBid], min_bid: u64, builder_boost_factor: u64, ) -> Option { - None + let best = bids_desc + .iter() + .find(|signed| bid_total_gwei(&signed.message) >= min_bid); + let Some(local) = local else { + return best.cloned().map(PayloadChoice::Bid); + }; + let Some(best) = best else { + return Some(PayloadChoice::Local); + }; + if local.should_override_builder { + return Some(PayloadChoice::Local); + } + let weighted_bid = u128::from(builder_boost_factor) * u128::from(bid_total_gwei(&best.message)); + let weighted_local = local.value_wei / WEI_PER_WEIGHTED_GWEI; + if weighted_bid > weighted_local { + Some(PayloadChoice::Bid(best.clone())) + } else { + Some(PayloadChoice::Local) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn bid(builder: u64, value: u64) -> SignedExecutionPayloadBid { + let mut signed = SignedExecutionPayloadBid::default(); + signed.message.builder_index = builder; + signed.message.value = value; + signed + } + + fn local(value_wei: u128) -> LocalCandidate { + LocalCandidate { + value_wei, + should_override_builder: false, + } + } + + fn picks_bid(choice: Option) -> bool { + matches!(choice, Some(PayloadChoice::Bid(_))) + } + + #[test] + fn a_tie_goes_to_the_local_build() { + // 1e9 wei is 1 gwei; factor 100 weights both sides to 100. + let bids = [bid(1, 1)]; + assert_eq!( + choose_payload(Some(&local(1_000_000_000)), &bids, 0, 100), + Some(PayloadChoice::Local) + ); + } + + #[test] + fn the_floor_at_ten_million_wei_decides_the_boundary() { + let bids = [bid(1, 1)]; + // One wei short of the tie: the local weighted value floors to 99. + assert!(picks_bid(choose_payload( + Some(&local(999_999_999)), + &bids, + 0, + 100 + ))); + // Exactly the tie, and above it. + for wei in [1_000_000_000, 1_000_000_001, 1_000_000_000_000] { + assert_eq!( + choose_payload(Some(&local(wei)), &bids, 0, 100), + Some(PayloadChoice::Local), + "{wei}" + ); + } + } + + #[test] + fn a_factor_of_zero_prefers_local_and_the_maximum_prefers_the_bid() { + let bids = [bid(1, 5)]; + assert_eq!( + choose_payload(Some(&local(1)), &bids, 0, 0), + Some(PayloadChoice::Local) + ); + assert!(picks_bid(choose_payload( + Some(&local(1_000_000_000_000_000)), + &bids, + 0, + u64::MAX + ))); + } + + #[test] + fn a_factor_of_zero_still_takes_the_bid_when_the_local_build_failed() { + let bids = [bid(1, 5)]; + assert!(picks_bid(choose_payload(None, &bids, 0, 0))); + } + + #[test] + fn the_min_bid_floor_skips_bids_below_it() { + let bids = [bid(1, 9), bid(2, 5), bid(3, 1)]; + // The best bid is below the floor, and so are the rest. + assert_eq!( + choose_payload(Some(&local(0)), &bids, 10, u64::MAX), + Some(PayloadChoice::Local) + ); + assert_eq!(choose_payload(None, &bids, 10, u64::MAX), None); + // The floor is inclusive. + let Some(PayloadChoice::Bid(chosen)) = choose_payload(None, &bids, 9, 0) else { + panic!("the bid at the floor is eligible") + }; + assert_eq!(chosen.message.builder_index, 1); + } + + #[test] + fn the_override_flag_keeps_the_local_build() { + let bids = [bid(1, u64::MAX)]; + let overriding = LocalCandidate { + value_wei: 0, + should_override_builder: true, + }; + assert_eq!( + choose_payload(Some(&overriding), &bids, 0, u64::MAX), + Some(PayloadChoice::Local) + ); + } + + #[test] + fn no_local_build_and_no_bid_is_nothing() { + assert_eq!(choose_payload(None, &[], 0, 100), None); + assert_eq!( + choose_payload(Some(&local(1)), &[], 0, 100), + Some(PayloadChoice::Local) + ); + } + + #[test] + fn the_total_counts_execution_payment_and_saturates() { + let mut signed = bid(1, u64::MAX); + signed.message.execution_payment = 5; + assert_eq!(bid_total_gwei(&signed.message), u64::MAX); + signed.message.value = 3; + assert_eq!(bid_total_gwei(&signed.message), 8); + } + + #[test] + fn the_weighted_comparison_cannot_overflow() { + let bids = [bid(1, u64::MAX)]; + assert!(picks_bid(choose_payload( + Some(&local(u128::MAX)), + &bids, + 0, + u64::MAX + ))); + } + + #[test] + fn wei_saturates_at_u128() { + assert_eq!(wei_u128(&Uint256::from_u128(42)), 42); + assert_eq!(wei_u128(&Uint256::from_u128(u128::MAX)), u128::MAX); + assert_eq!(wei_u128(&Uint256::MAX), u128::MAX); + let mut bytes = [0u8; 32]; + bytes[16] = 1; + assert_eq!( + wei_u128(ðlambda_types::beacon::primitives::U256(bytes)), + u128::MAX + ); + } } diff --git a/crates/net/rpc/src/beacon/bids.rs b/crates/net/rpc/src/beacon/bids.rs index 21d3ecacd..e7da89e78 100644 --- a/crates/net/rpc/src/beacon/bids.rs +++ b/crates/net/rpc/src/beacon/bids.rs @@ -2,12 +2,153 @@ //! node, validated with the gossip rules, pooled in the shared //! `BuilderMarket` and gossiped on `execution_payload_bid`. //! -//! Stub: no routes until filled. +//! The rules are the topic's own (`gossip::execution_payload_bid`), so a bid +//! that would draw a peer's penalty is refused here instead of being relayed. +//! A refusal is a 400 whatever the verdict: the specification has no 202 for +//! "valid but not forwarded", and a bid this node would ignore on gossip is one +//! it cannot vouch for to the network either. -use axum::Router; +use axum::{ + Extension, Router, + body::Bytes, + extract::State, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + routing::post, +}; +use ethlambda_network_api::RpcToP2PRef; +use ethlambda_state_transition::beacon::{ + builder_market::SharedBuilderMarket, + gossip::{ + IgnoreReason, Outcome, + execution_payload_bid::{ + MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE, cheap_checks, stateful_checks, + }, + }, +}; use ethlambda_storage::Store; +use ethlambda_types::beacon::{containers::gloas::SignedExecutionPayloadBid, fork::ForkName}; +use tracing::{debug, warn}; + +use crate::beacon::{ApiError, BodyEncoding}; -#[allow(dead_code)] // filled by Agent C pub(crate) fn routes() -> Router { - Router::new() + Router::new().route("/eth/v1/beacon/execution_payload_bids", post(post_bid)) +} + +/// A 400 whose message names the verdict, in the Beacon API's error shape. +pub(crate) fn bad_request(message: String) -> Response { + let body = serde_json::json!({ "code": 400, "message": message }); + let mut response = crate::json_response(body); + *response.status_mut() = StatusCode::BAD_REQUEST; + response +} + +/// `"{outcome}: {reason}"`, the label pair of a verdict. +pub(crate) fn describe(outcome: &Outcome) -> String { + let (outcome, reason) = outcome.labels(); + format!("{outcome}: {reason}") +} + +pub(crate) fn unix_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| elapsed.as_millis() as u64) + .unwrap_or(0) +} + +/// `Eth-Consensus-Version` is optional, and must name `allowed` when given. +pub(crate) fn require_version(headers: &HeaderMap, allowed: &[ForkName]) -> Result<(), ApiError> { + let Some(value) = headers.get("eth-consensus-version") else { + return Ok(()); + }; + match value.to_str().ok().and_then(ForkName::parse) { + Some(fork) if allowed.contains(&fork) => Ok(()), + _ => Err(ApiError::BadRequest( + "Eth-Consensus-Version names a fork this endpoint does not take", + )), + } +} + +/// `POST /eth/v1/beacon/execution_payload_bids`. +/// +/// 1. An identical bid already pooled is a success without republishing, so a +/// builder that retries does not flood the topic. +/// 2. The cheap rules run inline and the stateful ones (cached states, the +/// signature) on a blocking thread. +/// 3. An accepted bid is recorded in the market, which is where block +/// production reads it and where gossip's own seen rules look, then gossiped. +async fn post_bid( + State(store): State, + Extension(p2p): Extension, + Extension(market): Extension, + headers: HeaderMap, + body: Bytes, +) -> Response { + if let Err(err) = require_version(&headers, &[ForkName::Gloas]) { + return err.into_response(); + } + let encoding = match BodyEncoding::from_headers(&headers) { + Ok(encoding) => encoding, + Err(err) => return err.into_response(), + }; + if encoding == BodyEncoding::Ssz && body.len() > MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE { + return ApiError::BadRequest("the SignedExecutionPayloadBid exceeds its size bound") + .into_response(); + } + let Some(bid) = encoding.decode::(&body) else { + return ApiError::BadRequest("the body is not a gloas SignedExecutionPayloadBid") + .into_response(); + }; + + if market.contains_bid(&bid) { + debug!( + slot = bid.message.slot, + builder_index = bid.message.builder_index, + "Execution payload bid already pooled; not republishing" + ); + return StatusCode::OK.into_response(); + } + let verdict = match cheap_checks(&market, &store, &bid, unix_ms()) { + Ok(()) => { + let (store, market, bid) = (store.clone(), market.clone(), bid.clone()); + match tokio::task::spawn_blocking(move || stateful_checks(&store, &market, &bid)).await + { + Ok(verdict) => verdict, + Err(_) => { + return ApiError::Internal("validating the bid failed").into_response(); + } + } + } + Err(outcome) => outcome, + }; + if verdict != Outcome::Accept { + let (outcome, reason) = verdict.labels(); + warn!( + slot = bid.message.slot, + builder_index = bid.message.builder_index, + outcome, + reason, + "Refused a submitted execution payload bid" + ); + return bad_request(describe(&verdict)); + } + // The state a stateful check read can have moved on, and another bid can + // have taken the key meanwhile: recording re-runs the seen rules under the + // market's lock. + let slot = bid.message.slot; + let builder_index = bid.message.builder_index; + if !market.record_bid(bid.clone()) { + return bad_request(describe(&Outcome::Ignore(IgnoreReason::AlreadySeen))); + } + match p2p.publish_execution_payload_bid(bid) { + Ok(()) => { + debug!( + slot, + builder_index, "Accepted execution payload bid for gossip" + ); + StatusCode::OK.into_response() + } + Err(_) => ApiError::Internal("the network actor is not running").into_response(), + } } diff --git a/crates/net/rpc/src/beacon/builder_config.rs b/crates/net/rpc/src/beacon/builder_config.rs index 5408aa847..ebea79e12 100644 --- a/crates/net/rpc/src/beacon/builder_config.rs +++ b/crates/net/rpc/src/beacon/builder_config.rs @@ -1,53 +1,275 @@ //! The `BuilderConfig` a validator client sends with `produceBlockV4`. //! -//! Stubs until filled. Field types below are placeholders; the SSZ and JSON -//! forms are added with the real containers. +//! Its containers are the Beacon API's own (`types/gloas/builder_entry.yaml` +//! and `request_auth.yaml`), not consensus containers, so they live with the +//! API. Each has the SSZ form the specification gives and the JSON form with +//! quoted integers and hex byte strings. +//! +//! Only the top-level `min_bid` and `builder_boost_factor` are used today: they +//! govern the bids this node sees over p2p. The `builders` entries (bid +//! requests to a builder's URL) are decoded and left alone. -use axum::http::HeaderMap; +use axum::http::{HeaderMap, header}; +use ethlambda_types::beacon::primitives::{BlsPubkey, BlsSignature}; +use libssz_derive::{SszDecode, SszEncode}; +use libssz_types::SszList; +use serde::{Deserialize, Serialize}; use crate::beacon::ApiError; -#[allow(dead_code)] // filled by Agent C pub(crate) const MAX_BUILDER_ENTRIES: usize = 64; -#[allow(dead_code)] // filled by Agent C pub(crate) const MAX_BUILDER_URL_SIZE: usize = 2048; -#[allow(dead_code)] // filled by Agent C pub(crate) const MAX_BUILDER_PUBKEYS: usize = 64; -#[allow(dead_code)] // filled by Agent C pub(crate) const MAX_BUILDER_AUTH_DATA_SIZE: usize = 4096; -#[derive(Debug, Clone, Default)] -#[allow(dead_code)] // filled by Agent C +/// The builder-specs' `BuilderRequestAuth`: opaque authentication bytes and the +/// slot they authorize. +#[derive(Debug, Clone, Default, PartialEq, Eq, SszEncode, SszDecode, Serialize, Deserialize)] +pub(crate) struct BuilderRequestAuth { + #[serde(with = "ethlambda_types::beacon::serde_helpers::ssz_hex")] + pub(crate) data: SszList, + #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")] + pub(crate) slot: u64, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, SszEncode, SszDecode, Serialize, Deserialize)] +pub(crate) struct SignedBuilderRequestAuth { + pub(crate) message: BuilderRequestAuth, + pub(crate) signature: BlsSignature, +} + +/// The URL as the SSZ container holds it (UTF-8 bytes) and JSON writes it (a +/// string). +mod url_text { + use super::{MAX_BUILDER_URL_SIZE, SszList}; + + pub fn serialize( + value: &SszList, + serializer: S, + ) -> Result { + serializer.serialize_str(&String::from_utf8_lossy(value)) + } + + pub fn deserialize<'de, D: serde::Deserializer<'de>>( + deserializer: D, + ) -> Result, D::Error> { + let text = ::deserialize(deserializer)?; + SszList::try_from(text.into_bytes()) + .map_err(|_| serde::de::Error::custom("url exceeds MAX_BUILDER_URL_SIZE")) + } +} + +/// A per-builder bid request a validator client supplies. +#[derive(Debug, Clone, Default, PartialEq, Eq, SszEncode, SszDecode, Serialize, Deserialize)] pub(crate) struct BuilderEntry { - pub(crate) url: String, - pub(crate) auth_data: Vec, - pub(crate) auth_slot: u64, + #[serde(with = "url_text")] + pub(crate) url: SszList, + pub(crate) auth: SignedBuilderRequestAuth, + #[serde(with = "ethlambda_types::beacon::serde_helpers::seq")] + pub(crate) builder_pubkeys: SszList, + #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")] + pub(crate) max_execution_payment: u64, + #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")] pub(crate) min_bid: u64, + #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")] pub(crate) builder_boost_factor: u64, } -#[derive(Debug, Clone, Default)] -#[allow(dead_code)] // filled by Agent C +/// The resolved per-key builder config of one block-production request. +#[derive(Debug, Clone, Default, PartialEq, Eq, SszEncode, SszDecode, Serialize, Deserialize)] pub(crate) struct BuilderConfig { + /// Minimum total payment, in Gwei, accepted from a p2p bid. + #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")] pub(crate) min_bid: u64, + /// Percentage multiplier applied to a p2p bid against the local build. + #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")] pub(crate) builder_boost_factor: u64, - pub(crate) builders: Vec, + #[serde(with = "ethlambda_types::beacon::serde_helpers::seq")] + pub(crate) builders: SszList, } -#[allow(dead_code)] // filled by Agent C impl BuilderEntry { /// A non-empty url, non-empty `auth.data` and `auth.message.slot == slot`. - /// An unusable entry never fails the request. - pub(crate) fn is_usable_for(&self, _slot: u64) -> bool { - false + /// An unusable entry never fails the request: it yields no bid. + pub(crate) fn is_usable_for(&self, slot: u64) -> bool { + !self.url.is_empty() && !self.auth.message.data.is_empty() && self.auth.message.slot == slot + } +} + +impl BuilderConfig { + /// How many entries a builder request could be made for at `slot`. + pub(crate) fn usable_entries(&self, slot: u64) -> usize { + self.builders + .iter() + .filter(|entry| entry.is_usable_for(slot)) + .count() } } -/// A missing or undecodable body is a 400. -#[allow(dead_code)] // filled by Agent C +/// Decodes the request body as a `BuilderConfig`: SSZ for an +/// `application/octet-stream` body, JSON otherwise (what a client with no +/// `Content-Type` sends). A missing or undecodable body is a 400. pub(crate) fn decode_builder_config( - _headers: &HeaderMap, - _body: &[u8], + headers: &HeaderMap, + body: &[u8], ) -> Result { - Err(ApiError::BadRequest("the body is not a BuilderConfig")) + let invalid = || ApiError::BadRequest("the body is not a BuilderConfig"); + let ssz = headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.starts_with(crate::SSZ_CONTENT_TYPE)); + if ssz { + ::from_ssz_bytes(body).map_err(|_| invalid()) + } else { + serde_json::from_slice(body).map_err(|_| invalid()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use libssz::SszEncode as _; + + fn entry(url: &str, data: &[u8], slot: u64) -> BuilderEntry { + BuilderEntry { + url: SszList::try_from(url.as_bytes().to_vec()).unwrap(), + auth: SignedBuilderRequestAuth { + message: BuilderRequestAuth { + data: SszList::try_from(data.to_vec()).unwrap(), + slot, + }, + signature: BlsSignature::default(), + }, + builder_pubkeys: vec![BlsPubkey::default()].try_into().unwrap(), + max_execution_payment: 7, + min_bid: 5, + builder_boost_factor: 120, + } + } + + fn config(entries: Vec) -> BuilderConfig { + BuilderConfig { + min_bid: 10_000_000, + builder_boost_factor: 100, + builders: entries.try_into().unwrap(), + } + } + + fn ssz_headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert( + header::CONTENT_TYPE, + crate::SSZ_CONTENT_TYPE.parse().unwrap(), + ); + headers + } + + #[test] + fn json_and_ssz_round_trip_with_entries() { + let original = config(vec![ + entry("https://builder.example.com", b"auth", 9), + entry("https://other.example.com", b"x", 9), + ]); + let json = serde_json::to_vec(&original).unwrap(); + assert_eq!( + decode_builder_config(&HeaderMap::new(), &json).unwrap(), + original + ); + let ssz = original.to_ssz(); + assert_eq!( + decode_builder_config(&ssz_headers(), &ssz).unwrap(), + original + ); + } + + #[test] + fn the_json_form_quotes_integers_and_writes_bytes_as_hex() { + let json = + serde_json::to_value(config(vec![entry("https://b.example", b"\x12\x34", 9)])).unwrap(); + assert_eq!(json["min_bid"], "10000000"); + assert_eq!(json["builder_boost_factor"], "100"); + let builder = &json["builders"][0]; + assert_eq!(builder["url"], "https://b.example"); + assert_eq!(builder["auth"]["message"]["data"], "0x1234"); + assert_eq!(builder["auth"]["message"]["slot"], "9"); + assert_eq!(builder["max_execution_payment"], "7"); + assert_eq!(builder["min_bid"], "5"); + assert_eq!(builder["builder_boost_factor"], "120"); + } + + #[test] + fn an_unusable_entry_still_decodes() { + let slot = 9; + let unusable = [ + entry("", b"auth", slot), + entry("https://b.example", b"", slot), + entry("https://b.example", b"auth", slot + 1), + ]; + for bad in &unusable { + assert!(!bad.is_usable_for(slot)); + } + let usable = entry("https://b.example", b"auth", slot); + assert!(usable.is_usable_for(slot)); + let original = config(vec![ + unusable[0].clone(), + unusable[1].clone(), + unusable[2].clone(), + usable, + ]); + let json = serde_json::to_vec(&original).unwrap(); + let decoded = decode_builder_config(&HeaderMap::new(), &json).unwrap(); + assert_eq!(decoded, original); + assert_eq!(decoded.usable_entries(slot), 1); + let ssz = decode_builder_config(&ssz_headers(), &original.to_ssz()).unwrap(); + assert_eq!(ssz, original); + } + + #[test] + fn an_undecodable_or_oversized_body_is_a_400() { + for body in [ + &b""[..], + b"not json", + br#"{"min_bid": "1"}"#, + br#"{"min_bid": "x", "builder_boost_factor": "1", "builders": []}"#, + ] { + assert!(matches!( + decode_builder_config(&HeaderMap::new(), body), + Err(ApiError::BadRequest(_)) + )); + } + // SSZ: too short, and an offset that points nowhere. + for body in [&[][..], &[0u8; 19][..], &[1u8; 20][..]] { + assert!(decode_builder_config(&ssz_headers(), body).is_err()); + } + // More than MAX_BUILDER_ENTRIES entries. + let many = serde_json::json!({ + "min_bid": "0", + "builder_boost_factor": "0", + "builders": vec![ + serde_json::to_value(entry("https://b.example", b"a", 1)).unwrap(); + MAX_BUILDER_ENTRIES + 1 + ], + }); + assert!( + decode_builder_config(&HeaderMap::new(), &serde_json::to_vec(&many).unwrap()).is_err() + ); + // A url above the bound. + let long = "a".repeat(MAX_BUILDER_URL_SIZE + 1); + let mut json = serde_json::to_value(config(vec![entry("https://b", b"a", 1)])).unwrap(); + json["builders"][0]["url"] = long.into(); + assert!( + decode_builder_config(&HeaderMap::new(), &serde_json::to_vec(&json).unwrap()).is_err() + ); + } + + #[test] + fn the_empty_local_preferred_config_decodes() { + let body = br#"{"min_bid":"0","builder_boost_factor":"0","builders":[]}"#; + let decoded = decode_builder_config(&HeaderMap::new(), body).unwrap(); + assert_eq!(decoded, BuilderConfig::default()); + // Twenty bytes: the two integers and the offset of the empty list. + let ssz = + decode_builder_config(&ssz_headers(), &BuilderConfig::default().to_ssz()).unwrap(); + assert_eq!(ssz, decoded); + } } diff --git a/crates/net/rpc/src/beacon/builders.rs b/crates/net/rpc/src/beacon/builders.rs index 59995e4e0..36a5d8546 100644 --- a/crates/net/rpc/src/beacon/builders.rs +++ b/crates/net/rpc/src/beacon/builders.rs @@ -1,12 +1,336 @@ //! `POST /eth/v1/beacon/states/{state_id}/builders`: the builder registry of a //! gloas state, filtered by id and status. //! -//! Stub: no routes until filled. +//! The Beacon API defines only the POST form, so a builder list is a request +//! with a body rather than a query string. -use axum::Router; +use axum::{ + Router, + body::Bytes, + extract::{Path, State}, + response::{IntoResponse, Response}, + routing::post, +}; +use ethlambda_state_transition::beacon::helpers::gloas::is_active_builder; use ethlambda_storage::Store; +use ethlambda_types::beacon::{ + constants::FAR_FUTURE_EPOCH, + containers::{BeaconState, gloas::Builder}, + primitives::{BlsPubkey, ValidatorIndex}, +}; +use serde::{Deserialize, Serialize}; + +use crate::beacon::{ApiError, blocks::is_finalized, states::load}; -#[allow(dead_code)] // filled by Agent C pub(crate) fn routes() -> Router { - Router::new() + Router::new().route( + "/eth/v1/beacon/states/{state_id}/builders", + post(post_builders), + ) +} + +/// `api.yaml#BuilderStatus`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BuilderStatus { + Pending, + Active, + Exited, +} + +impl BuilderStatus { + fn name(self) -> &'static str { + match self { + Self::Pending => "pending", + Self::Active => "active", + Self::Exited => "exited", + } + } + + fn parse(text: &str) -> Option { + match text { + "pending" => Some(Self::Pending), + "active" => Some(Self::Active), + "exited" => Some(Self::Exited), + _ => None, + } + } +} + +/// A builder id: a registry index or a 48-byte public key. +enum BuilderId { + Index(u64), + Pubkey(BlsPubkey), +} + +impl BuilderId { + fn parse(text: &str) -> Result { + let invalid = || ApiError::BadRequest("invalid builder id"); + if let Some(digits) = text.strip_prefix("0x") { + let bytes: [u8; 48] = hex::decode(digits) + .map_err(|_| invalid())? + .try_into() + .map_err(|_| invalid())?; + return Ok(Self::Pubkey(BlsPubkey(bytes))); + } + text.parse().map(Self::Index).map_err(|_| invalid()) + } + + fn selects(&self, index: u64, builder: &Builder) -> bool { + match self { + Self::Index(wanted) => *wanted == index, + Self::Pubkey(wanted) => *wanted == builder.pubkey, + } + } +} + +/// The optional request body. Empty or absent filters select everything. +#[derive(Debug, Default, Deserialize)] +struct BuildersRequest { + #[serde(default)] + ids: Vec, + #[serde(default)] + statuses: Vec, +} + +#[derive(Debug, Serialize)] +struct BuilderEntry<'a> { + #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")] + index: ValidatorIndex, + status: &'static str, + builder: &'a Builder, +} + +/// `POST /eth/v1/beacon/states/{state_id}/builders`. +/// +/// An id naming no builder is omitted rather than failing the request. The +/// answer is in registry order, which the specification leaves unspecified. +async fn post_builders( + Path(state_id): Path, + State(store): State, + body: Bytes, +) -> Response { + let request = if body.is_empty() { + BuildersRequest::default() + } else { + match serde_json::from_slice(&body) { + Ok(request) => request, + Err(_) => return ApiError::BadRequest("invalid request body").into_response(), + } + }; + let ids = match request + .ids + .iter() + .map(|id| BuilderId::parse(id)) + .collect::, _>>() + { + Ok(ids) => ids, + Err(err) => return err.into_response(), + }; + let statuses = match request + .statuses + .iter() + .map(|status| BuilderStatus::parse(status).ok_or(ApiError::BadRequest("invalid status"))) + .collect::, _>>() + { + Ok(statuses) => statuses, + Err(err) => return err.into_response(), + }; + let (root, state) = match load(&store, &state_id) { + Ok(found) => found, + Err(err) => return err.into_response(), + }; + let BeaconState::Gloas(inner) = state.as_ref() else { + return ApiError::BadRequest("the requested state is prior to Gloas").into_response(); + }; + + let mut entries = Vec::new(); + for (index, builder) in inner.builders.iter().enumerate() { + let index = index as u64; + if !(ids.is_empty() || ids.iter().any(|id| id.selects(index, builder))) { + continue; + } + let status = if builder.withdrawable_epoch != FAR_FUTURE_EPOCH { + BuilderStatus::Exited + } else if is_active_builder(inner, index).unwrap_or(false) { + BuilderStatus::Active + } else { + BuilderStatus::Pending + }; + if !(statuses.is_empty() || statuses.contains(&status)) { + continue; + } + entries.push(BuilderEntry { + index, + status: status.name(), + builder, + }); + } + + crate::json_response(serde_json::json!({ + "execution_optimistic": crate::shared::optimistic::block_is_optimistic(&store, root), + "finalized": is_finalized(&store, state.slot()), + "data": entries, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::beacon_store_with_config; + use axum::{ + body::Body, + http::{Request, StatusCode}, + }; + use ethlambda_state_transition::beacon::helpers::test_state::with_signing_validators_at; + use ethlambda_types::beacon::{config::Config, fork::ForkName}; + use http_body_util::BodyExt as _; + use tower::ServiceExt as _; + + fn builder(seed: u8, deposit_epoch: u64, withdrawable_epoch: u64) -> Builder { + Builder { + pubkey: BlsPubkey([seed; 48]), + version: 3, + execution_address: Default::default(), + balance: 32_000_000_000 + u64::from(seed), + deposit_epoch, + withdrawable_epoch, + } + } + + /// A gloas head state whose registry holds an active builder (0), a + /// pending one (1, deposited at the finalized epoch) and an exited one (2). + fn gloas_state_with_builders() -> BeaconState { + let mut state = with_signing_validators_at(ForkName::Gloas, 64); + let BeaconState::Gloas(inner) = &mut state else { + unreachable!("built as gloas") + }; + inner.finalized_checkpoint.epoch = 10; + for builder in [ + builder(1, 3, FAR_FUTURE_EPOCH), + builder(2, 10, FAR_FUTURE_EPOCH), + builder(3, 3, 20), + ] { + inner.builders.push(builder); + } + state + } + + async fn post_to(store: Store, state_id: &str, body: &str) -> (StatusCode, serde_json::Value) { + let request = Request::post(format!("/eth/v1/beacon/states/{state_id}/builders")) + .body(Body::from(body.to_string())) + .unwrap(); + let response = routes().with_state(store).oneshot(request).await.unwrap(); + let status = response.status(); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + (status, serde_json::from_slice(&bytes).unwrap_or_default()) + } + + fn gloas_store() -> Store { + let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 0); + beacon_store_with_config(gloas_state_with_builders(), config).0 + } + + fn indices(json: &serde_json::Value) -> Vec { + json["data"] + .as_array() + .unwrap() + .iter() + .map(|entry| entry["index"].as_str().unwrap().to_string()) + .collect() + } + + #[tokio::test] + async fn every_builder_is_listed_in_registry_order_with_its_status() { + for body in ["", "{}", r#"{"ids":[],"statuses":[]}"#] { + let (status, json) = post_to(gloas_store(), "head", body).await; + assert_eq!(status, StatusCode::OK, "{body:?}"); + assert_eq!(indices(&json), ["0", "1", "2"]); + let statuses: Vec<_> = json["data"] + .as_array() + .unwrap() + .iter() + .map(|entry| entry["status"].as_str().unwrap().to_string()) + .collect(); + assert_eq!(statuses, ["active", "pending", "exited"]); + assert_eq!(json["execution_optimistic"], false); + assert!(json["finalized"].is_boolean()); + } + } + + #[tokio::test] + async fn numbers_are_quoted() { + let (_, json) = post_to(gloas_store(), "head", "").await; + let first = &json["data"][0]; + assert_eq!(first["index"], "0"); + assert_eq!(first["builder"]["version"], "3"); + assert_eq!(first["builder"]["balance"], "32000000001"); + assert_eq!(first["builder"]["deposit_epoch"], "3"); + assert_eq!( + first["builder"]["withdrawable_epoch"], + FAR_FUTURE_EPOCH.to_string() + ); + assert!( + first["builder"]["pubkey"] + .as_str() + .unwrap() + .starts_with("0x") + ); + } + + #[tokio::test] + async fn builders_are_selected_by_index_or_public_key_and_unknown_ids_are_omitted() { + let pubkey = format!("0x{}", "03".repeat(48)); + let body = format!(r#"{{"ids":["1","{pubkey}","99","0x{}"]}}"#, "ff".repeat(48)); + let (status, json) = post_to(gloas_store(), "head", &body).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(indices(&json), ["1", "2"]); + } + + #[tokio::test] + async fn the_status_filter_narrows_the_list() { + let (_, json) = post_to(gloas_store(), "head", r#"{"statuses":["active"]}"#).await; + assert_eq!(indices(&json), ["0"]); + let (_, json) = post_to( + gloas_store(), + "head", + r#"{"statuses":["pending","exited"]}"#, + ) + .await; + assert_eq!(indices(&json), ["1", "2"]); + let (_, json) = post_to( + gloas_store(), + "head", + r#"{"ids":["0","1"],"statuses":["exited"]}"#, + ) + .await; + assert!(indices(&json).is_empty()); + } + + #[tokio::test] + async fn a_bad_body_id_or_status_is_a_400() { + for body in [ + "not json", + r#"{"ids":["zero"]}"#, + r#"{"ids":["0x1234"]}"#, + r#"{"statuses":["retired"]}"#, + r#"{"ids":"0"}"#, + ] { + let (status, _) = post_to(gloas_store(), "head", body).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{body}"); + } + } + + #[tokio::test] + async fn a_pre_gloas_state_is_a_400_and_an_unknown_one_a_404() { + let state = with_signing_validators_at(ForkName::Fulu, 64); + let (store, _) = beacon_store_with_config(state, Config::mainnet()); + let (status, json) = post_to(store, "head", "").await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(json["message"], "the requested state is prior to Gloas"); + + let (status, _) = post_to(gloas_store(), "999999", "").await; + assert_eq!(status, StatusCode::NOT_FOUND); + let (status, _) = post_to(gloas_store(), "nonsense", "").await; + assert_eq!(status, StatusCode::BAD_REQUEST); + } } diff --git a/crates/net/rpc/src/beacon/gloas_proposal.rs b/crates/net/rpc/src/beacon/gloas_proposal.rs index 12a3a507d..d12245d05 100644 --- a/crates/net/rpc/src/beacon/gloas_proposal.rs +++ b/crates/net/rpc/src/beacon/gloas_proposal.rs @@ -15,9 +15,15 @@ //! serves the envelope back for a validator client that asked for the block //! with `include_payload=false`. //! -//! This node never takes a builder's bid: the `BuilderConfig` request body is -//! decoded, as the specification requires of a body that cannot be, and -//! otherwise ignored. +//! The node always builds a local payload when it has an execution client, and +//! may instead build on a bid it saw over p2p (`bid_selection` decides, from the +//! request's `BuilderConfig`). A bid-won block comes back bare: the builder +//! reveals the payload, so nothing is cached here and this node never signs or +//! publishes an envelope for it. The `builders` entries of the config (bid +//! requests to a builder's URL) are decoded and not consulted. +//! +//! A self-built payload's fee recipient and gas target come from the signed +//! proposer preferences for the slot when this node holds them. //! //! What `produceBlockV4` builds is kept in a small cache, keyed by slot and //! block root and holding the current and the previous slot only, so the @@ -48,15 +54,18 @@ use ethlambda_state_transition::beacon::{ attestation_pool::SharedAttestationPool, block_production::advance_to_slot, bls, + builder_market::SharedBuilderMarket, fork_choice::{ get_head_node, gloas_verify_data_column_sidecar, gloas_verify_data_column_sidecar_kzg_proofs, should_build_on_full, }, gloas_block_production::{ - GloasBlockInputs, GloasPayloadInputs, GloasProduced, assemble_gloas_block, + GloasBidBlockInputs, GloasBlockInputs, GloasPayloadInputs, GloasProduced, + assemble_gloas_block, assemble_gloas_block_on_bid, bid_is_includable, gloas_data_column_sidecars, gloas_payload_inputs, pack_gloas_attestations, pack_payload_attestations, parse_gloas_execution_requests, }, + gossip::proposer_preferences::dependent_root_at, helpers::{ accessors::{get_beacon_proposer_index, get_domain}, misc::compute_signing_root, @@ -74,7 +83,8 @@ use ethlambda_types::{ deneb::Blob, gloas::{ BeaconBlock, ExecutionPayloadEnvelope, ExecutionRequests, - SignedExecutionPayloadEnvelope, + SignedExecutionPayloadBid, SignedExecutionPayloadEnvelope, + SignedProposerPreferences, }, }, fork::ForkName, @@ -91,6 +101,8 @@ use tracing::{debug, info, warn}; use crate::beacon::{ ApiError, BodyEncoding, + bid_selection::{LocalCandidate, PayloadChoice, bid_total_gwei, choose_payload, wei_u128}, + builder_config::{BuilderConfig, decode_builder_config}, proposal::{Blobs, CellKzgProofs, decimal, require_gloas_slot}, validator::{FeeRecipients, head}, }; @@ -175,13 +187,6 @@ pub(crate) fn routes() -> Router { .layer(Extension(PayloadCache::default())) } -fn is_ssz(headers: &HeaderMap) -> bool { - headers - .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.starts_with(crate::SSZ_CONTENT_TYPE)) -} - fn consensus_version(headers: &HeaderMap) -> Option { headers .get("eth-consensus-version") @@ -189,41 +194,6 @@ fn consensus_version(headers: &HeaderMap) -> Option { .and_then(ForkName::parse) } -/// `BuilderConfig` as it arrives as JSON. Only decoded: this node takes no -/// builder bids, so nothing past the shape is read. -#[derive(Debug, Deserialize)] -struct BuilderConfigJson { - #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")] - #[allow(dead_code)] - min_bid: u64, - #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")] - #[allow(dead_code)] - builder_boost_factor: u64, - builders: Vec, -} - -/// Decodes the request body as a `BuilderConfig`, returning how many builder -/// entries it names. A body that does not decode is invalid per the -/// specification. -/// -/// The SSZ form is `min_bid`, `builder_boost_factor` and the offset of the -/// `builders` list, so a decodable body is at least that long and the offset -/// points just past those three fields; the entries themselves are not parsed. -fn decode_builder_config(headers: &HeaderMap, body: &[u8]) -> Result { - if is_ssz(headers) { - let offset = body - .get(16..20) - .map(|bytes| u32::from_le_bytes(bytes.try_into().expect("four bytes"))); - return match offset { - Some(20) => Ok(usize::from(body.len() > 20)), - _ => Err(ApiError::BadRequest("the body is not a BuilderConfig")), - }; - } - serde_json::from_slice::(body) - .map(|config| config.builders.len()) - .map_err(|_| ApiError::BadRequest("the body is not a BuilderConfig")) -} - #[derive(Debug, Deserialize)] struct ProduceQuery { randao_reveal: BlsSignature, @@ -246,6 +216,7 @@ async fn post_produce_block( Extension(engine): Extension>, Extension(pool): Extension, Extension(ptc_pool): Extension, + Extension(market): Extension, Extension(fee_recipients): Extension, Extension(custody): Extension, Extension(cache): Extension, @@ -273,24 +244,29 @@ async fn post_produce_block( ) { return err.into_response(); } - let builders = match decode_builder_config(&headers, &body) { - Ok(builders) => builders, + let builder_config = match decode_builder_config(&headers, &body) { + Ok(config) => config, Err(err) => return err.into_response(), }; + let builders = builder_config.usable_entries(slot); if builders > 0 { - debug!(%slot, builders, "Ignoring the builders of the block production request"); + debug!(%slot, builders, "Not consulting the builder entries of the block production request"); } - let Some(engine) = engine else { + // Without an execution client a block can only be built on a bid, so a node + // holding none for the slot has nothing to offer. + if engine.is_none() && !market.has_bids_for_slot(slot) { return ApiError::ServiceUnavailable( "no execution client configured to build a payload with", ) .into_response(); - }; + } let graffiti = query.graffiti.unwrap_or(Bytes32::ZERO); let produced = produce( &store, - &engine, + engine.as_ref(), + &market, + &builder_config, &pool, &ptc_pool, &fee_recipients, @@ -304,49 +280,59 @@ async fn post_produce_block( Ok(produced) => produced, Err(err) => return err.into_response(), }; - let Produced { - built, - block, - envelope, - payload_value, - } = produced; - - let block_root = block.hash_tree_root(); - cache.insert( - slot, - block_root, - CachedPayload { - envelope: envelope.clone(), - blobs: built.blobs_bundle.blobs.clone(), - cell_proofs: built.blobs_bundle.proofs.clone(), - }, - ); - let accept = headers.get(header::ACCEPT).and_then(|v| v.to_str().ok()); let encoding = Encoding::from_accept(accept); - let included = query.include_payload; - let mut response = if included { - let (Ok(kzg_proofs), Ok(blobs)) = ( - CellKzgProofs::try_from(built.blobs_bundle.proofs), - blobs_list(built.blobs_bundle.blobs), - ) else { - return ApiError::Internal("the blobs bundle exceeds the block's bounds") - .into_response(); - }; - let contents = GloasBlockContents { - block, - execution_payload_envelope: envelope, - kzg_proofs, - blobs, - }; - match encoding { - Encoding::Ssz => ssz_response(contents.to_ssz()), - Encoding::Json => block_json(&payload_value, true, &contents), + let (mut response, included, payload_value) = match produced { + Production::Bid { block, bid } => { + let payload_value = bid_payload_value(&bid); + let response = match encoding { + Encoding::Ssz => ssz_response(block.to_ssz()), + Encoding::Json => block_json(&payload_value, false, &block), + }; + (response, false, payload_value) } - } else { - match encoding { - Encoding::Ssz => ssz_response(block.to_ssz()), - Encoding::Json => block_json(&payload_value, false, &block), + Production::Local(Produced { + built, + block, + envelope, + payload_value, + }) => { + let block_root = block.hash_tree_root(); + cache.insert( + slot, + block_root, + CachedPayload { + envelope: envelope.clone(), + blobs: built.blobs_bundle.blobs.clone(), + cell_proofs: built.blobs_bundle.proofs.clone(), + }, + ); + let included = query.include_payload; + let response = if included { + let (Ok(kzg_proofs), Ok(blobs)) = ( + CellKzgProofs::try_from(built.blobs_bundle.proofs), + blobs_list(built.blobs_bundle.blobs), + ) else { + return ApiError::Internal("the blobs bundle exceeds the block's bounds") + .into_response(); + }; + let contents = GloasBlockContents { + block, + execution_payload_envelope: envelope, + kzg_proofs, + blobs, + }; + match encoding { + Encoding::Ssz => ssz_response(contents.to_ssz()), + Encoding::Json => block_json(&payload_value, true, &contents), + } + } else { + match encoding { + Encoding::Ssz => ssz_response(block.to_ssz()), + Encoding::Json => block_json(&payload_value, false, &block), + } + }; + (response, included, payload_value) } }; let response_headers = response.headers_mut(); @@ -357,12 +343,17 @@ async fn post_produce_block( if let Ok(value) = HeaderValue::from_str(&payload_value) { response_headers.insert("eth-execution-payload-value", value); } - // Not computed: nothing here reads it, and the builder comparison it - // exists for does not happen on this node. + // Not computed: nothing here reads it, and the bid comparison compares + // execution payload values only. response_headers.insert("eth-consensus-block-value", HeaderValue::from_static("0")); with_consensus_version(response, ForkName::Gloas) } +/// A bid's total payment in Wei, the unit `Eth-Execution-Payload-Value` is in. +fn bid_payload_value(bid: &SignedExecutionPayloadBid) -> String { + (u128::from(bid_total_gwei(&bid.message)) * 1_000_000_000).to_string() +} + fn block_json(payload_value: &str, included: bool, data: &T) -> Response { crate::json_response(serde_json::json!({ "version": ForkName::Gloas.as_str(), @@ -381,7 +372,19 @@ fn blobs_list(blobs: Vec>) -> Result { Blobs::try_from(blobs).map_err(|_| ()) } -/// Everything `produceBlockV4` built. +/// What `produceBlockV4` built for one block. +enum Production { + /// A block on this node's own payload, with the envelope that reveals it. + Local(Produced), + /// A block committing to a builder's bid. The builder reveals the payload, + /// so there is no envelope and nothing to cache. + Bid { + block: BeaconBlock, + bid: SignedExecutionPayloadBid, + }, +} + +/// A self-built block. struct Produced { built: BuiltGloasPayload, block: BeaconBlock, @@ -389,6 +392,12 @@ struct Produced { payload_value: String, } +/// This node's own payload for the slot, checked and ready to assemble. +struct LocalBuild { + built: BuiltGloasPayload, + execution_requests: ExecutionRequests, +} + /// What the build needs from the chain, read and advanced off the runtime. struct Prepared { state: BeaconState, @@ -399,12 +408,17 @@ struct Prepared { /// The parent envelope's requests when building on its full payload and /// the parent is gloas; empty otherwise. parent_requests: ExecutionRequests, + /// The proposer's signed preferences for the slot, when this node holds + /// them under the slot's dependent root. + preferences: Option, } #[allow(clippy::too_many_arguments)] async fn produce( store: &Store, - engine: &EngineClient, + engine: Option<&EngineClient>, + market: &SharedBuilderMarket, + builder_config: &BuilderConfig, pool: &SharedAttestationPool, ptc_pool: &SharedPayloadAttestationPool, fee_recipients: &FeeRecipients, @@ -412,27 +426,39 @@ async fn produce( slot: Slot, randao_reveal: BlsSignature, graffiti: Bytes32, -) -> Result { +) -> Result { let prepare_store = store.clone(); - let prepared = - tokio::task::spawn_blocking(move || prepare(&prepare_store, slot, randao_reveal)) - .await - .map_err(|_| ApiError::Internal("preparing the block failed"))??; - - let built = build_payload(store, engine, fee_recipients, custody, &prepared).await?; - let bundle = &built.blobs_bundle; - if bundle.commitments.len() != bundle.blobs.len() - || bundle.proofs.len() - != bundle.blobs.len() * ethlambda_types::beacon::preset::CELLS_PER_EXT_BLOB - { - warn!(%slot, "The execution client's blobs bundle is inconsistent"); - return Err(ApiError::ServiceUnavailable( - "the execution client returned an inconsistent blobs bundle", - )); - } - let execution_requests = parse_gloas_execution_requests(&built.execution_requests) - .map_err(|_| ApiError::Internal("the execution client's request list is malformed"))?; - let payload_value = decimal(&built.block_value); + let prepare_market = market.clone(); + let prepared = tokio::task::spawn_blocking(move || { + prepare(&prepare_store, &prepare_market, slot, randao_reveal) + }) + .await + .map_err(|_| ApiError::Internal("preparing the block failed"))??; + + // The local build first: bids keep arriving while the engine works, and + // are read once it is done. + let local = match engine { + Some(engine) => { + Some(build_local(store, engine, fee_recipients, custody, &prepared, slot).await) + } + None => None, + }; + let (local, local_error) = match local { + Some(Ok(local)) => (Some(local), None), + Some(Err(err)) => (None, Some(err)), + None => (None, None), + }; + let bids = candidate_bids(market, &prepared); + let candidate = local.as_ref().map(|local| LocalCandidate { + value_wei: wei_u128(&local.built.block_value), + should_override_builder: local.built.should_override_builder, + }); + let choice = choose_payload( + candidate.as_ref(), + &bids, + builder_config.min_bid, + builder_config.builder_boost_factor, + ); let candidates = pool .lock() @@ -443,12 +469,64 @@ async fn produce( .expect("payload attestation pool lock poisoned") .messages_for(slot.saturating_sub(1), prepared.head_root); let config = store.config(); + let prepared = Arc::new(prepared); + + if let Some(PayloadChoice::Bid(bid)) = choice { + let builder_index = bid.message.builder_index; + let value_gwei = bid_total_gwei(&bid.message); + let assembled = { + let (config, prepared, bid) = (config.clone(), prepared.clone(), bid.clone()); + let (candidates, messages) = (candidates.clone(), messages.clone()); + tokio::task::spawn_blocking(move || { + assemble_on_bid( + &config, + &prepared, + bid, + candidates, + messages, + randao_reveal, + graffiti, + ) + }) + .await + }; + match assembled { + Ok(Ok(block)) => { + info!( + %slot, + builder_index, + value_gwei, + "Produced gloas block on a builder bid" + ); + return Ok(Production::Bid { block, bid }); + } + Ok(Err(_)) | Err(_) if local.is_some() => { + warn!(%slot, builder_index, "The winning bid failed to build; using the local payload"); + } + Ok(Err(_)) | Err(_) => { + return Err(ApiError::ServiceUnavailable( + "the winning bid failed to build", + )); + } + } + } + + let Some(local) = local else { + return Err(local_error.unwrap_or(ApiError::ServiceUnavailable( + "no execution client and no viable builder bid for the slot", + ))); + }; + let LocalBuild { + built, + execution_requests, + } = local; + let payload_value = decimal(&built.block_value); let assembled = { let built = built.clone(); tokio::task::spawn_blocking(move || { assemble( &config, - prepared, + &prepared, built, execution_requests, candidates, @@ -469,17 +547,51 @@ async fn produce( blobs = built.blobs_bundle.blobs.len(), "Produced gloas block" ); - Ok(Produced { + Ok(Production::Local(Produced { built, block, envelope, payload_value, - }) + })) +} + +/// The pooled bids this node could build on at the prepared slot: on the same +/// parent payload the local build extends, with its `prev_randao`, packable +/// into a block on the advanced state, and, when the proposer's preferences are +/// held, paying the fee recipient they name. Value descending. +/// +/// Bids on the head's parent (a proposer reorg) are not considered. +fn candidate_bids( + market: &SharedBuilderMarket, + prepared: &Prepared, +) -> Vec { + market + .bids_for( + prepared.state.slot(), + prepared.head_root, + prepared.inputs.head_block_hash, + ) + .into_iter() + .filter(|signed| { + let bid = &signed.message; + bid.prev_randao == prepared.inputs.prev_randao + && prepared + .preferences + .as_ref() + .is_none_or(|prefs| prefs.message.fee_recipient == bid.fee_recipient) + && bid_is_includable(&prepared.state, signed, &prepared.parent_requests) + }) + .collect() } /// The chain-side half of production: pick the parent payload branch, advance /// the head state to `slot` and derive the payload inputs from it. -fn prepare(store: &Store, slot: Slot, randao_reveal: BlsSignature) -> Result { +fn prepare( + store: &Store, + market: &SharedBuilderMarket, + slot: Slot, + randao_reveal: BlsSignature, +) -> Result { let config = store.config(); let (head_root, head_state) = head(store)?; let head_slot = head_state.slot(); @@ -527,8 +639,18 @@ fn prepare(store: &Store, slot: Slot, randao_reveal: BlsSignature) -> Result
 Result
 Result {
+    let built = build_payload(store, engine, fee_recipients, custody, prepared).await?;
+    let bundle = &built.blobs_bundle;
+    if bundle.commitments.len() != bundle.blobs.len()
+        || bundle.proofs.len()
+            != bundle.blobs.len() * ethlambda_types::beacon::preset::CELLS_PER_EXT_BLOB
+    {
+        warn!(%slot, "The execution client's blobs bundle is inconsistent");
+        return Err(ApiError::ServiceUnavailable(
+            "the execution client returned an inconsistent blobs bundle",
+        ));
+    }
+    let execution_requests = parse_gloas_execution_requests(&built.execution_requests)
+        .map_err(|_| ApiError::Internal("the execution client's request list is malformed"))?;
+    Ok(LocalBuild {
+        built,
+        execution_requests,
+    })
+}
+
+/// Where a self-built payload's fee recipient comes from: the proposer's signed
+/// preferences, else `prepare_beacon_proposer`'s, else the zero address.
+fn fee_recipient_for(fee_recipients: &FeeRecipients, prepared: &Prepared) -> ExecutionAddress {
+    if let Some(preferences) = &prepared.preferences {
+        debug!(
+            proposer = prepared.proposer,
+            "Using the signed proposer preferences' fee recipient"
+        );
+        return preferences.message.fee_recipient;
+    }
+    let prepared_recipient = fee_recipients
+        .lock()
+        .expect("fee recipient lock poisoned")
+        .get(&prepared.proposer)
+        .copied();
+    match prepared_recipient {
+        Some(recipient) => {
+            debug!(
+                proposer = prepared.proposer,
+                "Using the fee recipient from prepare_beacon_proposer"
+            );
+            recipient
+        }
+        None => {
+            warn!(
+                proposer = prepared.proposer,
+                "No fee recipient for the proposer; using the zero address"
+            );
+            ExecutionAddress::ZERO
+        }
+    }
+}
+
 /// Ask the execution client to build on the chosen parent payload for the
 /// prepared slot, then collect what it built.
 ///
@@ -575,18 +760,7 @@ async fn build_payload(
     prepared: &Prepared,
 ) -> Result {
     let inputs = &prepared.inputs;
-    let fee_recipient = fee_recipients
-        .lock()
-        .expect("fee recipient lock poisoned")
-        .get(&prepared.proposer)
-        .copied()
-        .unwrap_or_else(|| {
-            warn!(
-                proposer = prepared.proposer,
-                "No fee recipient prepared for the proposer; using the zero address"
-            );
-            ExecutionAddress::ZERO
-        });
+    let fee_recipient = fee_recipient_for(fee_recipients, prepared);
     let forkchoice = ForkchoiceStateV1 {
         head_block_hash: inputs.head_block_hash,
         safe_block_hash: checkpoint_hash(store, store.beacon_justified_checkpoint().root),
@@ -628,7 +802,7 @@ async fn build_payload(
 #[allow(clippy::too_many_arguments)]
 fn assemble(
     config: &Config,
-    prepared: Prepared,
+    prepared: &Prepared,
     built: BuiltGloasPayload,
     execution_requests: ExecutionRequests,
     candidates: Vec,
@@ -643,9 +817,9 @@ fn assemble(
         parent_requests,
         ..
     } = prepared;
-    let attestations = pack_gloas_attestations(&state, candidates);
+    let attestations = pack_gloas_attestations(state, candidates);
     let payload_attestations =
-        pack_payload_attestations(&state, head_root, head_slot, messages, config);
+        pack_payload_attestations(state, *head_root, *head_slot, messages, config);
     let commitments = built.blobs_bundle.commitments;
     let inputs = |attestations, payload_attestations| GloasBlockInputs {
         randao_reveal,
@@ -658,14 +832,55 @@ fn assemble(
         execution_requests: execution_requests.clone(),
     };
     let operations = attestations.len() + payload_attestations.len();
-    match assemble_gloas_block(&state, inputs(attestations, payload_attestations), config) {
+    match assemble_gloas_block(state, inputs(attestations, payload_attestations), config) {
         Ok(produced) => Ok(produced),
         // The packers check every operation's signature against this state, so
         // this should not happen; but a block without them still earns the
         // proposal, and one that fails to build earns nothing.
         Err(err) if operations > 0 => {
             warn!(slot = state.slot(), %err, "Block with operations failed to build; retrying without");
-            assemble_gloas_block(&state, inputs(Vec::new(), Vec::new()), config)
+            assemble_gloas_block(state, inputs(Vec::new(), Vec::new()), config)
+                .map_err(|_| ApiError::Internal("the block failed to build"))
+        }
+        Err(_) => Err(ApiError::Internal("the block failed to build")),
+    }
+}
+
+/// [`assemble`] for a block that commits to a builder's bid: no payload, no
+/// envelope. `process_block` on the block enforces the bid's own rules.
+fn assemble_on_bid(
+    config: &Config,
+    prepared: &Prepared,
+    signed_bid: SignedExecutionPayloadBid,
+    candidates: Vec,
+    messages: Vec,
+    randao_reveal: BlsSignature,
+    graffiti: Bytes32,
+) -> Result {
+    let Prepared {
+        state,
+        head_root,
+        head_slot,
+        parent_requests,
+        ..
+    } = prepared;
+    let attestations = pack_gloas_attestations(state, candidates);
+    let payload_attestations =
+        pack_payload_attestations(state, *head_root, *head_slot, messages, config);
+    let inputs = |attestations, payload_attestations| GloasBidBlockInputs {
+        randao_reveal,
+        graffiti,
+        attestations,
+        payload_attestations,
+        parent_execution_requests: parent_requests.clone(),
+        signed_bid: signed_bid.clone(),
+    };
+    let operations = attestations.len() + payload_attestations.len();
+    match assemble_gloas_block_on_bid(state, inputs(attestations, payload_attestations), config) {
+        Ok(block) => Ok(block),
+        Err(err) if operations > 0 => {
+            warn!(slot = state.slot(), %err, "Bid block with operations failed to build; retrying without");
+            assemble_gloas_block_on_bid(state, inputs(Vec::new(), Vec::new()), config)
                 .map_err(|_| ApiError::Internal("the block failed to build"))
         }
         Err(_) => Err(ApiError::Internal("the block failed to build")),
@@ -922,6 +1137,7 @@ mod tests {
             .layer(Extension(engine))
             .layer(Extension(SharedAttestationPool::default()))
             .layer(Extension(SharedPayloadAttestationPool::default()))
+            .layer(Extension(SharedBuilderMarket::default()))
             .layer(Extension(FeeRecipients::default()))
             .layer(Extension(NodeCustodyColumns::default()))
     }
@@ -1017,7 +1233,10 @@ mod tests {
     fn a_builder_config_decodes_as_json_or_ssz() {
         let mut headers = HeaderMap::new();
         assert_eq!(
-            decode_builder_config(&headers, EMPTY_CONFIG.as_bytes()).unwrap(),
+            decode_builder_config(&headers, EMPTY_CONFIG.as_bytes())
+                .unwrap()
+                .builders
+                .len(),
             0
         );
         assert!(decode_builder_config(&headers, b"{}").is_err());
@@ -1029,7 +1248,13 @@ mod tests {
         // min_bid, boost factor, then the offset of an empty builders list.
         let mut ssz = vec![0u8; 16];
         ssz.extend_from_slice(&20u32.to_le_bytes());
-        assert_eq!(decode_builder_config(&headers, &ssz).unwrap(), 0);
+        assert_eq!(
+            decode_builder_config(&headers, &ssz)
+                .unwrap()
+                .builders
+                .len(),
+            0
+        );
         assert!(decode_builder_config(&headers, &ssz[..19]).is_err());
     }
 
diff --git a/crates/net/rpc/src/beacon/proposer_preferences.rs b/crates/net/rpc/src/beacon/proposer_preferences.rs
index 6dd900087..bd217f0b5 100644
--- a/crates/net/rpc/src/beacon/proposer_preferences.rs
+++ b/crates/net/rpc/src/beacon/proposer_preferences.rs
@@ -2,12 +2,143 @@
 //! from a validator client, validated with the gossip rules, cached in the
 //! shared `BuilderMarket` and gossiped on `proposer_preferences`.
 //!
-//! Stub: no routes until filled.
+//! Not gated on the sync status: the specification lists no 503, and a node
+//! that lacks the dependent block simply answers `ignore: unknown_block` per
+//! entry.
 
-use axum::Router;
+use axum::{
+    Extension, Router,
+    body::Bytes,
+    extract::State,
+    http::{HeaderMap, StatusCode},
+    response::{IntoResponse, Response},
+    routing::post,
+};
+use ethlambda_network_api::RpcToP2PRef;
+use ethlambda_state_transition::beacon::{
+    builder_market::SharedBuilderMarket,
+    gossip::{IgnoreReason, Outcome, proposer_preferences::validate},
+    preset,
+};
 use ethlambda_storage::Store;
+use ethlambda_types::beacon::{containers::gloas::SignedProposerPreferences, fork::ForkName};
+use serde::Serialize;
+use tracing::{debug, warn};
+
+use crate::beacon::{
+    ApiError,
+    bids::{describe, require_version, unix_ms},
+    decode_list,
+};
+
+/// The most entries one request may carry: the SSZ list's bound,
+/// `(MIN_SEED_LOOKAHEAD + 1) * SLOTS_PER_EPOCH`.
+const MAX_ENTRIES: usize = preset::PROPOSER_LOOKAHEAD_LENGTH;
 
-#[allow(dead_code)] // filled by Agent C
 pub(crate) fn routes() -> Router {
-    Router::new()
+    Router::new().route(
+        "/eth/v1/validator/proposer_preferences",
+        post(post_proposer_preferences),
+    )
+}
+
+/// One refused entry, in the Beacon API's `IndexedErrorMessage` shape.
+#[derive(Debug, Serialize)]
+struct Failure {
+    index: usize,
+    message: String,
+}
+
+/// `POST /eth/v1/validator/proposer_preferences`.
+///
+/// Each entry goes through the topic's rules (current slot window, a known
+/// dependent block, the proposer the lookahead names, the signature), since a
+/// message that fails them is one every peer would score this node down for
+/// relaying. Valid ones are cached for block production and for the bids judged
+/// against them, then gossiped; the others are reported by position and the
+/// rest still go out. An entry already cached, identical, is a success that is
+/// not republished.
+async fn post_proposer_preferences(
+    State(store): State,
+    Extension(p2p): Extension,
+    Extension(market): Extension,
+    headers: HeaderMap,
+    body: Bytes,
+) -> Response {
+    // A validator client submits during the epoch before gloas, when the
+    // consensus version it names is still fulu.
+    if let Err(err) = require_version(&headers, &[ForkName::Fulu, ForkName::Gloas]) {
+        return err.into_response();
+    }
+    let preferences = match decode_list::(&headers, &body) {
+        Ok(preferences) => preferences,
+        Err(err) => return err.into_response(),
+    };
+    if preferences.len() > MAX_ENTRIES {
+        return ApiError::BadRequest("too many signed proposer preferences").into_response();
+    }
+
+    let validated =
+        tokio::task::spawn_blocking(move || submit(&store, &p2p, &market, preferences)).await;
+    let failures = match validated {
+        Ok(failures) => failures,
+        Err(_) => return ApiError::Internal("validating the preferences failed").into_response(),
+    };
+    if failures.is_empty() {
+        return StatusCode::OK.into_response();
+    }
+    let body = serde_json::json!({
+        "code": 400,
+        "message": "some signed proposer preferences failed validation and were not published",
+        "failures": failures,
+    });
+    let mut response = crate::json_response(body);
+    *response.status_mut() = StatusCode::BAD_REQUEST;
+    response
+}
+
+/// Validate, record and publish each entry, in order.
+fn submit(
+    store: &Store,
+    p2p: &RpcToP2PRef,
+    market: &SharedBuilderMarket,
+    preferences: Vec,
+) -> Vec {
+    let now_ms = unix_ms();
+    let wall_slot = crate::beacon::node::wall_slot(store);
+    let mut failures = Vec::new();
+    for (index, signed) in preferences.into_iter().enumerate() {
+        let slot = signed.message.proposal_slot;
+        let validator = signed.message.validator_index;
+        let held = market.preferences(slot, signed.message.dependent_root);
+        if held.as_ref() == Some(&signed) {
+            debug!(%slot, validator, "Proposer preferences already cached; not republishing");
+            continue;
+        }
+        let verdict = validate(market, store, &signed, now_ms);
+        if verdict != Outcome::Accept {
+            let (outcome, reason) = verdict.labels();
+            warn!(%slot, validator, outcome, reason, "Refused submitted proposer preferences");
+            failures.push(Failure {
+                index,
+                message: describe(&verdict),
+            });
+            continue;
+        }
+        if !market.record_preferences(signed.clone(), wall_slot) {
+            failures.push(Failure {
+                index,
+                message: describe(&Outcome::Ignore(IgnoreReason::AlreadySeen)),
+            });
+            continue;
+        }
+        match p2p.publish_proposer_preferences(signed) {
+            Ok(()) => debug!(%slot, validator, "Accepted proposer preferences for gossip"),
+            Err(_) => failures.push(Failure {
+                index,
+                message: "the network actor is not running".to_string(),
+            }),
+        }
+    }
+    failures
 }
diff --git a/crates/net/rpc/src/beacon/states.rs b/crates/net/rpc/src/beacon/states.rs
index 5d307019a..66734684a 100644
--- a/crates/net/rpc/src/beacon/states.rs
+++ b/crates/net/rpc/src/beacon/states.rs
@@ -67,7 +67,10 @@ fn resolve_state_id(store: &Store, state_id: &str) -> Result {
 }
 
 /// Load the state a `state_id` names, or the response explaining why not.
-fn load(store: &Store, state_id: &str) -> Result<(H256, std::sync::Arc), ApiError> {
+pub(crate) fn load(
+    store: &Store,
+    state_id: &str,
+) -> Result<(H256, std::sync::Arc), ApiError> {
     let root = resolve_state_id(store, state_id)?;
     let state = store
         .get_state(&root)
diff --git a/crates/net/rpc/src/beacon/validator_client_tests.rs b/crates/net/rpc/src/beacon/validator_client_tests.rs
index 931960490..cb1de1820 100644
--- a/crates/net/rpc/src/beacon/validator_client_tests.rs
+++ b/crates/net/rpc/src/beacon/validator_client_tests.rs
@@ -109,6 +109,9 @@ async fn spawn_server(
         .layer(Extension(p2p))
         .layer(Extension(SharedAttestationPool::default()))
         .layer(Extension(payload_pool.clone()))
+        .layer(Extension(
+            ethlambda_state_transition::beacon::builder_market::SharedBuilderMarket::default(),
+        ))
         .layer(Extension(crate::CustodyColumns(Vec::new())))
         .layer(Extension(crate::beacon::validator::FeeRecipients::default()))
         .layer(Extension(engine));
diff --git a/docs/rpc.md b/docs/rpc.md
index 17fd4b807..acbd47209 100644
--- a/docs/rpc.md
+++ b/docs/rpc.md
@@ -252,10 +252,13 @@ surface rather than sitting beside it; a `/lean/v0` path on a beacon node is a
 | `GET` | `/eth/v2/validator/aggregate_attestation` | JSON | The pooled votes for a data root and committee, aggregated |
 | `POST` | `/eth/v2/validator/aggregate_and_proofs` | *(status only)* | Validate and gossip `SignedAggregateAndProof`s (JSON or SSZ body) |
 | `GET` | `/eth/v3/validator/blocks/{slot}` | SSZ or JSON | An unsigned fulu block built on the head (`produceBlockV3`) |
-| `POST` | `/eth/v4/validator/blocks/{slot}` | SSZ or JSON | An unsigned self-built gloas block, with its envelope and blobs when asked (`produceBlockV4`) |
+| `POST` | `/eth/v4/validator/blocks/{slot}` | SSZ or JSON | An unsigned gloas block: self-built, with its envelope and blobs when asked, or on a pooled builder bid (`produceBlockV4`) |
 | `GET` | `/eth/v1/validator/execution_payload_envelopes/{slot}/{beacon_block_root}` | SSZ or JSON | The unsigned envelope `produceBlockV4` built (gloas) |
 | `POST` | `/eth/v2/beacon/blocks` | *(status only)* | Gossip and import a signed fulu or gloas block (`publishBlockV2`; JSON or SSZ body) |
 | `POST` | `/eth/v1/beacon/execution_payload_envelopes` | *(status only)* | Gossip a signed envelope and its data columns (gloas; JSON or SSZ body) |
+| `POST` | `/eth/v1/beacon/execution_payload_bids` | *(status only)* | Validate, pool and gossip a builder's `SignedExecutionPayloadBid` (gloas; JSON or SSZ body) |
+| `POST` | `/eth/v1/validator/proposer_preferences` | *(status only)* | Validate, cache and gossip `SignedProposerPreferences` (gloas; JSON or SSZ body) |
+| `POST` | `/eth/v1/beacon/states/{state_id}/builders` | JSON | The gloas builder registry, filtered by id and status |
 | `POST` | `/eth/v1/validator/prepare_beacon_proposer` | *(status only)* | Acknowledged, not acted on (see below) |
 
 ### Validator endpoints
@@ -387,10 +390,12 @@ publication below builds and gossips the columns.
 ### Gloas block production
 
 A gloas proposer signs two things on their own, the block and the envelope that
-reveals its payload, and this node serves both halves. It only builds for
-itself: no builder bid is taken from gossip or a builder API, and no
-`SignedProposerPreferences` is read (see
-[Spec Deviations](./spec_deviations.md#self-build-only)).
+reveals its payload, and this node serves both halves for a block it builds
+itself. It may instead build on a builder's bid it holds in the shared builder
+market (see [Builder market](#builder-market)); then the builder, not this node,
+reveals the payload. The builder-API path (a bid requested from a builder's URL
+through the `BuilderConfig`'s `builders` entries) is not implemented: those
+entries are decoded and not consulted.
 
 **`POST /eth/v4/validator/blocks/{slot}`** (`produceBlockV4`):
 
@@ -398,18 +403,21 @@ itself: no builder bid is taken from gossip or a builder API, and no
 |---|---|
 | Query | `randao_reveal` (required), `include_payload` (required, `true` or `false`), `graffiti` (optional, 32-byte hex), `skip_randao_verification` (accepted, ignored) |
 | Request headers | `Eth-Consensus-Version` is optional but must be `gloas` when present. `Accept: application/octet-stream` for SSZ, JSON otherwise |
-| Body | A `BuilderConfig` (`min_bid`, `builder_boost_factor`, `builders`), JSON, or SSZ with `Content-Type: application/octet-stream`. It is decoded and otherwise ignored; a missing or undecodable one is a `400` |
-| `200` SSZ | With `include_payload=true`, `BlockContents` (`block`, `execution_payload_envelope`, `kzg_proofs`, `blobs`); with `false`, the bare `gloas::BeaconBlock` |
+| Body | A `BuilderConfig`, JSON or SSZ with `Content-Type: application/octet-stream`: `min_bid` and `builder_boost_factor` (govern the bids seen over p2p) and `builders` (up to 64 entries, decoded and ignored). Integers are quoted strings, as everywhere in the Beacon API. A missing or undecodable one is a `400` |
+| `200` SSZ | Self-built with `include_payload=true`: `BlockContents` (`block`, `execution_payload_envelope`, `kzg_proofs`, `blobs`). Otherwise, and for every bid-won block, the bare `gloas::BeaconBlock` |
 | `200` JSON | `{version: "gloas", consensus_block_value, execution_payload_value, execution_payload_included, data}`, where `data` is the same container as the SSZ body |
-| Response headers | `Eth-Consensus-Version: gloas`, `Eth-Execution-Payload-Included` (`true` or `false`), `Eth-Execution-Payload-Value` (wei, decimal), `Eth-Consensus-Block-Value` (always `0`: no builder comparison happens on this node) |
+| Response headers | `Eth-Consensus-Version: gloas`, `Eth-Execution-Payload-Included` (`true` or `false`; always `false` for a bid-won block), `Eth-Execution-Payload-Value` (wei, decimal; a bid's value times `10^9` when it won), `Eth-Consensus-Block-Value` (always `0`: the comparison is of execution payload values only). No `Eth-Builder-Url`: that is for builder-API bids |
 | `400` | A slot the schedule does not place at gloas, a missing or malformed query, a wrong `Eth-Consensus-Version`, a bad body, a slot not after the head block, or a `randao_reveal` that does not verify against the slot's proposer. The slot check precedes the execution-client check, so it is a `400` on any node |
-| `503` | No execution client configured, the execution client did not start or return a build, or the node is building on a FULL parent whose envelope it does not hold |
+| `503` | No execution client and no viable pooled bid for the slot, the execution client did not start or return a build (and no bid took its place), the node is building on a FULL parent whose envelope it does not hold, or the winning bid failed to build and there is no local payload to fall back on |
 
 The node advances the head state to the slot, and decides which parent payload
 to build on with `should_build_on_full` over the payload status fork choice
 recorded for the head. It then asks its execution client to build
-(`forkchoiceUpdatedV4` with `PayloadAttributesV4`, then `getPayloadV6`) with the
-proposer's `prepare_beacon_proposer` fee recipient. The body packs the
+(`forkchoiceUpdatedV4` with `PayloadAttributesV4`, then `getPayloadV6`). The
+fee recipient and gas target of that build come from the proposer's signed
+`ProposerPreferences` for the slot when the market holds them under the slot's
+dependent root, else from `prepare_beacon_proposer` (fee recipient, or the zero
+address with a warning) and the parent bid's gas limit. The body packs the
 attestation pool's best aggregates and the payload attestation pool's votes for
 the parent block (an aggregate that does not verify against the advanced state
 is dropped, since one bad operation fails the block), and the state root comes
@@ -417,6 +425,22 @@ from running the block through `process_block`. The bid is a zero-value
 self-build bid read off the built payload. What was built is cached by `(slot,
 block root)`, for the current and previous slot only.
 
+**Choosing between the local payload and a bid.** The bids considered are the
+market's for `(slot, head root, parent payload hash)` with this build's
+`prev_randao`, packable into a block on the advanced state, and paying the
+preferences' fee recipient when those are held. The best bid is the highest
+whose `value + execution_payment` is at least the config's `min_bid`. It wins iff
+`builder_boost_factor * bid_gwei > floor(local_value_wei / 10^7)` (the
+specification's weighting of the local value by 100, in integers), and the local
+payload wins a tie, so a factor of `0` prefers it and `2^64 - 1` prefers the bid.
+An engine that sets `shouldOverrideBuilder` keeps the local payload. Without a
+local payload (no engine, or the build failed) the best bid is taken whatever its
+weight. A bid-won block caches nothing, so the envelope `GET` below answers
+`404` for it, and `POST /eth/v1/beacon/execution_payload_envelopes` refuses a
+proposer-signed (self-build) envelope for it because the builder index differs
+from the bid's. The builder's own envelope goes through that endpoint as for any
+block.
+
 **`GET /eth/v1/validator/execution_payload_envelopes/{slot}/{beacon_block_root}`**
 serves the cached unsigned envelope, for a client that asked for the block with
 `include_payload=false`: `{version: "gloas", data}` as JSON, or the SSZ
@@ -450,6 +474,47 @@ envelope and all `NUMBER_OF_COLUMNS` gloas data column sidecars are handed to
 P2P, which gossips them together. A node that does not subscribe to a column's
 subnet publishes through gossipsub fanout.
 
+### Builder market
+
+Three endpoints serve the gloas builder market. Bids and preferences go through
+the same rules as the `execution_payload_bid` and `proposer_preferences` gossip
+topics and are held in one shared builder market that p2p also fills, so a
+message accepted from either side is seen by both.
+
+- **`POST /eth/v1/beacon/execution_payload_bids`** takes one
+  `SignedExecutionPayloadBid`, as JSON (or no `Content-Type`) or SSZ
+  (`application/octet-stream`, at most 196,932 bytes; any other type is `415`).
+  `Eth-Consensus-Version` is optional and must be `gloas` when present. An
+  identical bid already pooled is `200` and is not republished. Otherwise the
+  gossip rules run (a known parent, the preferences for the slot and their fee
+  recipient and gas limit, a known parent payload, an active funded builder, the
+  signature), and a bid that is anything but accepted is a `400` whose message is
+  `"{outcome}: {reason}"` (for instance `reject: bad_signature` or `ignore:
+  preferences_unseen`), because there is no status for a valid bid this node
+  would not relay. An accepted bid is pooled for block production and gossiped.
+  `500` when the network actor is down.
+- **`POST /eth/v1/validator/proposer_preferences`** takes a list of
+  `SignedProposerPreferences` (at most `(MIN_SEED_LOOKAHEAD + 1) *
+  SLOTS_PER_EPOCH`, 64 on mainnet), as a JSON array or an SSZ list.
+  `Eth-Consensus-Version` is optional and may be `fulu` or `gloas`, since a
+  validator client submits during the epoch before the fork. Each entry runs the
+  gossip rules in order; an entry already cached, identical, succeeds without a
+  republish. Accepted entries are cached (the first per `(proposal slot,
+  dependent root)` wins) and gossiped. If any entry fails the answer is `400`
+  `{code, message, failures: [{index, message}]}` and the others were still
+  published. The endpoint is not gated on the sync status. A node whose
+  dependent block is unknown answers `ignore: unknown_block` for that entry.
+- **`POST /eth/v1/beacon/states/{state_id}/builders`** takes an optional JSON
+  body `{ids?: [index | 0x pubkey], statuses?: [pending | active | exited]}`
+  (empty or absent selects everything) and answers `{execution_optimistic,
+  finalized, data: [{index, status, builder}]}` in registry order, integers
+  quoted. An id naming no builder is omitted. `exited` is a set
+  `withdrawable_epoch`, else `active` per `is_active_builder`, else `pending`.
+  `400` for a malformed body, id or status, or a pre-gloas state; `404` for an
+  unknown state. The Beacon API defines no `GET` form.
+
+`POST /eth/v1/validator/builder_preferences` is not served (`404`).
+
 ### Payload timeliness committee
 
 - **`POST /eth/v1/validator/duties/ptc/{epoch}`** takes a JSON array of quoted

From 449a8d5e98eb90065a1726a8c95690e94a13636c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Mon, 5 Oct 2026 22:59:09 -0300
Subject: [PATCH 119/132] test(rpc): drive the builder market endpoints and bid
 selection end to end

Cover the three endpoints, the verdicts they turn into 400s, idempotent
resubmission, SSZ bodies, produceBlockV4 choosing between a stand-in engine's
payload and a pooled bid, the preferences-driven fee recipient and gas target,
and the envelope endpoints' behaviour for a block built on a bid.
---
 .../rpc/src/beacon/builder_market_tests.rs    | 1225 +++++++++++++++++
 crates/net/rpc/src/beacon/gloas_proposal.rs   |    4 +
 2 files changed, 1229 insertions(+)
 create mode 100644 crates/net/rpc/src/beacon/builder_market_tests.rs

diff --git a/crates/net/rpc/src/beacon/builder_market_tests.rs b/crates/net/rpc/src/beacon/builder_market_tests.rs
new file mode 100644
index 000000000..b43efd66a
--- /dev/null
+++ b/crates/net/rpc/src/beacon/builder_market_tests.rs
@@ -0,0 +1,1225 @@
+//! The builder market endpoints and `produceBlockV4`'s bid selection, driven
+//! through the real router with a stand-in execution client.
+//!
+//! Tests that need the gossip rules, `dependent_root_at`, `bid_is_includable`
+//! or `assemble_gloas_block_on_bid` exercise those functions as they are
+//! implemented in `ethlambda-state-transition`; the ones that only check shapes
+//! (headers, status codes, routing, idempotency against a primed market) do not
+//! depend on them.
+
+use std::sync::{Arc, Mutex};
+
+use axum::{
+    Extension, Router,
+    body::Body,
+    http::{HeaderMap, Request, StatusCode},
+};
+use ethlambda_engine::{EngineClient, JwtSecret};
+use ethlambda_network_api::RpcToP2PRef;
+use ethlambda_state_transition::beacon::{
+    attestation_pool::SharedAttestationPool,
+    block_production::advance_to_slot,
+    builder_market::SharedBuilderMarket,
+    gloas_block_production::test_support::{
+        config as chain_config, parent_state, post_state, randao_reveal,
+    },
+    gossip::proposer_preferences::dependent_root_at,
+    helpers::{
+        accessors::get_domain,
+        misc::compute_signing_root,
+        test_state::{secret_key_for, sign_for},
+    },
+    payload_attestation_pool::SharedPayloadAttestationPool,
+};
+use ethlambda_storage::Store;
+use ethlambda_types::{
+    beacon::{
+        constants::{
+            BUILDER_INDEX_SELF_BUILD, DOMAIN_BEACON_BUILDER, DOMAIN_PROPOSER_PREFERENCES,
+            FAR_FUTURE_EPOCH, PAYLOAD_BUILDER_VERSION,
+        },
+        containers::{
+            BeaconState, SignedBeaconBlock,
+            gloas::{
+                BeaconBlock, Builder, ExecutionPayloadBid, ExecutionRequests, ProposerPreferences,
+                SignedExecutionPayloadBid, SignedExecutionPayloadEnvelope,
+                SignedProposerPreferences,
+            },
+        },
+        primitives::{BlsPubkey, BlsSignature, ExecutionAddress, HashTreeRoot as _},
+        signing::compute_epoch_at_slot,
+    },
+    primitives::H256,
+};
+use http_body_util::BodyExt as _;
+use libssz::{SszDecode as _, SszEncode as _};
+use tower::ServiceExt as _;
+
+use super::{Prepared, prepare, routes as produce_routes};
+use crate::{
+    CustodyColumns,
+    beacon::{bids, proposer_preferences, validator::FeeRecipients},
+    test_utils::{RecordingNetwork, beacon_store_with_head_block, gloas_beacon_block},
+};
+
+/// The slot the block is built for. The head block sits one slot earlier, and
+/// the clock is at the head's slot, so this one is the next slot: the one bids
+/// and preferences are accepted for.
+const SLOT: u64 = 33;
+const HEAD_SLOT: u64 = SLOT - 1;
+const GWEI: u64 = 1_000_000_000;
+
+// ---------------------------------------------------------------------------
+// Fake execution client
+// ---------------------------------------------------------------------------
+
+/// A stand-in execution client and what it was asked to build.
+struct FakeEngine {
+    client: EngineClient,
+    /// The payload attributes the last `forkchoiceUpdatedV4` carried.
+    attributes: Arc>>,
+}
+
+/// `forkchoiceUpdated` with attributes answers a payload id, and `getPayloadV6`
+/// a payload extending the requested head with exactly the requested
+/// attributes, worth `block_value_wei`.
+async fn fake_engine(block_value_wei: u64, should_override_builder: bool) -> FakeEngine {
+    use axum::{Json, routing::post};
+
+    let attributes: Arc>> = Arc::default();
+    let recorded = attributes.clone();
+    let handler = move |Json(request): Json| {
+        let recorded = recorded.clone();
+        async move {
+            let result = match request["method"].as_str() {
+                Some("engine_forkchoiceUpdatedV4") => {
+                    let mut attributes = request["params"][1].clone();
+                    attributes["parentHash"] = request["params"][0]["headBlockHash"].clone();
+                    *recorded.lock().unwrap() = Some(attributes);
+                    serde_json::json!({
+                        "payloadStatus": { "status": "VALID", "latestValidHash": null },
+                        "payloadId": "0x0000000000000001",
+                    })
+                }
+                Some("engine_getPayloadV6") => {
+                    let attributes = recorded.lock().unwrap().clone().unwrap();
+                    serde_json::json!({
+                        "executionPayload": {
+                            "parentHash": attributes["parentHash"],
+                            "feeRecipient": attributes["suggestedFeeRecipient"],
+                            "stateRoot": format!("0x{}", "00".repeat(32)),
+                            "receiptsRoot": format!("0x{}", "00".repeat(32)),
+                            "logsBloom": format!("0x{}", "00".repeat(256)),
+                            "prevRandao": attributes["prevRandao"],
+                            "blockNumber": "0x1",
+                            "gasLimit": attributes["targetGasLimit"],
+                            "gasUsed": "0x0",
+                            "timestamp": attributes["timestamp"],
+                            "extraData": "0x",
+                            "baseFeePerGas": "0x7",
+                            "blockHash": format!("0x{}", "ee".repeat(32)),
+                            "transactions": [],
+                            "withdrawals": attributes["withdrawals"],
+                            "blobGasUsed": "0x0",
+                            "excessBlobGas": "0x0",
+                            "blockAccessList": "0xc0",
+                            "slotNumber": attributes["slotNumber"],
+                        },
+                        "blockValue": format!("0x{block_value_wei:x}"),
+                        "blobsBundle": { "commitments": [], "proofs": [], "blobs": [] },
+                        "shouldOverrideBuilder": should_override_builder,
+                        "executionRequests": [],
+                    })
+                }
+                _ => serde_json::Value::Null,
+            };
+            Json(serde_json::json!({ "jsonrpc": "2.0", "id": request["id"], "result": result }))
+        }
+    };
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+    let router = Router::new().route("/", post(handler));
+    tokio::spawn(async move { axum::serve(listener, router).await });
+    FakeEngine {
+        client: EngineClient::new(format!("http://{address}"), JwtSecret::new([0x0f; 32])).unwrap(),
+        attributes,
+    }
+}
+
+/// An execution client nothing listens for, so every build fails.
+fn dead_engine() -> FakeEngine {
+    FakeEngine {
+        client: EngineClient::new("http://127.0.0.1:1".to_string(), JwtSecret::new([0x0f; 32]))
+            .unwrap(),
+        attributes: Arc::default(),
+    }
+}
+
+impl FakeEngine {
+    fn requested(&self) -> serde_json::Value {
+        self.attributes
+            .lock()
+            .unwrap()
+            .clone()
+            .expect("the engine was asked to build")
+    }
+}
+
+// ---------------------------------------------------------------------------
+// The chain
+// ---------------------------------------------------------------------------
+
+fn builder_pubkey() -> BlsPubkey {
+    BlsPubkey(secret_key_for(1000).sk_to_pk().to_bytes())
+}
+
+fn sign_as_builder(root: H256) -> BlsSignature {
+    BlsSignature(
+        secret_key_for(1000)
+            .sign(
+                root.as_slice(),
+                ethlambda_state_transition::beacon::bls::DST,
+                &[],
+            )
+            .to_bytes(),
+    )
+}
+
+/// The block at slot 0 whose root the head state names as the dependent root of
+/// every slot of the first two epochs.
+fn genesis_block() -> SignedBeaconBlock {
+    gloas_beacon_block(0, H256::ZERO, H256::ZERO, H256::repeat_byte(0x0a))
+}
+
+/// A gloas head state at slot 32 whose registry holds one funded, active
+/// builder (index 0, key `secret_key_for(1000)`) and whose block roots name the
+/// genesis block.
+fn chain_state() -> BeaconState {
+    let mut state = parent_state();
+    let genesis_root = genesis_block().message_hash_tree_root();
+    let BeaconState::Gloas(inner) = &mut state else {
+        unreachable!("built as gloas")
+    };
+    // `is_active_builder` wants the deposit epoch below the finalized one.
+    inner.finalized_checkpoint.epoch = 1;
+    inner.builders.push(Builder {
+        pubkey: builder_pubkey(),
+        version: PAYLOAD_BUILDER_VERSION,
+        execution_address: ExecutionAddress::ZERO,
+        balance: 100_000_000_000,
+        deposit_epoch: 0,
+        withdrawable_epoch: FAR_FUTURE_EPOCH,
+    });
+    inner.block_roots[0] = genesis_root;
+    state
+}
+
+struct World {
+    store: Store,
+    state: BeaconState,
+    head_root: H256,
+    market: SharedBuilderMarket,
+    network: Arc,
+    fee_recipients: FeeRecipients,
+}
+
+impl World {
+    /// The head block at slot 32 on a clock where that slot is running.
+    fn new() -> Self {
+        let state = chain_state();
+        let genesis = genesis_block();
+        let genesis_root = genesis.message_hash_tree_root();
+        let block = gloas_beacon_block(
+            state.slot(),
+            genesis_root,
+            H256::repeat_byte(0x01),
+            H256::repeat_byte(0x02),
+        );
+        // The root a block built on this state names as its parent.
+        let mut header = state.latest_block_header().clone();
+        header.state_root = state.hash_tree_root();
+        let head_root = header.hash_tree_root();
+        let mut store = beacon_store_with_head_block(
+            state.clone(),
+            chain_config(),
+            block,
+            head_root,
+            HEAD_SLOT,
+        );
+        store.insert_signed_block(genesis_root, genesis).unwrap();
+        store.insert_state(genesis_root, state.clone()).unwrap();
+        Self {
+            store,
+            state,
+            head_root,
+            market: SharedBuilderMarket::default(),
+            network: Arc::default(),
+            fee_recipients: FeeRecipients::default(),
+        }
+    }
+
+    fn app(&self, engine: Option<&FakeEngine>) -> Router {
+        let p2p: RpcToP2PRef = self.network.clone();
+        produce_routes()
+            .merge(bids::routes())
+            .merge(proposer_preferences::routes())
+            .with_state(self.store.clone())
+            .layer(Extension(p2p))
+            .layer(Extension(engine.map(|engine| engine.client.clone())))
+            .layer(Extension(SharedAttestationPool::default()))
+            .layer(Extension(SharedPayloadAttestationPool::default()))
+            .layer(Extension(self.market.clone()))
+            .layer(Extension(self.fee_recipients.clone()))
+            .layer(Extension(CustodyColumns::default()))
+    }
+
+    fn advanced(&self) -> BeaconState {
+        advance_to_slot(&self.state, SLOT, &chain_config()).unwrap()
+    }
+
+    /// What `produceBlockV4` reads off the chain for [`SLOT`].
+    fn prepared(&self) -> Prepared {
+        prepare(
+            &self.store,
+            &self.market,
+            SLOT,
+            randao_reveal(&self.advanced()),
+        )
+        .unwrap()
+    }
+
+    /// A bid of builder 0 on the parent payload `produceBlockV4` builds on,
+    /// signed under the builder domain.
+    fn bid(&self, value: u64, fee_recipient: ExecutionAddress) -> SignedExecutionPayloadBid {
+        let prepared = self.prepared();
+        let message = ExecutionPayloadBid {
+            parent_block_hash: prepared.inputs.head_block_hash,
+            parent_block_root: prepared.head_root,
+            block_hash: H256::repeat_byte(0xb1),
+            prev_randao: prepared.inputs.prev_randao,
+            fee_recipient,
+            gas_limit: prepared.inputs.target_gas_limit,
+            builder_index: 0,
+            slot: SLOT,
+            value,
+            execution_payment: 0,
+            blob_kzg_commitments: Default::default(),
+            execution_requests_root: ExecutionRequests::default().hash_tree_root(),
+        };
+        let domain = get_domain(&prepared.state, DOMAIN_BEACON_BUILDER, None);
+        let signing_root = compute_signing_root(message.hash_tree_root(), domain);
+        SignedExecutionPayloadBid {
+            message,
+            signature: sign_as_builder(signing_root),
+        }
+    }
+
+    /// The proposer's preferences for [`SLOT`], signed with its key.
+    fn preferences(&self, fee_recipient: ExecutionAddress, gas: u64) -> SignedProposerPreferences {
+        self.preferences_by(self.prepared().proposer, fee_recipient, gas)
+    }
+
+    /// Preferences naming `validator`, signed with that validator's key.
+    fn preferences_by(
+        &self,
+        validator: u64,
+        fee_recipient: ExecutionAddress,
+        gas: u64,
+    ) -> SignedProposerPreferences {
+        let dependent_root = dependent_root_at(&self.state, self.head_root, SLOT)
+            .expect("the head's chain gives the slot a dependent root");
+        let message = ProposerPreferences {
+            dependent_root,
+            proposal_slot: SLOT,
+            validator_index: validator,
+            fee_recipient,
+            target_gas_limit: gas,
+        };
+        let domain = get_domain(
+            &self.state,
+            DOMAIN_PROPOSER_PREFERENCES,
+            Some(compute_epoch_at_slot(SLOT)),
+        );
+        let signing_root = compute_signing_root(message.hash_tree_root(), domain);
+        SignedProposerPreferences {
+            signature: sign_for(validator as usize, signing_root),
+            message,
+        }
+    }
+
+    /// Preferences recorded straight into the market, without the gossip rules.
+    fn prime_preferences(&self, fee_recipient: ExecutionAddress, gas: u64) {
+        let signed = self.preferences(fee_recipient, gas);
+        assert!(self.market.record_preferences(signed, HEAD_SLOT));
+    }
+
+    fn prime_bid(&self, bid: &SignedExecutionPayloadBid) {
+        assert!(self.market.record_bid(bid.clone()));
+    }
+
+    /// The parent payload bids are judged against becomes known to gossip, as
+    /// its envelope arriving would make it.
+    fn reveal_parent_payload(&self) {
+        let prepared = self.prepared();
+        let mut envelope = crate::test_utils::gloas_envelope(self.head_root, HEAD_SLOT);
+        envelope.message.payload.block_hash = prepared.inputs.head_block_hash;
+        envelope.message.payload.gas_limit = 30_000_000;
+        self.market.record_execution_payload(&envelope.message);
+    }
+}
+
+struct Reply {
+    status: StatusCode,
+    headers: HeaderMap,
+    body: Vec,
+}
+
+impl Reply {
+    fn json(&self) -> serde_json::Value {
+        serde_json::from_slice(&self.body).unwrap_or_default()
+    }
+}
+
+async fn send(app: &Router, request: Request) -> Reply {
+    let response = app.clone().oneshot(request).await.unwrap();
+    let status = response.status();
+    let headers = response.headers().clone();
+    let body = response
+        .into_body()
+        .collect()
+        .await
+        .unwrap()
+        .to_bytes()
+        .to_vec();
+    Reply {
+        status,
+        headers,
+        body,
+    }
+}
+
+fn address(byte: u8) -> ExecutionAddress {
+    ExecutionAddress::repeat_byte(byte)
+}
+
+// ---------------------------------------------------------------------------
+// produceBlockV4
+// ---------------------------------------------------------------------------
+
+fn builder_config(min_bid: u64, factor: u64) -> String {
+    format!(r#"{{"min_bid":"{min_bid}","builder_boost_factor":"{factor}","builders":[]}}"#)
+}
+
+fn produce_request(
+    world: &World,
+    include_payload: bool,
+    config: &str,
+    accept_ssz: bool,
+) -> Request {
+    let reveal = randao_reveal(&world.advanced());
+    let mut request = Request::post(format!(
+        "/eth/v4/validator/blocks/{SLOT}?randao_reveal=0x{}&include_payload={include_payload}",
+        hex::encode(reveal.0)
+    ))
+    .header("eth-consensus-version", "gloas")
+    .header("content-type", "application/json");
+    if accept_ssz {
+        request = request.header("accept", "application/octet-stream");
+    }
+    request.body(Body::from(config.to_string())).unwrap()
+}
+
+async fn produce(
+    world: &World,
+    engine: Option<&FakeEngine>,
+    include_payload: bool,
+    config: &str,
+) -> Reply {
+    send(
+        &world.app(engine),
+        produce_request(world, include_payload, config, false),
+    )
+    .await
+}
+
+/// The bid the produced block commits to, from a JSON response either way: a
+/// bare block, or contents carrying it.
+fn produced_block_bid(reply: &Reply) -> ExecutionPayloadBid {
+    let data = &reply.json()["data"];
+    let block = if data.get("block").is_some() {
+        &data["block"]
+    } else {
+        data
+    };
+    serde_json::from_value(block["body"]["signed_execution_payload_bid"]["message"].clone())
+        .expect("a gloas block carries its bid")
+}
+
+fn self_built(reply: &Reply) -> bool {
+    produced_block_bid(reply).builder_index == BUILDER_INDEX_SELF_BUILD
+}
+
+#[tokio::test]
+async fn a_bid_worth_more_than_the_local_payload_is_built_on_and_returned_bare() {
+    let world = World::new();
+    let engine = fake_engine(GWEI, false).await;
+    let bid = world.bid(5, address(0xcc));
+    world.prime_bid(&bid);
+
+    // Factor 100 weights both sides evenly: 5 gwei against 1 gwei.
+    let reply = produce(&world, Some(&engine), true, &builder_config(0, 100)).await;
+
+    assert_eq!(
+        reply.status,
+        StatusCode::OK,
+        "{}",
+        String::from_utf8_lossy(&reply.body)
+    );
+    assert_eq!(produced_block_bid(&reply), bid.message);
+    // `include_payload=true` still comes back bare: the builder reveals.
+    let json = reply.json();
+    assert_eq!(json["execution_payload_included"], false);
+    assert_eq!(json["version"], "gloas");
+    assert!(json["data"].get("execution_payload_envelope").is_none());
+    assert_eq!(reply.headers["eth-execution-payload-included"], "false");
+    assert_eq!(
+        reply.headers["eth-execution-payload-value"],
+        (5 * GWEI).to_string().as_str()
+    );
+    assert_eq!(reply.headers["eth-consensus-block-value"], "0");
+    assert_eq!(reply.headers["eth-consensus-version"], "gloas");
+    assert!(reply.headers.get("eth-builder-url").is_none());
+}
+
+#[tokio::test]
+async fn a_bid_won_block_is_served_as_ssz_too() {
+    let world = World::new();
+    let engine = fake_engine(GWEI, false).await;
+    let bid = world.bid(5, address(0xcc));
+    world.prime_bid(&bid);
+
+    let reply = send(
+        &world.app(Some(&engine)),
+        produce_request(&world, false, &builder_config(0, 100), true),
+    )
+    .await;
+
+    assert_eq!(reply.status, StatusCode::OK);
+    assert_eq!(reply.headers["eth-execution-payload-included"], "false");
+    let block = BeaconBlock::from_ssz_bytes(&reply.body).unwrap();
+    assert_eq!(block.body.signed_execution_payload_bid, bid);
+}
+
+#[tokio::test]
+async fn nothing_is_cached_for_a_bid_block_and_a_self_build_envelope_for_it_is_refused() {
+    let world = World::new();
+    let engine = fake_engine(GWEI, false).await;
+    world.prime_bid(&world.bid(5, address(0xcc)));
+    let app = world.app(Some(&engine));
+    let reply = send(
+        &app,
+        produce_request(&world, false, &builder_config(0, 100), true),
+    )
+    .await;
+    assert_eq!(reply.status, StatusCode::OK);
+    let block = BeaconBlock::from_ssz_bytes(&reply.body).unwrap();
+    let root = block.hash_tree_root();
+
+    // No envelope was cached for the block.
+    let reply = send(
+        &app,
+        Request::get(format!(
+            "/eth/v1/validator/execution_payload_envelopes/{SLOT}/{root:?}"
+        ))
+        .body(Body::empty())
+        .unwrap(),
+    )
+    .await;
+    assert_eq!(reply.status, StatusCode::NOT_FOUND);
+
+    // The block is imported; the proposer's self-build envelope for it names
+    // a different builder than the bid, so it is refused.
+    let mut store = world.store.clone();
+    let post = post_state(&world.advanced(), &block);
+    store
+        .insert_signed_block(
+            root,
+            SignedBeaconBlock::Gloas(
+                ethlambda_types::beacon::containers::gloas::SignedBeaconBlock {
+                    message: block,
+                    signature: Default::default(),
+                },
+            ),
+        )
+        .unwrap();
+    store.insert_state(root, post).unwrap();
+    let mut envelope = crate::test_utils::gloas_envelope(root, SLOT);
+    envelope.message.builder_index = BUILDER_INDEX_SELF_BUILD;
+    let request = Request::post("/eth/v1/beacon/execution_payload_envelopes")
+        .header("eth-consensus-version", "gloas")
+        .header("eth-blob-data-included", "false")
+        .header("content-type", crate::SSZ_CONTENT_TYPE)
+        .body(Body::from(SignedExecutionPayloadEnvelope::to_ssz(
+            &envelope,
+        )))
+        .unwrap();
+    let reply = send(&app, request).await;
+    assert_eq!(reply.status, StatusCode::BAD_REQUEST);
+    assert_eq!(
+        reply.json()["message"],
+        "the envelope does not fulfill the block's bid"
+    );
+    assert!(world.network.envelopes.lock().unwrap().is_empty());
+}
+
+#[tokio::test]
+async fn the_local_payload_wins_ties_a_zero_factor_a_floor_and_an_override() {
+    // (local value in wei, should_override, min_bid in gwei, factor)
+    let cases = [
+        // 5 gwei each side at factor 100: a tie.
+        (5 * GWEI, false, 0, 100),
+        // Factor 0 prefers the local payload.
+        (1, false, 0, 0),
+        // The bid is below the config's floor.
+        (1, false, 6, u64::MAX),
+        // The engine insists on its own payload.
+        (1, true, 0, u64::MAX),
+    ];
+    for (local_wei, should_override, min_bid, factor) in cases {
+        let world = World::new();
+        let engine = fake_engine(local_wei, should_override).await;
+        world.prime_bid(&world.bid(5, address(0xcc)));
+
+        let reply = produce(
+            &world,
+            Some(&engine),
+            true,
+            &builder_config(min_bid, factor),
+        )
+        .await;
+
+        assert_eq!(
+            reply.status,
+            StatusCode::OK,
+            "{local_wei} {min_bid} {factor}"
+        );
+        assert!(
+            self_built(&reply),
+            "{local_wei} {should_override} {min_bid} {factor}"
+        );
+        assert_eq!(reply.json()["execution_payload_included"], true);
+        assert_eq!(reply.headers["eth-execution-payload-included"], "true");
+        assert!(
+            reply.json()["data"]
+                .get("execution_payload_envelope")
+                .is_some()
+        );
+    }
+}
+
+#[tokio::test]
+async fn a_bid_wins_when_the_local_build_is_below_it_by_one_unit_of_weight() {
+    // 1e9 wei is 1 gwei: a tie at factor 100. One wei less and the bid wins.
+    for (local_wei, bid_wins) in [(GWEI, false), (GWEI - 1, true)] {
+        let world = World::new();
+        let engine = fake_engine(local_wei, false).await;
+        world.prime_bid(&world.bid(1, address(0xcc)));
+
+        let reply = produce(&world, Some(&engine), false, &builder_config(0, 100)).await;
+
+        assert_eq!(reply.status, StatusCode::OK);
+        assert_eq!(self_built(&reply), !bid_wins, "{local_wei}");
+    }
+}
+
+#[tokio::test]
+async fn without_an_engine_a_viable_bid_is_built_on_and_none_is_a_503() {
+    let world = World::new();
+    let reply = produce(&world, None, true, &builder_config(0, 0)).await;
+    assert_eq!(reply.status, StatusCode::SERVICE_UNAVAILABLE);
+
+    let bid = world.bid(5, address(0xcc));
+    world.prime_bid(&bid);
+    // Even a factor of 0 takes the bid when there is nothing to prefer.
+    let reply = produce(&world, None, true, &builder_config(0, 0)).await;
+    assert_eq!(reply.status, StatusCode::OK);
+    assert_eq!(produced_block_bid(&reply), bid.message);
+
+    // A bid under the floor is not viable.
+    let reply = produce(&world, None, true, &builder_config(6, 100)).await;
+    assert_eq!(reply.status, StatusCode::SERVICE_UNAVAILABLE);
+}
+
+#[tokio::test]
+async fn a_failed_local_build_falls_back_to_a_viable_bid() {
+    let world = World::new();
+    let engine = dead_engine();
+    let reply = produce(&world, Some(&engine), true, &builder_config(0, 0)).await;
+    assert_eq!(reply.status, StatusCode::SERVICE_UNAVAILABLE);
+
+    let bid = world.bid(5, address(0xcc));
+    world.prime_bid(&bid);
+    let reply = produce(&world, Some(&engine), true, &builder_config(0, 0)).await;
+    assert_eq!(reply.status, StatusCode::OK);
+    assert_eq!(produced_block_bid(&reply), bid.message);
+}
+
+#[tokio::test]
+async fn bids_that_do_not_fit_the_slot_are_left_to_the_local_payload() {
+    // Each would win on value (factor MAX against a 1 wei local build).
+    let mismatches: [(&str, fn(&mut ExecutionPayloadBid)); 3] = [
+        ("randao", |bid| bid.prev_randao = H256::repeat_byte(0x99)),
+        ("parent hash", |bid| {
+            bid.parent_block_hash = H256::repeat_byte(0x98)
+        }),
+        ("parent root", |bid| {
+            bid.parent_block_root = H256::repeat_byte(0x97)
+        }),
+    ];
+    for (name, change) in mismatches {
+        let world = World::new();
+        let engine = fake_engine(1, false).await;
+        let mut bid = world.bid(5, address(0xcc));
+        change(&mut bid.message);
+        // Re-signing is beside the point: the pool is keyed on the fields that
+        // changed, and a bid that does not match never reaches a block.
+        world.market.record_bid(bid);
+
+        let reply = produce(&world, Some(&engine), true, &builder_config(0, u64::MAX)).await;
+
+        assert_eq!(reply.status, StatusCode::OK, "{name}");
+        assert!(self_built(&reply), "{name}");
+    }
+}
+
+#[tokio::test]
+async fn a_bid_paying_someone_other_than_the_preferred_recipient_is_skipped() {
+    let world = World::new();
+    let engine = fake_engine(1, false).await;
+    world.prime_preferences(address(0xaa), 30_000_000);
+    world.prime_bid(&world.bid(5, address(0xcc)));
+
+    let reply = produce(&world, Some(&engine), true, &builder_config(0, u64::MAX)).await;
+
+    assert_eq!(reply.status, StatusCode::OK);
+    assert!(self_built(&reply));
+}
+
+#[tokio::test]
+async fn a_bid_paying_the_preferred_recipient_is_taken() {
+    let world = World::new();
+    let engine = fake_engine(1, false).await;
+    world.prime_preferences(address(0xaa), 30_000_000);
+    let bid = world.bid(5, address(0xaa));
+    world.prime_bid(&bid);
+
+    let reply = produce(&world, Some(&engine), true, &builder_config(0, u64::MAX)).await;
+
+    assert_eq!(reply.status, StatusCode::OK);
+    assert_eq!(produced_block_bid(&reply), bid.message);
+}
+
+#[tokio::test]
+async fn the_self_build_takes_its_fee_recipient_and_gas_target_from_the_preferences() {
+    let world = World::new();
+    let engine = fake_engine(GWEI, false).await;
+    let proposer = world.prepared().proposer;
+    world
+        .fee_recipients
+        .lock()
+        .unwrap()
+        .insert(proposer, address(0xbb));
+    world.prime_preferences(address(0xaa), 31_000_000);
+
+    let reply = produce(&world, Some(&engine), true, &builder_config(0, 0)).await;
+
+    assert_eq!(reply.status, StatusCode::OK);
+    let requested = engine.requested();
+    assert_eq!(
+        requested["suggestedFeeRecipient"],
+        format!("0x{}", "aa".repeat(20))
+    );
+    assert_eq!(
+        requested["targetGasLimit"],
+        format!("0x{:x}", 31_000_000u64)
+    );
+    // The payload the engine built carries them into the block's bid.
+    let bid = produced_block_bid(&reply);
+    assert_eq!(bid.fee_recipient, address(0xaa));
+    assert_eq!(bid.gas_limit, 31_000_000);
+}
+
+#[tokio::test]
+async fn without_preferences_the_self_build_falls_back_to_the_prepared_recipient_and_parent_gas() {
+    let world = World::new();
+    let engine = fake_engine(GWEI, false).await;
+    let proposer = world.prepared().proposer;
+    world
+        .fee_recipients
+        .lock()
+        .unwrap()
+        .insert(proposer, address(0xbb));
+
+    let reply = produce(&world, Some(&engine), true, &builder_config(0, 0)).await;
+
+    assert_eq!(reply.status, StatusCode::OK);
+    let requested = engine.requested();
+    assert_eq!(
+        requested["suggestedFeeRecipient"],
+        format!("0x{}", "bb".repeat(20))
+    );
+    // The parent bid's gas limit, which is what `chain_state` gives it.
+    assert_eq!(
+        requested["targetGasLimit"],
+        format!("0x{:x}", 30_000_000u64)
+    );
+
+    // Neither: the zero address.
+    let world = World::new();
+    let engine = fake_engine(GWEI, false).await;
+    produce(&world, Some(&engine), true, &builder_config(0, 0)).await;
+    assert_eq!(
+        engine.requested()["suggestedFeeRecipient"],
+        format!("0x{}", "00".repeat(20))
+    );
+}
+
+#[tokio::test]
+async fn preferences_of_another_validator_are_not_the_proposers() {
+    let world = World::new();
+    let engine = fake_engine(GWEI, false).await;
+    let proposer = world.prepared().proposer;
+    let other = (proposer + 1) % 64;
+    let stranger = world.preferences_by(other, address(0xaa), 31_000_000);
+    assert!(world.market.record_preferences(stranger, HEAD_SLOT));
+
+    produce(&world, Some(&engine), true, &builder_config(0, 0)).await;
+
+    let requested = engine.requested();
+    assert_eq!(
+        requested["suggestedFeeRecipient"],
+        format!("0x{}", "00".repeat(20))
+    );
+    assert_eq!(
+        requested["targetGasLimit"],
+        format!("0x{:x}", 30_000_000u64)
+    );
+}
+
+#[tokio::test]
+async fn the_existing_error_paths_are_unchanged() {
+    let world = World::new();
+    let app = world.app(None);
+    // A pre-existing client sends the empty local-preferred config.
+    let reply = send(
+        &app,
+        produce_request(&world, true, &builder_config(0, 0), false),
+    )
+    .await;
+    assert_eq!(reply.status, StatusCode::SERVICE_UNAVAILABLE);
+    let reply = send(&app, produce_request(&world, true, "not a config", false)).await;
+    assert_eq!(reply.status, StatusCode::BAD_REQUEST);
+    assert_eq!(reply.json()["message"], "the body is not a BuilderConfig");
+}
+
+// ---------------------------------------------------------------------------
+// POST execution_payload_bids
+// ---------------------------------------------------------------------------
+
+fn bid_request(bid: &SignedExecutionPayloadBid) -> Request {
+    Request::post("/eth/v1/beacon/execution_payload_bids")
+        .header("eth-consensus-version", "gloas")
+        .header("content-type", "application/json")
+        .body(Body::from(serde_json::to_vec(bid).unwrap()))
+        .unwrap()
+}
+
+/// A world where a bid of builder 0 passes every rule: the preferences for the
+/// slot are cached and the parent payload is known.
+fn world_ready_for_bids() -> World {
+    let world = World::new();
+    world.prime_preferences(address(0xaa), 30_000_000);
+    world.reveal_parent_payload();
+    world
+}
+
+#[tokio::test]
+async fn a_valid_bid_is_pooled_and_published_once() {
+    let world = world_ready_for_bids();
+    let app = world.app(None);
+    let bid = world.bid(5, address(0xaa));
+
+    let reply = send(&app, bid_request(&bid)).await;
+
+    assert_eq!(
+        reply.status,
+        StatusCode::OK,
+        "{}",
+        String::from_utf8_lossy(&reply.body)
+    );
+    assert_eq!(*world.network.bids.lock().unwrap(), vec![bid.clone()]);
+    assert!(world.market.contains_bid(&bid));
+
+    // An identical resubmission is a success and is not gossiped again.
+    let reply = send(&app, bid_request(&bid)).await;
+    assert_eq!(reply.status, StatusCode::OK);
+    assert_eq!(world.network.bids.lock().unwrap().len(), 1);
+}
+
+#[tokio::test]
+async fn a_pooled_bid_resubmitted_is_a_200_without_a_publish() {
+    // Needs only the market: the identical bid short-circuits the rules.
+    let world = World::new();
+    let bid = world.bid(5, address(0xaa));
+    world.prime_bid(&bid);
+
+    let reply = send(&world.app(None), bid_request(&bid)).await;
+
+    assert_eq!(reply.status, StatusCode::OK);
+    assert!(world.network.bids.lock().unwrap().is_empty());
+}
+
+#[tokio::test]
+async fn a_bid_is_accepted_as_ssz() {
+    let world = world_ready_for_bids();
+    let bid = world.bid(5, address(0xaa));
+    let request = Request::post("/eth/v1/beacon/execution_payload_bids")
+        .header("eth-consensus-version", "gloas")
+        .header("content-type", crate::SSZ_CONTENT_TYPE)
+        .body(Body::from(bid.to_ssz()))
+        .unwrap();
+
+    let reply = send(&world.app(None), request).await;
+
+    assert_eq!(
+        reply.status,
+        StatusCode::OK,
+        "{}",
+        String::from_utf8_lossy(&reply.body)
+    );
+    assert_eq!(*world.network.bids.lock().unwrap(), vec![bid]);
+}
+
+#[tokio::test]
+async fn a_bid_with_a_bad_signature_is_a_400_naming_the_verdict() {
+    let world = world_ready_for_bids();
+    let mut bid = world.bid(5, address(0xaa));
+    bid.signature = BlsSignature::default();
+
+    let reply = send(&world.app(None), bid_request(&bid)).await;
+
+    assert_eq!(reply.status, StatusCode::BAD_REQUEST);
+    assert_eq!(reply.json()["code"], 400);
+    assert_eq!(reply.json()["message"], "reject: bad_signature");
+    assert!(world.network.bids.lock().unwrap().is_empty());
+    assert!(!world.market.contains_bid(&bid));
+}
+
+#[tokio::test]
+async fn a_bid_for_a_slot_without_preferences_is_a_400() {
+    let world = World::new();
+    world.reveal_parent_payload();
+    let bid = world.bid(5, address(0xaa));
+
+    let reply = send(&world.app(None), bid_request(&bid)).await;
+
+    assert_eq!(reply.status, StatusCode::BAD_REQUEST);
+    assert_eq!(reply.json()["message"], "ignore: preferences_unseen");
+    assert!(world.network.bids.lock().unwrap().is_empty());
+}
+
+#[tokio::test]
+async fn a_lower_bid_after_a_higher_one_is_refused_and_the_verdict_is_a_400() {
+    let world = world_ready_for_bids();
+    let app = world.app(None);
+    let high = world.bid(9, address(0xaa));
+    assert_eq!(send(&app, bid_request(&high)).await.status, StatusCode::OK);
+    let mut low = world.bid(5, address(0xaa));
+    low.message.builder_index = 0;
+
+    let reply = send(&app, bid_request(&low)).await;
+
+    assert_eq!(reply.status, StatusCode::BAD_REQUEST);
+    assert_eq!(world.network.bids.lock().unwrap().len(), 1);
+}
+
+#[tokio::test]
+async fn a_bid_with_a_wrong_header_or_content_type_or_body_is_refused() {
+    let world = world_ready_for_bids();
+    let app = world.app(None);
+    let bid = world.bid(5, address(0xaa));
+    let json = serde_json::to_vec(&bid).unwrap();
+
+    let wrong_fork = Request::post("/eth/v1/beacon/execution_payload_bids")
+        .header("eth-consensus-version", "fulu")
+        .body(Body::from(json.clone()))
+        .unwrap();
+    assert_eq!(send(&app, wrong_fork).await.status, StatusCode::BAD_REQUEST);
+
+    let plain = Request::post("/eth/v1/beacon/execution_payload_bids")
+        .header("content-type", "text/plain")
+        .body(Body::from(json.clone()))
+        .unwrap();
+    assert_eq!(
+        send(&app, plain).await.status,
+        StatusCode::UNSUPPORTED_MEDIA_TYPE
+    );
+
+    let garbage = Request::post("/eth/v1/beacon/execution_payload_bids")
+        .body(Body::from("not a bid"))
+        .unwrap();
+    assert_eq!(send(&app, garbage).await.status, StatusCode::BAD_REQUEST);
+
+    let oversized = Request::post("/eth/v1/beacon/execution_payload_bids")
+        .header("content-type", crate::SSZ_CONTENT_TYPE)
+        .body(Body::from(vec![0u8; 196_933]))
+        .unwrap();
+    assert_eq!(send(&app, oversized).await.status, StatusCode::BAD_REQUEST);
+
+    // No header at all is read leniently.
+    let lenient = Request::post("/eth/v1/beacon/execution_payload_bids")
+        .body(Body::from(json))
+        .unwrap();
+    assert_eq!(send(&app, lenient).await.status, StatusCode::OK);
+}
+
+// ---------------------------------------------------------------------------
+// POST proposer_preferences
+// ---------------------------------------------------------------------------
+
+fn preferences_request(
+    preferences: &[SignedProposerPreferences],
+    version: Option<&str>,
+) -> Request {
+    let mut request = Request::post("/eth/v1/validator/proposer_preferences")
+        .header("content-type", "application/json");
+    if let Some(version) = version {
+        request = request.header("eth-consensus-version", version);
+    }
+    request
+        .body(Body::from(serde_json::to_vec(preferences).unwrap()))
+        .unwrap()
+}
+
+#[tokio::test]
+async fn valid_preferences_are_cached_and_published_once() {
+    let world = World::new();
+    let app = world.app(None);
+    let signed = world.preferences(address(0xaa), 30_000_000);
+
+    let reply = send(&app, preferences_request(&[signed.clone()], Some("gloas"))).await;
+
+    assert_eq!(
+        reply.status,
+        StatusCode::OK,
+        "{}",
+        String::from_utf8_lossy(&reply.body)
+    );
+    assert_eq!(
+        *world.network.proposer_preferences.lock().unwrap(),
+        vec![signed.clone()]
+    );
+    assert_eq!(
+        world
+            .market
+            .preferences(SLOT, signed.message.dependent_root),
+        Some(signed.clone())
+    );
+
+    // The same again is a success without a second publish.
+    let reply = send(&app, preferences_request(&[signed], Some("gloas"))).await;
+    assert_eq!(reply.status, StatusCode::OK);
+    assert_eq!(world.network.proposer_preferences.lock().unwrap().len(), 1);
+}
+
+#[tokio::test]
+async fn cached_preferences_resubmitted_are_a_200_without_a_publish() {
+    // Needs only the market: an identical entry short-circuits the rules.
+    let world = World::new();
+    let signed = world.preferences(address(0xaa), 30_000_000);
+    assert!(world.market.record_preferences(signed.clone(), HEAD_SLOT));
+
+    let reply = send(&world.app(None), preferences_request(&[signed], None)).await;
+
+    assert_eq!(reply.status, StatusCode::OK);
+    assert!(
+        world
+            .network
+            .proposer_preferences
+            .lock()
+            .unwrap()
+            .is_empty()
+    );
+}
+
+#[tokio::test]
+async fn different_preferences_for_a_held_key_fail_with_their_position() {
+    let world = World::new();
+    let first = world.preferences(address(0xaa), 30_000_000);
+    let second = world.preferences(address(0xbb), 30_000_000);
+
+    let reply = send(
+        &world.app(None),
+        preferences_request(&[first.clone(), second], Some("gloas")),
+    )
+    .await;
+
+    assert_eq!(reply.status, StatusCode::BAD_REQUEST);
+    let json = reply.json();
+    assert_eq!(json["code"], 400);
+    assert_eq!(json["failures"].as_array().unwrap().len(), 1);
+    assert_eq!(json["failures"][0]["index"], 1);
+    assert_eq!(json["failures"][0]["message"], "ignore: already_seen");
+    // The first still went out.
+    assert_eq!(
+        *world.network.proposer_preferences.lock().unwrap(),
+        vec![first]
+    );
+}
+
+#[tokio::test]
+async fn preferences_by_the_wrong_proposer_or_signer_fail_and_the_rest_still_go_out() {
+    let world = World::new();
+    let proposer = world.prepared().proposer;
+    let wrong_proposer = world.preferences_by((proposer + 1) % 64, address(0xaa), 30_000_000);
+    let mut bad_signature = world.preferences(address(0xaa), 30_000_000);
+    bad_signature.signature = BlsSignature::default();
+    let good = world.preferences(address(0xcc), 30_000_000);
+
+    let reply = send(
+        &world.app(None),
+        preferences_request(
+            &[wrong_proposer, bad_signature, good.clone()],
+            Some("gloas"),
+        ),
+    )
+    .await;
+
+    assert_eq!(reply.status, StatusCode::BAD_REQUEST);
+    let json = reply.json();
+    let failures: Vec<(u64, String)> = json["failures"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|failure| {
+            (
+                failure["index"].as_u64().unwrap(),
+                failure["message"].as_str().unwrap().to_string(),
+            )
+        })
+        .collect();
+    assert_eq!(
+        failures,
+        vec![
+            (0, "reject: wrong_proposer".to_string()),
+            (1, "reject: bad_signature".to_string()),
+        ]
+    );
+    assert_eq!(
+        *world.network.proposer_preferences.lock().unwrap(),
+        vec![good]
+    );
+}
+
+#[tokio::test]
+async fn preferences_are_accepted_as_an_ssz_list_and_the_header_may_be_fulu() {
+    let world = World::new();
+    let signed = world.preferences(address(0xaa), 30_000_000);
+    let request = Request::post("/eth/v1/validator/proposer_preferences")
+        .header("content-type", crate::SSZ_CONTENT_TYPE)
+        .header("eth-consensus-version", "fulu")
+        .body(Body::from(vec![signed.clone()].to_ssz()))
+        .unwrap();
+
+    let reply = send(&world.app(None), request).await;
+
+    assert_eq!(
+        reply.status,
+        StatusCode::OK,
+        "{}",
+        String::from_utf8_lossy(&reply.body)
+    );
+    assert_eq!(
+        *world.network.proposer_preferences.lock().unwrap(),
+        vec![signed]
+    );
+}
+
+#[tokio::test]
+async fn preferences_with_a_bad_header_type_body_or_count_are_refused() {
+    let world = World::new();
+    let app = world.app(None);
+    let signed = world.preferences(address(0xaa), 30_000_000);
+
+    for version in ["electra", "nonsense"] {
+        let reply = send(&app, preferences_request(&[signed.clone()], Some(version))).await;
+        assert_eq!(reply.status, StatusCode::BAD_REQUEST, "{version}");
+    }
+    let plain = Request::post("/eth/v1/validator/proposer_preferences")
+        .header("content-type", "text/plain")
+        .body(Body::from("[]"))
+        .unwrap();
+    assert_eq!(
+        send(&app, plain).await.status,
+        StatusCode::UNSUPPORTED_MEDIA_TYPE
+    );
+    let garbage = Request::post("/eth/v1/validator/proposer_preferences")
+        .body(Body::from("not json"))
+        .unwrap();
+    assert_eq!(send(&app, garbage).await.status, StatusCode::BAD_REQUEST);
+
+    // One over the list bound is refused before any entry is looked at.
+    let many =
+        vec![signed; ethlambda_state_transition::beacon::preset::PROPOSER_LOOKAHEAD_LENGTH + 1];
+    let reply = send(&app, preferences_request(&many, Some("gloas"))).await;
+    assert_eq!(reply.status, StatusCode::BAD_REQUEST);
+    assert!(
+        world
+            .network
+            .proposer_preferences
+            .lock()
+            .unwrap()
+            .is_empty()
+    );
+}
+
+// ---------------------------------------------------------------------------
+// Routing
+// ---------------------------------------------------------------------------
+
+/// Every new route answers something other than a 500 through the layer set
+/// the production server applies, and the phase-2 route does not exist.
+#[tokio::test]
+async fn the_new_routes_are_wired_with_the_production_layers() {
+    use ethlambda_blockchain::{SyncStatusController, metrics::SyncStatus};
+
+    let world = World::new();
+    let router =
+        crate::build_beacon_api_router(world.store.clone(), "ethlambda/test", "peer".into())
+            .layer(Extension(SyncStatusController::new(SyncStatus::Synced)))
+            .layer(Extension::(world.network.clone()))
+            .layer(Extension(SharedAttestationPool::default()))
+            .layer(Extension(SharedPayloadAttestationPool::default()))
+            .layer(Extension(world.market.clone()))
+            .layer(Extension(CustodyColumns::default()))
+            .layer(Extension(FeeRecipients::default()))
+            .layer(Extension(None::));
+
+    for (uri, body) in [
+        ("/eth/v1/beacon/execution_payload_bids", "{}"),
+        ("/eth/v1/validator/proposer_preferences", "[]"),
+        ("/eth/v1/beacon/states/head/builders", ""),
+    ] {
+        let reply = send(&router, Request::post(uri).body(Body::from(body)).unwrap()).await;
+        assert_ne!(reply.status, StatusCode::INTERNAL_SERVER_ERROR, "{uri}");
+        assert_ne!(reply.status, StatusCode::NOT_FOUND, "{uri}");
+    }
+    let reply = send(
+        &router,
+        Request::post("/eth/v1/validator/builder_preferences")
+            .body(Body::from("[]"))
+            .unwrap(),
+    )
+    .await;
+    assert_eq!(reply.status, StatusCode::NOT_FOUND);
+}
diff --git a/crates/net/rpc/src/beacon/gloas_proposal.rs b/crates/net/rpc/src/beacon/gloas_proposal.rs
index d12245d05..3c9a3d1d6 100644
--- a/crates/net/rpc/src/beacon/gloas_proposal.rs
+++ b/crates/net/rpc/src/beacon/gloas_proposal.rs
@@ -1839,3 +1839,7 @@ mod tests {
         assert_eq!(rejected.envelopes.lock().unwrap().len(), 1);
     }
 }
+
+#[cfg(test)]
+#[path = "builder_market_tests.rs"]
+mod builder_market_tests;

From 834640e3abd83b723d71f28d1347015af51a0549 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Mon, 5 Oct 2026 23:00:55 -0300
Subject: [PATCH 120/132] style(rpc): satisfy clippy in the builder market code
 and tests

---
 crates/net/rpc/src/beacon/builder_market_tests.rs | 15 ++++++++++++---
 crates/net/rpc/src/beacon/gloas_proposal.rs       |  2 ++
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/crates/net/rpc/src/beacon/builder_market_tests.rs b/crates/net/rpc/src/beacon/builder_market_tests.rs
index b43efd66a..26876e6e8 100644
--- a/crates/net/rpc/src/beacon/builder_market_tests.rs
+++ b/crates/net/rpc/src/beacon/builder_market_tests.rs
@@ -667,7 +667,8 @@ async fn a_failed_local_build_falls_back_to_a_viable_bid() {
 #[tokio::test]
 async fn bids_that_do_not_fit_the_slot_are_left_to_the_local_payload() {
     // Each would win on value (factor MAX against a 1 wei local build).
-    let mismatches: [(&str, fn(&mut ExecutionPayloadBid)); 3] = [
+    type Change = fn(&mut ExecutionPayloadBid);
+    let mismatches: [(&str, Change); 3] = [
         ("randao", |bid| bid.prev_randao = H256::repeat_byte(0x99)),
         ("parent hash", |bid| {
             bid.parent_block_hash = H256::repeat_byte(0x98)
@@ -1007,7 +1008,11 @@ async fn valid_preferences_are_cached_and_published_once() {
     let app = world.app(None);
     let signed = world.preferences(address(0xaa), 30_000_000);
 
-    let reply = send(&app, preferences_request(&[signed.clone()], Some("gloas"))).await;
+    let reply = send(
+        &app,
+        preferences_request(std::slice::from_ref(&signed), Some("gloas")),
+    )
+    .await;
 
     assert_eq!(
         reply.status,
@@ -1152,7 +1157,11 @@ async fn preferences_with_a_bad_header_type_body_or_count_are_refused() {
     let signed = world.preferences(address(0xaa), 30_000_000);
 
     for version in ["electra", "nonsense"] {
-        let reply = send(&app, preferences_request(&[signed.clone()], Some(version))).await;
+        let reply = send(
+            &app,
+            preferences_request(std::slice::from_ref(&signed), Some(version)),
+        )
+        .await;
         assert_eq!(reply.status, StatusCode::BAD_REQUEST, "{version}");
     }
     let plain = Request::post("/eth/v1/validator/proposer_preferences")
diff --git a/crates/net/rpc/src/beacon/gloas_proposal.rs b/crates/net/rpc/src/beacon/gloas_proposal.rs
index 3c9a3d1d6..59d5b7605 100644
--- a/crates/net/rpc/src/beacon/gloas_proposal.rs
+++ b/crates/net/rpc/src/beacon/gloas_proposal.rs
@@ -373,6 +373,8 @@ fn blobs_list(blobs: Vec>) -> Result {
 }
 
 /// What `produceBlockV4` built for one block.
+// Short-lived (one per request), so boxing the larger variant buys nothing.
+#[allow(clippy::large_enum_variant)]
 enum Production {
     /// A block on this node's own payload, with the envelope that reveals it.
     Local(Produced),

From 3c6eb700eefa8c7cf915304c9d57418a541c50b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Mon, 5 Oct 2026 23:17:12 -0300
Subject: [PATCH 121/132] docs(claude): describe the gloas builder market and
 its two extra topics

CLAUDE.md still said gloas added two topics and that duties were self-build only.
---
 CLAUDE.md | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/CLAUDE.md b/CLAUDE.md
index 8bbb0375c..87203def7 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -334,9 +334,11 @@ actual_slot = finalized_slot + 1 + relative_index
   - Beacon wire: `validate_messages()` is on, so every beacon message waits for a verdict (~4.2s before gossipsub's cache evicts it). Rules in `state_transition::beacon::gossip` (cheap half inline, stateful half on a bounded `spawn_blocking` task); plumbing in `p2p/src/beacon/verdict.rs`. Lean gossip still auto-forwards
   - Data columns: every check runs in p2p. A column gossip did not accept (`Queue`/`Overloaded`), every fetched column, and parked columns replayed after their parent imports go through `column::chain_checks` in `p2p/src/beacon/column_checks.rs`. The chain actor stores what it gets unchecked; only debug builds re-run `chain_checks` there
   - Beacon subscribes seven global topics plus two node-id-derived subnet families: custody
-    columns and backbone attestation subnets. Gloas digests add two more topics,
-    `execution_payload` and `payload_attestation_message` (`BeaconTopics::for_fork`; earlier
-    digests never carry them). Gloas has its own rules for `beacon_block`,
+    columns and backbone attestation subnets. Gloas digests add four more topics,
+    `execution_payload`, `payload_attestation_message`, `execution_payload_bid` and
+    `proposer_preferences` (`BeaconTopics::for_fork`; earlier digests never carry them).
+    Bids and preferences are validated in p2p against one shared `BuilderMarket`
+    (`state_transition::beacon::builder_market`) and never reach the chain actor. Gloas has its own rules for `beacon_block`,
     `data_column_sidecar` (fork enum `DataColumnSidecar`, fork from the topic's digest),
     aggregates (`SignedAggregateAndProof::Gloas`, aggregation-bits length bounded before
     expansion) and attestations (`verify_attestation_payload_status`). Deliberate
@@ -768,8 +770,13 @@ transitions are in `ethlambda-types`, per the section above. Nothing above
   the parking of gloas column sidecars (they carry no signature). Fork-choice
   events are timed at arrival, not at the slot tick, and the head's payload
   status is kept with its root (`Store::head_payload_status`, recomputed after
-  a restart). The node also serves gloas validator duties, self-build only (no
-  bids, no proposer preferences; see `docs/spec_deviations.md`).
+  a restart). The node also serves gloas validator duties and the builder market:
+  it validates, pools and relays gossip bids and proposer preferences, serves
+  `POST /eth/v1/beacon/execution_payload_bids` and `/proposer_preferences` plus the
+  builder endpoints, and `produceBlockV4` weighs the best pooled p2p bid against
+  the local build per `BuilderConfig` (`min_bid`, `builder_boost_factor`; the local
+  build wins a tie). A winning bid returns the block only. The builder API (relay
+  or `builder_pubkeys` bids) is phase 2; see `docs/spec_deviations.md`.
   `state_transition/src/beacon/gloas_block_production.rs` assembles the block and
   its envelope (`gloas_payload_inputs`, `parse_gloas_execution_requests`,
   `pack_gloas_attestations`, `pack_payload_attestations`, `assemble_gloas_block`,

From 1e63598014e4634d88153a7ae2c2673c98285638 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Mon, 5 Oct 2026 23:21:30 -0300
Subject: [PATCH 122/132] test(p2p): make the second bid in the settle test
 outbid the first

The spec ignores a bid whose value does not exceed the best recorded for the
same (slot, parent hash, parent root), so a value-10 bid after a value-10 one
answered NotHighestBid, not Accept. The fixture was wrong; the rule is spec.
---
 crates/net/p2p/src/beacon/verdict.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/crates/net/p2p/src/beacon/verdict.rs b/crates/net/p2p/src/beacon/verdict.rs
index ef263492a..4810155d9 100644
--- a/crates/net/p2p/src/beacon/verdict.rs
+++ b/crates/net/p2p/src/beacon/verdict.rs
@@ -1224,8 +1224,10 @@ mod tests {
             settle(&mut server, Outcome::Accept, &object),
             Outcome::Ignore(IgnoreReason::AlreadySeen)
         );
-        // Anything but an accept records nothing.
-        let other = bid_object(&server, bid(5, 4, 10));
+        // Anything but an accept records nothing. The value must beat the
+        // best one recorded for the same (slot, parent), or the spec's
+        // highest-bid rule would ignore it on its own account.
+        let other = bid_object(&server, bid(5, 4, 11));
         assert_eq!(
             settle(
                 &mut server,

From 3ff21660025779f6d79f6425c95dad71a7fa75b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Mon, 5 Oct 2026 23:56:31 -0300
Subject: [PATCH 123/132] test(state-transition): start the builder scene at a
 reachable finalized epoch

The scene put a funded, active builder in an epoch-1 state: it is active only
once the finalized epoch passes its deposit epoch, but a chain never finalizes
past the previous epoch, so no real chain holds that state. Advancing it
through the end of epoch 1 underflows get_previous_epoch - finalized epoch,
which the spec treats as invalid and a stricter get_finality_delay now errors on.

scene_at builds the parent in a later epoch by advancing with nothing
finalized and finalizing epoch 1 afterwards. The epoch-crossing bid test uses
it with its slots and epochs shifted by one epoch; its assertions are unchanged.
---
 .../beacon/gossip/execution_payload_bid.rs    | 10 +--
 .../src/beacon/gossip/test_support.rs         | 61 +++++++++++++++----
 2 files changed, 56 insertions(+), 15 deletions(-)

diff --git a/crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs b/crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs
index 59ad8c922..5e5340942 100644
--- a/crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs
+++ b/crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs
@@ -962,8 +962,10 @@ mod tests {
 
     #[test]
     fn a_bid_across_an_epoch_uses_and_caches_the_checkpoint_state() {
-        let scene = scene();
-        let slot = 64;
+        // The parent is in epoch 2: a funded builder is active only after
+        // finality passes its deposit epoch, which an epoch-1 chain cannot have.
+        let scene = scene_at(64, |_| {});
+        let slot = 96;
         // Preferences for the later epoch's proposer.
         let dependent = dependent_root_at(&scene.state, PARENT, slot).expect("in the window");
         let proposer = fixed_proposer(&scene.state, slot).expect("in the lookahead window");
@@ -980,13 +982,13 @@ mod tests {
         assert!(scene.market.record_preferences(preferences, slot - 1));
         let bid = scene.signed(|bid| bid.slot = slot);
         let key = CacheKey::CheckpointState {
-            epoch: 2,
+            epoch: 3,
             root: PARENT,
         };
         assert!(scene.store.cached_state(key).is_none());
         assert_eq!(stateful(&scene, &bid), Outcome::Accept);
         let cached = scene.store.cached_state(key).expect("the advanced state");
-        assert_eq!(get_current_epoch(&cached), 2);
+        assert_eq!(get_current_epoch(&cached), 3);
         // A second bid finds it.
         assert_eq!(stateful(&scene, &bid), Outcome::Accept);
     }
diff --git a/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs b/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs
index 451320a6c..7d9bf6378 100644
--- a/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs
+++ b/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs
@@ -116,6 +116,13 @@ pub(crate) mod builder_scene {
 
     pub(crate) const PARENT: Root = Root::repeat_byte(0x50);
     pub(crate) const BID_SLOT: Slot = 34;
+    /// The slot the scene's parent sits at: epoch 1, whose state can hold a
+    /// funded builder only with a finalized epoch that a real chain cannot have
+    /// there (a builder is active once the finalized epoch passes its deposit
+    /// epoch, and the finalized epoch never exceeds the previous one). Fine for
+    /// a rule that reads the state as it is; a test that advances it through
+    /// an epoch's end needs [`scene_at`] with a parent in epoch 2 or later.
+    pub(crate) const PARENT_SLOT: Slot = 32;
     pub(crate) const PARENT_GAS_LIMIT: u64 = 30_000_000;
 
     pub(crate) fn fee_recipient() -> ExecutionAddress {
@@ -147,7 +154,7 @@ pub(crate) mod builder_scene {
 
     /// An empty store, gloas from genesis, anchored (and headed) at `PARENT`,
     /// whose block the caller must store.
-    pub(crate) fn gloas_store() -> Store {
+    fn gloas_store_at(parent_slot: Slot) -> Store {
         Store::init_beacon(
             Arc::new(InMemoryBackend::new()),
             GENESIS_TIME,
@@ -155,9 +162,9 @@ pub(crate) mod builder_scene {
             PARENT,
             Checkpoint {
                 root: PARENT,
-                slot: 32,
+                slot: parent_slot,
             },
-            32,
+            parent_slot,
         )
     }
 
@@ -170,10 +177,35 @@ pub(crate) mod builder_scene {
         pub bid: gloas::SignedExecutionPayloadBid,
     }
 
+    /// `PARENT`'s post-state at `parent_slot`, with an active funded builder 0.
+    /// Reached by advancing the epoch-1 state while nothing is finalized, so
+    /// every epoch transition on the way is one a real chain takes, then
+    /// finalizing epoch 1: reachable once `parent_slot` is in epoch 2 or later.
+    fn parent_state_at(parent_slot: Slot) -> BeaconState {
+        let mut state = test_support::gloas_state_with_builder(0, 100_000_000_000, 0);
+        if parent_slot == PARENT_SLOT {
+            return state;
+        }
+        let BeaconState::Gloas(inner) = &mut state else {
+            unreachable!("built as gloas")
+        };
+        inner.finalized_checkpoint.epoch = 0;
+        let config = gloas_support::config();
+        let mut state =
+            crate::beacon::block_production::advance_to_slot(&state, parent_slot, &config)
+                .expect("advance to the parent's slot");
+        let BeaconState::Gloas(inner) = &mut state else {
+            unreachable!("built as gloas")
+        };
+        inner.finalized_checkpoint.epoch = 1;
+        inner.latest_block_header.slot = inner.slot;
+        state
+    }
+
     impl Scene {
         /// 100 ms into the bid's slot.
         pub(crate) fn now_ms(&self) -> u64 {
-            slot_start_ms(&self.store, BID_SLOT) + 100
+            slot_start_ms(&self.store, self.bid.message.slot) + 100
         }
 
         /// The scene's bid after `edit`, re-signed by its builder.
@@ -190,13 +222,20 @@ pub(crate) mod builder_scene {
     /// The scene with `edit` applied to the parent's state before it is stored,
     /// and the prerequisites of a passing bid recorded in the market.
     pub(crate) fn scene_with(edit: impl FnOnce(&mut gloas::BeaconState)) -> Scene {
-        let mut state = test_support::gloas_state_with_builder(0, 100_000_000_000, 0);
+        scene_at(PARENT_SLOT, edit)
+    }
+
+    /// [`scene_with`] for a parent at `parent_slot`, whose bid is for the slot
+    /// two after it.
+    pub(crate) fn scene_at(parent_slot: Slot, edit: impl FnOnce(&mut gloas::BeaconState)) -> Scene {
+        let bid_slot = parent_slot + (BID_SLOT - PARENT_SLOT);
+        let mut state = parent_state_at(parent_slot);
         let BeaconState::Gloas(inner) = &mut state else {
             unreachable!("built as gloas")
         };
         edit(inner);
         state.apply_pending_mutations();
-        let mut store = gloas_store();
+        let mut store = gloas_store_at(parent_slot);
         store
             .insert_pending_block(PARENT, block_at(state.slot()))
             .expect("insert the parent");
@@ -208,20 +247,20 @@ pub(crate) mod builder_scene {
             .expect("move the head");
         store.set_head_payload_status(PARENT, PayloadStatus::Full);
 
-        let dependent_root = dependent_root_at(&state, PARENT, BID_SLOT).expect("in the window");
-        let proposer = crate::beacon::precheck::fixed_proposer(&state, BID_SLOT).expect("window");
+        let dependent_root = dependent_root_at(&state, PARENT, bid_slot).expect("in the window");
+        let proposer = crate::beacon::precheck::fixed_proposer(&state, bid_slot).expect("window");
         let market = BuilderMarket::default();
         let preferences = test_support::sign_preferences(
             &state,
             gloas::ProposerPreferences {
                 dependent_root,
-                proposal_slot: BID_SLOT,
+                proposal_slot: bid_slot,
                 validator_index: proposer,
                 fee_recipient: fee_recipient(),
                 target_gas_limit: PARENT_GAS_LIMIT,
             },
         );
-        assert!(market.record_preferences(preferences, BID_SLOT - 1));
+        assert!(market.record_preferences(preferences, bid_slot - 1));
         market.record_execution_payload(&test_support::envelope_with_gas_limit(
             parent_block_hash(),
             PARENT_GAS_LIMIT,
@@ -238,7 +277,7 @@ pub(crate) mod builder_scene {
                 fee_recipient: fee_recipient(),
                 gas_limit: PARENT_GAS_LIMIT,
                 builder_index: 0,
-                slot: BID_SLOT,
+                slot: bid_slot,
                 value: 1,
                 ..Default::default()
             },

From 194deb08f5c7a75d2155f3876c59d5960bb3ca0f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Wed, 7 Oct 2026 16:31:33 -0300
Subject: [PATCH 124/132] test(state-transition): derive the builder scene's
 slots from the preset

The bid and preferences scenes hardcoded mainnet slots (parent 32, bid 34,
epochs at 64 and 96), but their base state sits at SLOTS_PER_EPOCH. On the
minimal preset that is slot 8, so every bid slot fell outside the parent's
two-epoch proposer lookahead and 43 tests panicked or hit the wrong rule.
Express each slot through SLOTS_PER_EPOCH and the scene's own PARENT_SLOT,
as the other gossip tests do, and place the stray and stateless dependent
blocks before the dependent slot so they reach the rule they test.
---
 .../beacon/gossip/execution_payload_bid.rs    | 28 ++++++-----
 .../src/beacon/gossip/proposer_preferences.rs | 47 ++++++++++++-------
 .../src/beacon/gossip/test_support.rs         | 13 ++---
 3 files changed, 53 insertions(+), 35 deletions(-)

diff --git a/crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs b/crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs
index 5e5340942..8ee152be3 100644
--- a/crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs
+++ b/crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs
@@ -477,6 +477,7 @@ mod tests {
     use crate::beacon::gossip::test_support::{slot_start_ms, store};
     use crate::beacon::helpers::accessors::get_current_epoch;
     use crate::beacon::precheck::fixed_proposer;
+    use crate::beacon::preset::SLOTS_PER_EPOCH;
 
     fn ignore(reason: IgnoreReason) -> Outcome {
         Outcome::Ignore(reason)
@@ -583,7 +584,7 @@ mod tests {
     fn a_bid_is_timely_from_just_before_the_previous_slot_until_just_after_its_own() {
         let store = store(0);
         let config = store.config();
-        let slot = 34;
+        let slot = BID_SLOT;
         let earliest = slot_start_ms(&store, slot - 1) - 500;
         let latest = slot_start_ms(&store, slot + 1) + 500;
         assert!(is_current_or_next_slot(&config, slot, earliest));
@@ -634,7 +635,7 @@ mod tests {
     #[test]
     fn a_bid_for_a_distant_slot_is_not_current_or_next() {
         let scene = scene();
-        let far = scene.signed(|bid| bid.slot = 40);
+        let far = scene.signed(|bid| bid.slot = BID_SLOT + 6);
         assert_eq!(
             cheap(&scene, &far),
             Err(ignore(IgnoreReason::NotCurrentOrNextSlot))
@@ -702,7 +703,7 @@ mod tests {
     #[test]
     fn a_bid_not_after_its_parent_is_rejected() {
         let scene = scene();
-        let bid = scene.signed(|bid| bid.slot = 32);
+        let bid = scene.signed(|bid| bid.slot = PARENT_SLOT);
         assert_eq!(stateful(&scene, &bid), reject(RejectReason::NotAfterParent));
     }
 
@@ -712,7 +713,7 @@ mod tests {
         let stateless = Root::repeat_byte(0x60);
         scene
             .store
-            .insert_pending_block(stateless, block_at(10))
+            .insert_pending_block(stateless, block_at(PARENT_SLOT))
             .expect("insert the block");
         let bid = scene.signed(|bid| bid.parent_block_root = stateless);
         assert_eq!(
@@ -725,7 +726,7 @@ mod tests {
     fn a_bid_beyond_the_parents_lookahead_is_ignored() {
         let scene = scene();
         // The parent is in epoch 1, so epoch 3 is past its lookahead.
-        let bid = scene.signed(|bid| bid.slot = 96);
+        let bid = scene.signed(|bid| bid.slot = 3 * SLOTS_PER_EPOCH);
         assert_eq!(
             stateful(&scene, &bid),
             ignore(IgnoreReason::BeyondLookahead)
@@ -897,7 +898,7 @@ mod tests {
             message: envelope_with_gas_limit(parent_block_hash(), PARENT_GAS_LIMIT, PARENT, exits),
             signature: Default::default(),
         };
-        scene.store.insert_verified_payload(32, &envelope);
+        scene.store.insert_verified_payload(PARENT_SLOT, &envelope);
         assert_eq!(
             stateful(&scene, &scene.bid),
             ignore(IgnoreReason::BuilderMayExit)
@@ -964,8 +965,8 @@ mod tests {
     fn a_bid_across_an_epoch_uses_and_caches_the_checkpoint_state() {
         // The parent is in epoch 2: a funded builder is active only after
         // finality passes its deposit epoch, which an epoch-1 chain cannot have.
-        let scene = scene_at(64, |_| {});
-        let slot = 96;
+        let scene = scene_at(2 * SLOTS_PER_EPOCH, |_| {});
+        let slot = 3 * SLOTS_PER_EPOCH;
         // Preferences for the later epoch's proposer.
         let dependent = dependent_root_at(&scene.state, PARENT, slot).expect("in the window");
         let proposer = fixed_proposer(&scene.state, slot).expect("in the lookahead window");
@@ -1010,8 +1011,9 @@ mod tests {
             .with_fork_epoch(ForkName::Gloas, 2);
         let parent = Root::repeat_byte(0x70);
         let header_hash = ExecutionBlockHash::repeat_byte(0x41);
+        let parent_slot = 2 * SLOTS_PER_EPOCH - 2;
         let mut state = with_signing_validators_at(ForkName::Fulu, 64);
-        *state.slot_mut() = 62;
+        *state.slot_mut() = parent_slot;
         if let BeaconState::Fulu(fulu) = &mut state {
             fulu.latest_execution_payload_header.block_hash = header_hash;
             fulu.latest_execution_payload_header.gas_limit = 30_000_000;
@@ -1026,12 +1028,12 @@ mod tests {
             parent,
             ethlambda_types::checkpoint::Checkpoint {
                 root: parent,
-                slot: 62,
+                slot: parent_slot,
             },
-            62,
+            parent_slot,
         );
         store
-            .insert_pending_block(parent, block_at(62))
+            .insert_pending_block(parent, block_at(parent_slot))
             .expect("insert the parent");
         store
             .insert_state(parent, state.clone())
@@ -1042,7 +1044,7 @@ mod tests {
         // The status the chain actor records; a pre-gloas head has one node.
         store.set_head_payload_status(parent, PayloadStatus::Empty);
 
-        let slot = 64;
+        let slot = 2 * SLOTS_PER_EPOCH;
         let market = BuilderMarket::default();
         let proposer = fixed_proposer(&state, slot).expect("in the window");
         let dependent_root = dependent_root_at(&state, parent, slot).expect("in the window");
diff --git a/crates/blockchain/state_transition/src/beacon/gossip/proposer_preferences.rs b/crates/blockchain/state_transition/src/beacon/gossip/proposer_preferences.rs
index a3bb007a8..21abad0fa 100644
--- a/crates/blockchain/state_transition/src/beacon/gossip/proposer_preferences.rs
+++ b/crates/blockchain/state_transition/src/beacon/gossip/proposer_preferences.rs
@@ -265,14 +265,17 @@ mod tests {
     use crate::beacon::gossip::test_support::builder_scene::*;
     use crate::beacon::gossip::test_support::{slot_start_ms, store};
     use crate::beacon::helpers::test_state::{secret_key_for, with_signing_validators_at};
+    use crate::beacon::preset::SLOTS_PER_EPOCH;
     use crate::beacon::primitives::ValidatorIndex;
     use crate::beacon::stf;
 
     /// The epoch-2 slot the scene's proposer preferences name, and the block
     /// that fixed its proposer: the genesis-slot block, an ancestor of the
-    /// head (slot 32) whose `block_roots` entry covers slot 31.
-    const PROPOSAL_SLOT: Slot = 64;
+    /// head (`PARENT_SLOT`) whose `block_roots` entry covers `DEPENDENT_SLOT`.
+    const PROPOSAL_SLOT: Slot = 2 * SLOTS_PER_EPOCH;
     const DEPENDENT: Root = Root::repeat_byte(0x31);
+    /// `PROPOSAL_SLOT`'s shuffling dependent slot: the last of epoch 0.
+    const DEPENDENT_SLOT: Slot = PARENT_SLOT - 1;
 
     fn ignore(reason: IgnoreReason) -> Outcome {
         Outcome::Ignore(reason)
@@ -283,9 +286,9 @@ mod tests {
     }
 
     /// The builder scene, with the parent's `block_roots` naming `DEPENDENT` at
-    /// slot 31, and `DEPENDENT` stored (block and state) as the head's parent.
+    /// `DEPENDENT_SLOT`, and `DEPENDENT` stored (block and state) as the head's parent.
     fn dependent_scene() -> Scene {
-        let mut scene = scene_with(|state| state.block_roots[31] = DEPENDENT);
+        let mut scene = scene_with(|state| state.block_roots[DEPENDENT_SLOT as usize] = DEPENDENT);
         scene
             .store
             .insert_pending_block(DEPENDENT, block_at(0))
@@ -294,7 +297,7 @@ mod tests {
         // that a dependent block is possible reads the live chain.
         scene
             .store
-            .insert_signed_block(PARENT, block_with_parent(32, DEPENDENT))
+            .insert_signed_block(PARENT, block_with_parent(PARENT_SLOT, DEPENDENT))
             .expect("link the head to the dependent block");
         let mut dependent_state = scene.state.clone();
         *dependent_state.slot_mut() = 0;
@@ -353,7 +356,11 @@ mod tests {
     fn a_second_message_for_a_key_is_already_seen() {
         let scene = dependent_scene();
         let preferences = valid_preferences(&scene);
-        assert!(scene.market.record_preferences(preferences.clone(), 33));
+        assert!(
+            scene
+                .market
+                .record_preferences(preferences.clone(), PARENT_SLOT + 1)
+        );
         assert_eq!(
             cheap_checks(&scene.market, &scene.store, &preferences, now_ms(&scene)),
             Err(ignore(IgnoreReason::AlreadySeen))
@@ -385,9 +392,9 @@ mod tests {
     fn preferences_are_early_before_the_lookahead_opens() {
         let scene = dependent_scene();
         let mut preferences = valid_preferences(&scene);
-        // Epoch 3's lookahead opens with epoch 2, at slot 64.
-        preferences.message.proposal_slot = 96;
-        let opens = slot_start_ms(&scene.store, 64);
+        // Epoch 3's lookahead opens with epoch 2's first slot.
+        preferences.message.proposal_slot = 3 * SLOTS_PER_EPOCH;
+        let opens = slot_start_ms(&scene.store, 2 * SLOTS_PER_EPOCH);
         let check = |now| cheap_checks(&scene.market, &scene.store, &preferences, now);
         assert_eq!(check(opens - 500), Ok(()));
         assert_eq!(
@@ -414,7 +421,7 @@ mod tests {
         let stateless = Root::repeat_byte(0x62);
         scene
             .store
-            .insert_pending_block(stateless, block_at(20))
+            .insert_pending_block(stateless, block_at(DEPENDENT_SLOT - 1))
             .expect("insert the block");
         let preferences = preferences_for(&scene, stateless, 0);
         assert_eq!(
@@ -426,7 +433,7 @@ mod tests {
     #[test]
     fn a_dependent_block_after_the_dependent_slot_is_rejected() {
         let scene = dependent_scene();
-        // The head itself is at slot 32, after slot 31.
+        // The head itself is at `PARENT_SLOT`, after `DEPENDENT_SLOT`.
         let preferences = preferences_for(&scene, PARENT, 0);
         assert_eq!(
             stateful_checks(&scene.store, &preferences),
@@ -440,7 +447,7 @@ mod tests {
         let stray = Root::repeat_byte(0x63);
         scene
             .store
-            .insert_pending_block(stray, block_at(10))
+            .insert_pending_block(stray, block_at(DEPENDENT_SLOT - 1))
             .expect("insert the block");
         scene
             .store
@@ -451,8 +458,16 @@ mod tests {
             stateful_checks(&scene.store, &preferences),
             ignore(IgnoreReason::ImpossibleDependentRoot)
         );
-        assert!(!is_valid_dependent_root(&scene.store, stray, 31));
-        assert!(is_valid_dependent_root(&scene.store, PARENT, 31));
+        assert!(!is_valid_dependent_root(
+            &scene.store,
+            stray,
+            DEPENDENT_SLOT
+        ));
+        assert!(is_valid_dependent_root(
+            &scene.store,
+            PARENT,
+            DEPENDENT_SLOT
+        ));
     }
 
     #[test]
@@ -493,7 +508,7 @@ mod tests {
             .cached_state(CacheKey::BlockState(DEPENDENT))
             .expect("stored");
         let mut owned = (*advanced).clone();
-        stf::process_slots(&mut owned, 32, &scene.store.config()).expect("advance");
+        stf::process_slots(&mut owned, PARENT_SLOT, &scene.store.config()).expect("advance");
         advanced = std::sync::Arc::new(owned);
         let proposer =
             crate::beacon::precheck::fixed_proposer(&advanced, PROPOSAL_SLOT).expect("window");
@@ -584,7 +599,7 @@ mod tests {
 
         // A signature under either verifies; one under neither does not.
         let preferences = gloas::ProposerPreferences {
-            proposal_slot: 64,
+            proposal_slot: 2 * SLOTS_PER_EPOCH,
             ..Default::default()
         };
         let message_root = preferences.hash_tree_root();
diff --git a/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs b/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs
index 7d9bf6378..f7e8a985a 100644
--- a/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs
+++ b/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs
@@ -99,9 +99,9 @@ pub(crate) fn fulu_parent(proposer: ValidatorIndex) -> BeaconState {
 }
 
 /// The builder-market rules' shared scene: a gloas chain whose head `PARENT`
-/// (slot 32, epoch 1) is a FULL block with a funded active builder 0, the
-/// proposer preferences and parent payload gossip would have delivered, and a
-/// signed bid for slot 34 that passes every rule.
+/// (`PARENT_SLOT`, epoch 1) is a FULL block with a funded active builder 0,
+/// the proposer preferences and parent payload gossip would have delivered,
+/// and a signed bid for `BID_SLOT` that passes every rule.
 pub(crate) mod builder_scene {
     use ethlambda_storage::ForkCheckpoints;
     use ethlambda_types::beacon::containers::{SignedBeaconBlock, electra, gloas};
@@ -112,17 +112,18 @@ pub(crate) mod builder_scene {
     use crate::beacon::gloas_block_production::test_support as gloas_support;
     use crate::beacon::gossip::proposer_preferences::dependent_root_at;
     use crate::beacon::helpers::accessors::{get_current_epoch, get_randao_mix};
+    use crate::beacon::preset;
     use crate::beacon::primitives::{ExecutionAddress, ExecutionBlockHash};
 
     pub(crate) const PARENT: Root = Root::repeat_byte(0x50);
-    pub(crate) const BID_SLOT: Slot = 34;
-    /// The slot the scene's parent sits at: epoch 1, whose state can hold a
+    pub(crate) const BID_SLOT: Slot = PARENT_SLOT + 2;
+    /// The slot the scene's parent sits at: epoch 1's first, whose state can hold a
     /// funded builder only with a finalized epoch that a real chain cannot have
     /// there (a builder is active once the finalized epoch passes its deposit
     /// epoch, and the finalized epoch never exceeds the previous one). Fine for
     /// a rule that reads the state as it is; a test that advances it through
     /// an epoch's end needs [`scene_at`] with a parent in epoch 2 or later.
-    pub(crate) const PARENT_SLOT: Slot = 32;
+    pub(crate) const PARENT_SLOT: Slot = preset::SLOTS_PER_EPOCH;
     pub(crate) const PARENT_GAS_LIMIT: u64 = 30_000_000;
 
     pub(crate) fn fee_recipient() -> ExecutionAddress {

From 1c36c339ce4d61b4480361c304c46ee72ca124a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Mon, 19 Oct 2026 20:36:13 -0300
Subject: [PATCH 125/132] feat(rpc): honor skip_randao_verification on block
 production

beacon-APIs defines `skip_randao_verification` on produceBlockV3 and
produceBlockV4 as a flag with no value that asks the node to build on the
point-at-infinity reveal without checking it. We accepted the flag on v4
and ignored it, and v3 did not read it, so a request carrying it with the
infinity reveal was refused (v4: 400 `invalid randao_reveal`, v3: 500),
where Lighthouse and Prysm build the block.

The beacon state transition now takes a `BlockProcessingOptions` whose
`RandaoVerification` lets block production mix in a reveal it did not
check. `state_transition` always passes the default, so import still
verifies every reveal, and every other check on a produced block stays.
Parsing follows Lighthouse: a value on the flag, or the flag with a reveal
other than the point at infinity, is a 400.
---
 .../src/beacon/block_production.rs            |  87 +++++++++-
 .../src/beacon/gloas_block_production.rs      | 137 ++++++++++++++--
 .../src/beacon/stf/bellatrix.rs               |   3 +-
 .../state_transition/src/beacon/stf/block.rs  |  81 +++++++--
 .../src/beacon/stf/capella.rs                 |   3 +-
 .../state_transition/src/beacon/stf/deneb.rs  |   3 +-
 .../src/beacon/stf/electra.rs                 |   3 +-
 .../state_transition/src/beacon/stf/fulu.rs   |   3 +-
 .../state_transition/src/beacon/stf/gloas.rs  |   3 +-
 .../state_transition/src/beacon/stf/mod.rs    |   9 +-
 .../rpc/src/beacon/builder_market_tests.rs    |  45 +++++
 crates/net/rpc/src/beacon/gloas_proposal.rs   | 102 ++++++++++--
 crates/net/rpc/src/beacon/proposal.rs         | 155 +++++++++++++++++-
 .../rpc/src/beacon/validator_client_tests.rs  |   2 +-
 14 files changed, 583 insertions(+), 53 deletions(-)

diff --git a/crates/blockchain/state_transition/src/beacon/block_production.rs b/crates/blockchain/state_transition/src/beacon/block_production.rs
index 77b974543..726febd74 100644
--- a/crates/blockchain/state_transition/src/beacon/block_production.rs
+++ b/crates/blockchain/state_transition/src/beacon/block_production.rs
@@ -307,11 +307,14 @@ pub struct BlockInputs {
 /// sync aggregate. The block is run through `process_block` on a copy of
 /// `state` with an execution engine that accepts the payload, which is the
 /// node's own execution client's payload; that run is also what rejects a body
-/// the network would, before anything is signed.
+/// the network would, before anything is signed. `options` is what that run may
+/// leave unchecked: only a caller that set `skip_randao_verification` passes
+/// anything but the default.
 pub fn assemble_block(
     state: &BeaconState,
     inputs: BlockInputs,
     config: &Config,
+    options: stf::block::BlockProcessingOptions,
 ) -> Result {
     let body = BeaconBlockBody {
         randao_reveal: inputs.randao_reveal,
@@ -366,6 +369,7 @@ pub fn assemble_block(
         config,
         &ExecutionEngine::valid(),
         &CommitteeCache::default(),
+        options,
     )?;
     block.state_root = post.hash_tree_root();
     Ok(block)
@@ -379,6 +383,7 @@ mod tests {
     use crate::beacon::helpers::fulu::initialize_proposer_lookahead;
     use crate::beacon::helpers::misc::compute_signing_root;
     use crate::beacon::helpers::test_state::{sign_for, with_signing_validators_at};
+    use crate::beacon::stf::block::{BlockProcessingOptions, RandaoVerification};
     use ethlambda_types::beacon::containers::shared::{AttestationData, Checkpoint};
 
     /// A fulu state one epoch in, its lookahead and sync committee filled from
@@ -438,7 +443,13 @@ mod tests {
             blob_kzg_commitments: Vec::new(),
             execution_requests: ExecutionRequests::default(),
         };
-        let block = assemble_block(&state, inputs, &Config::mainnet()).unwrap();
+        let block = assemble_block(
+            &state,
+            inputs,
+            &Config::mainnet(),
+            BlockProcessingOptions::default(),
+        )
+        .unwrap();
 
         assert_eq!(block.slot, state.slot());
         assert_eq!(
@@ -458,6 +469,68 @@ mod tests {
             &Config::mainnet(),
             &ExecutionEngine::valid(),
             &CommitteeCache::default(),
+            BlockProcessingOptions::default(),
+        )
+        .unwrap();
+        assert_eq!(block.state_root, post.hash_tree_root());
+    }
+
+    fn skip_randao() -> BlockProcessingOptions {
+        BlockProcessingOptions {
+            randao: RandaoVerification::Skip,
+        }
+    }
+
+    fn inputs_with_infinity_reveal(state: &BeaconState) -> BlockInputs {
+        BlockInputs {
+            randao_reveal: BlsSignature(bls::G2_POINT_AT_INFINITY),
+            graffiti: Bytes32::ZERO,
+            attestations: Vec::new(),
+            execution_payload: payload_for(state),
+            blob_kzg_commitments: Vec::new(),
+            execution_requests: ExecutionRequests::default(),
+        }
+    }
+
+    #[test]
+    fn a_block_with_the_infinity_reveal_is_produced_only_when_randao_is_skipped() {
+        let state = state_to_build_on();
+        let config = Config::mainnet();
+        assert!(
+            assemble_block(
+                &state,
+                inputs_with_infinity_reveal(&state),
+                &config,
+                BlockProcessingOptions::default(),
+            )
+            .is_err()
+        );
+
+        let block = assemble_block(
+            &state,
+            inputs_with_infinity_reveal(&state),
+            &config,
+            skip_randao(),
+        )
+        .unwrap();
+        assert_eq!(
+            block.body.randao_reveal,
+            BlsSignature(bls::G2_POINT_AT_INFINITY)
+        );
+
+        // Importing it with the same option lands on the state root it names.
+        let mut post = state.clone();
+        let signed = SignedBeaconBlock::Fulu(electra::SignedBeaconBlock {
+            message: block.clone(),
+            signature: BlsSignature::default(),
+        });
+        stf::block::process_block(
+            &mut post,
+            &signed,
+            &config,
+            &ExecutionEngine::valid(),
+            &CommitteeCache::default(),
+            skip_randao(),
         )
         .unwrap();
         assert_eq!(block.state_root, post.hash_tree_root());
@@ -476,7 +549,15 @@ mod tests {
             blob_kzg_commitments: Vec::new(),
             execution_requests: ExecutionRequests::default(),
         };
-        assert!(assemble_block(&state, inputs, &Config::mainnet()).is_err());
+        assert!(
+            assemble_block(
+                &state,
+                inputs,
+                &Config::mainnet(),
+                BlockProcessingOptions::default(),
+            )
+            .is_err()
+        );
     }
 
     /// An attestation at `slot` voting for this state's own chain: sourced from
diff --git a/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs b/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs
index 1a0fa8edc..d0ad7ecf4 100644
--- a/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs
+++ b/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs
@@ -367,11 +367,13 @@ pub struct GloasBidBlockInputs {
 /// a copy fills `state_root` and enforces `process_execution_payload_bid`:
 /// the builder is active, covers the bid and signed it, and the bid's slot,
 /// parent hash and root and randao are the state's. There is no envelope: the
-/// builder reveals the payload itself.
+/// builder reveals the payload itself. `options` is what that run may leave
+/// unchecked, as in [`assemble_gloas_block`].
 pub fn assemble_gloas_block_on_bid(
     state: &BeaconState,
     inputs: GloasBidBlockInputs,
     config: &Config,
+    options: stf::block::BlockProcessingOptions,
 ) -> Result {
     verify(
         matches!(state, BeaconState::Gloas(_)),
@@ -396,7 +398,13 @@ pub fn assemble_gloas_block_on_bid(
         body,
     };
     let mut post = state.clone();
-    stf::gloas::process_block(&mut post, &block, config, &CommitteeCache::default())?;
+    stf::gloas::process_block(
+        &mut post,
+        &block,
+        config,
+        &CommitteeCache::default(),
+        options,
+    )?;
     block.state_root = post.hash_tree_root();
     Ok(block)
 }
@@ -459,11 +467,14 @@ pub struct GloasProduced {
 /// is also what rejects a body the network would, before anything is signed.
 /// The envelope is then checked against that post-state with every condition
 /// `verify_execution_payload_envelope` holds it to except the signature, so a
-/// malformed build fails here and not on the network.
+/// malformed build fails here and not on the network. `options` is what the
+/// `process_block` run may leave unchecked: only a caller that set
+/// `skip_randao_verification` passes anything but the default.
 pub fn assemble_gloas_block(
     state: &BeaconState,
     inputs: GloasBlockInputs,
     config: &Config,
+    options: stf::block::BlockProcessingOptions,
 ) -> Result {
     verify(
         matches!(state, BeaconState::Gloas(_)),
@@ -512,7 +523,13 @@ pub fn assemble_gloas_block(
     let BeaconState::Gloas(_) = &post else {
         unreachable!("checked above")
     };
-    stf::gloas::process_block(&mut post, &block, config, &CommitteeCache::default())?;
+    stf::gloas::process_block(
+        &mut post,
+        &block,
+        config,
+        &CommitteeCache::default(),
+        options,
+    )?;
     block.state_root = post.hash_tree_root();
 
     let envelope = ExecutionPayloadEnvelope {
@@ -604,6 +621,7 @@ pub mod test_support {
     use crate::beacon::helpers::gloas::compute_ptc;
     use crate::beacon::helpers::misc::compute_signing_root;
     use crate::beacon::helpers::test_state::{sign_for, with_signing_validators_at};
+    use crate::beacon::stf::block::BlockProcessingOptions;
     use ethlambda_types::beacon::containers::bellatrix::{ExtraData, LogsBloom};
     use ethlambda_types::beacon::primitives::Uint256;
 
@@ -712,13 +730,21 @@ pub mod test_support {
                 execution_requests: ExecutionRequests::default(),
             },
             &config(),
+            BlockProcessingOptions::default(),
         )
     }
 
     /// The state after `block` is applied to `state`, as the chain stores it.
     pub fn post_state(state: &BeaconState, block: &BeaconBlock) -> BeaconState {
         let mut post = state.clone();
-        stf::gloas::process_block(&mut post, block, &config(), &CommitteeCache::default()).unwrap();
+        stf::gloas::process_block(
+            &mut post,
+            block,
+            &config(),
+            &CommitteeCache::default(),
+            BlockProcessingOptions::default(),
+        )
+        .unwrap();
         post
     }
 
@@ -754,6 +780,7 @@ mod gloas_block_production_tests {
     use crate::beacon::helpers::accessors::get_domain;
     use crate::beacon::helpers::misc::compute_signing_root;
     use crate::beacon::helpers::test_state::sign_for;
+    use crate::beacon::stf::block::{BlockProcessingOptions, RandaoVerification};
 
     /// A vote on the parent block (slot 32) signed by `validator`.
     fn ptc_message(
@@ -816,6 +843,7 @@ mod gloas_block_production_tests {
             &config(),
             &ExecutionEngine::valid(),
             &CommitteeCache::default(),
+            BlockProcessingOptions::default(),
         )
         .unwrap();
         assert_eq!(block.state_root, post.hash_tree_root());
@@ -906,6 +934,52 @@ mod gloas_block_production_tests {
         assert_eq!(parent_hash(&on_empty), empty.head_block_hash);
     }
 
+    #[test]
+    fn a_gloas_block_with_the_infinity_reveal_is_produced_only_when_randao_is_skipped() {
+        let state = state_to_build_on();
+        let skip = BlockProcessingOptions {
+            randao: RandaoVerification::Skip,
+        };
+        let produce_with = |options| {
+            let requests = ExecutionRequests::default();
+            let inputs = gloas_payload_inputs(&state, true, &requests, &config()).unwrap();
+            assemble_gloas_block(
+                &state,
+                GloasBlockInputs {
+                    randao_reveal: BlsSignature(bls::G2_POINT_AT_INFINITY),
+                    graffiti: Bytes32::ZERO,
+                    attestations: Vec::new(),
+                    payload_attestations: Vec::new(),
+                    parent_execution_requests: requests,
+                    execution_payload: payload_for(&inputs),
+                    blob_kzg_commitments: Vec::new(),
+                    execution_requests: ExecutionRequests::default(),
+                },
+                &config(),
+                options,
+            )
+        };
+        assert!(produce_with(BlockProcessingOptions::default()).is_err());
+
+        let produced = produce_with(skip).unwrap();
+        assert_eq!(
+            produced.block.body.randao_reveal,
+            BlsSignature(bls::G2_POINT_AT_INFINITY)
+        );
+
+        // Importing it with the same option lands on the state root it names.
+        let mut post = state.clone();
+        stf::gloas::process_block(
+            &mut post,
+            &produced.block,
+            &config(),
+            &CommitteeCache::default(),
+            skip,
+        )
+        .unwrap();
+        assert_eq!(produced.block.state_root, post.hash_tree_root());
+    }
+
     #[test]
     fn a_payload_on_the_wrong_parent_is_refused_before_signing() {
         let state = state_to_build_on();
@@ -926,6 +1000,7 @@ mod gloas_block_production_tests {
                 execution_requests: ExecutionRequests::default(),
             },
             &config(),
+            BlockProcessingOptions::default(),
         );
         assert!(result.is_err());
     }
@@ -956,6 +1031,7 @@ mod gloas_block_production_tests {
                 execution_requests: ExecutionRequests::default(),
             },
             &config(),
+            BlockProcessingOptions::default(),
         );
         assert!(produced.is_err());
     }
@@ -1109,6 +1185,7 @@ mod gloas_block_production_tests {
                 execution_requests: ExecutionRequests::default(),
             },
             &config(),
+            BlockProcessingOptions::default(),
         )
         .unwrap();
     }
@@ -1183,6 +1260,7 @@ mod bid_assembly_tests {
     use crate::beacon::builder_market::test_support::{
         builder_secret, gloas_state_with_builder, sign_bid,
     };
+    use crate::beacon::stf::block::BlockProcessingOptions;
     use ethlambda_types::beacon::primitives::{BlsPubkey, ExecutionAddress};
 
     /// A state at slot 33 with a funded, active builder 0, and a bid on its
@@ -1233,8 +1311,13 @@ mod bid_assembly_tests {
             &bid,
             &ExecutionRequests::default()
         ));
-        let block =
-            assemble_gloas_block_on_bid(&state, inputs(&state, bid.clone()), &config()).unwrap();
+        let block = assemble_gloas_block_on_bid(
+            &state,
+            inputs(&state, bid.clone()),
+            &config(),
+            BlockProcessingOptions::default(),
+        )
+        .unwrap();
         assert_eq!(block.body.signed_execution_payload_bid, bid);
         assert_eq!(block.slot, state.slot());
         assert_ne!(block.state_root, Root::ZERO);
@@ -1262,7 +1345,15 @@ mod bid_assembly_tests {
             &bid,
             &ExecutionRequests::default()
         ));
-        assert!(assemble_gloas_block_on_bid(&state, inputs(&state, bid), &config()).is_err());
+        assert!(
+            assemble_gloas_block_on_bid(
+                &state,
+                inputs(&state, bid),
+                &config(),
+                BlockProcessingOptions::default(),
+            )
+            .is_err()
+        );
     }
 
     #[test]
@@ -1276,7 +1367,15 @@ mod bid_assembly_tests {
             &bid,
             &ExecutionRequests::default()
         ));
-        assert!(assemble_gloas_block_on_bid(&state, inputs(&state, bid), &config()).is_err());
+        assert!(
+            assemble_gloas_block_on_bid(
+                &state,
+                inputs(&state, bid),
+                &config(),
+                BlockProcessingOptions::default(),
+            )
+            .is_err()
+        );
     }
 
     #[test]
@@ -1288,7 +1387,15 @@ mod bid_assembly_tests {
             &bid,
             &ExecutionRequests::default()
         ));
-        assert!(assemble_gloas_block_on_bid(&state, inputs(&state, bid), &config()).is_err());
+        assert!(
+            assemble_gloas_block_on_bid(
+                &state,
+                inputs(&state, bid),
+                &config(),
+                BlockProcessingOptions::default(),
+            )
+            .is_err()
+        );
     }
 
     #[test]
@@ -1301,7 +1408,15 @@ mod bid_assembly_tests {
             &bid,
             &ExecutionRequests::default()
         ));
-        assert!(assemble_gloas_block_on_bid(&state, inputs(&state, bid), &config()).is_err());
+        assert!(
+            assemble_gloas_block_on_bid(
+                &state,
+                inputs(&state, bid),
+                &config(),
+                BlockProcessingOptions::default(),
+            )
+            .is_err()
+        );
     }
 
     #[test]
diff --git a/crates/blockchain/state_transition/src/beacon/stf/bellatrix.rs b/crates/blockchain/state_transition/src/beacon/stf/bellatrix.rs
index ddb320905..3a9c9de85 100644
--- a/crates/blockchain/state_transition/src/beacon/stf/bellatrix.rs
+++ b/crates/blockchain/state_transition/src/beacon/stf/bellatrix.rs
@@ -67,6 +67,7 @@ pub fn process_block(
     config: &Config,
     engine: &ExecutionEngine,
     committees: &CommitteeCache,
+    options: super::block::BlockProcessingOptions,
 ) -> Result<()> {
     super::block::process_block_header(
         state,
@@ -78,7 +79,7 @@ pub fn process_block(
     if is_execution_enabled(state, &block.body.execution_payload)? {
         process_execution_payload(state, &block.body.execution_payload, config, engine)?;
     }
-    super::block::process_randao(state, &block.body.randao_reveal)?;
+    super::block::process_randao(state, &block.body.randao_reveal, options.randao)?;
     super::block::process_eth1_data(state, &block.body.eth1_data)?;
     super::operations::process_operations(
         state,
diff --git a/crates/blockchain/state_transition/src/beacon/stf/block.rs b/crates/blockchain/state_transition/src/beacon/stf/block.rs
index bacb60dfa..6831c5460 100644
--- a/crates/blockchain/state_transition/src/beacon/stf/block.rs
+++ b/crates/blockchain/state_transition/src/beacon/stf/block.rs
@@ -46,34 +46,35 @@ pub fn process_block(
     config: &Config,
     engine: &ExecutionEngine,
     committees: &CommitteeCache,
+    options: BlockProcessingOptions,
 ) -> Result<()> {
     match signed_block {
         containers::SignedBeaconBlock::Phase0(signed) => {
-            process_block_phase0(state, &signed.message, config, committees)
+            process_block_phase0(state, &signed.message, config, committees, options)
         }
         containers::SignedBeaconBlock::Altair(signed) => {
-            process_block_altair(state, &signed.message, config, committees)
+            process_block_altair(state, &signed.message, config, committees, options)
         }
         containers::SignedBeaconBlock::Bellatrix(signed) => {
-            bellatrix::process_block(state, &signed.message, config, engine, committees)
+            bellatrix::process_block(state, &signed.message, config, engine, committees, options)
         }
         containers::SignedBeaconBlock::Capella(signed) => {
-            capella::process_block(state, &signed.message, config, engine, committees)
+            capella::process_block(state, &signed.message, config, engine, committees, options)
         }
         containers::SignedBeaconBlock::Deneb(signed) => {
-            deneb::process_block(state, &signed.message, config, engine, committees)
+            deneb::process_block(state, &signed.message, config, engine, committees, options)
         }
         containers::SignedBeaconBlock::Electra(signed) => {
-            electra::process_block(state, &signed.message, config, engine, committees)
+            electra::process_block(state, &signed.message, config, engine, committees, options)
         }
         containers::SignedBeaconBlock::Fulu(signed) => {
-            fulu::process_block(state, &signed.message, config, engine, committees)
+            fulu::process_block(state, &signed.message, config, engine, committees, options)
         }
         // ePBS (EIP-7732) restructures block processing around a builder's
         // bid rather than an embedded payload: a gloas block carries no
         // payload for `engine` to answer for, so this arm never reaches it.
         containers::SignedBeaconBlock::Gloas(signed) => {
-            gloas::process_block(state, &signed.message, config, committees)
+            gloas::process_block(state, &signed.message, config, committees, options)
         }
         containers::SignedBeaconBlock::Lean(_) => {
             crate::beacon::lean_block_unreachable("process_block")
@@ -88,6 +89,7 @@ pub fn process_block_phase0(
     block: &phase0::BeaconBlock,
     config: &Config,
     committees: &CommitteeCache,
+    options: BlockProcessingOptions,
 ) -> Result<()> {
     process_block_header(
         state,
@@ -96,7 +98,7 @@ pub fn process_block_phase0(
         block.parent_root,
         block.body.hash_tree_root(),
     )?;
-    process_randao(state, &block.body.randao_reveal)?;
+    process_randao(state, &block.body.randao_reveal, options.randao)?;
     process_eth1_data(state, &block.body.eth1_data)?;
     crate::beacon::stf::operations::process_operations(
         state,
@@ -122,6 +124,7 @@ pub fn process_block_altair(
     block: &altair::BeaconBlock,
     config: &Config,
     committees: &CommitteeCache,
+    options: BlockProcessingOptions,
 ) -> Result<()> {
     process_block_header(
         state,
@@ -130,7 +133,7 @@ pub fn process_block_altair(
         block.parent_root,
         block.body.hash_tree_root(),
     )?;
-    process_randao(state, &block.body.randao_reveal)?;
+    process_randao(state, &block.body.randao_reveal, options.randao)?;
     process_eth1_data(state, &block.body.eth1_data)?;
     crate::beacon::stf::operations::process_operations(
         state,
@@ -204,24 +207,48 @@ pub fn process_block_header(
     Ok(())
 }
 
-/// Verifies the proposer's RANDAO reveal and mixes it into the current epoch's
-/// randao mix.
+/// What `process_block` may leave unchecked. The default checks everything the
+/// specification does, and is the only value block import uses.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct BlockProcessingOptions {
+    pub randao: RandaoVerification,
+}
+
+/// Whether `process_randao` verifies the proposer's reveal before mixing it in.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub enum RandaoVerification {
+    #[default]
+    Verify,
+    /// Block production for a caller that set `skip_randao_verification`: the
+    /// reveal is the point at infinity, which verifies against no key, so the
+    /// state root can only be computed by mixing it in unchecked.
+    Skip,
+}
+
+/// Verifies the proposer's RANDAO reveal (unless `verification` skips it) and
+/// mixes it into the current epoch's randao mix.
 ///
 /// The reveal signs the epoch rather than the slot, so the same signature would
 /// verify for every slot of the epoch: the mix it feeds is per-epoch, not
 /// per-slot, and a skipped slot leaves nothing for the next proposer to reveal
 /// differently.
-pub fn process_randao(state: &mut BeaconState, randao_reveal: &BlsSignature) -> Result<()> {
+pub fn process_randao(
+    state: &mut BeaconState,
+    randao_reveal: &BlsSignature,
+    verification: RandaoVerification,
+) -> Result<()> {
     let epoch = get_current_epoch(state);
     let proposer_index = get_beacon_proposer_index(state)?;
     let domain = get_domain(state, constants::DOMAIN_RANDAO, None);
     let signing_root = compute_signing_root(epoch.hash_tree_root(), domain);
 
     let proposer = state.validator(proposer_index)?;
-    verify(
-        bls::verify(&proposer.pubkey, signing_root, randao_reveal),
-        "RANDAO reveal signature",
-    )?;
+    if verification == RandaoVerification::Verify {
+        verify(
+            bls::verify(&proposer.pubkey, signing_root, randao_reveal),
+            "RANDAO reveal signature",
+        )?;
+    }
 
     let mix = xor(get_randao_mix(state, epoch), hash(&randao_reveal.0));
     let position = epoch as usize % preset::EPOCHS_PER_HISTORICAL_VECTOR;
@@ -324,4 +351,24 @@ mod tests {
             .is_err()
         );
     }
+
+    #[test]
+    fn skipping_randao_verification_mixes_in_the_infinity_reveal() {
+        let mut state = crate::beacon::helpers::test_state::with_validators(8);
+        let epoch = get_current_epoch(&state);
+        let reveal = BlsSignature(bls::G2_POINT_AT_INFINITY);
+        let expected = xor(get_randao_mix(&state, epoch), hash(&reveal.0));
+
+        process_randao(&mut state, &reveal, RandaoVerification::Skip).unwrap();
+
+        assert_eq!(get_randao_mix(&state, epoch), expected);
+    }
+
+    #[test]
+    fn verifying_randao_rejects_the_infinity_reveal() {
+        let mut state = crate::beacon::helpers::test_state::with_validators(8);
+        let reveal = BlsSignature(bls::G2_POINT_AT_INFINITY);
+
+        assert!(process_randao(&mut state, &reveal, RandaoVerification::Verify).is_err());
+    }
 }
diff --git a/crates/blockchain/state_transition/src/beacon/stf/capella.rs b/crates/blockchain/state_transition/src/beacon/stf/capella.rs
index 9fedd3649..2c075659b 100644
--- a/crates/blockchain/state_transition/src/beacon/stf/capella.rs
+++ b/crates/blockchain/state_transition/src/beacon/stf/capella.rs
@@ -76,6 +76,7 @@ pub fn process_block(
     config: &Config,
     engine: &ExecutionEngine,
     committees: &CommitteeCache,
+    options: super::block::BlockProcessingOptions,
 ) -> Result<()> {
     super::block::process_block_header(
         state,
@@ -86,7 +87,7 @@ pub fn process_block(
     )?;
     process_withdrawals(state, &block.body.execution_payload)?;
     process_execution_payload(state, &block.body.execution_payload, config, engine)?;
-    super::block::process_randao(state, &block.body.randao_reveal)?;
+    super::block::process_randao(state, &block.body.randao_reveal, options.randao)?;
     super::block::process_eth1_data(state, &block.body.eth1_data)?;
     process_operations(
         state,
diff --git a/crates/blockchain/state_transition/src/beacon/stf/deneb.rs b/crates/blockchain/state_transition/src/beacon/stf/deneb.rs
index e90ca6a6a..895c033fa 100644
--- a/crates/blockchain/state_transition/src/beacon/stf/deneb.rs
+++ b/crates/blockchain/state_transition/src/beacon/stf/deneb.rs
@@ -77,6 +77,7 @@ pub fn process_block(
     config: &Config,
     engine: &ExecutionEngine,
     committees: &CommitteeCache,
+    options: super::block::BlockProcessingOptions,
 ) -> Result<()> {
     super::block::process_block_header(
         state,
@@ -93,7 +94,7 @@ pub fn process_block(
         config,
         engine,
     )?;
-    super::block::process_randao(state, &block.body.randao_reveal)?;
+    super::block::process_randao(state, &block.body.randao_reveal, options.randao)?;
     super::block::process_eth1_data(state, &block.body.eth1_data)?;
     process_operations(
         state,
diff --git a/crates/blockchain/state_transition/src/beacon/stf/electra.rs b/crates/blockchain/state_transition/src/beacon/stf/electra.rs
index 104b2dd23..bc7bf0224 100644
--- a/crates/blockchain/state_transition/src/beacon/stf/electra.rs
+++ b/crates/blockchain/state_transition/src/beacon/stf/electra.rs
@@ -1836,6 +1836,7 @@ pub fn process_block(
     config: &Config,
     engine: &ExecutionEngine,
     committees: &CommitteeCache,
+    options: super::block::BlockProcessingOptions,
 ) -> Result<()> {
     super::block::process_block_header(
         state,
@@ -1846,7 +1847,7 @@ pub fn process_block(
     )?;
     process_withdrawals(state, &block.body.execution_payload)?;
     process_execution_payload(state, &block.body, config, engine)?;
-    super::block::process_randao(state, &block.body.randao_reveal)?;
+    super::block::process_randao(state, &block.body.randao_reveal, options.randao)?;
     super::block::process_eth1_data(state, &block.body.eth1_data)?;
     process_operations(state, &block.body, config, committees)?;
     super::altair::process_sync_aggregate(state, &block.body.sync_aggregate)?;
diff --git a/crates/blockchain/state_transition/src/beacon/stf/fulu.rs b/crates/blockchain/state_transition/src/beacon/stf/fulu.rs
index 3d720ac33..f2618905e 100644
--- a/crates/blockchain/state_transition/src/beacon/stf/fulu.rs
+++ b/crates/blockchain/state_transition/src/beacon/stf/fulu.rs
@@ -65,6 +65,7 @@ pub fn process_block(
     config: &Config,
     engine: &ExecutionEngine,
     committees: &CommitteeCache,
+    options: super::block::BlockProcessingOptions,
 ) -> Result<()> {
     super::block::process_block_header(
         state,
@@ -75,7 +76,7 @@ pub fn process_block(
     )?;
     super::electra::process_withdrawals(state, &block.body.execution_payload)?;
     process_execution_payload(state, &block.body, config, engine)?;
-    super::block::process_randao(state, &block.body.randao_reveal)?;
+    super::block::process_randao(state, &block.body.randao_reveal, options.randao)?;
     super::block::process_eth1_data(state, &block.body.eth1_data)?;
     process_operations(state, &block.body, config, committees)?;
     super::altair::process_sync_aggregate(state, &block.body.sync_aggregate)?;
diff --git a/crates/blockchain/state_transition/src/beacon/stf/gloas.rs b/crates/blockchain/state_transition/src/beacon/stf/gloas.rs
index fe982ed14..2ae164fc2 100644
--- a/crates/blockchain/state_transition/src/beacon/stf/gloas.rs
+++ b/crates/blockchain/state_transition/src/beacon/stf/gloas.rs
@@ -102,6 +102,7 @@ pub fn process_block(
     block: &gloas::BeaconBlock,
     config: &Config,
     committees: &CommitteeCache,
+    options: super::block::BlockProcessingOptions,
 ) -> Result<()> {
     // [New in Gloas:EIP7732]
     let parent_slot = state.latest_block_header().slot;
@@ -120,7 +121,7 @@ pub fn process_block(
     // [Modified in Gloas:EIP7732] Removed `process_execution_payload`.
     // [New in Gloas:EIP7732]
     process_execution_payload_bid(state, &block.body.signed_execution_payload_bid, config)?;
-    super::block::process_randao(state, &block.body.randao_reveal)?;
+    super::block::process_randao(state, &block.body.randao_reveal, options.randao)?;
     super::block::process_eth1_data(state, &block.body.eth1_data)?;
     // [Modified in Gloas:EIP7732]
     process_operations(state, &block.body, parent_slot, config, committees)?;
diff --git a/crates/blockchain/state_transition/src/beacon/stf/mod.rs b/crates/blockchain/state_transition/src/beacon/stf/mod.rs
index ea23506a1..6687d6f59 100644
--- a/crates/blockchain/state_transition/src/beacon/stf/mod.rs
+++ b/crates/blockchain/state_transition/src/beacon/stf/mod.rs
@@ -155,7 +155,14 @@ pub fn state_transition(
         )?;
     }
 
-    block::process_block(state, signed_block, config, engine, committees)?;
+    block::process_block(
+        state,
+        signed_block,
+        config,
+        engine,
+        committees,
+        block::BlockProcessingOptions::default(),
+    )?;
     // As in `process_slot`: the post-state's root, checked below and needed
     // again in the next slot, is then computed on the state's own nodes.
     state.apply_pending_mutations();
diff --git a/crates/net/rpc/src/beacon/builder_market_tests.rs b/crates/net/rpc/src/beacon/builder_market_tests.rs
index 26876e6e8..d70ef45ea 100644
--- a/crates/net/rpc/src/beacon/builder_market_tests.rs
+++ b/crates/net/rpc/src/beacon/builder_market_tests.rs
@@ -30,6 +30,7 @@ use ethlambda_state_transition::beacon::{
         test_state::{secret_key_for, sign_for},
     },
     payload_attestation_pool::SharedPayloadAttestationPool,
+    stf::block::RandaoVerification,
 };
 use ethlambda_storage::Store;
 use ethlambda_types::{
@@ -284,6 +285,7 @@ impl World {
             &self.market,
             SLOT,
             randao_reveal(&self.advanced()),
+            RandaoVerification::Verify,
         )
         .unwrap()
     }
@@ -442,6 +444,19 @@ async fn produce(
     .await
 }
 
+/// A request for [`SLOT`] carrying `reveal` and `flag` (a raw query suffix such
+/// as `&skip_randao_verification`), with a self-built bare block wanted back.
+fn produce_request_with(reveal: BlsSignature, flag: &str) -> Request {
+    Request::post(format!(
+        "/eth/v4/validator/blocks/{SLOT}?randao_reveal=0x{}&include_payload=false{flag}",
+        hex::encode(reveal.0)
+    ))
+    .header("eth-consensus-version", "gloas")
+    .header("content-type", "application/json")
+    .body(Body::from(builder_config(0, 100)))
+    .unwrap()
+}
+
 /// The bid the produced block commits to, from a JSON response either way: a
 /// bare block, or contents carrying it.
 fn produced_block_bid(reply: &Reply) -> ExecutionPayloadBid {
@@ -1232,3 +1247,33 @@ async fn the_new_routes_are_wired_with_the_production_layers() {
     .await;
     assert_eq!(reply.status, StatusCode::NOT_FOUND);
 }
+
+#[tokio::test]
+async fn the_infinity_reveal_is_produced_only_when_the_flag_skips_its_verification() {
+    use ethlambda_state_transition::beacon::bls::G2_POINT_AT_INFINITY;
+
+    let world = World::new();
+    let engine = fake_engine(GWEI, false).await;
+    let app = world.app(Some(&engine));
+    let infinity = BlsSignature(G2_POINT_AT_INFINITY);
+
+    for flag in ["&skip_randao_verification=", "&skip_randao_verification"] {
+        let reply = send(&app, produce_request_with(infinity, flag)).await;
+        assert_eq!(
+            reply.status,
+            StatusCode::OK,
+            "{flag}: {}",
+            String::from_utf8_lossy(&reply.body)
+        );
+        assert_eq!(
+            reply.json()["data"]["body"]["randao_reveal"],
+            format!("0x{}", hex::encode(G2_POINT_AT_INFINITY)),
+            "{flag}"
+        );
+    }
+
+    // Without the flag the same reveal is the caller's mistake, as before.
+    let reply = send(&app, produce_request_with(infinity, "")).await;
+    assert_eq!(reply.status, StatusCode::BAD_REQUEST);
+    assert_eq!(reply.json()["message"], "invalid randao_reveal");
+}
diff --git a/crates/net/rpc/src/beacon/gloas_proposal.rs b/crates/net/rpc/src/beacon/gloas_proposal.rs
index 59d5b7605..b556998c0 100644
--- a/crates/net/rpc/src/beacon/gloas_proposal.rs
+++ b/crates/net/rpc/src/beacon/gloas_proposal.rs
@@ -71,7 +71,10 @@ use ethlambda_state_transition::beacon::{
         misc::compute_signing_root,
     },
     payload_attestation_pool::SharedPayloadAttestationPool,
-    stf::gloas::verify_execution_payload_envelope_signature,
+    stf::{
+        block::{BlockProcessingOptions, RandaoVerification},
+        gloas::verify_execution_payload_envelope_signature,
+    },
 };
 use ethlambda_storage::Store;
 use ethlambda_types::{
@@ -103,7 +106,7 @@ use crate::beacon::{
     ApiError, BodyEncoding,
     bid_selection::{LocalCandidate, PayloadChoice, bid_total_gwei, choose_payload, wei_u128},
     builder_config::{BuilderConfig, decode_builder_config},
-    proposal::{Blobs, CellKzgProofs, decimal, require_gloas_slot},
+    proposal::{Blobs, CellKzgProofs, decimal, randao_verification, require_gloas_slot},
     validator::{FeeRecipients, head},
 };
 use crate::shared::content::{Encoding, ssz_response, with_consensus_version};
@@ -200,10 +203,10 @@ struct ProduceQuery {
     #[serde(default)]
     graffiti: Option,
     include_payload: bool,
-    /// Accepted for compatibility. Not honored: the block's state root comes
-    /// from running it through the state transition, which verifies the reveal.
+    /// A flag with no value, validated by [`randao_verification`] rather than
+    /// parsed here, so a bad value gets its own message and not the generic
+    /// query rejection.
     #[serde(default)]
-    #[allow(dead_code)]
     skip_randao_verification: Option,
 }
 
@@ -235,6 +238,15 @@ async fn post_produce_block(
     if consensus_version(&headers).is_some_and(|fork| fork != ForkName::Gloas) {
         return ApiError::BadRequest("Eth-Consensus-Version must be gloas").into_response();
     }
+    // Ahead of the engine check, like the slot check below: a bad flag is the
+    // caller's mistake, a 400 on any node.
+    let randao = match randao_verification(
+        query.skip_randao_verification.as_deref(),
+        &query.randao_reveal,
+    ) {
+        Ok(randao) => randao,
+        Err(err) => return err.into_response(),
+    };
     // Ahead of the engine check: a slot this node cannot build is a 400 on any
     // node, not a 503 on one with no execution client.
     if let Err(err) = require_gloas_slot(
@@ -273,6 +285,7 @@ async fn post_produce_block(
         &custody,
         slot,
         query.randao_reveal,
+        randao,
         graffiti,
     )
     .await;
@@ -413,6 +426,9 @@ struct Prepared {
     /// The proposer's signed preferences for the slot, when this node holds
     /// them under the slot's dependent root.
     preferences: Option,
+    /// What the block's state transition may leave unchecked: the reveal, when
+    /// the caller set `skip_randao_verification`.
+    options: BlockProcessingOptions,
 }
 
 #[allow(clippy::too_many_arguments)]
@@ -427,12 +443,13 @@ async fn produce(
     custody: &NodeCustodyColumns,
     slot: Slot,
     randao_reveal: BlsSignature,
+    randao: RandaoVerification,
     graffiti: Bytes32,
 ) -> Result {
     let prepare_store = store.clone();
     let prepare_market = market.clone();
     let prepared = tokio::task::spawn_blocking(move || {
-        prepare(&prepare_store, &prepare_market, slot, randao_reveal)
+        prepare(&prepare_store, &prepare_market, slot, randao_reveal, randao)
     })
     .await
     .map_err(|_| ApiError::Internal("preparing the block failed"))??;
@@ -593,6 +610,7 @@ fn prepare(
     market: &SharedBuilderMarket,
     slot: Slot,
     randao_reveal: BlsSignature,
+    randao: RandaoVerification,
 ) -> Result {
     let config = store.config();
     let (head_root, head_state) = head(store)?;
@@ -640,7 +658,9 @@ fn prepare(
         .map_err(|_| ApiError::Internal("advancing the head state failed"))?;
     let proposer = get_beacon_proposer_index(&state)
         .map_err(|_| ApiError::Internal("no proposer for the slot"))?;
-    verify_randao_reveal(&state, proposer, randao_reveal)?;
+    if randao == RandaoVerification::Verify {
+        verify_randao_reveal(&state, proposer, randao_reveal)?;
+    }
     let mut inputs = gloas_payload_inputs(&state, build_on_full, &parent_requests, &config)
         .map_err(|_| ApiError::Internal("computing the payload attributes failed"))?;
 
@@ -661,6 +681,7 @@ fn prepare(
         head_slot,
         parent_requests,
         preferences,
+        options: BlockProcessingOptions { randao },
     })
 }
 
@@ -817,6 +838,7 @@ fn assemble(
         head_root,
         head_slot,
         parent_requests,
+        options,
         ..
     } = prepared;
     let attestations = pack_gloas_attestations(state, candidates);
@@ -834,14 +856,19 @@ fn assemble(
         execution_requests: execution_requests.clone(),
     };
     let operations = attestations.len() + payload_attestations.len();
-    match assemble_gloas_block(state, inputs(attestations, payload_attestations), config) {
+    match assemble_gloas_block(
+        state,
+        inputs(attestations, payload_attestations),
+        config,
+        *options,
+    ) {
         Ok(produced) => Ok(produced),
         // The packers check every operation's signature against this state, so
         // this should not happen; but a block without them still earns the
         // proposal, and one that fails to build earns nothing.
         Err(err) if operations > 0 => {
             warn!(slot = state.slot(), %err, "Block with operations failed to build; retrying without");
-            assemble_gloas_block(state, inputs(Vec::new(), Vec::new()), config)
+            assemble_gloas_block(state, inputs(Vec::new(), Vec::new()), config, *options)
                 .map_err(|_| ApiError::Internal("the block failed to build"))
         }
         Err(_) => Err(ApiError::Internal("the block failed to build")),
@@ -864,6 +891,7 @@ fn assemble_on_bid(
         head_root,
         head_slot,
         parent_requests,
+        options,
         ..
     } = prepared;
     let attestations = pack_gloas_attestations(state, candidates);
@@ -878,11 +906,16 @@ fn assemble_on_bid(
         signed_bid: signed_bid.clone(),
     };
     let operations = attestations.len() + payload_attestations.len();
-    match assemble_gloas_block_on_bid(state, inputs(attestations, payload_attestations), config) {
+    match assemble_gloas_block_on_bid(
+        state,
+        inputs(attestations, payload_attestations),
+        config,
+        *options,
+    ) {
         Ok(block) => Ok(block),
         Err(err) if operations > 0 => {
             warn!(slot = state.slot(), %err, "Bid block with operations failed to build; retrying without");
-            assemble_gloas_block_on_bid(state, inputs(Vec::new(), Vec::new()), config)
+            assemble_gloas_block_on_bid(state, inputs(Vec::new(), Vec::new()), config, *options)
                 .map_err(|_| ApiError::Internal("the block failed to build"))
         }
         Err(_) => Err(ApiError::Internal("the block failed to build")),
@@ -1200,6 +1233,51 @@ mod tests {
         assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE);
     }
 
+    #[tokio::test]
+    async fn the_randao_skip_flag_is_refused_with_a_value_or_a_real_reveal() {
+        let (store, head_slot) = gloas_scheduled_store();
+        let slot = head_slot + 1;
+        let infinity = hex::encode(bls::G2_POINT_AT_INFINITY);
+        let with_reveal = |reveal: &str, flag: &str| {
+            Request::post(format!(
+                "/eth/v4/validator/blocks/{slot}?randao_reveal=0x{reveal}&include_payload=true{flag}"
+            ))
+            .header("eth-consensus-version", "gloas")
+            .header(header::CONTENT_TYPE, "application/json")
+            .body(Body::from(EMPTY_CONFIG))
+            .unwrap()
+        };
+
+        // A reveal that is not the point at infinity cannot ask to skip.
+        let (status, json) = respond(
+            produce_app(store.clone(), None),
+            with_reveal(&"00".repeat(96), "&skip_randao_verification"),
+        )
+        .await;
+        assert_eq!(status, StatusCode::BAD_REQUEST);
+        assert_eq!(
+            json["message"],
+            "randao_reveal must be the point at infinity when skip_randao_verification is set"
+        );
+
+        // The flag takes no value, `true` included.
+        let (status, json) = respond(
+            produce_app(store.clone(), None),
+            with_reveal(&infinity, "&skip_randao_verification=true"),
+        )
+        .await;
+        assert_eq!(status, StatusCode::BAD_REQUEST);
+        assert_eq!(
+            json["message"],
+            "skip_randao_verification does not take a value"
+        );
+
+        // Without the flag the infinity reveal is checked like any other, and
+        // the request gets as far as the engine check.
+        let (status, _) = respond(produce_app(store, None), with_reveal(&infinity, "")).await;
+        assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE);
+    }
+
     #[tokio::test]
     async fn a_missing_include_payload_or_an_undecodable_body_is_a_400() {
         let (store, head_slot) = gloas_scheduled_store();
@@ -1416,6 +1494,7 @@ mod tests {
                 execution_requests: ExecutionRequests::default(),
             },
             &config(),
+            ethlambda_state_transition::beacon::stf::block::BlockProcessingOptions::default(),
         )
         .unwrap()
     }
@@ -1664,6 +1743,7 @@ mod tests {
                 execution_requests: ExecutionRequests::default(),
             },
             &config(),
+            ethlambda_state_transition::beacon::stf::block::BlockProcessingOptions::default(),
         )
         .unwrap();
         WithBlob {
diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs
index 475c00052..1e2be2d7f 100644
--- a/crates/net/rpc/src/beacon/proposal.rs
+++ b/crates/net/rpc/src/beacon/proposal.rs
@@ -32,7 +32,11 @@ use ethlambda_state_transition::beacon::{
         BlockInputs, advance_to_slot, assemble_block, pack_attestations, parse_execution_requests,
         payload_inputs,
     },
-    stf::verify_block_signature,
+    bls,
+    stf::{
+        block::{BlockProcessingOptions, RandaoVerification},
+        verify_block_signature,
+    },
 };
 use ethlambda_storage::Store;
 use ethlambda_types::{
@@ -264,6 +268,39 @@ struct ProduceQuery {
     randao_reveal: BlsSignature,
     #[serde(default)]
     graffiti: Option,
+    /// A flag with no value, validated by [`randao_verification`].
+    #[serde(default)]
+    skip_randao_verification: Option,
+}
+
+/// Whether the block's RANDAO reveal is verified, from the request's
+/// `skip_randao_verification` flag, the way Lighthouse reads it.
+///
+/// The flag takes no value, so `?skip_randao_verification` and
+/// `?skip_randao_verification=` set it and any other value is refused rather
+/// than guessed at. Skipping is only for a reveal that cannot verify against
+/// any key (`bls::G2_POINT_AT_INFINITY`), so a caller that sets the flag with a
+/// real reveal has misunderstood it, and the block would otherwise be produced
+/// against a reveal that was never checked.
+pub(super) fn randao_verification(
+    skip: Option<&str>,
+    randao_reveal: &BlsSignature,
+) -> Result {
+    match skip {
+        None => Ok(RandaoVerification::Verify),
+        Some("") => {
+            if randao_reveal.0 == bls::G2_POINT_AT_INFINITY {
+                Ok(RandaoVerification::Skip)
+            } else {
+                Err(ApiError::BadRequest(
+                    "randao_reveal must be the point at infinity when skip_randao_verification is set",
+                ))
+            }
+        }
+        Some(_) => Err(ApiError::BadRequest(
+            "skip_randao_verification does not take a value",
+        )),
+    }
 }
 
 /// Refuses a slot the schedule does not place at fulu, before anything
@@ -296,6 +333,15 @@ async fn get_block(
     let Ok(slot) = slot.parse::() else {
         return ApiError::BadRequest("invalid slot").into_response();
     };
+    // Ahead of the engine check, like the slot check below: a bad flag is the
+    // caller's mistake, a 400 on any node.
+    let randao = match randao_verification(
+        query.skip_randao_verification.as_deref(),
+        &query.randao_reveal,
+    ) {
+        Ok(randao) => randao,
+        Err(err) => return err.into_response(),
+    };
     // Ahead of the engine check: a slot this node cannot build is a 400 on
     // any node, not a 503 on one with no execution client.
     if let Err(err) = require_fulu_slot(
@@ -319,6 +365,7 @@ async fn get_block(
         &fee_recipients,
         slot,
         query.randao_reveal,
+        BlockProcessingOptions { randao },
         graffiti,
     )
     .await;
@@ -359,6 +406,7 @@ async fn get_block(
 
 /// The block for `slot`, the payload's value in wei as a decimal string, and
 /// the block's fork.
+#[allow(clippy::too_many_arguments)]
 async fn produce(
     store: &Store,
     engine: &EngineClient,
@@ -366,6 +414,7 @@ async fn produce(
     fee_recipients: &FeeRecipients,
     slot: Slot,
     randao_reveal: BlsSignature,
+    options: BlockProcessingOptions,
     graffiti: Bytes32,
 ) -> Result<(BeaconBlock, String, ForkName), ApiError> {
     let config = store.config();
@@ -405,14 +454,14 @@ async fn produce(
         execution_requests: execution_requests.clone(),
     };
     let attestation_count = attestations.len();
-    let block = match assemble_block(&state, inputs(attestations), &config) {
+    let block = match assemble_block(&state, inputs(attestations), &config, options) {
         Ok(block) => block,
         // `pack_attestations` checks every attestation's signature against this
         // state, so this should not happen; but a block without them still
         // earns the proposal, and one that fails to build earns nothing.
         Err(err) if attestation_count > 0 => {
             warn!(%slot, %err, "Block with attestations failed to build; retrying without");
-            assemble_block(&state, inputs(Vec::new()), &config)
+            assemble_block(&state, inputs(Vec::new()), &config, options)
                 .map_err(|_| ApiError::Internal("the block failed to build"))?
         }
         Err(_) => return Err(ApiError::Internal("the block failed to build")),
@@ -533,6 +582,106 @@ mod tests {
         (status, serde_json::from_slice(&body).unwrap_or_default())
     }
 
+    /// A fulu head with its proposer lookahead and sync committee filled in,
+    /// which a block built on it needs, under a schedule that puts every slot
+    /// at fulu.
+    fn fulu_store() -> (Store, Slot) {
+        use ethlambda_state_transition::beacon::helpers::{
+            altair::get_next_sync_committee, fulu::initialize_proposer_lookahead,
+        };
+        let mut state = with_signing_validators_at(ForkName::Fulu, 64);
+        let lookahead = initialize_proposer_lookahead(&state).unwrap();
+        let sync_committee = get_next_sync_committee(&state).unwrap();
+        let BeaconState::Fulu(inner) = &mut state else {
+            unreachable!("built as fulu")
+        };
+        inner.proposer_lookahead = lookahead.try_into().unwrap();
+        inner.current_sync_committee = sync_committee.clone();
+        inner.next_sync_committee = sync_committee;
+        let head_slot = state.slot();
+        let (store, _root) =
+            beacon_store_with_config(state, Config::mainnet().with_fork_epoch(ForkName::Fulu, 0));
+        (store, head_slot)
+    }
+
+    fn produce_app(store: Store, engine: Option) -> Router {
+        routes()
+            .with_state(store)
+            .layer(Extension(engine))
+            .layer(Extension(SharedAttestationPool::default()))
+            .layer(Extension(FeeRecipients::default()))
+    }
+
+    fn infinity_reveal() -> String {
+        hex::encode(bls::G2_POINT_AT_INFINITY)
+    }
+
+    fn produce_uri(slot: Slot, reveal: &str, flag: &str) -> String {
+        format!("/eth/v3/validator/blocks/{slot}?randao_reveal=0x{reveal}{flag}")
+    }
+
+    #[tokio::test]
+    async fn the_infinity_reveal_is_produced_only_when_the_flag_skips_its_verification() {
+        let (store, head_slot) = fulu_store();
+        let slot = head_slot + 1;
+        let engine = super::super::validator_client_tests::fake_execution_client().await;
+
+        for flag in ["&skip_randao_verification=", "&skip_randao_verification"] {
+            let app = produce_app(store.clone(), Some(engine.clone()));
+            let uri = produce_uri(slot, &infinity_reveal(), flag);
+            let (status, json) = respond(app, Request::get(uri).body(Body::empty()).unwrap()).await;
+            assert_eq!(status, StatusCode::OK, "{flag}: {json}");
+            assert_eq!(
+                json["data"]["block"]["body"]["randao_reveal"],
+                format!("0x{}", infinity_reveal()),
+                "{flag}"
+            );
+        }
+
+        // Without the flag the reveal is verified, and the infinity one fails.
+        let app = produce_app(store, Some(engine));
+        let uri = produce_uri(slot, &infinity_reveal(), "");
+        let (status, _) = respond(app, Request::get(uri).body(Body::empty()).unwrap()).await;
+        assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR);
+    }
+
+    #[tokio::test]
+    async fn the_randao_skip_flag_is_refused_with_a_value_or_a_real_reveal() {
+        let (store, head_slot) = fulu_store();
+        let slot = head_slot + 1;
+        let get = |uri: String| Request::get(uri).body(Body::empty()).unwrap();
+
+        let (status, json) = respond(
+            produce_app(store.clone(), None),
+            get(produce_uri(
+                slot,
+                &"00".repeat(96),
+                "&skip_randao_verification",
+            )),
+        )
+        .await;
+        assert_eq!(status, StatusCode::BAD_REQUEST);
+        assert_eq!(
+            json["message"],
+            "randao_reveal must be the point at infinity when skip_randao_verification is set"
+        );
+
+        let (status, json) = respond(
+            produce_app(store, None),
+            get(produce_uri(
+                slot,
+                &infinity_reveal(),
+                "&skip_randao_verification=true",
+            )),
+        )
+        .await;
+        assert_eq!(status, StatusCode::BAD_REQUEST);
+        assert_eq!(
+            json["message"],
+            "skip_randao_verification does not take a value"
+        );
+    }
+
     #[tokio::test]
     async fn producing_a_gloas_slot_is_a_400_even_with_no_execution_client() {
         let (store, head_slot) = gloas_scheduled_store();
diff --git a/crates/net/rpc/src/beacon/validator_client_tests.rs b/crates/net/rpc/src/beacon/validator_client_tests.rs
index cb1de1820..443af4355 100644
--- a/crates/net/rpc/src/beacon/validator_client_tests.rs
+++ b/crates/net/rpc/src/beacon/validator_client_tests.rs
@@ -291,7 +291,7 @@ async fn block_production_without_an_execution_client_is_retryable() {
 /// exactly the requested attributes, which is all a real one's payload has to
 /// agree with for the block to verify. Answers both generations: V3/V5 for
 /// fulu and V4/V6 (`PayloadAttributesV4`, `ExecutionPayloadV4`) for gloas.
-async fn fake_execution_client() -> ethlambda_engine::EngineClient {
+pub(super) async fn fake_execution_client() -> ethlambda_engine::EngineClient {
     fake_execution_client_with_blob(None).await
 }
 

From 66320554f9b60f6c74d50cb4fdf3cb5bf326e59f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Thu, 22 Oct 2026 19:31:40 -0300
Subject: [PATCH 126/132] fix(beacon): skip a proposer boost outside the
 justified subtree

Main now prunes the beacon LiveChain below the finalized block (#627). A
block can hold the boost while sitting outside the justified subtree (the
boost rule only asks that it share the pre-import head's shuffling
dependent root), and a finalization advance later in the same slot can
prune its parent's index row. The gloas gate then reads that missing
parent and fails the whole head computation until the next slot clears
the boost.

Such a boost changes no comparison the descent makes, so
compute_node_weights now skips it under either fork's rules
(boost_reaches_the_descent) before the gate runs. The walk-bound docs
say why the bound still matters with a pruned index, and the test for a
boosted finalized block goes, since that boost is now skipped. Also drops
#628's second anchor_slot binding in get_forkchoice_store, which shadowed
the one this branch already takes from the anchor block.
---
 .../src/beacon/fork_choice.rs                 | 196 +++++++++++++-----
 docs/spec_deviations.md                       |  16 +-
 2 files changed, 152 insertions(+), 60 deletions(-)

diff --git a/crates/blockchain/state_transition/src/beacon/fork_choice.rs b/crates/blockchain/state_transition/src/beacon/fork_choice.rs
index 1d275f3fd..1a326707c 100644
--- a/crates/blockchain/state_transition/src/beacon/fork_choice.rs
+++ b/crates/blockchain/state_transition/src/beacon/fork_choice.rs
@@ -1118,11 +1118,6 @@ pub fn get_forkchoice_store(
     // this file's only outright whole-`BeaconState` clone. `checkpoint_state`
     // now derives that second copy on demand instead of caching it, so the
     // anchor is written once.
-    // Captured before `anchor_state` moves into `insert_state` below:
-    // `set_unrealized_justification` needs the anchor's own slot to persist
-    // alongside its checkpoint (see `Table::BeaconUnrealizedJustifications`),
-    // and `insert_state` takes the state by value.
-    let anchor_slot = anchor_state.slot();
     store
         .insert_signed_block(anchor_root, anchor_block)
         .expect("insert");
@@ -1841,6 +1836,36 @@ fn parent_node(parent_root: Root, parent_status: PayloadStatus) -> ForkChoiceNod
     }
 }
 
+/// Whether the proposer boost on `boost_root` can reach a node the head
+/// descent compares: whether `boost_root` is `justified_root` or a block under
+/// it in `index`.
+///
+/// The boost adds weight to the boosted block and its ancestors only, and the
+/// descent compares only blocks under the justified root, so a boosted block
+/// outside that subtree changes no comparison, whatever the gloas gate would
+/// answer. Such a block can hold the boost: the boost rule asks only that it
+/// share the pre-import head's shuffling dependent root, which can sit below
+/// the justified block, and a finalization advance later in the same slot can
+/// leave it off the finalized chain, its parent pruned from `index`. Skipping
+/// it keeps the gate from reading that parent, which would raise and fail the
+/// head computation until the next slot clears the boost. A boosted block
+/// whose own row is gone (`invalidate_subtree`) is skipped the same way.
+///
+/// A justified root missing from `index` answers `true`, leaving the boost to
+/// its own rules: on a consistent store the justified block sits at or above
+/// the finalized one and keeps its row, and the descent could not start from
+/// it anyway.
+fn boost_reaches_the_descent(
+    index: &HashMap,
+    justified_root: Root,
+    boost_root: Root,
+) -> bool {
+    let Some(&(justified_slot, _)) = index.get(&justified_root) else {
+        return true;
+    };
+    get_ancestor_or_missing(index, boost_root, justified_slot) == Ok(justified_root)
+}
+
 /// The lowest slot [`compute_node_weights`] weighs: the least of the finalized
 /// block's slot, the justified block's slot and, when a proposer boost root is
 /// set and indexed, its parent's slot. All three come from `index`.
@@ -1867,16 +1892,19 @@ fn walk_bound(store: &Store, index: &HashMap) -> Slot {
 /// `rules` selects the two fork-dependent rules: how a block's payload
 /// dimension is read (see [`PayloadTree`]) and whether the proposer boost is
 /// gated by `should_apply_proposer_boost` (gloas) or applies whenever a boost
-/// root is set (pre-gloas).
+/// root is set (pre-gloas). Under either, a boosted block outside the justified
+/// subtree gets no boost at all; see [`boost_reaches_the_descent`].
 ///
 /// # The bound
 ///
 /// Vote placement, the fold and the boost chain all stop at [`walk_bound`]'s
 /// slot: a block below it gets no weight and is never read, so a head
 /// computation covers the unfinalized window, not the history since the
-/// anchor. (A beacon store never prunes its block index, so without the bound
-/// every call would fold, and read the payload link of, every block down to
-/// the anchor.) The bound is the least of the finalized block's slot, the
+/// anchor. (Finality pruning drops the index rows below the finalized block's
+/// own slot, so on a live node the index already starts there; the bound
+/// holds the walk to that window whatever the index still keeps below it,
+/// such as the rows of a prune skipped for a finalized root with no block
+/// entry.) The bound is the least of the finalized block's slot, the
 /// justified block's slot and the boosted block's parent's slot, and it
 /// changes no answer the head reads, whichever of the three is least:
 /// - The descent compares only descendants of the justified root, so every
@@ -1892,9 +1920,10 @@ fn walk_bound(store: &Store, index: &HashMap) -> Slot {
 ///   construction.
 ///
 /// On a live node the least of the three is the finalized block's slot: the
-/// justified block is at or above it, and the boosted block is a current-slot
-/// block, which descends from the finalized root, so its parent is too. That
-/// is what keeps the walk to the unfinalized window.
+/// justified block is at or above it, and a boost applies only to a block
+/// under the justified root (a current-slot block, so never that root itself),
+/// whose parent is therefore at or above the justified block too. That is
+/// what keeps the walk to the unfinalized window.
 ///
 /// The weight of a node below the bound is not computed, and reads as `0`.
 pub fn compute_node_weights(
@@ -2008,18 +2037,27 @@ pub fn compute_node_weights(
     // The boost. Gloas asks `should_apply_proposer_boost`, fed the parent's
     // score from the table just built; before gloas it applies whenever a boost
     // root is set and still indexed, the tolerance `compute_weights` documents.
+    // Neither runs for a boosted block outside the justified subtree, which no
+    // node the descent compares would gain from.
     let boost_root = store.proposer_boost_root();
-    let boost_applies = match rules {
-        ForkRules::PreGloas => !boost_root.is_zero() && in_window(&boost_root),
-        ForkRules::Gloas => {
-            should_apply_proposer_boost_with(store, config, committees, index, &state, |parent| {
-                Ok(weights.raw(ForkChoiceNode {
-                    root: parent,
-                    payload_status: PayloadStatus::Pending,
-                }))
-            })?
-        }
-    };
+    let justified_root = store.beacon_justified_checkpoint().root;
+    let boost_applies = boost_reaches_the_descent(index, justified_root, boost_root)
+        && match rules {
+            ForkRules::PreGloas => !boost_root.is_zero() && in_window(&boost_root),
+            ForkRules::Gloas => should_apply_proposer_boost_with(
+                store,
+                config,
+                committees,
+                index,
+                &state,
+                |parent| {
+                    Ok(weights.raw(ForkChoiceNode {
+                        root: parent,
+                        payload_status: PayloadStatus::Pending,
+                    }))
+                },
+            )?,
+        };
     if boost_applies {
         let proposer_score = get_proposer_score(store, config)?;
         let mut root = boost_root;
@@ -7784,6 +7822,73 @@ mod tests {
             .expect("the same tolerance must hold through the whole head walk");
     }
 
+    /// A boosted block outside the justified subtree gets no boost under
+    /// either fork's rules, and the gloas gate is never asked about it. Its
+    /// parent has lost its index row here, as a finalization advance after
+    /// the boost was set prunes it, so the gate itself raises: the walk must
+    /// not reach it.
+    #[test]
+    fn a_boost_outside_the_justified_subtree_is_skipped_without_its_gate() {
+        let config = Config::active();
+        let (mut store, anchor_root, anchor_slot) =
+            anchored_store(preset::SLOTS_PER_EPOCH as usize);
+        let committees = CommitteeCache::default();
+
+        // anchor ─┬─ justified (anchor + 1)
+        //         └─ parent (anchor + 1, row pruned) ─ boosted (anchor + 2)
+        let justified_root = Root::repeat_byte(0xa1);
+        let parent_root = Root::repeat_byte(0xa2);
+        let boost_root = Root::repeat_byte(0xa3);
+        store
+            .insert_signed_block(justified_root, block(anchor_slot + 1, anchor_root))
+            .unwrap();
+        store
+            .insert_signed_block(parent_root, block(anchor_slot + 1, anchor_root))
+            .unwrap();
+        store
+            .insert_signed_block(boost_root, block(anchor_slot + 2, parent_root))
+            .unwrap();
+        store.delete_live_chain_entries(&[(anchor_slot + 1, parent_root)]);
+
+        let justified = Checkpoint {
+            epoch: store.beacon_justified_checkpoint().epoch + 1,
+            root: justified_root,
+        };
+        // The weights read the justified checkpoint's state, cached here so
+        // that no slot processing is needed to reach it.
+        let justified_state = store.get_state(&anchor_root).unwrap().unwrap();
+        store.cache_state(
+            CacheKey::CheckpointState {
+                epoch: justified.epoch,
+                root: justified.root,
+            },
+            justified_state,
+        );
+        let finalized = store.beacon_finalized_checkpoint();
+        update_checkpoints(&mut store, justified, finalized);
+        store.set_proposer_boost_root(boost_root);
+
+        assert_eq!(
+            failed_assertion(should_apply_proposer_boost(&store, &config, &committees)),
+            Some("parent_root in store.blocks"),
+            "the gate reads the boosted block's parent, which is gone"
+        );
+        let index = store.block_index();
+        let boosted = ForkChoiceNode {
+            root: boost_root,
+            payload_status: PayloadStatus::Pending,
+        };
+        for rules in [ForkRules::PreGloas, ForkRules::Gloas] {
+            let weights = compute_node_weights(&store, &index, &config, &committees, rules)
+                .unwrap_or_else(|err| panic!("{rules:?}: the gate must not be reached: {err:?}"));
+            assert_eq!(
+                weights.weight(boosted, anchor_slot + 2),
+                0,
+                "{rules:?}: a block outside the justified subtree gets no boost"
+            );
+        }
+    }
+
     // -----------------------------------------------------------------------
     // Gloas handlers
     // -----------------------------------------------------------------------
@@ -8404,13 +8509,17 @@ mod tests {
         branches_filtered_out: usize,
     }
 
-    /// Whether `root` is `justified` or a block below it in `index`.
+    /// Whether `root` is `justified` or a block below it in `index`. A root
+    /// `index` no longer holds, as finality pruning leaves one, is neither.
     fn descends_from_justified(
         index: &HashMap,
         root: Root,
         justified: Root,
     ) -> bool {
         let justified_slot = index[&justified].0;
+        if !index.contains_key(&root) {
+            return false;
+        }
         let mut cursor = root;
         while index[&cursor].0 > justified_slot {
             cursor = index[&cursor].1;
@@ -8427,7 +8536,9 @@ mod tests {
     /// Node weights are compared only for nodes at or above `walk_bound`, which
     /// can be lower than the finalized slot: the walk weighs nothing below it
     /// (see `compute_node_weights`' bound), so those nodes read as `0` where
-    /// the references count their votes.
+    /// the references count their votes. They are also compared only inside
+    /// the justified subtree, the nodes the descent compares, since the walk
+    /// gives no boost to a boosted block outside it.
     fn check_walk_against_references(
         seed: u64,
         tree: &mut RandomTree,
@@ -8466,11 +8577,14 @@ mod tests {
                 "seed {seed} slot {current_slot}: head differs from gloas_get_head"
             );
 
+            // Weights of the nodes the descent compares, the justified subtree:
+            // a boost root outside it is skipped by the walk but not by
+            // `gloas_get_weight`, which reaches no node inside it either way.
             let weights =
                 compute_node_weights(store, &index, &config, &committees, ForkRules::Gloas)
                     .unwrap_or_else(|err| panic!("seed {seed}: weights failed: {err:?}"));
             for (&root, &slot) in tree.roots.iter().zip(&tree.slots) {
-                if slot < bound {
+                if slot < bound || !descends_from_justified(&index, root, justified) {
                     continue;
                 }
                 for payload_status in [
@@ -8874,36 +8988,6 @@ mod tests {
         );
     }
 
-    /// The bound is the least of three slots so that it holds without assuming
-    /// the boosted block descends from the finalized one: with the boost on the
-    /// finalized block itself, the boost gate reads the score of a parent below
-    /// the finalized slot, and the walk still agrees with the unbounded
-    /// references on the head and on every node at or above the bound.
-    #[test]
-    fn the_bound_covers_a_boosted_finalized_block() {
-        let mut stats = WalkStats::default();
-        let mut boosted_finalized = 0;
-        for seed in 0..200 {
-            let mut tree = random_tree(seed, Arc::new(InMemoryBackend::new()));
-            if !tree.justified_moved {
-                continue;
-            }
-            let justified = finalize_at_justified(&mut tree);
-            tree.store.set_proposer_boost_root(justified.root);
-            if boost_gate_scans_for_equivocation(&tree) {
-                continue;
-            }
-            boosted_finalized += 1;
-            for current_slot in comparison_slots(&tree) {
-                check_walk_against_references(seed, &mut tree, current_slot, &mut stats);
-            }
-        }
-        assert!(
-            boosted_finalized > 0,
-            "the finalized block was never boosted"
-        );
-    }
-
     /// After a restart no link is recorded. Each cycle of the actor's sequence
     /// (record the finalized root's link, prune, compute the head) derives what
     /// it finds missing once, so the first cycle reads blocks and the next two
diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md
index 23fc7cd93..71809cdce 100644
--- a/docs/spec_deviations.md
+++ b/docs/spec_deviations.md
@@ -270,10 +270,18 @@ node, and each call walks every latest message and every ancestor step.
   slot and the boosted block's parent's slot, which on a live node is the
   finalized block's. The other two terms make the bound safe without assuming
   that the justified block and the boosted block's parent are at or above the
-  finalized block. The block index of a beacon store is never pruned, so
-  without the bound each head computation would fold, and read the link of,
-  every block since the anchor. Links below the finalized block are pruned
-  because nothing reads them.
+  finalized block. Finality pruning drops the block index rows below the
+  finalized block's own slot, so on a live node the index already starts
+  there; the bound holds the walk to that window whatever the index still
+  keeps below it. Links below the finalized block are pruned because nothing
+  reads them.
+- **No boost outside the justified subtree:** the specification evaluates
+  `should_apply_proposer_boost` whenever a boost root is set. The walk skips
+  the boost, gate included, when the boosted block is not under the justified
+  root: the boost reaches only that block's ancestors, none of which the
+  descent compares, so the head is the same. It also keeps the gate from
+  reading a parent that finality pruning removed after the boost was set,
+  which would otherwise fail the head computation until the next slot.
 - **Cost:** work is proportional to the latest messages plus the indexed
   blocks at or above the finalized block, the unfinalized window. The
   spec-literal form is proportional to the messages times the

From a8707ea2e477f3aa8a567834097c2cbd7d7bf05b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Thu, 22 Oct 2026 20:46:51 -0300
Subject: [PATCH 127/132] test(state-transition): move justified up with a
 fixture's finality override

The builder market gossip vectors finalize epoch 1 through the runner's
finalized_checkpoint override, then walk the head from the justified
root. Since main's #627, moving finality prunes LiveChain below the
finalized block, which left the genesis justified checkpoint pointing at a
pruned row: every bid and preferences case failed with "computing the
head: SpecAssert(\"block_root in store.blocks\")". A store's justified
checkpoint is never behind its finalized one, so for those cases the
override now moves it up too.
---
 .../state_transition/tests/beacon_spec/gossip.rs       | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs
index 1d3e32ca2..a9607efdf 100644
--- a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs
+++ b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs
@@ -391,8 +391,16 @@ fn build_store(
         let head = store
             .head()
             .map_err(|err| format!("reading the head: {err}"))?;
+        // The builder market cases walk the head next, from the justified
+        // root, and moving finality prunes `LiveChain` below the finalized
+        // block, so a justified checkpoint left behind the override would
+        // start that walk from a pruned row. A store's justified checkpoint is
+        // never behind its finalized one, so it moves up with the override.
+        let justified = (builder_market
+            && store.beacon_justified_checkpoint().epoch < finalized.epoch)
+            .then_some(checkpoint);
         store
-            .update_checkpoints(ForkCheckpoints::new(head, None, Some(checkpoint)))
+            .update_checkpoints(ForkCheckpoints::new(head, justified, Some(checkpoint)))
             .map_err(|err| format!("overriding the finalized checkpoint: {err}"))?;
         if builder_market {
             // The bid generators activate their builders by finalizing epoch 1

From e8f0e0512e5fcca8a5b7ce810869fa02db4a2ad6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Fri, 23 Oct 2026 19:28:35 -0300
Subject: [PATCH 128/132] =?UTF-8?q?chore(beacon):=20refresh=20Plat=C3=A5be?=
 =?UTF-8?q?rget's=20bootnodes=20from=20upstream?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

ethpandaops re-signed every devnet-8 bootnode ENR on 2026-10-09
(glamsterdam-devnets f271042f, "bump erigon"). Two of the twenty hosts
(168.144.43.157, 64.23.215.76) also rotated their node keys, so the
embedded entries named peer ids those hosts no longer answer to, both
for the static dial and as discv5 seeds. The other eighteen keep their
key, address and ports and only carry a higher seq.

Every refreshed ENR now carries the gloas digest, so the comment on the
fulu digest pin no longer holds for the embedded file. The pin stays;
the comment now points at the upstream revision that published it.
---
 .../assets/plataberget/bootstrap_nodes.yaml   | 40 +++++++++----------
 bin/ethlambda/src/network/built_in.rs         | 21 +++++-----
 2 files changed, 32 insertions(+), 29 deletions(-)

diff --git a/bin/ethlambda/assets/plataberget/bootstrap_nodes.yaml b/bin/ethlambda/assets/plataberget/bootstrap_nodes.yaml
index 9760301f4..d4a8922f8 100644
--- a/bin/ethlambda/assets/plataberget/bootstrap_nodes.yaml
+++ b/bin/ethlambda/assets/plataberget/bootstrap_nodes.yaml
@@ -1,20 +1,20 @@
-- enr:-Nm4QPXGASzUuGms8SNcTssTcr7grNKan3z7icEA4DCyBEslO6hYExCa1RTeJO1ixQZgysYNWX13AuNxJ0rNtvXzl_qGAZ_6u353h2F0dG5ldHOI__________-DY2djgYCEZXRoMpCYyRDPgHMxg___________gmlkgnY0gmlwhI5djeSDbmZkhAAAAACEcXVpY4IyyIlzZWNwMjU2azGhAt7VH1W8VHb4u-uKSNx_bkuwlG53pOYmZacvWOLy_E6viHN5bmNuZXRzAIN0Y3CCIyiDdWRwgiMo
-- enr:-Nm4QEVFVQbiiXSwlJCdX5v6LkcrloL51ehKNOJPflznWDKYODvDKgoCgKMq58NwIE9VysVCvxMmt9D86ZCtWSqLbdeGAZ_60r5sh2F0dG5ldHOI__________-DY2djgYCEZXRoMpCYyRDPgHMxg___________gmlkgnY0gmlwhKiQtFiDbmZkhAAAAACEcXVpY4IyyIlzZWNwMjU2azGhAn8t3459ORe6Z7XWmK-sBrRgbIvtDjp7EiYuzrsc9iXGiHN5bmNuZXRzAIN0Y3CCIyiDdWRwgiMo
-- enr:-QEXuEBiDkCfc3hB4T9PixYuIYzbiR5O5_4VntYekTaCTWtM5y3o8ak_LjsEWy5VF6qyljQz4ux7A_qfZYKzRfDbwp6dRodhdHRuZXRziP__________g2NnYwSGY2xpZW50z4hHcmFuZGluZYUyLjAuNYRldGgykJjJEM-AczGD__________-CaWSCdjSCaXCEqkCk9INpcDaQJABhgAAQAgAAAAAA28iAC4NuZmSEAAAAAIRxdWljgiMphXF1aWM2giNbiXNlY3AyNTZrMaEDf0VGB3-ABOw6d0mj2IxXi8s-fNHZSaMAHnO1SerUpZiIc3luY25ldHMPg3RjcIIjKIR0Y3A2giNag3VkcIIjKIR1ZHA2giNa
-- enr:-QEZuEAIY_Gyc62AY753YbfjXvJC1bJ-iiMjuiNXQSGS9bE2UAaVN5DHoBixt_nfo7mei1nDvL9tJvO1NoM0sbIGXo4zRYdhdHRuZXRziP__________g2NnYwSGY2xpZW500YpMaWdodGhvdXNlhTguMi4xhGV0aDKQXJQ4B4BzMYMABgAAAAAAAIJpZIJ2NIJpcIQuZUmFg2lwNpAqA7DAAAEA4AAAAAGlQMALg25mZISYyRDPhHF1aWOCIymFcXVpYzaCIymJc2VjcDI1NmsxoQIDtUPCwK4eWR0IJb5l-3zwOxxrE0TTAIRwWPkCWTiWgYhzeW5jbmV0cw-DdGNwgiMohHRjcDaCIyiDdWRwgiMohHVkcDaCIyg
-- enr:-QEZuEARbbwpYtSzfihstpGzZG5I_r8F7jmvn6n91Ims9cC0SncA75PmaWRxcDLPDQMrMM2wd2zPXdoVZXkq-4Qk6BL4QYdhdHRuZXRziP__________g2NnYwSGY2xpZW500YpMaWdodGhvdXNlhTguMi4yhGV0aDKQmMkQz4BzMYP__________4JpZIJ2NIJpcISokBA9g2lwNpAkAGGAAQAA0AAAAAGBCbALg25mZIQAAAAAhHF1aWOCIymFcXVpYzaCIymJc2VjcDI1NmsxoQIRNN_qOxRAVi5vowWakdp4Bes8spZtIQVWhh5tof2goYhzeW5jbmV0cwCDdGNwgiMohHRjcDaCIyiDdWRwgiMohHVkcDaCIyg
-- enr:-QEAuEConRusQCS4SqkO5fqwFNQJXhbK2R86q9AqpJmLg49UAXvqKSyXAUBDPaDcN95W70VzFnyJ2ZzPy_4EDKWvMuayBodhdHRuZXRziAAAAAADAAAAg2NnYwSEZXRoMpCYyRDPgHMxg___________gmlkgnY0gmlwhKpAhXODaXA2kCQAYYAAEAIAAAAAANvIkAuDbmZkhAAAAACEcXVpY4IjKYVxdWljNoIjK4lzZWNwMjU2azGhArqsAqRydR0Okd4qw_kJPvx3uUwb_b3NIAj9JZ3DOegwiHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjKoN1ZHCCIyiEdWRwNoIjKg
-- enr:-QEAuEAMEaYw8LFo5LX16MPNsQOvIHH7eAgcLz4gk2x_ZPhDDBquq2ALyWfmjnK1Qgq-Yt__H-U9DZ4Sfh5Jf_qk36l0BodhdHRuZXRziAAAAAAAAMAAg2NnYwSEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhKpA04yDaXA2kCQAYYAAEAIAAAAAANjJMAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjK4lzZWNwMjU2azGhA7tBIf5SwD2Tzmy25-uslQAsAFL-zM6DX0G78yFef4TgiHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjKoN1ZHCCIyiEdWRwNoIjKg
-- enr:-Mi4QJf9GKh8dkheSoE9gOPgBZ_zSkP0zzwyXF_pwT9Gl3ZCA44FTuuna7BAbbKkdWchlOrRcDmvjhURMbAkDspKnTYDh2F0dG5ldHOI__________-DY2djBIRldGgykJjJEM-AczGD__________-CaWSCdjSCaXCEqJArnYNuZmSEAAAAAIRxdWljgiMpiXNlY3AyNTZrMaECHBw0gzzECxIZ5JxwBLbqG15dAQai3x56jGkr_lK60qeDdGNwgiMog3VkcIIjKA
-- enr:-Ni4QK-MdSOl5Zam_puaMNb0VJ5QouxEn-BRM3cNNQ2L_3IdVsoCElRa1olJT5H5VyJGxz-f7oyqH9dzg2sihoqquSuGAZ_65aq1h2F0dG5ldHOI__________-DY2djBIRldGgykFyUOAeAczGDAAYAAAAAAACCaWSCdjSCaXCEQBfXTINuZmSEmMkQz4RxdWljgjLIiXNlY3AyNTZrMaEDDu_e1O_fd1uc9-gJ0WgVILH0Ix85MtY-HxsAws_LrWCIc3luY25ldHMAg3RjcIIjKIN1ZHCCIyg
-- enr:-Li4QFuQCN3eqEsUwmOYpGC16gSFhQ7HtgFhWrfUwQGEO8uucqwSrf9EsGY5JCZViicQ9q7BwPTbFAId4N0S5SUcGqaBm4NjZ2MEhGV0aDKQmMkQz4BzMYP__________4JpZIJ2NIJpcIREt8lQg25mZIQAAAAAhHF1aWOCIymJc2VjcDI1NmsxoQKaWbx7I0HUqpOYKs7-goPu_myo7e8vFNG4CrSBnNqFoYN0Y3CCIyiDdWRwgiMo
-- enr:-QEYuECJkYqGfhFmM8alfBJOp0VUaBXWu-uBq8O6EDCZt4IUlRWUOpq9FYXue1gjIXr1pEkPK2_QEpaIaLnob_bq4mUGAYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhJ_LBKmDaXA2kCYEqIAMrQDQAAAAAaN4cAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhAjPAcyQXBn8ISq63R_3Sg3gu6SC6IpbtiNCSMrFwRdSFiHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg
-- enr:-QEYuED8Hm5EW_x3Pg9tERZtRu1syR87Bb4BGHGeDHEiDXQJHmDD5vXol6DKNh8avwqe1kAn7SsshN-Vrflqgc31eo4ZAYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhJ31aSGDaXA2kCQAYYABAADQAAAAAXl_QAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhAgvWQ3QMaNmKNx_mR1YVY1kFyKYnFJfuYBv_BBbQCZy4iHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg
-- enr:-QEYuEAMENdhgbfS7XRUcl3k-O2_f9tjc0PBHnVaXiC8vaXd9zZpmSQYpYI4PY1Anr5thI1br-rNoUl1FbbhouKq4XGzAYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhIbHn9ODaXA2kCQAYYAAEAIAAAAAANjJIAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhA4EKEgHSBAmKqEjWVZGCjn1TmPdbv0VO8AbjpcocDeQ6iHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg
-- enr:-QEYuEApS_2JUFBGwhRgg3mS4n-RxjevE-4XqNcSkUSfWnVAuTzqWBbjbbFgk_f9d_ZNrhDTAcf3N-ibRGleTxDbDVdbAYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhC5l_2aDaXA2kCoDsMAAAwDwAAAAAs2MAAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhA_Q9CjhlGYywa7fylxY5Qi6HMubCdDzwavcDq4ctRd2hiHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg
-- enr:-QEYuEB882HfZiAfsbCvoadDpm2iCAqpy64tlbvYuYSLAQ0yGksn7tiADjU62Tw22GaXx96fz2-LVYlS38hg_5taKNS0AYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhIs7IY-DaXA2kCQAYYABAADQAAAAAXl_gAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhAoVD1N5sqhIE4Z2MO5_iEz5fgsW5HvjenNUu97i0gV5RiHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg
-- enr:-QEYuEDebd1bjfQYHTgOj85z5RUwdd9KxU1tQkGzNbqSCCZri2ZdU1SFk6Tv-p2F5pq5qtS8_44keTOn6FirGEIfqytwAYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhIm4emeDaXA2kCYEqIAABAHQAAAAA0hsEAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhAi_CwnwYpvgM9GKZv-oDRkoSwC1hRmrou2BOFeN4lxzfiHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg
-- enr:-QEYuECNqtrKxqKB5dF-8rYKDoogeNCgHrZ9FR2g9KEhVnYCTFuk-o-NrCDXXKWql4gBK9iMmHerLG_6OhwVJL8otgO0AYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhKiQtOyDaXA2kCQAYYABAADQAAAAAXl-8AuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhA56KySRQoku4PbUzYWuqzkTTulEt574e9qkmdEBeAzFiiHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg
-- enr:-QEYuEC7v0dZOZswuKbzVXUcD3Vkx2sxCDWthxh-uExjhlsj-UXiSOY66BG4yebK0tdyBpt30gTdzwMN8kIirFRqhB5KAYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhKpAoDGDaXA2kCQAYYAAEAIAAAAAANjJEAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhAkzN2TBr3faHcjF0W-L_p5FdSvk1MbeFIFEG3NXduY7RiHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg
-- enr:-QEYuEAtmWBixpnqPbdELk5e6P9ZAUK7kEs37UtkhFc-59RC6Ged9iatnPgkWt3jki9rDZln7AKZIcs3BgfAJfOTWh4MAYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhKiQdc2DaXA2kCQAYYABAADQAAAAAXl_MAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhAgFphB9XXsRdGLS3LZkjy0TTx3o0EdHIwsY5D0nI-NKliHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg
-- enr:-QEYuEBpJHYFTuleF94yNA-eJXNOq4ZQuuGb4vs7uMfGxKXFjQDk5-hyqpKGDh2GSh7rYMHYdml8QPq1jj1_KDSH65btAYdhdHRuZXRziAAAAAAAAAAAg2NnY4GAhmNsaWVudM-IR3JhbmRpbmWFMi4wLjWEZXRoMpBclDgHgHMxgwAGAAAAAAAAgmlkgnY0gmlwhI_GaY2DaXA2kCYEqIAABAHQAAAAA0hsQAuDbmZkhJjJEM-EcXVpY4IjKYVxdWljNoIjW4lzZWNwMjU2azGhA7mE6wvAxEMRBo5iIclDEOMWhCudz8dSYx6N4pxKrXroiHN5bmNuZXRzAIN0Y3CCIyiEdGNwNoIjWoN1ZHCCIyiEdWRwNoIjWg
\ No newline at end of file
+- enr:-Nm4QDKxpDtUJAw7S4QnA3pxuixIZ6NRIGYcLEmLwfW6eCzgLbC5X4D82j5XqmAvTTOIFL6AEbl5o9hhFOFCj08fr-aGAZ_6u36mh2F0dG5ldHOI__________-DY2djgYCEZXRoMpCYyRDPgHMxg___________gmlkgnY0gmlwhI5djeSDbmZkhAAAAACEcXVpY4IyyIlzZWNwMjU2azGhAt7VH1W8VHb4u-uKSNx_bkuwlG53pOYmZacvWOLy_E6viHN5bmNuZXRzAoN0Y3CCIyiDdWRwgiMo
+- enr:-Nm4QL7bUbj6KtMwA4vl3JhcBKR23oDoiRvGKOVWR3HGiy2HVFUyFB1LV4i102NVBgY1C-nO7HuTVKKujwFy30HTI3uGAZ_60r6Eh2F0dG5ldHOI__________-DY2djgYCEZXRoMpCYyRDPgHMxg___________gmlkgnY0gmlwhKiQtFiDbmZkhAAAAACEcXVpY4IyyIlzZWNwMjU2azGhAn8t3459ORe6Z7XWmK-sBrRgbIvtDjp7EiYuzrsc9iXGiHN5bmNuZXRzAIN0Y3CCIyiDdWRwgiMo
+- enr:-QEZuECFiAupr9KMT-4bX1AhI_OH6U7gURBxXlOkKE9B7yYXxW8QVasuiJbMS6pIAHhBOOnT5bsg0-xWfmSbDeyeDgHvgvsJh2F0dG5ldHOI__________-DY2djZIZjbGllbnTPiEdyYW5kaW5lhTMuMC4whGV0aDKQmMkQz4BzMYP__________4JpZIJ2NIJpcISqQKT0g2lwNpAkAGGAABACAAAAAADbyIALg25mZIQAAAAAhHF1aWOCIymFcXVpYzaCI1uJc2VjcDI1NmsxoQN_RUYHf4AE7Dp3SaPYjFeLyz580dlJowAec7VJ6tSlmIhzeW5jbmV0cw-DdGNwgiMohHRjcDaCI1qDdWRwgiMohHVkcDaCI1o
+- enr:-QEguEB4-T__2QJ-KlF3lFeHDfMwHZQ226345Kghy7abrLW2akpyxWxYbfRL7SZZfc41tLOvdLDIWxYBGJYCNlR2OfUQggybh2F0dG5ldHOI__________-DY2djZIZjbGllbnTWikxpZ2h0aG91c2WKOC4zLjAtcmMuMIRldGgykJjJEM-AczGD__________-CaWSCdjSCaXCELmVJhYNpcDaQKgOwwAABAOAAAAABpUDAC4NuZmSEAAAAAIRxdWljgiMphXF1aWM2giMpiXNlY3AyNTZrMaECA7VDwsCuHlkdCCW-Zft88DscaxNE0wCEcFj5Alk4loGIc3luY25ldHMPg3RjcIIjKIR0Y3A2giMog3VkcIIjKIR1ZHA2giMo
+- enr:-QEguEBOwRuCe3SL3QqTDtQPh_zZ7M2KdwDP-OE5he-cM1brN14PygPrmF1HsT8j68KSjJNkAfZNdj9Di3QT3WeNHJnJggiZh2F0dG5ldHOI__________-DY2djZIZjbGllbnTWikxpZ2h0aG91c2WKOC4zLjAtcmMuMIRldGgykJjJEM-AczGD__________-CaWSCdjSCaXCEqJAQPYNpcDaQJABhgAEAANAAAAABgQmwC4NuZmSEAAAAAIRxdWljgiMphXF1aWM2giMpiXNlY3AyNTZrMaECETTf6jsUQFYub6MFmpHaeAXrPLKWbSEFVoYebaH9oKGIc3luY25ldHMPg3RjcIIjKIR0Y3A2giMog3VkcIIjKIR1ZHA2giMo
+- enr:-QEBuECv5ALBYCZlW_qfX7peq2Pt4OVJj8uC-ytkBwp1UwIPCl8o5Hj3qfBr29RF273xkDkc9KeRbfz5yBVFVgDGp84VgbWHYXR0bmV0c4gAAAAwAAAAAINjZ2MEhGV0aDKQmMkQz4BzMYP__________4JpZIJ2NIJpcISqQIVzg2lwNpAkAGGAABACAAAAAADbyJALg25mZIQAAAAAhHF1aWOCIymFcXVpYzaCIyuJc2VjcDI1NmsxoQK6rAKkcnUdDpHeKsP5CT78d7lMG_29zSAI_SWdwznoMIhzeW5jbmV0cwCDdGNwgiMohHRjcDaCIyqDdWRwgiMohHVkcDaCIyo
+- enr:-QECuEBheB4e3QifHJVY9megfM3S4klfzwWeTB2K0KINKVqUqhH8D4raENTYVFC1mDh3OC3JkqtOnDAIxqzWBfC6zD77ggFQh2F0dG5ldHOIAAAAAACAAQCDY2djBIRldGgykJjJEM-AczGD__________-CaWSCdjSCaXCEqkDTjINpcDaQJABhgAAQAgAAAAAA2MkwC4NuZmSEAAAAAIRxdWljgiMphXF1aWM2giMriXNlY3AyNTZrMaEDu0Eh_lLAPZPObLbn66yVACwAUv7MzoNfQbvzIV5_hOCIc3luY25ldHMAg3RjcIIjKIR0Y3A2giMqg3VkcIIjKIR1ZHA2giMq
+- enr:-NK4QCnqBLt1P5rIhDPzuHtdUo0ri5ztkyj8vlZyakP8g5NgZt5buCzlET8gH5Vi71_3_Or-O0epiy0uoNctc36IaiEEh2F0dG5ldHOI__________-DY2djBIRldGgykJjJEM-AczGD__________-CaWSCdjSCaXCEqJArnYNuZmSEAAAAAIRxdWljgiMpiXNlY3AyNTZrMaEDLJZP1-rhVThA7FQiIAjbjvCSPE9GfgrYVRZjfV3cMpaIc3luY25ldHMPg3RjcIIjKIN1ZHCCIyg
+- enr:-Ni4QMqK8DNL9eGd3-FCROCaauUqPhtmzJH55D6Z5qh4AUWARyJNkAh6wi4yFQlIoNEf_roxL-fB7mwxfHfLdIuU9H2GAaCfqZ93h2F0dG5ldHOI__________-DY2djCIRldGgykJjJEM-AczGD__________-CaWSCdjSCaXCEQBfXTINuZmSEAAAAAIRxdWljgjLIiXNlY3AyNTZrMaEC0m0CmX5YW_W2Bnbo3tacWvS1OSJz9TshWpcmhUtAMEeIc3luY25ldHMAg3RjcIIjKIN1ZHCCIyg
+- enr:-NS4QFTfV5GXLuJMUo5kB_ZEG4aerNvh_miYafZqN-L4hgRjMRzl8Ajsg4LTyO-A7Jp5ps10XWKAr_s-DpT5bp8JQsaCAbiHYXR0bmV0c4j__________4NjZ2NkhGV0aDKQmMkQz4BzMYP__________4JpZIJ2NIJpcIREt8lQg25mZIQAAAAAhHF1aWOCIymJc2VjcDI1NmsxoQKaWbx7I0HUqpOYKs7-goPu_myo7e8vFNG4CrSBnNqFoYhzeW5jbmV0cw-DdGNwgiMog3VkcIIjKA
+- enr:-QEauEC4UfvS6oNO4lQqlu5c5iKfequKaRSSPT1Qy_mBR-cDKjBnvczL4LPP5iJX5TzCeC61wOQr3Ptx5M102z1-DQjcgg05h2F0dG5ldHOIAAAAABgAAACDY2djgYCGY2xpZW50z4hHcmFuZGluZYUzLjAuMIRldGgykJjJEM-AczGD__________-CaWSCdjSCaXCEn8sEqYNpcDaQJgSogAytANAAAAABo3hwC4NuZmSEAAAAAIRxdWljgiMphXF1aWM2giNbiXNlY3AyNTZrMaECM8BzJBcGfwhKrrdH_dKDeC7pILoilu2I0JIysXBF1IWIc3luY25ldHMAg3RjcIIjKIR0Y3A2giNag3VkcIIjKIR1ZHA2giNa
+- enr:-QEauEDeO3Y6dEmPihey51SA0zt9xHGB8exc9xhWYg7f9QCj71J7z-o7Esz67xBmwfEIswuTEzcwnlnhb_3gyEiY2hWQggugh2F0dG5ldHOIAAMAAAAAAACDY2djgYCGY2xpZW50z4hHcmFuZGluZYUzLjAuMIRldGgykJjJEM-AczGD__________-CaWSCdjSCaXCEnfVpIYNpcDaQJABhgAEAANAAAAABeX9AC4NuZmSEAAAAAIRxdWljgiMphXF1aWM2giNbiXNlY3AyNTZrMaECC9ZDdAxo2Yo3H-ZHVhVjWQXIpicUl-5gG_8EFtAJnLiIc3luY25ldHMAg3RjcIIjKIR0Y3A2giNag3VkcIIjKIR1ZHA2giNa
+- enr:-QEauED1rFByj4HGJG7nGjaTUPqvi2BAZ3yNcxjvLKDuq0t8ngibGqroHMmgXCdnInb8ePC1-eOpoq8y-ivvHT3XUOeRggyYh2F0dG5ldHOIAAAAAAAGAACDY2djgYCGY2xpZW50z4hHcmFuZGluZYUzLjAuMIRldGgykJjJEM-AczGD__________-CaWSCdjSCaXCEhsef04NpcDaQJABhgAAQAgAAAAAA2MkgC4NuZmSEAAAAAIRxdWljgiMphXF1aWM2giNbiXNlY3AyNTZrMaEDgQoSAdIECYqoSNZVkYKOfVOY91u_RU7wBuOlyhwN5DqIc3luY25ldHMCg3RjcIIjKIR0Y3A2giNag3VkcIIjKIR1ZHA2giNa
+- enr:-QEauEC-_nB7qhoY-S-l0qmIMtWA-fBrKcsmiVLLKB6RAXdgDxfLswMD6-0Zh9asOeelmtu5piMdY9vcdjM8cbP93hR9gg0Yh2F0dG5ldHOIAAAAYAAAAACDY2djgYCGY2xpZW50z4hHcmFuZGluZYUzLjAuMIRldGgykJjJEM-AczGD__________-CaWSCdjSCaXCELmX_ZoNpcDaQKgOwwAADAPAAAAACzYwAC4NuZmSEAAAAAIRxdWljgiMphXF1aWM2giNbiXNlY3AyNTZrMaED9D0KOGUZjLBrt_KXFjlCLocy5sJ0PPBq9wOrhy1F3aGIc3luY25ldHMCg3RjcIIjKIR0Y3A2giNag3VkcIIjKIR1ZHA2giNa
+- enr:-QEauEB5os7E3hpkFut2HgRe6_uwwXnGWPvWXc-35E1_3Zmw-k1Tye9y586UKMvAkgRMH1wWlXapbXrSjGKmOvf4GgXdggYoh2F0dG5ldHOIAAAAAAAABgCDY2djgYCGY2xpZW50z4hHcmFuZGluZYUzLjAuMIRldGgykJjJEM-AczGD__________-CaWSCdjSCaXCEizshj4NpcDaQJABhgAEAANAAAAABeX-AC4NuZmSEAAAAAIRxdWljgiMphXF1aWM2giNbiXNlY3AyNTZrMaEChUPU3myqEgThnYw7n-ITPl-Cxbke-N6c1S73uLSBXlGIc3luY25ldHMAg3RjcIIjKIR0Y3A2giNag3VkcIIjKIR1ZHA2giNa
+- enr:-QEauEAedUyBEcnyp7_PgAD9HHOO2fzbbFceMbmU4e_7wJ7-4Xk6_tHTIjI-6aniO6W042qFD8unKUMK3lb3MAMRKwDuggdfh2F0dG5ldHOIAAAAAAAGAACDY2djgYCGY2xpZW50z4hHcmFuZGluZYUzLjAuMIRldGgykJjJEM-AczGD__________-CaWSCdjSCaXCEibh6Z4NpcDaQJgSogAAEAdAAAAADSGwQC4NuZmSEAAAAAIRxdWljgiMphXF1aWM2giNbiXNlY3AyNTZrMaECL8LCfBim-Az0Ypm_6gNGShLALWFGaui7YE4V43iXHN-Ic3luY25ldHMCg3RjcIIjKIR0Y3A2giNag3VkcIIjKIR1ZHA2giNa
+- enr:-QEauECYTdsTbdckjcOMUsjdSqvcdu8Un-ajm7bsNAyV-m_vjXcK7lEmT8QIbS6kAvyrCq0JcJAHX4PBidlPSimc9Cgiggs1h2F0dG5ldHOIAACAAQAAAACDY2djgYCGY2xpZW50z4hHcmFuZGluZYUzLjAuMIRldGgykJjJEM-AczGD__________-CaWSCdjSCaXCEqJC07INpcDaQJABhgAEAANAAAAABeX7wC4NuZmSEAAAAAIRxdWljgiMphXF1aWM2giNbiXNlY3AyNTZrMaEDnorJJFCiS7g9tTNha6rORNO6US3nvh72qSZ0QF4DMWKIc3luY25ldHMAg3RjcIIjKIR0Y3A2giNag3VkcIIjKIR1ZHA2giNa
+- enr:-QEauEDDb1j_FhnQjFdzQL1Gg6EMkQ8upvyS9pZgJEYc6uCRZkZCA_D_WyxBpKLdkGfsbRtRgzbFUipuLYW070bYLmAGggw6h2F0dG5ldHOIAAAAAAAGAACDY2djgYCGY2xpZW50z4hHcmFuZGluZYUzLjAuMIRldGgykJjJEM-AczGD__________-CaWSCdjSCaXCEqkCgMYNpcDaQJABhgAAQAgAAAAAA2MkQC4NuZmSEAAAAAIRxdWljgiMphXF1aWM2giNbiXNlY3AyNTZrMaECTM3ZMGvd9odyMXRb4v-nkV1K-TUxt4UgUQbc1d25jtGIc3luY25ldHMAg3RjcIIjKIR0Y3A2giNag3VkcIIjKIR1ZHA2giNa
+- enr:-QEauEA4OcV6enoxMc1Q61gdy-D3OBzNZtqn8ugCRvsowfaUVg4y4cexjsSAu_SZu5khKAPED6DmP6RuN27sgBnA6PmSggrqh2F0dG5ldHOIAwAAAAAAAACDY2djgYCGY2xpZW50z4hHcmFuZGluZYUzLjAuMIRldGgykJjJEM-AczGD__________-CaWSCdjSCaXCEqJB1zYNpcDaQJABhgAEAANAAAAABeX8wC4NuZmSEAAAAAIRxdWljgiMphXF1aWM2giNbiXNlY3AyNTZrMaECAWmEH1dexF0YtLctmSPLRNPHejQR0cjCxjkPScj40qWIc3luY25ldHMAg3RjcIIjKIR0Y3A2giNag3VkcIIjKIR1ZHA2giNa
+- enr:-QEauEB8JTZG8pIm-tnDwxRc4z2hM8_5FRQyHluIcNtEmPxuDBaPY31jJJOeKKUBmAxymwCqV6UJs-U5R9xhOuRzPWWHggvKh2F0dG5ldHOIAAAAAAAAABiDY2djgYCGY2xpZW50z4hHcmFuZGluZYUzLjAuMIRldGgykJjJEM-AczGD__________-CaWSCdjSCaXCEj8ZpjYNpcDaQJgSogAAEAdAAAAADSGxAC4NuZmSEAAAAAIRxdWljgiMphXF1aWM2giNbiXNlY3AyNTZrMaEDuYTrC8DEQxEGjmIhyUMQ4xaEK53Px1JjHo3inEqteuiIc3luY25ldHMAg3RjcIIjKIR0Y3A2giNag3VkcIIjKIR1ZHA2giNa
\ No newline at end of file
diff --git a/bin/ethlambda/src/network/built_in.rs b/bin/ethlambda/src/network/built_in.rs
index d3cf05f8f..8133a765d 100644
--- a/bin/ethlambda/src/network/built_in.rs
+++ b/bin/ethlambda/src/network/built_in.rs
@@ -283,9 +283,10 @@ mod tests {
     /// as a test fixture (mainnet's is, and `beacon::tests` checks mainnet's
     /// constants against it). What pins them is a digest someone else computed
     /// from it: each expected value below is the `eth2` entry of a bootnode
-    /// ENR in the network's own `bootstrap_nodes.yaml`, published while that
-    /// fork was current. The digest mixes the fork version with the root, so
-    /// a wrong root or a wrong fork version fails here.
+    /// ENR in the network's own `bootstrap_nodes.yaml` (or an earlier revision
+    /// of it), published while that fork was current. The digest mixes the
+    /// fork version with the root, so a wrong root or a wrong fork version
+    /// fails here.
     #[test]
     fn the_genesis_roots_reproduce_published_digests() {
         // Sepolia's last listed bootnode, signed during bellatrix, before
@@ -309,12 +310,14 @@ mod tests {
             [0xd2, 0xf1, 0x99, 0x7f]
         );
 
-        // Platåberget's bootnodes come from both sides of its gloas fork.
-        // Those signed before it name gloas as their next fork, at its
-        // `GLOAS_FORK_EPOCH`; genesis was fulu, so the digest also mixes in
-        // the blob schedule's first entry. Those signed after it repeat
-        // gloas's version with FAR_FUTURE_EPOCH, so they pin the gloas fork
-        // version as well.
+        // Platåberget's fulu digest is not in the embedded file anymore:
+        // upstream re-signed every bootnode after the gloas fork (in
+        // `bump erigon`, 2026-10-09). It comes from the revision before that
+        // (`add new buildoors`, 2026-08-26), whose ENRs signed before the fork
+        // name gloas as their next fork, at its `GLOAS_FORK_EPOCH`; genesis
+        // was fulu, so the digest also mixes in the blob schedule's first
+        // entry. Every current ENR repeats gloas's version with
+        // FAR_FUTURE_EPOCH, so it pins the gloas fork version as well.
         let plataberget = BuiltInNetwork::Plataberget.resolve().unwrap();
         let root = plataberget.genesis.genesis_validators_root;
         assert_eq!(

From 360f977ac37e138859d56063e507ded2a7636e6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Mon, 26 Oct 2026 13:23:33 -0300
Subject: [PATCH 129/132] feat(beacon): gloas builder market: bid and
 preference gossip, endpoints, building on bids (#669)

## Motivation

Gloas makes payload building a market: builders gossip signed bids on
`execution_payload_bid`, proposers gossip signed `proposer_preferences`
(fee recipient, gas target) so builders can bid for their slots, and a
proposer may build on the best bid instead of its own payload. Our node
subscribed to neither topic, served none of the endpoints (lighthouse
and prysm validator clients already call `POST
/eth/v1/validator/proposer_preferences` and got a 404), and always
self-built. This adds the node side of the market. Stacked on #667.

## Changes

| Piece | What |
|---|---|
| Gossip | `execution_payload_bid` and `proposer_preferences` under
gloas digests: every rule of gloas `p2p-interface.md`, seen caches,
their own validation permit pool, relayed on accept. Never queued, never
sent to the chain actor. |
| State | One `SharedBuilderMarket` shared by p2p and rpc: proposer
preferences per proposal slot, the best bids per slot and parent, and
the known execution payloads the bid rules check parents against (gossip
envelopes plus the node's own). Bounded and pruned. |
| Endpoints | `POST /eth/v1/beacon/execution_payload_bids` (JSON or SSZ,
gossip-validated, published), `POST
/eth/v1/validator/proposer_preferences` (verified, cached, published),
`POST /eth/v1/beacon/states/{state_id}/builders` (filterable builder
registry). |
| `produceBlockV4` | Honors `BuilderConfig`: the best gossiped bid
competes with the local build under `min_bid` and `builder_boost_factor`
(local wins ties, `shouldOverrideBuilder` respected). A winning bid
returns the block only (no envelope, `Eth-Execution-Payload-Included:
false`); the builder releases the envelope. A bid that fails to assemble
falls back to the local build. |
| Self-build | From gloas, the fee recipient and gas target come from
the proposer's signed preferences, then `prepare_beacon_proposer`, as
beacon-APIs requires. |
| Types | `Builder` integers are quoted in JSON (they were bare, also in
the gloas state JSON); preferences deserialize. |

Phase 2 (not here): the builder API (bids fetched from
`BuilderConfig.builders` URLs, `POST
/eth/v1/validator/builder_preferences`, forwarding the block to the
winning builder). That route is a 404 for now.

## Deviations (in `docs/spec_deviations.md`)

- Preference signatures are accepted under the spec's lookahead-state
domain and the schedule's fork at `P - MIN_SEED_LOOKAHEAD` and at `P`,
which coincide outside a fork's first epoch: lighthouse signs with the
fork at `P`, the spec with `P - 1`, and refusing either would drop
honest preferences at the gloas boundary.
- Stateful checks read cached states only (a miss is IGNORE), and the
head fast path serves preferences whose dependent block's state was
evicted.
- A pre-gloas parent's payload counts as known.

## Validation

- Spec gossip vectors v1.7.0-beta.2 mainnet:
`gossip_execution_payload_bid` 45/45, `gossip_proposer_preferences`
27/27 (previously ignored). Minimal not run locally.
- Unit tests for the market, every rule (including fulu-parent boundary
cases and the domains), assembly on a bid (`process_block` accepts it),
topics, decode, verdict settle, publish, bid selection, `BuilderConfig`,
and router tests driving every endpoint and `produceBlockV4` against a
fake execution client.
- `--lib`: state-transition 544, types 221, p2p 312, rpc 234, engine 27,
validator 269, blockchain 236, storage 152; clippy and fmt clean.
---
 CLAUDE.md                                     |   17 +-
 bin/ethlambda/src/main.rs                     |    6 +
 .../src/beacon/builder_market.rs              |  623 +++++++++
 .../src/beacon/gloas_block_production.rs      |  290 +++-
 .../beacon/gossip/execution_payload_bid.rs    | 1110 +++++++++++++++
 .../state_transition/src/beacon/gossip/mod.rs |   62 +
 .../src/beacon/gossip/proposer_preferences.rs |  642 +++++++++
 .../src/beacon/gossip/test_support.rs         |  199 +++
 .../state_transition/src/beacon/mod.rs        |    1 +
 .../tests/beacon_spec/gossip.rs               |  145 +-
 .../types/src/beacon/containers/gloas.rs      |   37 +-
 .../types/src/beacon/containers/json_tests.rs |   81 ++
 crates/net/api/src/lib.rs                     |   18 +-
 crates/net/engine/src/building.rs             |   30 +
 crates/net/p2p/src/beacon/builder_market.rs   |  414 ++++++
 crates/net/p2p/src/beacon/decode.rs           |   53 +
 crates/net/p2p/src/beacon/mod.rs              |    1 +
 crates/net/p2p/src/beacon/topics.rs           |   28 +-
 crates/net/p2p/src/beacon/verdict.rs          |  252 +++-
 crates/net/p2p/src/gossipsub/handler.rs       |    9 +
 crates/net/p2p/src/gossipsub/mod.rs           |    1 +
 crates/net/p2p/src/lib.rs                     |   42 +-
 crates/net/p2p/src/req_resp/handlers.rs       |    4 +
 crates/net/rpc/src/beacon/bid_selection.rs    |  230 +++
 crates/net/rpc/src/beacon/bids.rs             |  154 ++
 crates/net/rpc/src/beacon/builder_config.rs   |  275 ++++
 .../rpc/src/beacon/builder_market_tests.rs    | 1234 +++++++++++++++++
 crates/net/rpc/src/beacon/builders.rs         |  336 +++++
 crates/net/rpc/src/beacon/gloas_proposal.rs   |  513 +++++--
 crates/net/rpc/src/beacon/mod.rs              |    8 +
 .../rpc/src/beacon/proposer_preferences.rs    |  144 ++
 crates/net/rpc/src/beacon/states.rs           |    5 +-
 .../rpc/src/beacon/validator_client_tests.rs  |    3 +
 crates/net/rpc/src/lib.rs                     |   29 +-
 docs/beacon_wire.md                           |   24 +-
 docs/metrics.md                               |   29 +-
 docs/rpc.md                                   |   87 +-
 docs/spec_deviations.md                       |  173 ++-
 38 files changed, 7068 insertions(+), 241 deletions(-)
 create mode 100644 crates/blockchain/state_transition/src/beacon/builder_market.rs
 create mode 100644 crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs
 create mode 100644 crates/blockchain/state_transition/src/beacon/gossip/proposer_preferences.rs
 create mode 100644 crates/net/p2p/src/beacon/builder_market.rs
 create mode 100644 crates/net/rpc/src/beacon/bid_selection.rs
 create mode 100644 crates/net/rpc/src/beacon/bids.rs
 create mode 100644 crates/net/rpc/src/beacon/builder_config.rs
 create mode 100644 crates/net/rpc/src/beacon/builder_market_tests.rs
 create mode 100644 crates/net/rpc/src/beacon/builders.rs
 create mode 100644 crates/net/rpc/src/beacon/proposer_preferences.rs

diff --git a/CLAUDE.md b/CLAUDE.md
index cc08b89cb..71a75fd18 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -334,9 +334,11 @@ actual_slot = finalized_slot + 1 + relative_index
   - Beacon wire: `validate_messages()` is on, so every beacon message waits for a verdict (~4.2s before gossipsub's cache evicts it). Rules in `state_transition::beacon::gossip` (cheap half inline, stateful half on a bounded `spawn_blocking` task); plumbing in `p2p/src/beacon/verdict.rs`. Lean gossip still auto-forwards
   - Data columns: every check runs in p2p. A column gossip did not accept (`Queue`/`Overloaded`), every fetched column, and parked columns replayed after their parent imports go through `column::chain_checks` in `p2p/src/beacon/column_checks.rs`. The chain actor stores what it gets unchecked; only debug builds re-run `chain_checks` there
   - Beacon subscribes seven global topics plus two node-id-derived subnet families: custody
-    columns and backbone attestation subnets. Gloas digests add two more topics,
-    `execution_payload` and `payload_attestation_message` (`BeaconTopics::for_fork`; earlier
-    digests never carry them). Gloas has its own rules for `beacon_block`,
+    columns and backbone attestation subnets. Gloas digests add four more topics,
+    `execution_payload`, `payload_attestation_message`, `execution_payload_bid` and
+    `proposer_preferences` (`BeaconTopics::for_fork`; earlier digests never carry them).
+    Bids and preferences are validated in p2p against one shared `BuilderMarket`
+    (`state_transition::beacon::builder_market`) and never reach the chain actor. Gloas has its own rules for `beacon_block`,
     `data_column_sidecar` (fork enum `DataColumnSidecar`, fork from the topic's digest),
     aggregates (`SignedAggregateAndProof::Gloas`, aggregation-bits length bounded before
     expansion) and attestations (`verify_attestation_payload_status`). Deliberate
@@ -769,8 +771,13 @@ transitions are in `ethlambda-types`, per the section above. Nothing above
   the parking of gloas column sidecars (they carry no signature). Fork-choice
   events are timed at arrival, not at the slot tick, and the head's payload
   status is kept with its root (`Store::head_payload_status`, recomputed after
-  a restart). The node also serves gloas validator duties, self-build only (no
-  bids, no proposer preferences; see `docs/spec_deviations.md`).
+  a restart). The node also serves gloas validator duties and the builder market:
+  it validates, pools and relays gossip bids and proposer preferences, serves
+  `POST /eth/v1/beacon/execution_payload_bids` and `/proposer_preferences` plus the
+  builder endpoints, and `produceBlockV4` weighs the best pooled p2p bid against
+  the local build per `BuilderConfig` (`min_bid`, `builder_boost_factor`; the local
+  build wins a tie). A winning bid returns the block only. The builder API (relay
+  or `builder_pubkeys` bids) is phase 2; see `docs/spec_deviations.md`.
   `state_transition/src/beacon/gloas_block_production.rs` assembles the block and
   its envelope (`gloas_payload_inputs`, `parse_gloas_execution_requests`,
   `pack_gloas_attestations`, `pack_payload_attestations`, `assemble_gloas_block`,
diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs
index edd6cc207..0d1fc34e7 100644
--- a/bin/ethlambda/src/main.rs
+++ b/bin/ethlambda/src/main.rs
@@ -724,6 +724,10 @@ async fn run_node(options: Options) -> eyre::Result<()> {
     // by block production and `GET .../pool/payload_attestations`.
     let payload_attestation_pool =
         ethlambda_state_transition::beacon::payload_attestation_pool::SharedPayloadAttestationPool::default();
+    // Bids, proposer preferences and known payloads: filled by gossip and the
+    // Beacon API's bid and preferences endpoints, read by block production.
+    let builder_market =
+        ethlambda_state_transition::beacon::builder_market::SharedBuilderMarket::default();
     let p2p = P2P::spawn(
         built,
         setup.store.clone(),
@@ -731,6 +735,7 @@ async fn run_node(options: Options) -> eyre::Result<()> {
         discovery,
         attestation_pool.clone(),
         payload_attestation_pool.clone(),
+        builder_market.clone(),
     )
     .await
     .wrap_err("failed to start discv5 discovery")?;
@@ -772,6 +777,7 @@ async fn run_node(options: Options) -> eyre::Result<()> {
                     p2p: rpc_p2p,
                     attestation_pool: attestation_pool.clone(),
                     payload_attestation_pool: payload_attestation_pool.clone(),
+                    builder_market: builder_market.clone(),
                     custody_columns: rpc_custody_columns,
                     engine: rpc_engine,
                     events: rpc_events,
diff --git a/crates/blockchain/state_transition/src/beacon/builder_market.rs b/crates/blockchain/state_transition/src/beacon/builder_market.rs
new file mode 100644
index 000000000..6eda031bd
--- /dev/null
+++ b/crates/blockchain/state_transition/src/beacon/builder_market.rs
@@ -0,0 +1,623 @@
+//! The shared state behind the gloas builder market: bids seen on
+//! `execution_payload_bid` (or posted to the Beacon API) and pooled for block
+//! production, the proposer preferences those bids are judged against, and the
+//! execution payloads gossip has revealed.
+//!
+//! One [`SharedBuilderMarket`] exists per node. p2p validates against it and
+//! the Beacon API reads it, like [`super::payload_attestation_pool`]. Gossip's
+//! stateful checks run on blocking threads, so none of this can be owned by the
+//! chain actor.
+
+use std::{
+    collections::{BTreeMap, BTreeSet},
+    num::NonZeroUsize,
+    sync::{Arc, Mutex, MutexGuard},
+};
+
+use lru::LruCache;
+
+use super::containers::gloas;
+use super::gossip::IgnoreReason;
+use super::primitives::{BlsPubkey, ExecutionAddress, ExecutionBlockHash, Root, Slot};
+
+/// Exactly one per node.
+pub type SharedBuilderMarket = Arc;
+
+/// Bids pooled per `(slot, parent hash, parent root)`, top values kept.
+pub const MAX_BIDS_PER_PARENT: usize = 16;
+/// A full slot refuses new keys: `record_bid` answers `false`.
+pub const MAX_SEEN_BID_KEYS_PER_SLOT: usize = 4096;
+/// Over the cap, the lowest `proposal_slot` is dropped first.
+pub const MAX_PREFERENCES: usize = 1024;
+/// Known payloads, evicted least recently used first, by block hash.
+pub const KNOWN_PAYLOADS_CAPACITY: NonZeroUsize = NonZeroUsize::new(256).unwrap();
+
+/// What gossip learned about an execution payload from its envelope.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct KnownPayload {
+    pub gas_limit: u64,
+    /// The block whose envelope revealed it.
+    pub beacon_block_root: Root,
+    /// `(pubkey, source_address)` of every builder exit request it carries.
+    pub builder_exits: Vec<(BlsPubkey, ExecutionAddress)>,
+}
+
+/// Bids for one `(parent_block_hash, parent_block_root)` of a slot.
+#[derive(Debug, Default)]
+struct ParentBids {
+    /// The highest value recorded, which a later bid must strictly beat. Kept
+    /// apart from `bids` because the pool is truncated and the bar is not.
+    best_value: Option,
+    /// Value descending, then builder index ascending.
+    bids: Vec,
+}
+
+type ParentKey = (ExecutionBlockHash, Root);
+
+#[derive(Debug, Default)]
+struct SlotBids {
+    /// The spec's `seen.execution_payload_bids`: one bid per builder per
+    /// `(slot, parent_hash, parent_root)`.
+    seen: BTreeSet<(ParentKey, u64)>,
+    parents: BTreeMap,
+}
+
+#[derive(Debug, Default)]
+struct BidPool {
+    slots: BTreeMap,
+}
+
+impl BidPool {
+    fn check(&self, bid: &gloas::ExecutionPayloadBid) -> Result<(), IgnoreReason> {
+        let Some(slot) = self.slots.get(&bid.slot) else {
+            return Ok(());
+        };
+        let parent = (bid.parent_block_hash, bid.parent_block_root);
+        if slot.seen.contains(&(parent, bid.builder_index)) {
+            return Err(IgnoreReason::AlreadySeen);
+        }
+        if let Some(best) = slot.parents.get(&parent).and_then(|p| p.best_value)
+            && bid.value <= best
+        {
+            return Err(IgnoreReason::NotHighestBid);
+        }
+        Ok(())
+    }
+}
+
+/// Preferences by `(proposal_slot, dependent_root)`: the first valid one wins.
+type PreferencesCache = BTreeMap<(Slot, Root), gloas::SignedProposerPreferences>;
+
+#[derive(Debug)]
+pub struct BuilderMarket {
+    bids: Mutex,
+    preferences: Mutex,
+    payloads: Mutex>,
+}
+
+impl Default for BuilderMarket {
+    fn default() -> Self {
+        Self {
+            bids: Mutex::default(),
+            preferences: Mutex::default(),
+            payloads: Mutex::new(LruCache::new(KNOWN_PAYLOADS_CAPACITY)),
+        }
+    }
+}
+
+/// A poisoned lock means a panic elsewhere interrupted an update, but every
+/// mutation here is a single insert or remove, so the data is still consistent
+/// and gossip should keep working.
+fn lock(mutex: &Mutex) -> MutexGuard<'_, T> {
+    mutex
+        .lock()
+        .unwrap_or_else(|poisoned| poisoned.into_inner())
+}
+
+impl BuilderMarket {
+    // Bids: seen.execution_payload_bids + seen.best_execution_payload_bid + pool.
+
+    /// The spec's two seen rules, in order: `(slot, parent_hash, parent_root,
+    /// builder)` recorded -> `AlreadySeen`; value <= best for `(slot,
+    /// parent_hash, parent_root)` -> `NotHighestBid`.
+    pub fn check_bid_seen(&self, bid: &gloas::ExecutionPayloadBid) -> Result<(), IgnoreReason> {
+        lock(&self.bids).check(bid)
+    }
+
+    /// Re-runs [`Self::check_bid_seen`] under the lock. If it passes, records
+    /// both seen keys and pools the bid. `false` = not recorded (race, or the
+    /// per-slot key cap). Prunes slots below `bid.slot - 1`.
+    pub fn record_bid(&self, signed: gloas::SignedExecutionPayloadBid) -> bool {
+        let mut pool = lock(&self.bids);
+        if pool.check(&signed.message).is_err() {
+            return false;
+        }
+        let bid_slot = signed.message.slot;
+        let slot = pool.slots.entry(bid_slot).or_default();
+        if slot.seen.len() >= MAX_SEEN_BID_KEYS_PER_SLOT {
+            return false;
+        }
+        let parent = (
+            signed.message.parent_block_hash,
+            signed.message.parent_block_root,
+        );
+        slot.seen.insert((parent, signed.message.builder_index));
+        let entry = slot.parents.entry(parent).or_default();
+        entry.best_value = Some(signed.message.value);
+        entry.bids.push(signed);
+        entry.bids.sort_by(|a, b| {
+            b.message
+                .value
+                .cmp(&a.message.value)
+                .then(a.message.builder_index.cmp(&b.message.builder_index))
+        });
+        entry.bids.truncate(MAX_BIDS_PER_PARENT);
+        let keep_from = bid_slot.saturating_sub(1);
+        pool.slots = pool.slots.split_off(&keep_from);
+        true
+    }
+
+    /// The identical message and signature is pooled (API idempotency).
+    pub fn contains_bid(&self, signed: &gloas::SignedExecutionPayloadBid) -> bool {
+        let pool = lock(&self.bids);
+        pool.slots
+            .get(&signed.message.slot)
+            .and_then(|slot| {
+                slot.parents.get(&(
+                    signed.message.parent_block_hash,
+                    signed.message.parent_block_root,
+                ))
+            })
+            .is_some_and(|parent| parent.bids.contains(signed))
+    }
+
+    /// Pooled bids for the key: value descending, then builder index ascending.
+    pub fn bids_for(
+        &self,
+        slot: Slot,
+        parent_block_root: Root,
+        parent_block_hash: ExecutionBlockHash,
+    ) -> Vec {
+        lock(&self.bids)
+            .slots
+            .get(&slot)
+            .and_then(|s| s.parents.get(&(parent_block_hash, parent_block_root)))
+            .map(|p| p.bids.clone())
+            .unwrap_or_default()
+    }
+
+    pub fn has_bids_for_slot(&self, slot: Slot) -> bool {
+        lock(&self.bids)
+            .slots
+            .get(&slot)
+            .is_some_and(|s| s.parents.values().any(|p| !p.bids.is_empty()))
+    }
+
+    pub fn prune_bids_before(&self, slot: Slot) {
+        let mut pool = lock(&self.bids);
+        pool.slots = pool.slots.split_off(&slot);
+    }
+
+    // Proposer preferences: seen.proposer_preferences.
+
+    pub fn preferences(
+        &self,
+        proposal_slot: Slot,
+        dependent_root: Root,
+    ) -> Option {
+        lock(&self.preferences)
+            .get(&(proposal_slot, dependent_root))
+            .cloned()
+    }
+
+    /// The first valid preferences per key win. `false` if one is held. Prunes
+    /// `proposal_slot < current_slot`.
+    pub fn record_preferences(
+        &self,
+        signed: gloas::SignedProposerPreferences,
+        current_slot: Slot,
+    ) -> bool {
+        let mut cache = lock(&self.preferences);
+        let key = (signed.message.proposal_slot, signed.message.dependent_root);
+        if cache.contains_key(&key) {
+            return false;
+        }
+        *cache = cache.split_off(&(current_slot, Root::ZERO));
+        cache.insert(key, signed);
+        while cache.len() > MAX_PREFERENCES {
+            cache.pop_first();
+        }
+        cache.contains_key(&key)
+    }
+
+    pub fn prune_preferences_before(&self, slot: Slot) {
+        let mut cache = lock(&self.preferences);
+        *cache = cache.split_off(&(slot, Root::ZERO));
+    }
+
+    // Known payloads: seen.execution_payloads.
+
+    pub fn record_execution_payload(&self, envelope: &gloas::ExecutionPayloadEnvelope) {
+        let builder_exits = envelope
+            .execution_requests
+            .builder_exits
+            .iter()
+            .map(|exit| (exit.pubkey, exit.source_address))
+            .collect();
+        let known = KnownPayload {
+            gas_limit: envelope.payload.gas_limit,
+            beacon_block_root: envelope.beacon_block_root,
+            builder_exits,
+        };
+        lock(&self.payloads).put(envelope.payload.block_hash, known);
+    }
+
+    pub fn known_payload(&self, block_hash: ExecutionBlockHash) -> Option {
+        lock(&self.payloads).get(&block_hash).cloned()
+    }
+}
+
+/// Builder-market fixtures shared by this module's tests, the gossip rules'
+/// and (`test-utils` feature) the Beacon API's and p2p's: a gloas state with a
+/// registered builder, and signatures that state verifies.
+#[cfg(any(test, feature = "test-utils"))]
+pub mod test_support {
+    use ethlambda_types::beacon::containers::bellatrix::{ExtraData, LogsBloom};
+    use ethlambda_types::beacon::primitives::{BlsSignature, Bytes32, Uint256};
+
+    use super::*;
+    use crate::beacon::containers::BeaconState;
+    use crate::beacon::gloas_block_production::test_support::parent_state;
+    use crate::beacon::helpers::accessors::get_domain;
+    use crate::beacon::helpers::misc::{compute_epoch_at_slot, compute_signing_root};
+    use crate::beacon::helpers::test_state::{secret_key_for, sign_for};
+    use crate::beacon::primitives::HashTreeRoot as _;
+    use crate::beacon::{constants, preset};
+
+    /// The secret key behind builder `index`'s registered pubkey. Offset from
+    /// the validators' keys so the two never collide.
+    pub fn builder_secret(index: usize) -> blst::min_pk::SecretKey {
+        secret_key_for(1000 + index)
+    }
+
+    /// `gloas_block_production::test_support::parent_state` with builders
+    /// `0..=builder_index` registered (each funded with `balance`, deposited at
+    /// `deposit_epoch`), and the finalized checkpoint one epoch past
+    /// `deposit_epoch` so they are active.
+    pub fn gloas_state_with_builder(
+        builder_index: u64,
+        balance: u64,
+        deposit_epoch: u64,
+    ) -> BeaconState {
+        let mut state = parent_state();
+        let BeaconState::Gloas(inner) = &mut state else {
+            unreachable!("built as gloas")
+        };
+        for index in 0..=builder_index {
+            let builder = gloas::Builder {
+                pubkey: BlsPubkey(builder_secret(index as usize).sk_to_pk().to_bytes()),
+                version: constants::PAYLOAD_BUILDER_VERSION,
+                execution_address: ExecutionAddress::repeat_byte(index as u8 + 1),
+                balance,
+                deposit_epoch,
+                withdrawable_epoch: constants::FAR_FUTURE_EPOCH,
+            };
+            inner.builders.push(builder);
+        }
+        inner.finalized_checkpoint.epoch = deposit_epoch + 1;
+        state
+    }
+
+    /// `bid` signed by builder `builder_index` under `state`'s builder domain.
+    pub fn sign_bid(
+        state: &BeaconState,
+        bid: gloas::ExecutionPayloadBid,
+        builder_index: u64,
+    ) -> gloas::SignedExecutionPayloadBid {
+        let domain = get_domain(state, constants::DOMAIN_BEACON_BUILDER, None);
+        let root = compute_signing_root(bid.hash_tree_root(), domain);
+        let signature = builder_secret(builder_index as usize).sign(
+            root.as_slice(),
+            crate::beacon::bls::DST,
+            &[],
+        );
+        gloas::SignedExecutionPayloadBid {
+            message: bid,
+            signature: BlsSignature(signature.to_bytes()),
+        }
+    }
+
+    /// `prefs` signed by its own `validator_index` under `state`'s preferences
+    /// domain at the proposal slot's epoch (the spec's).
+    pub fn sign_preferences(
+        state: &BeaconState,
+        prefs: gloas::ProposerPreferences,
+    ) -> gloas::SignedProposerPreferences {
+        let epoch = compute_epoch_at_slot(prefs.proposal_slot);
+        let domain = get_domain(state, constants::DOMAIN_PROPOSER_PREFERENCES, Some(epoch));
+        let root = compute_signing_root(prefs.hash_tree_root(), domain);
+        let signature = sign_for(prefs.validator_index as usize, root);
+        gloas::SignedProposerPreferences {
+            message: prefs,
+            signature,
+        }
+    }
+
+    /// An envelope revealing payload `block_hash` with `gas_limit`, for the
+    /// block `beacon_block_root`, carrying one exit request per `exits` entry.
+    pub fn envelope_with_gas_limit(
+        block_hash: ExecutionBlockHash,
+        gas_limit: u64,
+        beacon_block_root: Root,
+        exits: Vec<(BlsPubkey, ExecutionAddress)>,
+    ) -> gloas::ExecutionPayloadEnvelope {
+        let payload = gloas::ExecutionPayload {
+            parent_hash: ExecutionBlockHash::ZERO,
+            fee_recipient: Default::default(),
+            state_root: Bytes32::repeat_byte(1),
+            receipts_root: Bytes32::repeat_byte(2),
+            logs_bloom: LogsBloom::try_from(vec![0u8; preset::BYTES_PER_LOGS_BLOOM]).unwrap(),
+            prev_randao: Default::default(),
+            block_number: 7,
+            gas_limit,
+            gas_used: 0,
+            timestamp: 0,
+            extra_data: ExtraData::default(),
+            base_fee_per_gas: Uint256::from_u128(7),
+            block_hash,
+            transactions: Default::default(),
+            withdrawals: Default::default(),
+            blob_gas_used: 0,
+            excess_blob_gas: 0,
+            block_access_list: Default::default(),
+            slot_number: 0,
+        };
+        let builder_exits: Vec<_> = exits
+            .into_iter()
+            .map(|(pubkey, source_address)| gloas::BuilderExitRequest {
+                source_address,
+                pubkey,
+            })
+            .collect();
+        let execution_requests = gloas::ExecutionRequests {
+            builder_exits: builder_exits.into(),
+            ..Default::default()
+        };
+        gloas::ExecutionPayloadEnvelope {
+            payload,
+            execution_requests,
+            builder_index: 0,
+            beacon_block_root,
+            parent_beacon_block_root: Root::ZERO,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::test_support::envelope_with_gas_limit;
+    use super::*;
+
+    fn hash(byte: u8) -> ExecutionBlockHash {
+        ExecutionBlockHash::repeat_byte(byte)
+    }
+
+    fn bid(slot: Slot, builder: u64, value: u64) -> gloas::SignedExecutionPayloadBid {
+        gloas::SignedExecutionPayloadBid {
+            message: gloas::ExecutionPayloadBid {
+                slot,
+                builder_index: builder,
+                value,
+                parent_block_hash: hash(1),
+                parent_block_root: Root::repeat_byte(2),
+                block_hash: hash(3),
+                ..Default::default()
+            },
+            signature: Default::default(),
+        }
+    }
+
+    fn prefs(slot: Slot, dependent: u8, validator: u64) -> gloas::SignedProposerPreferences {
+        gloas::SignedProposerPreferences {
+            message: gloas::ProposerPreferences {
+                dependent_root: Root::repeat_byte(dependent),
+                proposal_slot: slot,
+                validator_index: validator,
+                ..Default::default()
+            },
+            signature: Default::default(),
+        }
+    }
+
+    #[test]
+    fn a_builder_bids_once_per_parent() {
+        let market = BuilderMarket::default();
+        assert!(market.record_bid(bid(10, 1, 5)));
+        assert_eq!(
+            market.check_bid_seen(&bid(10, 1, 9).message),
+            Err(IgnoreReason::AlreadySeen)
+        );
+        assert!(!market.record_bid(bid(10, 1, 9)));
+        // Another parent hash is another key.
+        let mut other = bid(10, 1, 9);
+        other.message.parent_block_hash = hash(9);
+        assert!(market.record_bid(other));
+    }
+
+    #[test]
+    fn a_bid_must_strictly_beat_the_best() {
+        let market = BuilderMarket::default();
+        assert!(market.record_bid(bid(10, 1, 5)));
+        assert_eq!(
+            market.check_bid_seen(&bid(10, 2, 5).message),
+            Err(IgnoreReason::NotHighestBid)
+        );
+        assert_eq!(
+            market.check_bid_seen(&bid(10, 2, 4).message),
+            Err(IgnoreReason::NotHighestBid)
+        );
+        assert_eq!(market.check_bid_seen(&bid(10, 2, 6).message), Ok(()));
+        assert!(market.record_bid(bid(10, 2, 6)));
+        // A lower bid never reached the pool, the bar stays at the best.
+        assert!(!market.record_bid(bid(10, 3, 6)));
+    }
+
+    #[test]
+    fn a_stale_check_fails_the_record() {
+        let market = BuilderMarket::default();
+        let racing = bid(10, 2, 7);
+        assert_eq!(market.check_bid_seen(&racing.message), Ok(()));
+        assert!(market.record_bid(bid(10, 1, 8)));
+        assert!(!market.record_bid(racing));
+    }
+
+    #[test]
+    fn a_full_slot_refuses_new_keys() {
+        let market = BuilderMarket::default();
+        for builder in 0..MAX_SEEN_BID_KEYS_PER_SLOT as u64 {
+            assert!(market.record_bid(bid(10, builder, builder + 1)));
+        }
+        let next = MAX_SEEN_BID_KEYS_PER_SLOT as u64;
+        assert!(!market.record_bid(bid(10, next, next + 1)));
+        // Other slots are unaffected.
+        assert!(market.record_bid(bid(11, next, 1)));
+    }
+
+    #[test]
+    fn bids_for_orders_by_value_and_keeps_the_top() {
+        let market = BuilderMarket::default();
+        let total = MAX_BIDS_PER_PARENT as u64 + 4;
+        for builder in 0..total {
+            assert!(market.record_bid(bid(10, builder, builder + 1)));
+        }
+        let pooled = market.bids_for(10, Root::repeat_byte(2), hash(1));
+        assert_eq!(pooled.len(), MAX_BIDS_PER_PARENT);
+        let values: Vec = pooled.iter().map(|b| b.message.value).collect();
+        let expected: Vec = (5..=total).rev().collect();
+        assert_eq!(values, expected);
+        // The truncated bar still holds: the best value is the latest.
+        assert_eq!(
+            market.check_bid_seen(&bid(10, 99, total).message),
+            Err(IgnoreReason::NotHighestBid)
+        );
+        assert!(market.bids_for(10, Root::ZERO, hash(1)).is_empty());
+    }
+
+    #[test]
+    fn contains_bid_matches_the_exact_message() {
+        let market = BuilderMarket::default();
+        let signed = bid(10, 1, 5);
+        assert!(!market.contains_bid(&signed));
+        assert!(market.record_bid(signed.clone()));
+        assert!(market.contains_bid(&signed));
+        let mut other = signed.clone();
+        other.signature.0[0] = 1;
+        assert!(!market.contains_bid(&other));
+        assert!(market.has_bids_for_slot(10));
+        assert!(!market.has_bids_for_slot(11));
+    }
+
+    #[test]
+    fn bids_prune_below_the_previous_slot() {
+        let market = BuilderMarket::default();
+        assert!(market.record_bid(bid(10, 1, 5)));
+        assert!(market.record_bid(bid(11, 1, 5)));
+        assert!(market.has_bids_for_slot(10));
+        assert!(market.record_bid(bid(12, 1, 5)));
+        assert!(!market.has_bids_for_slot(10));
+        assert!(market.has_bids_for_slot(11));
+        market.prune_bids_before(12);
+        assert!(!market.has_bids_for_slot(11));
+        assert!(market.has_bids_for_slot(12));
+    }
+
+    #[test]
+    fn the_first_preferences_win() {
+        let market = BuilderMarket::default();
+        let first = prefs(40, 1, 3);
+        assert!(market.record_preferences(first.clone(), 38));
+        assert!(!market.record_preferences(prefs(40, 1, 4), 38));
+        assert_eq!(market.preferences(40, Root::repeat_byte(1)), Some(first));
+        // Another dependent root is another key.
+        assert!(market.record_preferences(prefs(40, 2, 3), 38));
+        assert_eq!(market.preferences(41, Root::repeat_byte(1)), None);
+    }
+
+    #[test]
+    fn preferences_prune_by_proposal_slot() {
+        let market = BuilderMarket::default();
+        assert!(market.record_preferences(prefs(40, 1, 3), 38));
+        assert!(market.record_preferences(prefs(41, 1, 3), 41));
+        assert!(market.preferences(40, Root::repeat_byte(1)).is_none());
+        assert!(market.preferences(41, Root::repeat_byte(1)).is_some());
+        market.prune_preferences_before(42);
+        assert!(market.preferences(41, Root::repeat_byte(1)).is_none());
+    }
+
+    #[test]
+    fn preferences_over_the_cap_evict_the_lowest_slot() {
+        let market = BuilderMarket::default();
+        for slot in 0..MAX_PREFERENCES as u64 {
+            assert!(market.record_preferences(prefs(100 + slot, 1, 3), 0));
+        }
+        let high = 100 + MAX_PREFERENCES as u64;
+        assert!(market.record_preferences(prefs(high, 1, 3), 0));
+        assert!(market.preferences(100, Root::repeat_byte(1)).is_none());
+        assert!(market.preferences(101, Root::repeat_byte(1)).is_some());
+        // A message below everything held is itself the one dropped.
+        assert!(!market.record_preferences(prefs(50, 1, 3), 0));
+    }
+
+    #[test]
+    fn a_revealed_payload_is_known_with_its_exits() {
+        let market = BuilderMarket::default();
+        let pubkey = BlsPubkey([7; 48]);
+        let source = ExecutionAddress::repeat_byte(9);
+        let envelope = envelope_with_gas_limit(
+            hash(5),
+            36_000_000,
+            Root::repeat_byte(4),
+            vec![(pubkey, source)],
+        );
+        assert!(market.known_payload(hash(5)).is_none());
+        market.record_execution_payload(&envelope);
+        assert_eq!(
+            market.known_payload(hash(5)),
+            Some(KnownPayload {
+                gas_limit: 36_000_000,
+                beacon_block_root: Root::repeat_byte(4),
+                builder_exits: vec![(pubkey, source)],
+            })
+        );
+    }
+
+    #[test]
+    fn known_payloads_are_a_bounded_lru() {
+        let market = BuilderMarket::default();
+        let capacity = KNOWN_PAYLOADS_CAPACITY.get();
+        for index in 0..capacity {
+            let mut block_hash = ExecutionBlockHash::ZERO;
+            block_hash.0[..8].copy_from_slice(&(index as u64).to_le_bytes());
+            market.record_execution_payload(&envelope_with_gas_limit(
+                block_hash,
+                1,
+                Root::ZERO,
+                vec![],
+            ));
+        }
+        let mut first = ExecutionBlockHash::ZERO;
+        first.0[..8].copy_from_slice(&0u64.to_le_bytes());
+        // Touch the oldest so the second becomes the least recently used.
+        assert!(market.known_payload(first).is_some());
+        market.record_execution_payload(&envelope_with_gas_limit(
+            hash(0xff),
+            1,
+            Root::ZERO,
+            vec![],
+        ));
+        let mut second = ExecutionBlockHash::ZERO;
+        second.0[..8].copy_from_slice(&1u64.to_le_bytes());
+        assert!(market.known_payload(first).is_some());
+        assert!(market.known_payload(second).is_none());
+    }
+}
diff --git a/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs b/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs
index a9800cba9..1a0fa8edc 100644
--- a/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs
+++ b/crates/blockchain/state_transition/src/beacon/gloas_block_production.rs
@@ -29,8 +29,8 @@ use super::helpers::accessors::{
     CommitteeCache, get_beacon_proposer_index, get_current_epoch, get_randao_mix,
 };
 use super::helpers::gloas::{
-    get_indexed_payload_attestation, get_ptc, gloas_state_ref, is_attestation_same_slot,
-    is_valid_indexed_payload_attestation,
+    can_builder_cover_bid, get_indexed_payload_attestation, get_ptc, gloas_state_ref,
+    is_active_builder, is_attestation_same_slot, is_valid_indexed_payload_attestation,
 };
 use super::stf;
 use ethlambda_types::beacon::{
@@ -347,6 +347,102 @@ pub struct GloasBlockInputs {
     pub execution_requests: ExecutionRequests,
 }
 
+/// What a gloas block body carries when it commits to another builder's bid
+/// rather than to a payload this node built.
+#[derive(Debug, Clone)]
+pub struct GloasBidBlockInputs {
+    pub randao_reveal: BlsSignature,
+    pub graffiti: Bytes32,
+    pub attestations: Vec,
+    pub payload_attestations: Vec,
+    pub parent_execution_requests: ExecutionRequests,
+    pub signed_bid: SignedExecutionPayloadBid,
+}
+
+/// The unsigned block for `state.slot()` committing to `inputs.signed_bid`, a
+/// builder's bid rather than a payload this node built.
+///
+/// The body is [`assemble_gloas_block`]'s (empty sync aggregate, the state's own
+/// `eth1_data`), with the bid in place of the self-build one. `process_block` on
+/// a copy fills `state_root` and enforces `process_execution_payload_bid`:
+/// the builder is active, covers the bid and signed it, and the bid's slot,
+/// parent hash and root and randao are the state's. There is no envelope: the
+/// builder reveals the payload itself.
+pub fn assemble_gloas_block_on_bid(
+    state: &BeaconState,
+    inputs: GloasBidBlockInputs,
+    config: &Config,
+) -> Result {
+    verify(
+        matches!(state, BeaconState::Gloas(_)),
+        "gloas block production runs on a gloas state",
+    )?;
+    let body = BeaconBlockBody {
+        randao_reveal: inputs.randao_reveal,
+        eth1_data: state.eth1_data().clone(),
+        graffiti: inputs.graffiti,
+        attestations: inputs.attestations.into(),
+        sync_aggregate: empty_sync_aggregate(),
+        signed_execution_payload_bid: inputs.signed_bid,
+        payload_attestations: inputs.payload_attestations.into(),
+        parent_execution_requests: inputs.parent_execution_requests,
+        ..BeaconBlockBody::empty()
+    };
+    let mut block = BeaconBlock {
+        slot: state.slot(),
+        proposer_index: get_beacon_proposer_index(state)?,
+        parent_root: state.latest_block_header().hash_tree_root(),
+        state_root: Root::ZERO,
+        body,
+    };
+    let mut post = state.clone();
+    stf::gloas::process_block(&mut post, &block, config, &CommitteeCache::default())?;
+    block.state_root = post.hash_tree_root();
+    Ok(block)
+}
+
+/// Cheap pre-filter on the state advanced to the slot: whether the bid could
+/// be packed into a block on `state`, so the producer skips a bid whose block
+/// would fail rather than learning it from a full `process_block`.
+///
+/// Checks the bid is for `state`'s slot, parent root, previous randao and
+/// parent payload (full: the parent bid's block hash; empty: the state's
+/// `latest_block_hash`), that the builder exists, is a payload builder, is
+/// active and covers the value, that the commitment count is within the preset
+/// bound, and that the parent payload does not exit the builder when the bid
+/// builds on it (`process_parent_execution_payload` runs before the bid, so a
+/// builder that exits there is no longer active). The signature is not checked
+/// here; `process_block` does.
+pub fn bid_is_includable(
+    state: &BeaconState,
+    signed_bid: &SignedExecutionPayloadBid,
+    parent_requests: &ExecutionRequests,
+) -> bool {
+    let BeaconState::Gloas(inner) = state else {
+        return false;
+    };
+    let bid = &signed_bid.message;
+    let parent_is_full = bid.parent_block_hash == inner.latest_execution_payload_bid.block_hash;
+    let builds_on_known_parent = parent_is_full || bid.parent_block_hash == inner.latest_block_hash;
+    let Some(builder) = inner.builders.get(bid.builder_index as usize) else {
+        return false;
+    };
+    let exited_by_parent = parent_is_full
+        && parent_requests.builder_exits.iter().any(|exit| {
+            exit.pubkey == builder.pubkey && exit.source_address == builder.execution_address
+        });
+    bid.slot == state.slot()
+        && bid.parent_block_root == state.latest_block_header().hash_tree_root()
+        && bid.prev_randao == get_randao_mix(state, get_current_epoch(state))
+        && builds_on_known_parent
+        && bid.block_hash != bid.parent_block_hash
+        && builder.version == constants::PAYLOAD_BUILDER_VERSION
+        && bid.blob_kzg_commitments.len() <= preset::MAX_BLOB_COMMITMENTS_PER_BLOCK
+        && !exited_by_parent
+        && is_active_builder(inner, bid.builder_index).unwrap_or(false)
+        && can_builder_cover_bid(inner, bid.builder_index, bid.value).unwrap_or(false)
+}
+
 /// A produced block and the unsigned envelope that reveals its payload.
 #[derive(Debug, Clone)]
 pub struct GloasProduced {
@@ -1077,3 +1173,193 @@ mod gloas_block_production_tests {
         assert!(parse_gloas_execution_requests(&[vec![0x7f, 0]]).is_err());
     }
 }
+
+#[cfg(test)]
+mod bid_assembly_tests {
+    use super::test_support::*;
+    use super::*;
+    use crate::beacon::ForkName;
+    use crate::beacon::block_production::advance_to_slot;
+    use crate::beacon::builder_market::test_support::{
+        builder_secret, gloas_state_with_builder, sign_bid,
+    };
+    use ethlambda_types::beacon::primitives::{BlsPubkey, ExecutionAddress};
+
+    /// A state at slot 33 with a funded, active builder 0, and a bid on its
+    /// full parent that a block can carry.
+    fn scene() -> (BeaconState, SignedExecutionPayloadBid) {
+        let parent = gloas_state_with_builder(0, 100_000_000_000, 0);
+        let state = advance_to_slot(&parent, parent.slot() + 1, &config()).unwrap();
+        let bid = bid_for(&state, |_| {});
+        (state, bid)
+    }
+
+    /// A bid on `state`'s full parent payload, signed by builder 0 after `edit`.
+    fn bid_for(
+        state: &BeaconState,
+        edit: impl FnOnce(&mut ExecutionPayloadBid),
+    ) -> SignedExecutionPayloadBid {
+        let mut bid = ExecutionPayloadBid {
+            parent_block_hash: ExecutionBlockHash::repeat_byte(PARENT_BLOCK_HASH),
+            parent_block_root: state.latest_block_header().hash_tree_root(),
+            block_hash: ExecutionBlockHash::repeat_byte(0x33),
+            prev_randao: get_randao_mix(state, get_current_epoch(state)),
+            gas_limit: 30_000_000,
+            builder_index: 0,
+            slot: state.slot(),
+            value: 7,
+            ..Default::default()
+        };
+        edit(&mut bid);
+        sign_bid(state, bid, 0)
+    }
+
+    fn inputs(state: &BeaconState, signed_bid: SignedExecutionPayloadBid) -> GloasBidBlockInputs {
+        GloasBidBlockInputs {
+            randao_reveal: randao_reveal(state),
+            graffiti: Bytes32::repeat_byte(7),
+            attestations: Vec::new(),
+            payload_attestations: Vec::new(),
+            parent_execution_requests: ExecutionRequests::default(),
+            signed_bid,
+        }
+    }
+
+    #[test]
+    fn a_funded_signed_bid_becomes_a_block_that_carries_it() {
+        let (state, bid) = scene();
+        assert!(bid_is_includable(
+            &state,
+            &bid,
+            &ExecutionRequests::default()
+        ));
+        let block =
+            assemble_gloas_block_on_bid(&state, inputs(&state, bid.clone()), &config()).unwrap();
+        assert_eq!(block.body.signed_execution_payload_bid, bid);
+        assert_eq!(block.slot, state.slot());
+        assert_ne!(block.state_root, Root::ZERO);
+        // The block goes through the state transition as the network would run it.
+        let post = post_state(&state, &block);
+        let BeaconState::Gloas(inner) = &post else {
+            unreachable!("gloas")
+        };
+        assert_eq!(inner.latest_execution_payload_bid, bid.message);
+        assert_eq!(block.state_root, post.hash_tree_root());
+    }
+
+    #[test]
+    fn an_inactive_builder_cannot_have_a_block_built() {
+        // Deposited at the finalized epoch itself: not yet active.
+        let parent = gloas_state_with_builder(0, 100_000_000_000, 0);
+        let mut state = advance_to_slot(&parent, parent.slot() + 1, &config()).unwrap();
+        let BeaconState::Gloas(inner) = &mut state else {
+            unreachable!("gloas")
+        };
+        inner.builders[0].deposit_epoch = inner.finalized_checkpoint.epoch;
+        let bid = bid_for(&state, |_| {});
+        assert!(!bid_is_includable(
+            &state,
+            &bid,
+            &ExecutionRequests::default()
+        ));
+        assert!(assemble_gloas_block_on_bid(&state, inputs(&state, bid), &config()).is_err());
+    }
+
+    #[test]
+    fn a_bid_on_the_wrong_parent_hash_is_refused() {
+        let (state, _) = scene();
+        let bid = bid_for(&state, |bid| {
+            bid.parent_block_hash = ExecutionBlockHash::repeat_byte(0x77)
+        });
+        assert!(!bid_is_includable(
+            &state,
+            &bid,
+            &ExecutionRequests::default()
+        ));
+        assert!(assemble_gloas_block_on_bid(&state, inputs(&state, bid), &config()).is_err());
+    }
+
+    #[test]
+    fn a_bid_with_the_wrong_randao_is_refused() {
+        let (state, _) = scene();
+        let bid = bid_for(&state, |bid| bid.prev_randao = Bytes32::repeat_byte(9));
+        assert!(!bid_is_includable(
+            &state,
+            &bid,
+            &ExecutionRequests::default()
+        ));
+        assert!(assemble_gloas_block_on_bid(&state, inputs(&state, bid), &config()).is_err());
+    }
+
+    #[test]
+    fn a_bid_with_a_bad_signature_is_built_into_nothing() {
+        let (state, mut bid) = scene();
+        bid.signature.0[3] ^= 1;
+        // The cheap filter does not verify signatures, `process_block` does.
+        assert!(bid_is_includable(
+            &state,
+            &bid,
+            &ExecutionRequests::default()
+        ));
+        assert!(assemble_gloas_block_on_bid(&state, inputs(&state, bid), &config()).is_err());
+    }
+
+    #[test]
+    fn a_bid_for_another_slot_or_parent_root_is_not_includable() {
+        let (state, _) = scene();
+        let requests = ExecutionRequests::default();
+        let later = bid_for(&state, |bid| bid.slot += 1);
+        assert!(!bid_is_includable(&state, &later, &requests));
+        let other_root = bid_for(&state, |bid| bid.parent_block_root = Root::repeat_byte(5));
+        assert!(!bid_is_includable(&state, &other_root, &requests));
+        let same_hash = bid_for(&state, |bid| bid.block_hash = bid.parent_block_hash);
+        assert!(!bid_is_includable(&state, &same_hash, &requests));
+    }
+
+    #[test]
+    fn an_unknown_or_overdrawn_builder_is_not_includable() {
+        let (state, _) = scene();
+        let requests = ExecutionRequests::default();
+        let unknown = bid_for(&state, |bid| bid.builder_index = 4);
+        assert!(!bid_is_includable(&state, &unknown, &requests));
+        let overdrawn = bid_for(&state, |bid| bid.value = 200_000_000_000);
+        assert!(!bid_is_includable(&state, &overdrawn, &requests));
+    }
+
+    #[test]
+    fn a_builder_exited_by_the_parents_requests_is_not_includable() {
+        let (state, bid) = scene();
+        let pubkey = BlsPubkey(builder_secret(0).sk_to_pk().to_bytes());
+        let exit = |source_address| ExecutionRequests {
+            builder_exits: vec![gloas::BuilderExitRequest {
+                source_address,
+                pubkey,
+            }]
+            .into(),
+            ..Default::default()
+        };
+        // The builder's execution address is `index + 1` repeated.
+        assert!(!bid_is_includable(
+            &state,
+            &bid,
+            &exit(ExecutionAddress::repeat_byte(1))
+        ));
+        // A request from another address does not exit it.
+        assert!(bid_is_includable(
+            &state,
+            &bid,
+            &exit(ExecutionAddress::repeat_byte(9))
+        ));
+    }
+
+    #[test]
+    fn a_non_gloas_state_has_no_includable_bid() {
+        let (_, bid) = scene();
+        let fulu = crate::beacon::helpers::test_state::with_validators_at(ForkName::Fulu, 8);
+        assert!(!bid_is_includable(
+            &fulu,
+            &bid,
+            &ExecutionRequests::default()
+        ));
+    }
+}
diff --git a/crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs b/crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs
new file mode 100644
index 000000000..8ee152be3
--- /dev/null
+++ b/crates/blockchain/state_transition/src/beacon/gossip/execution_payload_bid.rs
@@ -0,0 +1,1110 @@
+//! Gossip validation for the gloas `execution_payload_bid` topic: a builder's
+//! `SignedExecutionPayloadBid`, the commitment a proposer may choose in place
+//! of building its own payload.
+//!
+//! The rules are the specification's `validate_execution_payload_bid_gossip`
+//! (`specs/gloas/p2p-interface.md`), split like [`super::envelope`]:
+//! [`cheap_checks`] reads only the message, the market's seen state and the
+//! clock, so the p2p actor runs it inline; [`stateful_checks`] reads cached
+//! states and verifies the signature, so it runs on a blocking thread. The
+//! caller records the bid in the [`BuilderMarket`] on `Accept`.
+//!
+//! Deliberate departures from the specification (`docs/spec_deviations.md`):
+//!
+//! - Never queues. Every "MAY be queued" is IGNORE, and a state that is not
+//!   cached is IGNORE rather than rebuilt from disk, so a verdict gossipsub
+//!   waits on is never stalled by a replay.
+//! - `store.block_states[parent]` advanced with `process_slots` to `bid.slot`
+//!   becomes the parent's cached post-state when the bid is in the parent's
+//!   own epoch (gloas's `process_slot` touches none of the fields rules 17 to
+//!   22 read), and the cached checkpoint state of the bid's epoch otherwise.
+//! - `get_head(store)` is the head the chain actor recorded, with a fresh walk
+//!   only when none is recorded.
+//! - `seen.execution_payloads` is the market's known payloads: envelopes gossip
+//!   accepted or this node published, plus a pre-gloas parent's own payload.
+
+use std::sync::Arc;
+
+use ethlambda_storage::CacheKey;
+
+use super::{
+    IgnoreReason, Outcome, RejectReason, is_current_slot, is_gloas_slot,
+    proposer_preferences::dependent_root_at,
+};
+use crate::beacon::builder_market::{BuilderMarket, KnownPayload};
+use crate::beacon::config::Config;
+use crate::beacon::constants::PAYLOAD_BUILDER_VERSION;
+use crate::beacon::containers::{BeaconState, SignedBeaconBlock, gloas};
+use crate::beacon::fork_choice::{self, ForkChoiceNode, PayloadStatus, Store};
+use crate::beacon::helpers::accessors::{get_current_epoch, get_randao_mix};
+use crate::beacon::helpers::gloas::{can_builder_cover_bid, is_active_builder};
+use crate::beacon::helpers::misc::{compute_epoch_at_slot, compute_start_slot_at_epoch};
+use crate::beacon::lean_boundary::lean_state_unreachable;
+use crate::beacon::preset;
+use crate::beacon::primitives::{Epoch, ExecutionBlockHash, Root, Slot};
+use crate::beacon::stf;
+use crate::beacon::stf::gloas::verify_execution_payload_bid_signature;
+
+/// Gloas p2p preset: the largest decompressed `SignedExecutionPayloadBid`.
+pub const MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE: usize = 196_932;
+
+/// The spec's `is_gas_limit_target_compatible`: whether `gas_limit` is what
+/// the EIP-1559 transition rule from `parent_gas_limit` allows when steering
+/// towards `target_gas_limit`.
+pub fn is_gas_limit_target_compatible(
+    parent_gas_limit: u64,
+    gas_limit: u64,
+    target_gas_limit: u64,
+) -> bool {
+    let max_difference = (parent_gas_limit / 1024).saturating_sub(1);
+    let min_gas_limit = parent_gas_limit - max_difference;
+    let max_gas_limit = parent_gas_limit.saturating_add(max_difference);
+    if target_gas_limit < min_gas_limit {
+        return gas_limit == min_gas_limit;
+    }
+    if target_gas_limit > max_gas_limit {
+        return gas_limit == max_gas_limit;
+    }
+    gas_limit == target_gas_limit
+}
+
+/// The spec's `is_current_or_next_slot`.
+pub(crate) fn is_current_or_next_slot(config: &Config, slot: Slot, now_ms: u64) -> bool {
+    is_current_slot(config, slot, now_ms)
+        || slot
+            .checked_sub(1)
+            .is_some_and(|previous| is_current_slot(config, previous, now_ms))
+}
+
+/// The rules that read only the message, the market's seen state and the
+/// clock.
+pub fn cheap_checks(
+    market: &BuilderMarket,
+    store: &Store,
+    signed: &gloas::SignedExecutionPayloadBid,
+    now_ms: u64,
+) -> Result<(), Outcome> {
+    let bid = &signed.message;
+    let config = store.config();
+    // [IGNORE] The first bid for this slot, parent and builder, and [IGNORE]
+    // the highest value seen for the slot and parent.
+    market.check_bid_seen(bid).map_err(Outcome::Ignore)?;
+    // [IGNORE] The bid's slot is the current slot or the next slot.
+    if !is_current_or_next_slot(&config, bid.slot, now_ms) {
+        return Err(Outcome::Ignore(IgnoreReason::NotCurrentOrNextSlot));
+    }
+    // [REJECT] The bid's execution payment is zero.
+    if bid.execution_payment != 0 {
+        return Err(Outcome::Reject(RejectReason::ExecutionPaymentNonZero));
+    }
+    // [REJECT] The bid's block hash is not its parent block hash.
+    if bid.block_hash == bid.parent_block_hash {
+        return Err(Outcome::Reject(RejectReason::BlockHashEqualsParent));
+    }
+    // [REJECT] The commitment count is within the epoch's blob limit.
+    let proposal_epoch = compute_epoch_at_slot(bid.slot);
+    if bid.blob_kzg_commitments.len() as u64 > config.max_blobs_per_block(proposal_epoch) {
+        return Err(Outcome::Reject(RejectReason::TooManyBlobs));
+    }
+    // Ours: a slot before the fork has no bids. Placed after the rejects so
+    // a malformed message is still penalized, whichever fork it names.
+    if !is_gloas_slot(&config, bid.slot) {
+        return Err(Outcome::Ignore(IgnoreReason::PreGloasSlot));
+    }
+    Ok(())
+}
+
+/// A pre-gloas state's execution payload header, as far as a bid needs it:
+/// `(block_hash, gas_limit)`. `None` for a state with no payload header.
+fn pre_gloas_payload(state: &BeaconState) -> Option<(ExecutionBlockHash, u64)> {
+    match state {
+        BeaconState::Bellatrix(s) => {
+            let header = &s.latest_execution_payload_header;
+            Some((header.block_hash, header.gas_limit))
+        }
+        BeaconState::Capella(s) => {
+            let header = &s.latest_execution_payload_header;
+            Some((header.block_hash, header.gas_limit))
+        }
+        BeaconState::Deneb(s) => {
+            let header = &s.latest_execution_payload_header;
+            Some((header.block_hash, header.gas_limit))
+        }
+        BeaconState::Electra(s) => {
+            let header = &s.latest_execution_payload_header;
+            Some((header.block_hash, header.gas_limit))
+        }
+        BeaconState::Fulu(s) => {
+            let header = &s.latest_execution_payload_header;
+            Some((header.block_hash, header.gas_limit))
+        }
+        BeaconState::Phase0(_) | BeaconState::Altair(_) | BeaconState::Gloas(_) => None,
+        BeaconState::Lean(_) => lean_state_unreachable("execution_payload_bid::pre_gloas_payload"),
+    }
+}
+
+/// The head's payload-relevant facts, off whichever source answers.
+enum HeadView {
+    Gloas {
+        node: ForkChoiceNode,
+        parent_root: Root,
+        bid_parent_hash: ExecutionBlockHash,
+        bid_block_hash: ExecutionBlockHash,
+    },
+    PreGloas {
+        root: Root,
+        block_hash: ExecutionBlockHash,
+    },
+}
+
+/// The head, from the chain actor's record (a fresh walk only when no status
+/// is recorded), and its payload hashes from the cached post-state (the
+/// decoded block when that is not cached).
+fn head_view(store: &Store) -> Result {
+    let config = store.config();
+    let internal = || Outcome::Ignore(IgnoreReason::Internal);
+    let node = match (store.head().ok(), store.head_payload_status()) {
+        (Some(root), Some(payload_status)) => ForkChoiceNode {
+            root,
+            payload_status,
+        },
+        _ => fork_choice::get_head_node(store, &config).map_err(|_| internal())?,
+    };
+    if let Some(state) = store.cached_state(CacheKey::BlockState(node.root)) {
+        return match &*state {
+            BeaconState::Gloas(inner) => Ok(HeadView::Gloas {
+                node,
+                parent_root: inner.latest_block_header.parent_root,
+                bid_parent_hash: inner.latest_execution_payload_bid.parent_block_hash,
+                bid_block_hash: inner.latest_execution_payload_bid.block_hash,
+            }),
+            other => match pre_gloas_payload(other) {
+                Some((block_hash, _)) => Ok(HeadView::PreGloas {
+                    root: node.root,
+                    block_hash,
+                }),
+                None => Err(Outcome::Ignore(IgnoreReason::NotOnHeadBranch)),
+            },
+        };
+    }
+    let block = store
+        .get_signed_block(&node.root)
+        .map_err(|_| internal())?
+        .ok_or(Outcome::Ignore(IgnoreReason::StateUnavailable))?;
+    match block {
+        SignedBeaconBlock::Gloas(block) => {
+            let bid = &block.message.body.signed_execution_payload_bid.message;
+            Ok(HeadView::Gloas {
+                node,
+                parent_root: block.message.parent_root,
+                bid_parent_hash: bid.parent_block_hash,
+                bid_block_hash: bid.block_hash,
+            })
+        }
+        other => match other.execution_block_hash() {
+            Some(block_hash) => Ok(HeadView::PreGloas {
+                root: node.root,
+                block_hash,
+            }),
+            None => Err(Outcome::Ignore(IgnoreReason::NotOnHeadBranch)),
+        },
+    }
+}
+
+/// The spec's `is_bid_compatible_with_head`, against the recorded head.
+///
+/// A pre-gloas head has no bid, so a bid is compatible when it builds on that
+/// head and its payload (the boundary rule: pre-gloas parent payloads count as
+/// full).
+pub fn is_bid_compatible_with_head(
+    store: &Store,
+    bid: &gloas::ExecutionPayloadBid,
+) -> Result {
+    match head_view(store)? {
+        HeadView::PreGloas { root, block_hash } => {
+            Ok(bid.parent_block_root == root && bid.parent_block_hash == block_hash)
+        }
+        HeadView::Gloas {
+            node,
+            parent_root,
+            bid_parent_hash,
+            bid_block_hash,
+        } => {
+            let builds_on_parent_block = bid.parent_block_root == parent_root;
+            let builds_on_parent_payload = bid.parent_block_hash == bid_parent_hash;
+            if builds_on_parent_block && builds_on_parent_payload {
+                return Ok(true);
+            }
+            if bid.parent_block_root != node.root {
+                return Ok(false);
+            }
+            let builds_on_head_payload = bid.parent_block_hash == bid_block_hash;
+            // The head's status can only be PENDING if a caller recorded a
+            // walk's intermediate node, which `get_head_node` never returns.
+            debug_assert_ne!(node.payload_status, PayloadStatus::Pending);
+            let build_on_full = fork_choice::should_build_on_full(store, node, bid.slot)
+                .map_err(|_| Outcome::Ignore(IgnoreReason::Internal))?;
+            Ok(if build_on_full {
+                builds_on_head_payload
+            } else {
+                builds_on_parent_payload
+            })
+        }
+    }
+}
+
+/// Cached-only: a `CheckpointState{epoch, root}` hit; else the cached
+/// `BlockState(root)` advanced to the epoch start and cached. `None` = miss.
+///
+/// The rebuild-from-disk `fork_choice::checkpoint_state` does on a miss would
+/// stall a gossip verdict, so a state that is not cached is a miss here.
+pub(crate) fn cached_checkpoint_state(
+    store: &Store,
+    epoch: Epoch,
+    root: Root,
+) -> Option> {
+    let key = CacheKey::CheckpointState { epoch, root };
+    if let Some(state) = store.cached_state(key) {
+        return Some(state);
+    }
+    let state = store.cached_state(CacheKey::BlockState(root))?;
+    let target_slot = compute_start_slot_at_epoch(epoch);
+    let state = if state.slot() < target_slot {
+        let mut advanced = (*state).clone();
+        stf::process_slots(&mut advanced, target_slot, &store.config()).ok()?;
+        Arc::new(advanced)
+    } else {
+        state
+    };
+    store.cache_state(key, state.clone());
+    Some(state)
+}
+
+/// The exits a parent payload carried, for rule 21: the market's known payload
+/// when it is the parent block's own, else the verified envelope in the store.
+/// `None` when neither is available.
+fn parent_payload_exits(
+    store: &Store,
+    market: &BuilderMarket,
+    bid: &gloas::ExecutionPayloadBid,
+) -> Option<
+    Vec<(
+        crate::beacon::primitives::BlsPubkey,
+        crate::beacon::primitives::ExecutionAddress,
+    )>,
+> {
+    if let Some(KnownPayload {
+        beacon_block_root,
+        builder_exits,
+        ..
+    }) = market.known_payload(bid.parent_block_hash)
+        && beacon_block_root == bid.parent_block_root
+    {
+        return Some(builder_exits);
+    }
+    let envelope = store
+        .get_execution_payload_envelope(&bid.parent_block_root)
+        .ok()??;
+    Some(
+        envelope
+            .message
+            .execution_requests
+            .builder_exits
+            .iter()
+            .map(|exit| (exit.pubkey, exit.source_address))
+            .collect(),
+    )
+}
+
+/// The rules that need states, then the signature. Runs on a blocking thread.
+pub fn stateful_checks(
+    store: &Store,
+    market: &BuilderMarket,
+    signed: &gloas::SignedExecutionPayloadBid,
+) -> Outcome {
+    match stateful_rules(store, market, signed) {
+        Ok(()) => Outcome::Accept,
+        Err(outcome) => outcome,
+    }
+}
+
+fn stateful_rules(
+    store: &Store,
+    market: &BuilderMarket,
+    signed: &gloas::SignedExecutionPayloadBid,
+) -> Result<(), Outcome> {
+    let bid = &signed.message;
+    let ignore = |reason| Outcome::Ignore(reason);
+    let reject = |reason| Outcome::Reject(reason);
+    let proposal_epoch = compute_epoch_at_slot(bid.slot);
+
+    // [IGNORE] The parent block is known (never queued).
+    if !store.has_block(&bid.parent_block_root) {
+        return Err(ignore(IgnoreReason::UnknownBlock));
+    }
+    // [REJECT] The bid is for a higher slot than its parent.
+    let (parent_slot, _) = store
+        .block_entry(&bid.parent_block_root)
+        .ok_or(ignore(IgnoreReason::UnknownBlock))?;
+    if bid.slot <= parent_slot {
+        return Err(reject(RejectReason::NotAfterParent));
+    }
+    // [IGNORE] The parent has been imported (its post-state is cached).
+    let parent_state = store
+        .cached_state(CacheKey::BlockState(bid.parent_block_root))
+        .ok_or(ignore(IgnoreReason::StateUnavailable))?;
+    // [IGNORE] The bid's slot is within the parent's proposer lookahead.
+    if proposal_epoch > get_current_epoch(&parent_state) + preset::MIN_SEED_LOOKAHEAD {
+        return Err(ignore(IgnoreReason::BeyondLookahead));
+    }
+    // [IGNORE] The matching proposer preferences have been seen. Rule 10
+    // keeps the dependent slot inside the parent state's `block_roots`.
+    let dependent_root = dependent_root_at(&parent_state, bid.parent_block_root, bid.slot)
+        .ok_or(ignore(IgnoreReason::AncestryUnknown))?;
+    let preferences = market
+        .preferences(bid.slot, dependent_root)
+        .ok_or(ignore(IgnoreReason::PreferencesUnseen))?
+        .message;
+    // [IGNORE] The fee recipient matches the proposer's preference.
+    if bid.fee_recipient != preferences.fee_recipient {
+        return Err(ignore(IgnoreReason::FeeRecipientMismatch));
+    }
+    // [IGNORE] The parent block hash is a known execution payload. Across the
+    // fork boundary, a pre-gloas parent's own payload counts.
+    let parent_gas_limit = match market.known_payload(bid.parent_block_hash) {
+        Some(known) => known.gas_limit,
+        None => match pre_gloas_payload(&parent_state) {
+            Some((block_hash, gas_limit)) if block_hash == bid.parent_block_hash => gas_limit,
+            _ => return Err(ignore(IgnoreReason::ParentPayloadUnknown)),
+        },
+    };
+    // [IGNORE] The gas limit is compatible with the proposer's target.
+    if !is_gas_limit_target_compatible(
+        parent_gas_limit,
+        bid.gas_limit,
+        preferences.target_gas_limit,
+    ) {
+        return Err(ignore(IgnoreReason::GasLimitIncompatible));
+    }
+    // [IGNORE] The bid is compatible with the head branch.
+    if !is_bid_compatible_with_head(store, bid)? {
+        return Err(ignore(IgnoreReason::NotOnHeadBranch));
+    }
+    // [REJECT] The previous randao is the parent state's.
+    if bid.prev_randao != get_randao_mix(&parent_state, get_current_epoch(&parent_state)) {
+        return Err(reject(RejectReason::PrevRandao));
+    }
+
+    // The parent state advanced to the bid's slot. Within the parent's own
+    // epoch the parent state answers identically, see the module docs.
+    let state = if proposal_epoch == get_current_epoch(&parent_state) {
+        parent_state.clone()
+    } else {
+        cached_checkpoint_state(store, proposal_epoch, bid.parent_block_root)
+            .ok_or(ignore(IgnoreReason::StateUnavailable))?
+    };
+    let BeaconState::Gloas(inner) = &*state else {
+        // A bid in a gloas epoch is advanced into gloas by the epoch
+        // transition; anything else is a state this cannot judge.
+        return Err(ignore(IgnoreReason::StateUnavailable));
+    };
+
+    // [REJECT] The builder index is valid, [REJECT] it is a payload builder
+    // and [REJECT] active.
+    let builder = inner
+        .builders
+        .get(bid.builder_index as usize)
+        .ok_or(reject(RejectReason::UnknownBuilder))?;
+    if builder.version != PAYLOAD_BUILDER_VERSION {
+        return Err(reject(RejectReason::NotPayloadBuilder));
+    }
+    if !is_active_builder(inner, bid.builder_index).unwrap_or(false) {
+        return Err(reject(RejectReason::InactiveBuilder));
+    }
+    // [IGNORE] The builder can cover the bid.
+    if !can_builder_cover_bid(inner, bid.builder_index, bid.value).unwrap_or(false) {
+        return Err(ignore(IgnoreReason::BuilderCannotCover));
+    }
+    // [IGNORE] The parent's payload does not try to exit the builder. Only a
+    // gloas parent has an envelope to carry the request.
+    let parent_is_gloas = matches!(&*parent_state, BeaconState::Gloas(_));
+    if parent_is_gloas && bid.parent_block_hash == inner.latest_execution_payload_bid.block_hash {
+        let exits = parent_payload_exits(store, market, bid)
+            .ok_or(ignore(IgnoreReason::ParentPayloadUnverified))?;
+        if exits.iter().any(|(pubkey, source)| {
+            *pubkey == builder.pubkey && *source == builder.execution_address
+        }) {
+            return Err(ignore(IgnoreReason::BuilderMayExit));
+        }
+    }
+    // [REJECT] The signature is valid. An error (an index the state lacks)
+    // is a signature that cannot be.
+    if !matches!(
+        verify_execution_payload_bid_signature(&state, signed),
+        Ok(true)
+    ) {
+        return Err(reject(RejectReason::BadSignature));
+    }
+    Ok(())
+}
+
+/// Both halves. The caller records the bid on `Accept`: the specification's
+/// `validate_execution_payload_bid_gossip`.
+pub fn validate(
+    market: &BuilderMarket,
+    store: &Store,
+    signed: &gloas::SignedExecutionPayloadBid,
+    now_ms: u64,
+) -> Outcome {
+    if let Err(outcome) = cheap_checks(market, store, signed, now_ms) {
+        return outcome;
+    }
+    stateful_checks(store, market, signed)
+}
+
+#[cfg(test)]
+mod tests {
+    use ethlambda_types::beacon::primitives::{
+        BlsPubkey, Bytes32, ExecutionAddress, KzgCommitment,
+    };
+
+    use super::*;
+    use crate::beacon::builder_market::test_support::{
+        builder_secret, envelope_with_gas_limit, sign_preferences,
+    };
+    use crate::beacon::gossip::proposer_preferences::dependent_root_at;
+    use crate::beacon::gossip::test_support::builder_scene::*;
+    use crate::beacon::gossip::test_support::{slot_start_ms, store};
+    use crate::beacon::helpers::accessors::get_current_epoch;
+    use crate::beacon::precheck::fixed_proposer;
+    use crate::beacon::preset::SLOTS_PER_EPOCH;
+
+    fn ignore(reason: IgnoreReason) -> Outcome {
+        Outcome::Ignore(reason)
+    }
+
+    fn reject(reason: RejectReason) -> Outcome {
+        Outcome::Reject(reason)
+    }
+
+    /// `stateful_checks` on the scene's market and store.
+    fn stateful(scene: &Scene, bid: &gloas::SignedExecutionPayloadBid) -> Outcome {
+        stateful_checks(&scene.store, &scene.market, bid)
+    }
+
+    fn builder_pubkey() -> BlsPubkey {
+        BlsPubkey(builder_secret(0).sk_to_pk().to_bytes())
+    }
+
+    #[test]
+    fn the_scenes_bid_is_accepted() {
+        let scene = scene();
+        let outcome = validate(&scene.market, &scene.store, &scene.bid, scene.now_ms());
+        assert_eq!(outcome, Outcome::Accept);
+    }
+
+    // ---- is_gas_limit_target_compatible ----
+
+    #[test]
+    fn a_gas_limit_may_step_towards_a_nearby_target() {
+        let parent = 60_000_000;
+        // max_difference = 60_000_000 / 1024 - 1 = 58_592.
+        assert!(is_gas_limit_target_compatible(
+            parent,
+            parent + 100,
+            parent + 100
+        ));
+        assert!(is_gas_limit_target_compatible(
+            parent,
+            parent - 10,
+            parent - 10
+        ));
+        assert!(is_gas_limit_target_compatible(parent, parent, parent));
+        assert!(!is_gas_limit_target_compatible(
+            parent,
+            parent + 99,
+            parent + 100
+        ));
+    }
+
+    #[test]
+    fn a_gas_limit_is_pinned_to_the_step_limit_beyond_it() {
+        let parent = 60_000_000;
+        assert!(is_gas_limit_target_compatible(
+            parent,
+            parent + 58_592,
+            100_000_000
+        ));
+        assert!(!is_gas_limit_target_compatible(
+            parent,
+            parent + 58_593,
+            100_000_000
+        ));
+        assert!(!is_gas_limit_target_compatible(
+            parent,
+            parent + 58_591,
+            100_000_000
+        ));
+        assert!(is_gas_limit_target_compatible(
+            parent,
+            parent - 58_592,
+            30_000_000
+        ));
+        assert!(!is_gas_limit_target_compatible(
+            parent,
+            parent - 58_593,
+            30_000_000
+        ));
+        // The edge itself: a target exactly at the limit is no longer beyond it.
+        assert!(is_gas_limit_target_compatible(
+            parent,
+            parent + 58_592,
+            parent + 58_592
+        ));
+    }
+
+    #[test]
+    fn a_small_parent_cannot_move() {
+        // 1023 / 1024 = 0, so the allowed difference saturates to zero.
+        assert!(is_gas_limit_target_compatible(1023, 1023, 5_000));
+        assert!(!is_gas_limit_target_compatible(1023, 1024, 5_000));
+        assert!(is_gas_limit_target_compatible(0, 0, 100));
+    }
+
+    #[test]
+    fn the_step_limit_saturates_near_the_top_of_the_range() {
+        let parent = u64::MAX - 5;
+        assert!(is_gas_limit_target_compatible(parent, u64::MAX, u64::MAX));
+        assert!(is_gas_limit_target_compatible(parent, parent, parent));
+    }
+
+    // ---- the clock ----
+
+    #[test]
+    fn a_bid_is_timely_from_just_before_the_previous_slot_until_just_after_its_own() {
+        let store = store(0);
+        let config = store.config();
+        let slot = BID_SLOT;
+        let earliest = slot_start_ms(&store, slot - 1) - 500;
+        let latest = slot_start_ms(&store, slot + 1) + 500;
+        assert!(is_current_or_next_slot(&config, slot, earliest));
+        assert!(!is_current_or_next_slot(&config, slot, earliest - 1));
+        assert!(is_current_or_next_slot(&config, slot, latest));
+        assert!(!is_current_or_next_slot(&config, slot, latest + 1));
+        // Slot zero has no previous slot.
+        assert!(is_current_or_next_slot(
+            &config,
+            0,
+            slot_start_ms(&store, 0)
+        ));
+    }
+
+    // ---- cheap rules ----
+
+    fn cheap(scene: &Scene, bid: &gloas::SignedExecutionPayloadBid) -> Result<(), Outcome> {
+        cheap_checks(&scene.market, &scene.store, bid, scene.now_ms())
+    }
+
+    #[test]
+    fn a_builders_second_bid_for_a_parent_is_already_seen() {
+        let scene = scene();
+        assert!(scene.market.record_bid(scene.bid.clone()));
+        assert_eq!(
+            cheap(&scene, &scene.signed(|bid| bid.value = 9)),
+            Err(ignore(IgnoreReason::AlreadySeen))
+        );
+    }
+
+    #[test]
+    fn a_bid_that_does_not_beat_the_best_is_not_highest() {
+        let scene = scene();
+        let mut other = scene.bid.clone();
+        other.message.builder_index = 1;
+        other.message.value = 5;
+        assert!(scene.market.record_bid(other));
+        assert_eq!(
+            cheap(&scene, &scene.bid),
+            Err(ignore(IgnoreReason::NotHighestBid))
+        );
+        assert_eq!(
+            cheap(&scene, &scene.signed(|bid| bid.value = 5)),
+            Err(ignore(IgnoreReason::NotHighestBid))
+        );
+    }
+
+    #[test]
+    fn a_bid_for_a_distant_slot_is_not_current_or_next() {
+        let scene = scene();
+        let far = scene.signed(|bid| bid.slot = BID_SLOT + 6);
+        assert_eq!(
+            cheap(&scene, &far),
+            Err(ignore(IgnoreReason::NotCurrentOrNextSlot))
+        );
+    }
+
+    #[test]
+    fn a_bid_with_an_execution_payment_is_rejected() {
+        let scene = scene();
+        let paid = scene.signed(|bid| bid.execution_payment = 1);
+        assert_eq!(
+            cheap(&scene, &paid),
+            Err(reject(RejectReason::ExecutionPaymentNonZero))
+        );
+    }
+
+    #[test]
+    fn a_bid_whose_block_hash_is_its_parents_is_rejected() {
+        let scene = scene();
+        let same = scene.signed(|bid| bid.block_hash = bid.parent_block_hash);
+        assert_eq!(
+            cheap(&scene, &same),
+            Err(reject(RejectReason::BlockHashEqualsParent))
+        );
+    }
+
+    #[test]
+    fn a_bid_with_too_many_commitments_is_rejected() {
+        let scene = scene();
+        let limit = scene.store.config().max_blobs_per_block(1) as usize;
+        let at_limit = scene.signed(|bid| {
+            bid.blob_kzg_commitments = vec![KzgCommitment([0; 48]); limit].into();
+        });
+        assert_eq!(cheap(&scene, &at_limit), Ok(()));
+        let over = scene.signed(|bid| {
+            bid.blob_kzg_commitments = vec![KzgCommitment([0; 48]); limit + 1].into();
+        });
+        assert_eq!(
+            cheap(&scene, &over),
+            Err(reject(RejectReason::TooManyBlobs))
+        );
+    }
+
+    #[test]
+    fn a_bid_before_the_fork_is_ignored() {
+        // A fulu-only store has no gloas slot at all.
+        let fulu = store(0);
+        let scene = scene();
+        let now_ms = slot_start_ms(&fulu, BID_SLOT) + 100;
+        assert_eq!(
+            cheap_checks(&scene.market, &fulu, &scene.signed(|_| {}), now_ms),
+            Err(ignore(IgnoreReason::PreGloasSlot))
+        );
+    }
+
+    // ---- stateful rules ----
+
+    #[test]
+    fn a_bid_on_an_unknown_parent_is_ignored() {
+        let scene = scene();
+        let bid = scene.signed(|bid| bid.parent_block_root = Root::repeat_byte(9));
+        assert_eq!(stateful(&scene, &bid), ignore(IgnoreReason::UnknownBlock));
+    }
+
+    #[test]
+    fn a_bid_not_after_its_parent_is_rejected() {
+        let scene = scene();
+        let bid = scene.signed(|bid| bid.slot = PARENT_SLOT);
+        assert_eq!(stateful(&scene, &bid), reject(RejectReason::NotAfterParent));
+    }
+
+    #[test]
+    fn a_parent_without_a_cached_state_is_ignored() {
+        let mut scene = scene();
+        let stateless = Root::repeat_byte(0x60);
+        scene
+            .store
+            .insert_pending_block(stateless, block_at(PARENT_SLOT))
+            .expect("insert the block");
+        let bid = scene.signed(|bid| bid.parent_block_root = stateless);
+        assert_eq!(
+            stateful(&scene, &bid),
+            ignore(IgnoreReason::StateUnavailable)
+        );
+    }
+
+    #[test]
+    fn a_bid_beyond_the_parents_lookahead_is_ignored() {
+        let scene = scene();
+        // The parent is in epoch 1, so epoch 3 is past its lookahead.
+        let bid = scene.signed(|bid| bid.slot = 3 * SLOTS_PER_EPOCH);
+        assert_eq!(
+            stateful(&scene, &bid),
+            ignore(IgnoreReason::BeyondLookahead)
+        );
+    }
+
+    #[test]
+    fn a_bid_without_preferences_is_ignored() {
+        let scene = scene();
+        let empty = BuilderMarket::default();
+        assert_eq!(
+            stateful_checks(&scene.store, &empty, &scene.bid),
+            ignore(IgnoreReason::PreferencesUnseen)
+        );
+    }
+
+    #[test]
+    fn a_bid_for_another_fee_recipient_is_ignored() {
+        let scene = scene();
+        let bid = scene.signed(|bid| bid.fee_recipient = ExecutionAddress::repeat_byte(0x99));
+        assert_eq!(
+            stateful(&scene, &bid),
+            ignore(IgnoreReason::FeeRecipientMismatch)
+        );
+    }
+
+    #[test]
+    fn a_bid_on_an_unknown_payload_is_ignored() {
+        let scene = scene();
+        let bid = scene.signed(|bid| bid.parent_block_hash = ExecutionBlockHash::repeat_byte(0x77));
+        assert_eq!(
+            stateful(&scene, &bid),
+            ignore(IgnoreReason::ParentPayloadUnknown)
+        );
+    }
+
+    #[test]
+    fn a_gas_limit_the_target_cannot_reach_is_ignored() {
+        let scene = scene();
+        let bid = scene.signed(|bid| bid.gas_limit = PARENT_GAS_LIMIT + 1_000);
+        assert_eq!(
+            stateful(&scene, &bid),
+            ignore(IgnoreReason::GasLimitIncompatible)
+        );
+    }
+
+    #[test]
+    fn a_bid_off_the_head_branch_is_ignored() {
+        let scene = scene();
+        // A known payload, but neither the head's nor its parent's.
+        let other = ExecutionBlockHash::repeat_byte(0x44);
+        scene
+            .market
+            .record_execution_payload(&envelope_with_gas_limit(
+                other,
+                PARENT_GAS_LIMIT,
+                PARENT,
+                vec![],
+            ));
+        let bid = scene.signed(|bid| bid.parent_block_hash = other);
+        assert_eq!(
+            stateful(&scene, &bid),
+            ignore(IgnoreReason::NotOnHeadBranch)
+        );
+    }
+
+    #[test]
+    fn a_wrong_previous_randao_is_rejected() {
+        let scene = scene();
+        let bid = scene.signed(|bid| bid.prev_randao = Bytes32::repeat_byte(9));
+        assert_eq!(stateful(&scene, &bid), reject(RejectReason::PrevRandao));
+    }
+
+    #[test]
+    fn an_unregistered_builder_is_rejected() {
+        let scene = scene();
+        let bid = scene.signed(|bid| bid.builder_index = 5);
+        assert_eq!(stateful(&scene, &bid), reject(RejectReason::UnknownBuilder));
+    }
+
+    #[test]
+    fn a_builder_of_another_version_is_rejected() {
+        let scene = scene_with(|state| state.builders[0].version = 7);
+        assert_eq!(
+            stateful(&scene, &scene.bid),
+            reject(RejectReason::NotPayloadBuilder)
+        );
+    }
+
+    #[test]
+    fn an_inactive_builder_is_rejected() {
+        let exiting = scene_with(|state| state.builders[0].withdrawable_epoch = 5);
+        assert_eq!(
+            stateful(&exiting, &exiting.bid),
+            reject(RejectReason::InactiveBuilder)
+        );
+        // Deposited at or after the finalized epoch.
+        let young = scene_with(|state| state.builders[0].deposit_epoch = 1);
+        assert_eq!(
+            stateful(&young, &young.bid),
+            reject(RejectReason::InactiveBuilder)
+        );
+    }
+
+    #[test]
+    fn a_builder_that_cannot_cover_the_bid_is_ignored() {
+        let scene = scene();
+        let bid = scene.signed(|bid| bid.value = 200_000_000_000);
+        assert_eq!(
+            stateful(&scene, &bid),
+            ignore(IgnoreReason::BuilderCannotCover)
+        );
+    }
+
+    #[test]
+    fn a_builder_the_parent_payload_exits_is_ignored() {
+        let scene = scene();
+        let exits = vec![(builder_pubkey(), ExecutionAddress::repeat_byte(1))];
+        scene
+            .market
+            .record_execution_payload(&envelope_with_gas_limit(
+                parent_block_hash(),
+                PARENT_GAS_LIMIT,
+                PARENT,
+                exits,
+            ));
+        assert_eq!(
+            stateful(&scene, &scene.bid),
+            ignore(IgnoreReason::BuilderMayExit)
+        );
+    }
+
+    #[test]
+    fn an_exit_for_another_builder_does_not_matter() {
+        let scene = scene();
+        let wrong_key = (BlsPubkey([3; 48]), ExecutionAddress::repeat_byte(1));
+        let wrong_source = (builder_pubkey(), ExecutionAddress::repeat_byte(9));
+        scene
+            .market
+            .record_execution_payload(&envelope_with_gas_limit(
+                parent_block_hash(),
+                PARENT_GAS_LIMIT,
+                PARENT,
+                vec![wrong_key, wrong_source],
+            ));
+        assert_eq!(stateful(&scene, &scene.bid), Outcome::Accept);
+    }
+
+    #[test]
+    fn exits_are_read_from_the_stored_envelope_when_the_known_payload_is_another_blocks() {
+        let mut scene = scene();
+        // The market knows the hash from some other block, so it cannot answer
+        // for the parent's own envelope.
+        scene
+            .market
+            .record_execution_payload(&envelope_with_gas_limit(
+                parent_block_hash(),
+                PARENT_GAS_LIMIT,
+                Root::repeat_byte(0x61),
+                vec![],
+            ));
+        assert_eq!(
+            stateful(&scene, &scene.bid),
+            ignore(IgnoreReason::ParentPayloadUnverified)
+        );
+
+        let exits = vec![(builder_pubkey(), ExecutionAddress::repeat_byte(1))];
+        let envelope = gloas::SignedExecutionPayloadEnvelope {
+            message: envelope_with_gas_limit(parent_block_hash(), PARENT_GAS_LIMIT, PARENT, exits),
+            signature: Default::default(),
+        };
+        scene.store.insert_verified_payload(PARENT_SLOT, &envelope);
+        assert_eq!(
+            stateful(&scene, &scene.bid),
+            ignore(IgnoreReason::BuilderMayExit)
+        );
+    }
+
+    #[test]
+    fn a_bad_signature_is_rejected() {
+        let scene = scene();
+        let mut bid = scene.bid.clone();
+        bid.signature.0[5] ^= 1;
+        assert_eq!(stateful(&scene, &bid), reject(RejectReason::BadSignature));
+        // Signed by another builder's key.
+        let forged = test_support_sign_by(&scene, 1);
+        assert_eq!(
+            stateful(&scene, &forged),
+            reject(RejectReason::BadSignature)
+        );
+    }
+
+    fn test_support_sign_by(scene: &Scene, secret: u64) -> gloas::SignedExecutionPayloadBid {
+        crate::beacon::builder_market::test_support::sign_bid(
+            &scene.state,
+            scene.bid.message.clone(),
+            secret,
+        )
+    }
+
+    // ---- the parent's state stands in for the advanced one ----
+
+    #[test]
+    fn a_state_advanced_within_its_epoch_keeps_what_the_rules_read() {
+        let scene = scene();
+        let config = scene.store.config();
+        let mut advanced = scene.state.clone();
+        stf::process_slots(&mut advanced, BID_SLOT, &config).expect("advance");
+        assert_eq!(advanced.slot(), BID_SLOT);
+        let (BeaconState::Gloas(before), BeaconState::Gloas(after)) = (&scene.state, &advanced)
+        else {
+            unreachable!("gloas states")
+        };
+        assert_eq!(before.builders, after.builders);
+        assert_eq!(before.finalized_checkpoint, after.finalized_checkpoint);
+        assert_eq!(before.fork, after.fork);
+        assert_eq!(
+            before.builder_pending_payments,
+            after.builder_pending_payments
+        );
+        assert_eq!(
+            before.builder_pending_withdrawals,
+            after.builder_pending_withdrawals
+        );
+        assert_eq!(
+            before.latest_execution_payload_bid,
+            after.latest_execution_payload_bid
+        );
+        assert_eq!(
+            get_randao_mix(&scene.state, get_current_epoch(&scene.state)),
+            get_randao_mix(&advanced, get_current_epoch(&advanced))
+        );
+    }
+
+    #[test]
+    fn a_bid_across_an_epoch_uses_and_caches_the_checkpoint_state() {
+        // The parent is in epoch 2: a funded builder is active only after
+        // finality passes its deposit epoch, which an epoch-1 chain cannot have.
+        let scene = scene_at(2 * SLOTS_PER_EPOCH, |_| {});
+        let slot = 3 * SLOTS_PER_EPOCH;
+        // Preferences for the later epoch's proposer.
+        let dependent = dependent_root_at(&scene.state, PARENT, slot).expect("in the window");
+        let proposer = fixed_proposer(&scene.state, slot).expect("in the lookahead window");
+        let preferences = sign_preferences(
+            &scene.state,
+            gloas::ProposerPreferences {
+                dependent_root: dependent,
+                proposal_slot: slot,
+                validator_index: proposer,
+                fee_recipient: fee_recipient(),
+                target_gas_limit: PARENT_GAS_LIMIT,
+            },
+        );
+        assert!(scene.market.record_preferences(preferences, slot - 1));
+        let bid = scene.signed(|bid| bid.slot = slot);
+        let key = CacheKey::CheckpointState {
+            epoch: 3,
+            root: PARENT,
+        };
+        assert!(scene.store.cached_state(key).is_none());
+        assert_eq!(stateful(&scene, &bid), Outcome::Accept);
+        let cached = scene.store.cached_state(key).expect("the advanced state");
+        assert_eq!(get_current_epoch(&cached), 3);
+        // A second bid finds it.
+        assert_eq!(stateful(&scene, &bid), Outcome::Accept);
+    }
+
+    /// A fulu parent one slot before gloas's first epoch: the rules reach the
+    /// advanced state (which the upgrade has made gloas, with no builders yet)
+    /// only if the parent's own payload counted as a known one.
+    fn fulu_parent_scene(
+        edit: impl FnOnce(&mut crate::beacon::containers::BeaconState),
+    ) -> (Store, BuilderMarket, gloas::SignedExecutionPayloadBid) {
+        use ethlambda_storage::ForkCheckpoints;
+
+        use crate::beacon::fork::ForkName;
+        use crate::beacon::gossip::test_support::{GENESIS_TIME, builder_scene::block_at};
+        use crate::beacon::helpers::test_state::with_signing_validators_at;
+
+        let config = Config::mainnet()
+            .with_fork_epoch(ForkName::Fulu, 0)
+            .with_fork_epoch(ForkName::Gloas, 2);
+        let parent = Root::repeat_byte(0x70);
+        let header_hash = ExecutionBlockHash::repeat_byte(0x41);
+        let parent_slot = 2 * SLOTS_PER_EPOCH - 2;
+        let mut state = with_signing_validators_at(ForkName::Fulu, 64);
+        *state.slot_mut() = parent_slot;
+        if let BeaconState::Fulu(fulu) = &mut state {
+            fulu.latest_execution_payload_header.block_hash = header_hash;
+            fulu.latest_execution_payload_header.gas_limit = 30_000_000;
+        }
+        edit(&mut state);
+        state.apply_pending_mutations();
+
+        let mut store = Store::init_beacon(
+            Arc::new(ethlambda_storage::backend::InMemoryBackend::new()),
+            GENESIS_TIME,
+            config,
+            parent,
+            ethlambda_types::checkpoint::Checkpoint {
+                root: parent,
+                slot: parent_slot,
+            },
+            parent_slot,
+        );
+        store
+            .insert_pending_block(parent, block_at(parent_slot))
+            .expect("insert the parent");
+        store
+            .insert_state(parent, state.clone())
+            .expect("insert the parent state");
+        store
+            .update_checkpoints(ForkCheckpoints::head_only(parent))
+            .expect("move the head");
+        // The status the chain actor records; a pre-gloas head has one node.
+        store.set_head_payload_status(parent, PayloadStatus::Empty);
+
+        let slot = 2 * SLOTS_PER_EPOCH;
+        let market = BuilderMarket::default();
+        let proposer = fixed_proposer(&state, slot).expect("in the window");
+        let dependent_root = dependent_root_at(&state, parent, slot).expect("in the window");
+        let preferences = sign_preferences(
+            &state,
+            gloas::ProposerPreferences {
+                dependent_root,
+                proposal_slot: slot,
+                validator_index: proposer,
+                fee_recipient: fee_recipient(),
+                target_gas_limit: 30_000_000,
+            },
+        );
+        assert!(market.record_preferences(preferences, slot - 1));
+        let bid = gloas::SignedExecutionPayloadBid {
+            message: gloas::ExecutionPayloadBid {
+                parent_block_hash: header_hash,
+                parent_block_root: parent,
+                block_hash: ExecutionBlockHash::repeat_byte(0x33),
+                prev_randao: get_randao_mix(&state, get_current_epoch(&state)),
+                fee_recipient: fee_recipient(),
+                gas_limit: 30_000_000,
+                builder_index: 0,
+                slot,
+                value: 1,
+                ..Default::default()
+            },
+            signature: Default::default(),
+        };
+        (store, market, bid)
+    }
+
+    #[test]
+    fn a_pre_gloas_parents_payload_counts_as_known_with_its_header_gas_limit() {
+        let (store, market, bid) = fulu_parent_scene(|_| {});
+        // Past the payload, gas, head and randao rules, the advanced state is
+        // gloas's first and has no builders yet.
+        assert_eq!(
+            stateful_checks(&store, &market, &bid),
+            reject(RejectReason::UnknownBuilder)
+        );
+    }
+
+    #[test]
+    fn a_pre_gloas_parents_header_gas_limit_bounds_the_bid() {
+        let (store, market, mut bid) = fulu_parent_scene(|_| {});
+        bid.message.gas_limit += 1_000;
+        assert_eq!(
+            stateful_checks(&store, &market, &bid),
+            ignore(IgnoreReason::GasLimitIncompatible)
+        );
+    }
+
+    #[test]
+    fn only_the_pre_gloas_parents_own_payload_hash_counts() {
+        let (store, market, mut bid) = fulu_parent_scene(|_| {});
+        bid.message.parent_block_hash = ExecutionBlockHash::repeat_byte(0x42);
+        assert_eq!(
+            stateful_checks(&store, &market, &bid),
+            ignore(IgnoreReason::ParentPayloadUnknown)
+        );
+    }
+}
diff --git a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs
index c8e047f57..65a6d32c0 100644
--- a/crates/blockchain/state_transition/src/beacon/gossip/mod.rs
+++ b/crates/blockchain/state_transition/src/beacon/gossip/mod.rs
@@ -11,7 +11,9 @@ pub mod attestation;
 pub mod block;
 pub mod column;
 pub mod envelope;
+pub mod execution_payload_bid;
 pub mod payload_attestation;
+pub mod proposer_preferences;
 #[cfg(test)]
 pub(crate) mod test_support;
 
@@ -140,6 +142,32 @@ pub enum IgnoreReason {
     FinalizedNotAncestor,
     /// An ancestor lies outside what the state's `block_roots` can answer.
     AncestryUnknown,
+    /// A gloas bid or proposer preferences names a slot before the fork.
+    PreGloasSlot,
+    /// A bid's slot is neither the current nor the next slot.
+    NotCurrentOrNextSlot,
+    /// A bid's value does not beat the best already seen for its slot and parent.
+    NotHighestBid,
+    /// A bid or preferences name a slot beyond the proposer lookahead.
+    BeyondLookahead,
+    /// No proposer preferences are known for a bid's slot and dependent root.
+    PreferencesUnseen,
+    /// A bid's fee recipient is not the one in the proposer's preferences.
+    FeeRecipientMismatch,
+    /// A bid's parent block hash is not a known execution payload.
+    ParentPayloadUnknown,
+    /// A bid's gas limit cannot reach the preferences' target from the parent's.
+    GasLimitIncompatible,
+    /// A bid is not compatible with this node's head branch.
+    NotOnHeadBranch,
+    /// A bid's builder cannot cover its value.
+    BuilderCannotCover,
+    /// A bid's builder exits in the parent payload.
+    BuilderMayExit,
+    /// Proposer preferences arrived after their proposal slot began.
+    SlotStarted,
+    /// Proposer preferences name a dependent root no chain here can have.
+    ImpossibleDependentRoot,
     /// A gloas block builds on its parent's full payload branch, but the
     /// parent's envelope has not been seen and verified (the specification
     /// lets it be queued until it is).
@@ -176,6 +204,19 @@ impl IgnoreReason {
             Self::StateUnavailable => "state_unavailable",
             Self::FinalizedNotAncestor => "finalized_not_ancestor",
             Self::AncestryUnknown => "ancestry_unknown",
+            Self::PreGloasSlot => "pre_gloas_slot",
+            Self::NotCurrentOrNextSlot => "not_current_or_next_slot",
+            Self::NotHighestBid => "not_highest_bid",
+            Self::BeyondLookahead => "beyond_lookahead",
+            Self::PreferencesUnseen => "preferences_unseen",
+            Self::FeeRecipientMismatch => "fee_recipient_mismatch",
+            Self::ParentPayloadUnknown => "parent_payload_unknown",
+            Self::GasLimitIncompatible => "gas_limit_incompatible",
+            Self::NotOnHeadBranch => "not_on_head_branch",
+            Self::BuilderCannotCover => "builder_cannot_cover",
+            Self::BuilderMayExit => "builder_may_exit",
+            Self::SlotStarted => "slot_started",
+            Self::ImpossibleDependentRoot => "impossible_dependent_root",
             Self::ParentPayloadUnverified => "parent_payload_unverified",
             Self::PayloadEnvelopeUnseen => "payload_envelope_unseen",
             Self::PayloadOptimistic => "payload_optimistic",
@@ -233,6 +274,20 @@ pub enum RejectReason {
     AggregateSignature,
     /// The target is not the voted block's ancestor at the target epoch.
     TargetNotAncestor,
+    /// A bid promises a payment outside the bid value.
+    ExecutionPaymentNonZero,
+    /// A bid's block hash is its parent's.
+    BlockHashEqualsParent,
+    /// A bid's `prev_randao` is not the parent state's current mix.
+    PrevRandao,
+    /// A bid's builder index is not in the registry.
+    UnknownBuilder,
+    /// A bid's builder is not a payload builder.
+    NotPayloadBuilder,
+    /// A bid's builder is not active.
+    InactiveBuilder,
+    /// Preferences' dependent block is later than the shuffling's dependent slot.
+    DependentRootTooLate,
     /// A gloas block body (or its parent execution requests) carries more of
     /// an operation than its limit, or any deposit.
     OperationLimit,
@@ -293,6 +348,13 @@ impl RejectReason {
             Self::AggregatorSignature => "aggregator_signature",
             Self::AggregateSignature => "aggregate_signature",
             Self::TargetNotAncestor => "target_not_ancestor",
+            Self::ExecutionPaymentNonZero => "execution_payment_nonzero",
+            Self::BlockHashEqualsParent => "block_hash_equals_parent",
+            Self::PrevRandao => "prev_randao",
+            Self::UnknownBuilder => "unknown_builder",
+            Self::NotPayloadBuilder => "not_payload_builder",
+            Self::InactiveBuilder => "inactive_builder",
+            Self::DependentRootTooLate => "dependent_root_too_late",
             Self::OperationLimit => "operation_limit",
             Self::BidParentMismatch => "bid_parent_mismatch",
             Self::BidNotOnParentHead => "bid_not_on_parent_head",
diff --git a/crates/blockchain/state_transition/src/beacon/gossip/proposer_preferences.rs b/crates/blockchain/state_transition/src/beacon/gossip/proposer_preferences.rs
new file mode 100644
index 000000000..21abad0fa
--- /dev/null
+++ b/crates/blockchain/state_transition/src/beacon/gossip/proposer_preferences.rs
@@ -0,0 +1,642 @@
+//! Gossip validation for the gloas `proposer_preferences` topic: a proposer's
+//! `SignedProposerPreferences` (fee recipient and gas target) for a slot, which
+//! bids on that slot are judged against.
+//!
+//! The rules are the specification's `validate_proposer_preferences_gossip`
+//! (`specs/gloas/p2p-interface.md`), split like [`super::envelope`]:
+//! [`cheap_checks`] inline in the p2p actor, [`stateful_checks`] on a blocking
+//! thread, reading cached states only. The caller records the preferences in
+//! the [`BuilderMarket`] on `Accept`.
+//!
+//! Deliberate departures from the specification (`docs/spec_deviations.md`):
+//!
+//! - Never queues: an unseen dependent block is IGNORE, and a state that is
+//!   not cached is IGNORE, not rebuilt from disk.
+//! - The lookahead comes from the cached head state when the head shares the
+//!   dependent root (the whole canonical case, and a dependent block about an
+//!   epoch old is usually evicted from the state cache), else from the cached
+//!   checkpoint state of the epoch before the proposal's.
+//! - The signature verifies under any of three domains, since clients disagree
+//!   at the fork boundary: see [`proposer_preferences_domains`].
+
+use ethlambda_storage::CacheKey;
+
+use super::execution_payload_bid::cached_checkpoint_state;
+use super::{
+    IgnoreReason, Outcome, RejectReason, ancestor_at, is_future_slot, is_gloas_slot, slot_start_ms,
+};
+use crate::beacon::bls;
+use crate::beacon::builder_market::BuilderMarket;
+use crate::beacon::config::Config;
+use crate::beacon::constants::{DOMAIN_PROPOSER_PREFERENCES, MAXIMUM_GOSSIP_CLOCK_DISPARITY};
+use crate::beacon::containers::{BeaconState, gloas};
+use crate::beacon::fork_choice::{
+    Store, compute_shuffling_dependent_slot, compute_shuffling_lookahead_start_slot,
+};
+use crate::beacon::helpers::accessors::get_domain;
+use crate::beacon::helpers::misc::{compute_domain, compute_epoch_at_slot, compute_signing_root};
+use crate::beacon::precheck::fixed_proposer;
+use crate::beacon::preset;
+use crate::beacon::primitives::{Domain, Epoch, HashTreeRoot as _, Root, Slot};
+
+/// The rules that read only the message, the market's seen state and the
+/// clock.
+pub fn cheap_checks(
+    market: &BuilderMarket,
+    store: &Store,
+    signed: &gloas::SignedProposerPreferences,
+    now_ms: u64,
+) -> Result<(), Outcome> {
+    let preferences = &signed.message;
+    let config = store.config();
+    // [IGNORE] The first valid preferences for this dependent root and slot.
+    if market
+        .preferences(preferences.proposal_slot, preferences.dependent_root)
+        .is_some()
+    {
+        return Err(Outcome::Ignore(IgnoreReason::AlreadySeen));
+    }
+    // [IGNORE] The proposal epoch is after the gloas upgrade.
+    if !is_gloas_slot(&config, preferences.proposal_slot) {
+        return Err(Outcome::Ignore(IgnoreReason::PreGloasSlot));
+    }
+    // [IGNORE] The proposal slot has not started yet.
+    if is_past_slot(&config, preferences.proposal_slot, now_ms) {
+        return Err(Outcome::Ignore(IgnoreReason::SlotStarted));
+    }
+    // [IGNORE] The proposer for the proposal slot is known.
+    let proposal_epoch = compute_epoch_at_slot(preferences.proposal_slot);
+    let lookahead_start_slot = compute_shuffling_lookahead_start_slot(proposal_epoch);
+    if is_future_slot(&config, lookahead_start_slot, now_ms) {
+        return Err(Outcome::Ignore(IgnoreReason::BeyondLookahead));
+    }
+    Ok(())
+}
+
+/// The rules that need the dependent block and a state, then the signature.
+/// Runs on a blocking thread.
+pub fn stateful_checks(store: &Store, signed: &gloas::SignedProposerPreferences) -> Outcome {
+    match stateful_rules(store, signed) {
+        Ok(()) => Outcome::Accept,
+        Err(outcome) => outcome,
+    }
+}
+
+fn stateful_rules(store: &Store, signed: &gloas::SignedProposerPreferences) -> Result<(), Outcome> {
+    let preferences = &signed.message;
+    let config = store.config();
+    let proposal_epoch = compute_epoch_at_slot(preferences.proposal_slot);
+    let dependent_slot = compute_shuffling_dependent_slot(proposal_epoch);
+
+    // [IGNORE] The dependent block has been seen (never queued).
+    if !store.has_block(&preferences.dependent_root) {
+        return Err(Outcome::Ignore(IgnoreReason::UnknownBlock));
+    }
+    // [IGNORE] The dependent block passes validation, i.e. has a post-state.
+    if !store
+        .has_state(&preferences.dependent_root)
+        .unwrap_or(false)
+    {
+        return Err(Outcome::Ignore(IgnoreReason::StateUnavailable));
+    }
+    // [REJECT] The dependent block is not after the shuffling dependent slot.
+    let (block_slot, _) = store
+        .block_entry(&preferences.dependent_root)
+        .ok_or(Outcome::Ignore(IgnoreReason::UnknownBlock))?;
+    if block_slot > dependent_slot {
+        return Err(Outcome::Reject(RejectReason::DependentRootTooLate));
+    }
+    // [IGNORE] The dependent block is a possible dependent block.
+    if !is_valid_dependent_root(store, preferences.dependent_root, dependent_slot) {
+        return Err(Outcome::Ignore(IgnoreReason::ImpossibleDependentRoot));
+    }
+
+    let state = lookahead_state(store, preferences, proposal_epoch)
+        .ok_or(Outcome::Ignore(IgnoreReason::StateUnavailable))?;
+
+    // [REJECT] The validator is the proposer for the slot in the lookahead.
+    if fixed_proposer(&state, preferences.proposal_slot) != Some(preferences.validator_index) {
+        return Err(Outcome::Reject(RejectReason::WrongProposer));
+    }
+    // [REJECT] The signature is valid, under any candidate domain.
+    let pubkey = state
+        .validator(preferences.validator_index)
+        .map_err(|_| Outcome::Reject(RejectReason::WrongProposer))?
+        .pubkey;
+    let message_root = preferences.hash_tree_root();
+    let valid = proposer_preferences_domains(&state, &config, proposal_epoch)
+        .into_iter()
+        .any(|domain| {
+            bls::verify(
+                &pubkey,
+                compute_signing_root(message_root, domain),
+                &signed.signature,
+            )
+        });
+    if !valid {
+        return Err(Outcome::Reject(RejectReason::BadSignature));
+    }
+    Ok(())
+}
+
+/// A cached state whose proposer lookahead answers for `preferences`: the
+/// head's when the head shares the dependent root, else the dependent block's
+/// state advanced to the epoch before the proposal's.
+fn lookahead_state(
+    store: &Store,
+    preferences: &gloas::ProposerPreferences,
+    proposal_epoch: Epoch,
+) -> Option> {
+    if let Ok(head_root) = store.head()
+        && let Some(head_state) = store.cached_state(CacheKey::BlockState(head_root))
+    {
+        // A state in the epoch before the proposal's, or in the proposal's
+        // own, holds the proposal slot in its two-epoch window.
+        let state_epoch = compute_epoch_at_slot(head_state.slot());
+        let covers = state_epoch == proposal_epoch
+            || state_epoch + preset::MIN_SEED_LOOKAHEAD == proposal_epoch;
+        if covers
+            && dependent_root_at(&head_state, head_root, preferences.proposal_slot)
+                == Some(preferences.dependent_root)
+        {
+            return Some(head_state);
+        }
+    }
+    cached_checkpoint_state(
+        store,
+        proposal_epoch.saturating_sub(preset::MIN_SEED_LOOKAHEAD),
+        preferences.dependent_root,
+    )
+}
+
+/// Both halves. The caller records the preferences on `Accept`: the
+/// specification's `validate_proposer_preferences_gossip`.
+pub fn validate(
+    market: &BuilderMarket,
+    store: &Store,
+    signed: &gloas::SignedProposerPreferences,
+    now_ms: u64,
+) -> Outcome {
+    if let Err(outcome) = cheap_checks(market, store, signed, now_ms) {
+        return outcome;
+    }
+    stateful_checks(store, signed)
+}
+
+/// The spec's `is_valid_dependent_root`: `root == store.head()`, or some block
+/// in the index has `parent_root == root` and `slot > dependent_slot`.
+pub fn is_valid_dependent_root(store: &Store, root: Root, dependent_slot: Slot) -> bool {
+    if store.head().is_ok_and(|head| head == root) {
+        return true;
+    }
+    store
+        .block_index()
+        .values()
+        .any(|&(slot, parent_root)| parent_root == root && slot > dependent_slot)
+}
+
+/// `ancestor_at(state, state_block_root, compute_shuffling_dependent_slot(
+/// epoch(proposal_slot)))`: the block that fixed the proposal slot's proposer
+/// shuffling on the chain `state` is the post-state of. Used by `produceBlockV4`
+/// and the validator client.
+pub fn dependent_root_at(
+    state: &BeaconState,
+    state_block_root: Root,
+    proposal_slot: Slot,
+) -> Option {
+    let dependent_slot = compute_shuffling_dependent_slot(compute_epoch_at_slot(proposal_slot));
+    ancestor_at(state, state_block_root, dependent_slot)
+}
+
+/// The distinct candidate signing domains, tried in order: `get_domain(
+/// lookahead_state, DOMAIN_PROPOSER_PREFERENCES, Some(P))` (the specification's);
+/// the schedule's fork version at `P - MIN_SEED_LOOKAHEAD` (saturating); and the
+/// schedule's at `P`.
+///
+/// Outside the first epoch of a fork the three coincide. Across a boundary
+/// the specification's gives the version of the epoch before the proposal's,
+/// while lighthouse signs with the proposal epoch's, so accepting both avoids
+/// rejecting an honest client's messages around the gloas upgrade.
+pub fn proposer_preferences_domains(
+    lookahead_state: &BeaconState,
+    config: &Config,
+    proposal_epoch: Epoch,
+) -> Vec {
+    let genesis_validators_root = lookahead_state.genesis_validators_root();
+    let at = |epoch: Epoch| {
+        compute_domain(
+            DOMAIN_PROPOSER_PREFERENCES,
+            config.fork_version(config.fork_at_epoch(epoch)),
+            genesis_validators_root,
+        )
+    };
+    let domains = [
+        get_domain(
+            lookahead_state,
+            DOMAIN_PROPOSER_PREFERENCES,
+            Some(proposal_epoch),
+        ),
+        at(proposal_epoch.saturating_sub(preset::MIN_SEED_LOOKAHEAD)),
+        at(proposal_epoch),
+    ];
+    let mut distinct: Vec = Vec::with_capacity(domains.len());
+    for domain in domains {
+        if !distinct.contains(&domain) {
+            distinct.push(domain);
+        }
+    }
+    distinct
+}
+
+/// `now > slot_start + MAXIMUM_GOSSIP_CLOCK_DISPARITY`: the specification's
+/// `is_past_slot`.
+pub(crate) fn is_past_slot(config: &Config, slot: Slot, now_ms: u64) -> bool {
+    now_ms > slot_start_ms(config, slot).saturating_add(MAXIMUM_GOSSIP_CLOCK_DISPARITY)
+}
+
+#[cfg(test)]
+mod tests {
+    use ethlambda_storage::ForkCheckpoints;
+    use ethlambda_types::beacon::containers::Fork;
+
+    use super::*;
+    use crate::beacon::builder_market::test_support::sign_preferences;
+    use crate::beacon::fork::ForkName;
+    use crate::beacon::gossip::test_support::builder_scene::*;
+    use crate::beacon::gossip::test_support::{slot_start_ms, store};
+    use crate::beacon::helpers::test_state::{secret_key_for, with_signing_validators_at};
+    use crate::beacon::preset::SLOTS_PER_EPOCH;
+    use crate::beacon::primitives::ValidatorIndex;
+    use crate::beacon::stf;
+
+    /// The epoch-2 slot the scene's proposer preferences name, and the block
+    /// that fixed its proposer: the genesis-slot block, an ancestor of the
+    /// head (`PARENT_SLOT`) whose `block_roots` entry covers `DEPENDENT_SLOT`.
+    const PROPOSAL_SLOT: Slot = 2 * SLOTS_PER_EPOCH;
+    const DEPENDENT: Root = Root::repeat_byte(0x31);
+    /// `PROPOSAL_SLOT`'s shuffling dependent slot: the last of epoch 0.
+    const DEPENDENT_SLOT: Slot = PARENT_SLOT - 1;
+
+    fn ignore(reason: IgnoreReason) -> Outcome {
+        Outcome::Ignore(reason)
+    }
+
+    fn reject(reason: RejectReason) -> Outcome {
+        Outcome::Reject(reason)
+    }
+
+    /// The builder scene, with the parent's `block_roots` naming `DEPENDENT` at
+    /// `DEPENDENT_SLOT`, and `DEPENDENT` stored (block and state) as the head's parent.
+    fn dependent_scene() -> Scene {
+        let mut scene = scene_with(|state| state.block_roots[DEPENDENT_SLOT as usize] = DEPENDENT);
+        scene
+            .store
+            .insert_pending_block(DEPENDENT, block_at(0))
+            .expect("insert the dependent block");
+        // The head, as a live-chain child of the dependent block: the rule
+        // that a dependent block is possible reads the live chain.
+        scene
+            .store
+            .insert_signed_block(PARENT, block_with_parent(PARENT_SLOT, DEPENDENT))
+            .expect("link the head to the dependent block");
+        let mut dependent_state = scene.state.clone();
+        *dependent_state.slot_mut() = 0;
+        scene
+            .store
+            .insert_state(DEPENDENT, dependent_state)
+            .expect("insert the dependent state");
+        scene
+    }
+
+    fn preferences_for(
+        scene: &Scene,
+        dependent_root: Root,
+        proposer: ValidatorIndex,
+    ) -> gloas::SignedProposerPreferences {
+        sign_preferences(
+            &scene.state,
+            gloas::ProposerPreferences {
+                dependent_root,
+                proposal_slot: PROPOSAL_SLOT,
+                validator_index: proposer,
+                fee_recipient: fee_recipient(),
+                target_gas_limit: 30_000_000,
+            },
+        )
+    }
+
+    /// Valid preferences from the head's lookahead.
+    fn valid_preferences(scene: &Scene) -> gloas::SignedProposerPreferences {
+        let proposer =
+            crate::beacon::precheck::fixed_proposer(&scene.state, PROPOSAL_SLOT).expect("window");
+        preferences_for(scene, DEPENDENT, proposer)
+    }
+
+    fn now_ms(scene: &Scene) -> u64 {
+        scene.now_ms()
+    }
+
+    #[test]
+    fn valid_preferences_are_accepted_from_the_head_state() {
+        let scene = dependent_scene();
+        let preferences = valid_preferences(&scene);
+        let outcome = validate(&scene.market, &scene.store, &preferences, now_ms(&scene));
+        assert_eq!(outcome, Outcome::Accept);
+        // The head's own lookahead answered: nothing was advanced and cached.
+        let key = CacheKey::CheckpointState {
+            epoch: 1,
+            root: DEPENDENT,
+        };
+        assert!(scene.store.cached_state(key).is_none());
+    }
+
+    // ---- cheap rules ----
+
+    #[test]
+    fn a_second_message_for_a_key_is_already_seen() {
+        let scene = dependent_scene();
+        let preferences = valid_preferences(&scene);
+        assert!(
+            scene
+                .market
+                .record_preferences(preferences.clone(), PARENT_SLOT + 1)
+        );
+        assert_eq!(
+            cheap_checks(&scene.market, &scene.store, &preferences, now_ms(&scene)),
+            Err(ignore(IgnoreReason::AlreadySeen))
+        );
+    }
+
+    #[test]
+    fn preferences_before_the_fork_are_ignored() {
+        let scene = dependent_scene();
+        let fulu = store(0);
+        let preferences = valid_preferences(&scene);
+        assert_eq!(
+            cheap_checks(&scene.market, &fulu, &preferences, now_ms(&scene)),
+            Err(ignore(IgnoreReason::PreGloasSlot))
+        );
+    }
+
+    #[test]
+    fn preferences_are_late_once_the_slot_began_by_more_than_the_disparity() {
+        let scene = dependent_scene();
+        let preferences = valid_preferences(&scene);
+        let started = slot_start_ms(&scene.store, PROPOSAL_SLOT);
+        let check = |now| cheap_checks(&scene.market, &scene.store, &preferences, now);
+        assert_eq!(check(started + 500), Ok(()));
+        assert_eq!(check(started + 501), Err(ignore(IgnoreReason::SlotStarted)));
+    }
+
+    #[test]
+    fn preferences_are_early_before_the_lookahead_opens() {
+        let scene = dependent_scene();
+        let mut preferences = valid_preferences(&scene);
+        // Epoch 3's lookahead opens with epoch 2's first slot.
+        preferences.message.proposal_slot = 3 * SLOTS_PER_EPOCH;
+        let opens = slot_start_ms(&scene.store, 2 * SLOTS_PER_EPOCH);
+        let check = |now| cheap_checks(&scene.market, &scene.store, &preferences, now);
+        assert_eq!(check(opens - 500), Ok(()));
+        assert_eq!(
+            check(opens - 501),
+            Err(ignore(IgnoreReason::BeyondLookahead))
+        );
+    }
+
+    // ---- stateful rules ----
+
+    #[test]
+    fn preferences_naming_an_unseen_dependent_block_are_ignored() {
+        let scene = dependent_scene();
+        let preferences = preferences_for(&scene, Root::repeat_byte(0x99), 0);
+        assert_eq!(
+            stateful_checks(&scene.store, &preferences),
+            ignore(IgnoreReason::UnknownBlock)
+        );
+    }
+
+    #[test]
+    fn a_dependent_block_without_a_state_is_ignored() {
+        let mut scene = dependent_scene();
+        let stateless = Root::repeat_byte(0x62);
+        scene
+            .store
+            .insert_pending_block(stateless, block_at(DEPENDENT_SLOT - 1))
+            .expect("insert the block");
+        let preferences = preferences_for(&scene, stateless, 0);
+        assert_eq!(
+            stateful_checks(&scene.store, &preferences),
+            ignore(IgnoreReason::StateUnavailable)
+        );
+    }
+
+    #[test]
+    fn a_dependent_block_after_the_dependent_slot_is_rejected() {
+        let scene = dependent_scene();
+        // The head itself is at `PARENT_SLOT`, after `DEPENDENT_SLOT`.
+        let preferences = preferences_for(&scene, PARENT, 0);
+        assert_eq!(
+            stateful_checks(&scene.store, &preferences),
+            reject(RejectReason::DependentRootTooLate)
+        );
+    }
+
+    #[test]
+    fn a_dependent_block_no_chain_can_use_is_ignored() {
+        let mut scene = dependent_scene();
+        let stray = Root::repeat_byte(0x63);
+        scene
+            .store
+            .insert_pending_block(stray, block_at(DEPENDENT_SLOT - 1))
+            .expect("insert the block");
+        scene
+            .store
+            .insert_state(stray, scene.state.clone())
+            .expect("insert the state");
+        let preferences = preferences_for(&scene, stray, 0);
+        assert_eq!(
+            stateful_checks(&scene.store, &preferences),
+            ignore(IgnoreReason::ImpossibleDependentRoot)
+        );
+        assert!(!is_valid_dependent_root(
+            &scene.store,
+            stray,
+            DEPENDENT_SLOT
+        ));
+        assert!(is_valid_dependent_root(
+            &scene.store,
+            PARENT,
+            DEPENDENT_SLOT
+        ));
+    }
+
+    #[test]
+    fn a_validator_that_is_not_the_proposer_is_rejected() {
+        let scene = dependent_scene();
+        let proposer =
+            crate::beacon::precheck::fixed_proposer(&scene.state, PROPOSAL_SLOT).expect("window");
+        let other = (proposer + 1) % 8;
+        let preferences = preferences_for(&scene, DEPENDENT, other);
+        assert_eq!(
+            stateful_checks(&scene.store, &preferences),
+            reject(RejectReason::WrongProposer)
+        );
+    }
+
+    #[test]
+    fn a_bad_signature_is_rejected() {
+        let scene = dependent_scene();
+        let mut preferences = valid_preferences(&scene);
+        preferences.signature.0[5] ^= 1;
+        assert_eq!(
+            stateful_checks(&scene.store, &preferences),
+            reject(RejectReason::BadSignature)
+        );
+    }
+
+    #[test]
+    fn a_dependent_block_off_the_head_uses_the_cached_checkpoint_state() {
+        let mut scene = dependent_scene();
+        // The dependent block is the head, whose own state is epoch 0, so the
+        // lookahead has to come from its state advanced to epoch 1.
+        scene
+            .store
+            .update_checkpoints(ForkCheckpoints::head_only(DEPENDENT))
+            .expect("move the head");
+        let mut advanced = scene
+            .store
+            .cached_state(CacheKey::BlockState(DEPENDENT))
+            .expect("stored");
+        let mut owned = (*advanced).clone();
+        stf::process_slots(&mut owned, PARENT_SLOT, &scene.store.config()).expect("advance");
+        advanced = std::sync::Arc::new(owned);
+        let proposer =
+            crate::beacon::precheck::fixed_proposer(&advanced, PROPOSAL_SLOT).expect("window");
+        let preferences = preferences_for(&scene, DEPENDENT, proposer);
+        let key = CacheKey::CheckpointState {
+            epoch: 1,
+            root: DEPENDENT,
+        };
+        assert!(scene.store.cached_state(key).is_none());
+        assert_eq!(stateful_checks(&scene.store, &preferences), Outcome::Accept);
+        assert!(scene.store.cached_state(key).is_some());
+    }
+
+    #[test]
+    fn a_dependent_state_evicted_from_the_cache_is_ignored_not_rebuilt() {
+        let mut scene = dependent_scene();
+        scene
+            .store
+            .update_checkpoints(ForkCheckpoints::head_only(DEPENDENT))
+            .expect("move the head");
+        let preferences = valid_preferences(&scene);
+        // Fill the bounded state cache with other blocks' states until the
+        // dependent block's is pushed out. The state still exists (`has_state`),
+        // but reading it would mean a rebuild from disk.
+        for byte in 0..64u8 {
+            let mut root = Root::repeat_byte(0xA0);
+            root.0[1] = byte;
+            scene
+                .store
+                .insert_state(root, scene.state.clone())
+                .expect("insert a filler state");
+        }
+        assert!(
+            scene
+                .store
+                .cached_state(CacheKey::BlockState(DEPENDENT))
+                .is_none()
+        );
+        assert!(scene.store.has_state(&DEPENDENT).expect("has_state"));
+        assert_eq!(
+            stateful_checks(&scene.store, &preferences),
+            ignore(IgnoreReason::StateUnavailable)
+        );
+    }
+
+    // ---- dependent roots and domains ----
+
+    #[test]
+    fn the_dependent_root_is_the_ancestor_at_the_shuffling_dependent_slot() {
+        let scene = dependent_scene();
+        assert_eq!(
+            dependent_root_at(&scene.state, PARENT, PROPOSAL_SLOT),
+            Some(DEPENDENT)
+        );
+        // At genesis the slot saturates to the state's own block.
+        let mut genesis = scene.state.clone();
+        *genesis.slot_mut() = 0;
+        assert_eq!(dependent_root_at(&genesis, PARENT, 5), Some(PARENT));
+    }
+
+    fn boundary_config() -> Config {
+        Config::mainnet()
+            .with_fork_epoch(ForkName::Fulu, 0)
+            .with_fork_epoch(ForkName::Gloas, 2)
+    }
+
+    fn fulu_state_with_fork(config: &Config) -> BeaconState {
+        let mut state = with_signing_validators_at(ForkName::Fulu, 8);
+        *state.fork_mut() = Fork {
+            previous_version: config.fork_version(ForkName::Electra),
+            current_version: config.fork_version(ForkName::Fulu),
+            epoch: 0,
+        };
+        state
+    }
+
+    #[test]
+    fn across_the_fork_both_neighbouring_versions_are_candidates() {
+        let config = boundary_config();
+        let state = fulu_state_with_fork(&config);
+        let root = state.genesis_validators_root();
+        let at =
+            |fork| compute_domain(DOMAIN_PROPOSER_PREFERENCES, config.fork_version(fork), root);
+        // Proposal epoch 2 is gloas's first: the lookahead state's fork (fulu)
+        // and the epoch before the proposal's agree, the proposal epoch's is gloas.
+        let domains = proposer_preferences_domains(&state, &config, 2);
+        assert_eq!(domains, vec![at(ForkName::Fulu), at(ForkName::Gloas)]);
+
+        // A signature under either verifies; one under neither does not.
+        let preferences = gloas::ProposerPreferences {
+            proposal_slot: 2 * SLOTS_PER_EPOCH,
+            ..Default::default()
+        };
+        let message_root = preferences.hash_tree_root();
+        let sign = |domain| {
+            let signature = secret_key_for(0).sign(
+                compute_signing_root(message_root, domain).as_slice(),
+                crate::beacon::bls::DST,
+                &[],
+            );
+            crate::beacon::primitives::BlsSignature(signature.to_bytes())
+        };
+        let pubkey = state.validator(0).expect("validator 0").pubkey;
+        let accepted = |signature| {
+            domains.iter().any(|domain| {
+                bls::verify(
+                    &pubkey,
+                    compute_signing_root(message_root, *domain),
+                    &signature,
+                )
+            })
+        };
+        assert!(accepted(sign(at(ForkName::Fulu))));
+        assert!(accepted(sign(at(ForkName::Gloas))));
+        assert!(!accepted(sign(at(ForkName::Electra))));
+    }
+
+    #[test]
+    fn away_from_a_fork_there_is_one_candidate_domain() {
+        let config = boundary_config();
+        let mut state = fulu_state_with_fork(&config);
+        *state.fork_mut() = Fork {
+            previous_version: config.fork_version(ForkName::Fulu),
+            current_version: config.fork_version(ForkName::Gloas),
+            epoch: 2,
+        };
+        assert_eq!(proposer_preferences_domains(&state, &config, 5).len(), 1);
+        // And at epoch 0 the saturating subtraction does not wrap.
+        assert!(!proposer_preferences_domains(&state, &config, 0).is_empty());
+    }
+}
diff --git a/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs b/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs
index 4b9e6b40a..ed79a2400 100644
--- a/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs
+++ b/crates/blockchain/state_transition/src/beacon/gossip/test_support.rs
@@ -188,3 +188,202 @@ pub(crate) fn fulu_parent(proposer: ValidatorIndex) -> BeaconState {
     state.apply_pending_mutations();
     state
 }
+
+/// The builder-market rules' shared scene: a gloas chain whose head `PARENT`
+/// (`PARENT_SLOT`, epoch 1) is a FULL block with a funded active builder 0,
+/// the proposer preferences and parent payload gossip would have delivered,
+/// and a signed bid for `BID_SLOT` that passes every rule.
+pub(crate) mod builder_scene {
+    use ethlambda_storage::ForkCheckpoints;
+    use ethlambda_types::beacon::containers::{SignedBeaconBlock, electra, gloas};
+
+    use super::*;
+    use crate::beacon::builder_market::{BuilderMarket, test_support};
+    use crate::beacon::fork_choice::PayloadStatus;
+    use crate::beacon::gloas_block_production::test_support as gloas_support;
+    use crate::beacon::gossip::proposer_preferences::dependent_root_at;
+    use crate::beacon::helpers::accessors::{get_current_epoch, get_randao_mix};
+    use crate::beacon::preset;
+    use crate::beacon::primitives::{ExecutionAddress, ExecutionBlockHash};
+
+    pub(crate) const PARENT: Root = Root::repeat_byte(0x50);
+    pub(crate) const BID_SLOT: Slot = PARENT_SLOT + 2;
+    /// The slot the scene's parent sits at: epoch 1's first, whose state can hold a
+    /// funded builder only with a finalized epoch that a real chain cannot have
+    /// there (a builder is active once the finalized epoch passes its deposit
+    /// epoch, and the finalized epoch never exceeds the previous one). Fine for
+    /// a rule that reads the state as it is; a test that advances it through
+    /// an epoch's end needs [`scene_at`] with a parent in epoch 2 or later.
+    pub(crate) const PARENT_SLOT: Slot = preset::SLOTS_PER_EPOCH;
+    pub(crate) const PARENT_GAS_LIMIT: u64 = 30_000_000;
+
+    pub(crate) fn fee_recipient() -> ExecutionAddress {
+        ExecutionAddress::repeat_byte(0x11)
+    }
+
+    /// The hash of the FULL parent's payload, which the bid builds on.
+    pub(crate) fn parent_block_hash() -> ExecutionBlockHash {
+        ExecutionBlockHash::repeat_byte(gloas_support::PARENT_BLOCK_HASH)
+    }
+
+    /// A fulu-shaped block at `slot`: the store only reads its slot and parent.
+    pub(crate) fn block_at(slot: Slot) -> SignedBeaconBlock {
+        block_with_parent(slot, Root::ZERO)
+    }
+
+    pub(crate) fn block_with_parent(slot: Slot, parent_root: Root) -> SignedBeaconBlock {
+        SignedBeaconBlock::Fulu(electra::SignedBeaconBlock {
+            message: electra::BeaconBlock {
+                slot,
+                proposer_index: 0,
+                parent_root,
+                state_root: Root::ZERO,
+                body: electra::BeaconBlockBody::empty(),
+            },
+            signature: Default::default(),
+        })
+    }
+
+    /// An empty store, gloas from genesis, anchored (and headed) at `PARENT`,
+    /// whose block the caller must store.
+    fn gloas_store_at(parent_slot: Slot) -> Store {
+        Store::init_beacon(
+            Arc::new(InMemoryBackend::new()),
+            GENESIS_TIME,
+            gloas_support::config(),
+            PARENT,
+            Checkpoint {
+                root: PARENT,
+                slot: parent_slot,
+            },
+            parent_slot,
+        )
+    }
+
+    pub(crate) struct Scene {
+        pub store: Store,
+        pub market: BuilderMarket,
+        /// `PARENT`'s post-state.
+        pub state: BeaconState,
+        /// A signed bid that passes every rule at [`Scene::now_ms`].
+        pub bid: gloas::SignedExecutionPayloadBid,
+    }
+
+    /// `PARENT`'s post-state at `parent_slot`, with an active funded builder 0.
+    /// Reached by advancing the epoch-1 state while nothing is finalized, so
+    /// every epoch transition on the way is one a real chain takes, then
+    /// finalizing epoch 1: reachable once `parent_slot` is in epoch 2 or later.
+    fn parent_state_at(parent_slot: Slot) -> BeaconState {
+        let mut state = test_support::gloas_state_with_builder(0, 100_000_000_000, 0);
+        if parent_slot == PARENT_SLOT {
+            return state;
+        }
+        let BeaconState::Gloas(inner) = &mut state else {
+            unreachable!("built as gloas")
+        };
+        inner.finalized_checkpoint.epoch = 0;
+        let config = gloas_support::config();
+        let mut state =
+            crate::beacon::block_production::advance_to_slot(&state, parent_slot, &config)
+                .expect("advance to the parent's slot");
+        let BeaconState::Gloas(inner) = &mut state else {
+            unreachable!("built as gloas")
+        };
+        inner.finalized_checkpoint.epoch = 1;
+        inner.latest_block_header.slot = inner.slot;
+        state
+    }
+
+    impl Scene {
+        /// 100 ms into the bid's slot.
+        pub(crate) fn now_ms(&self) -> u64 {
+            slot_start_ms(&self.store, self.bid.message.slot) + 100
+        }
+
+        /// The scene's bid after `edit`, re-signed by its builder.
+        pub(crate) fn signed(
+            &self,
+            edit: impl FnOnce(&mut gloas::ExecutionPayloadBid),
+        ) -> gloas::SignedExecutionPayloadBid {
+            let mut bid = self.bid.message.clone();
+            edit(&mut bid);
+            test_support::sign_bid(&self.state, bid, self.bid.message.builder_index)
+        }
+    }
+
+    /// The scene with `edit` applied to the parent's state before it is stored,
+    /// and the prerequisites of a passing bid recorded in the market.
+    pub(crate) fn scene_with(edit: impl FnOnce(&mut gloas::BeaconState)) -> Scene {
+        scene_at(PARENT_SLOT, edit)
+    }
+
+    /// [`scene_with`] for a parent at `parent_slot`, whose bid is for the slot
+    /// two after it.
+    pub(crate) fn scene_at(parent_slot: Slot, edit: impl FnOnce(&mut gloas::BeaconState)) -> Scene {
+        let bid_slot = parent_slot + (BID_SLOT - PARENT_SLOT);
+        let mut state = parent_state_at(parent_slot);
+        let BeaconState::Gloas(inner) = &mut state else {
+            unreachable!("built as gloas")
+        };
+        edit(inner);
+        state.apply_pending_mutations();
+        let mut store = gloas_store_at(parent_slot);
+        store
+            .insert_pending_block(PARENT, block_at(state.slot()))
+            .expect("insert the parent");
+        store
+            .insert_state(PARENT, state.clone())
+            .expect("insert the parent state");
+        store
+            .update_checkpoints(ForkCheckpoints::head_only(PARENT))
+            .expect("move the head");
+        store.set_head_payload_status(PARENT, PayloadStatus::Full);
+
+        let dependent_root = dependent_root_at(&state, PARENT, bid_slot).expect("in the window");
+        let proposer = crate::beacon::precheck::fixed_proposer(&state, bid_slot).expect("window");
+        let market = BuilderMarket::default();
+        let preferences = test_support::sign_preferences(
+            &state,
+            gloas::ProposerPreferences {
+                dependent_root,
+                proposal_slot: bid_slot,
+                validator_index: proposer,
+                fee_recipient: fee_recipient(),
+                target_gas_limit: PARENT_GAS_LIMIT,
+            },
+        );
+        assert!(market.record_preferences(preferences, bid_slot - 1));
+        market.record_execution_payload(&test_support::envelope_with_gas_limit(
+            parent_block_hash(),
+            PARENT_GAS_LIMIT,
+            PARENT,
+            vec![],
+        ));
+        let bid = test_support::sign_bid(
+            &state,
+            gloas::ExecutionPayloadBid {
+                parent_block_hash: parent_block_hash(),
+                parent_block_root: PARENT,
+                block_hash: ExecutionBlockHash::repeat_byte(0x33),
+                prev_randao: get_randao_mix(&state, get_current_epoch(&state)),
+                fee_recipient: fee_recipient(),
+                gas_limit: PARENT_GAS_LIMIT,
+                builder_index: 0,
+                slot: bid_slot,
+                value: 1,
+                ..Default::default()
+            },
+            0,
+        );
+        Scene {
+            store,
+            market,
+            state,
+            bid,
+        }
+    }
+
+    pub(crate) fn scene() -> Scene {
+        scene_with(|_| {})
+    }
+}
diff --git a/crates/blockchain/state_transition/src/beacon/mod.rs b/crates/blockchain/state_transition/src/beacon/mod.rs
index 1a4471fa6..6a5457c81 100644
--- a/crates/blockchain/state_transition/src/beacon/mod.rs
+++ b/crates/blockchain/state_transition/src/beacon/mod.rs
@@ -72,6 +72,7 @@ pub mod aggregate;
 pub mod attestation_pool;
 pub mod block_production;
 pub mod bls;
+pub mod builder_market;
 pub mod das;
 pub mod fork_choice;
 pub mod genesis;
diff --git a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs
index e98ec5ed2..a9607efdf 100644
--- a/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs
+++ b/crates/blockchain/state_transition/tests/beacon_spec/gossip.rs
@@ -14,6 +14,7 @@ use std::num::NonZeroUsize;
 use std::sync::Arc;
 
 use ethlambda_state_transition::beacon::ForkName;
+use ethlambda_state_transition::beacon::builder_market::BuilderMarket;
 use ethlambda_state_transition::beacon::config::Config;
 use ethlambda_state_transition::beacon::containers::{
     BeaconState, Checkpoint, DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock,
@@ -43,6 +44,8 @@ const HANDLERS: &[&str] = &[
     "gossip_beacon_attestation",
     "gossip_execution_payload_envelope",
     "gossip_payload_attestation_message",
+    "gossip_execution_payload_bid",
+    "gossip_proposer_preferences",
 ];
 
 /// The forks each of [`HANDLERS`] validates. A case from any other fork is
@@ -61,9 +64,10 @@ fn validated_forks(handler: &str) -> &'static [ForkName] {
         "gossip_beacon_aggregate_and_proof" | "gossip_beacon_attestation" => {
             &[ForkName::Fulu, ForkName::Gloas]
         }
-        "gossip_execution_payload_envelope" | "gossip_payload_attestation_message" => {
-            &[ForkName::Gloas]
-        }
+        "gossip_execution_payload_envelope"
+        | "gossip_payload_attestation_message"
+        | "gossip_execution_payload_bid"
+        | "gossip_proposer_preferences" => &[ForkName::Gloas],
         other => panic!("{other} is not in HANDLERS, so it has no validated forks"),
     }
 }
@@ -77,20 +81,14 @@ fn validated_forks(handler: &str) -> &'static [ForkName] {
 /// list, the same way [`super::UNMODELED_FORKS`] forces a decision on a new
 /// fork directory.
 ///
-/// `gossip_execution_payload_bid` and `gossip_proposer_preferences` are
-/// gloas's own topics (EIP-7732 ePBS): the builder's bid and a builder's
-/// advertised preferences, respectively. (Its envelope and payload
-/// attestation topics are in [`HANDLERS`].) None of them existed
-/// until gloas's fixture directory started parsing (`ForkName::Gloas`), so
-/// they land here rather than silently in `unknown` the first time this
-/// runner sees them. Alphabetized with the rest rather than kept together.
+/// Gloas's own topics (EIP-7732 ePBS) are all in [`HANDLERS`] now: the
+/// envelope, the payload attestation, the builder's bid and the proposer's
+/// preferences.
 const IGNORED_HANDLERS: &[&str] = &[
     "gossip_attester_slashing",
     "gossip_blob_sidecar",
     "gossip_bls_to_execution_change",
-    "gossip_execution_payload_bid",
     "gossip_partial_data_column_sidecar",
-    "gossip_proposer_preferences",
     "gossip_proposer_slashing",
     "gossip_sync_committee_contribution_and_proof",
     "gossip_sync_committee_message",
@@ -259,17 +257,28 @@ fn case_config(case: &Case, state: &BeaconState) -> Config {
 
 /// Delivers `entry`'s execution payload envelope, if it lists one, so
 /// `is_payload_verified` answers true for its block.
+///
+/// `trusted` records the envelope the way the specification's own bid and
+/// preferences generators do (`store.payloads[root] = envelope.message`, no
+/// verification): their gas-limit cases deliberately give the head an envelope
+/// whose payload gas limit differs from the block's bid, which
+/// `verify_execution_payload_envelope` would refuse.
 fn deliver_payload(
     store: &mut Store,
     case: &Case,
     entry: &StoreBlock,
     config: &Config,
+    trusted: bool,
 ) -> Result<(), String> {
     let Some(name) = &entry.payload else {
         return Ok(());
     };
     let envelope = gloas::SignedExecutionPayloadEnvelope::from_ssz_bytes(&case.ssz_bytes(name))
         .map_err(|err| format!("decoding {name}: {err:?}"))?;
+    if trusted {
+        fork_choice::accept_execution_payload_envelope(store, &envelope);
+        return Ok(());
+    }
     // No sampled columns are named, so the empty retrieval reads as available,
     // as in the fork-choice runner's envelope step.
     fork_choice::on_execution_payload_envelope(
@@ -282,6 +291,11 @@ fn deliver_payload(
     .map_err(|err| format!("delivering {name}: {err:?}"))
 }
 
+/// The two topics whose cases share one store setup and one market.
+fn is_builder_market_topic(topic: &str) -> bool {
+    matches!(topic, "execution_payload_bid" | "proposer_preferences")
+}
+
 /// The store the case describes: its anchor, then each listed block.
 fn build_store(
     case: &Case,
@@ -297,6 +311,9 @@ fn build_store(
     let backend = Arc::new(ethlambda_storage::backend::InMemoryBackend::new());
     let mut store = fork_choice::get_forkchoice_store(backend, state, anchor_block, config)
         .map_err(|err| format!("get_forkchoice_store: {err:?}"))?;
+    let builder_market = is_builder_market_topic(&meta.topic);
+    let trusted = builder_market;
+    let mut imported = Vec::new();
 
     // The store's clock at the case's base time, so `on_block` accepts every
     // listed block; a block from a slot past that time advances it to that
@@ -304,7 +321,7 @@ fn build_store(
     // slot a clock-disparity case sends its sidecar for.
     let mut clock_s = (config.genesis_time_ms() + meta.current_time_ms) / 1000;
     fork_choice::on_tick(&mut store, clock_s, config);
-    deliver_payload(&mut store, case, anchor, config)?;
+    deliver_payload(&mut store, case, anchor, config, trusted)?;
 
     for entry in rest {
         let block = decode_block(case, &entry.block)?;
@@ -348,7 +365,8 @@ fn build_store(
             &CommitteeCache::default(),
         )
         .map_err(|err| format!("importing {}: {err:?}", entry.block))?;
-        deliver_payload(&mut store, case, entry, config)?;
+        deliver_payload(&mut store, case, entry, config, trusted)?;
+        imported.push(root);
         if gloas && let Some(status) = entry.payload_status.as_deref() {
             let status = match status {
                 "VALID" => PayloadStatusEnum::Valid,
@@ -373,9 +391,38 @@ fn build_store(
         let head = store
             .head()
             .map_err(|err| format!("reading the head: {err}"))?;
+        // The builder market cases walk the head next, from the justified
+        // root, and moving finality prunes `LiveChain` below the finalized
+        // block, so a justified checkpoint left behind the override would
+        // start that walk from a pruned row. A store's justified checkpoint is
+        // never behind its finalized one, so it moves up with the override.
+        let justified = (builder_market
+            && store.beacon_justified_checkpoint().epoch < finalized.epoch)
+            .then_some(checkpoint);
         store
-            .update_checkpoints(ForkCheckpoints::new(head, None, Some(checkpoint)))
+            .update_checkpoints(ForkCheckpoints::new(head, justified, Some(checkpoint)))
             .map_err(|err| format!("overriding the finalized checkpoint: {err}"))?;
+        if builder_market {
+            // The bid generators activate their builders by finalizing epoch 1
+            // in the store *and* in the head's post-state (a replayed chain of
+            // empty blocks never finalizes), and say so through this override.
+            for block_root in &imported {
+                let Some(state) = store
+                    .get_state(block_root)
+                    .map_err(|err| format!("reading a state: {err}"))?
+                else {
+                    continue;
+                };
+                let mut state = (*state).clone();
+                *state.finalized_checkpoint_mut() = Checkpoint {
+                    epoch: finalized.epoch,
+                    root,
+                };
+                store
+                    .insert_state(*block_root, state)
+                    .map_err(|err| format!("overriding a state's finalized checkpoint: {err}"))?;
+            }
+        }
     }
 
     Ok(store)
@@ -406,7 +453,14 @@ fn run_case(case: &Case) -> Result<(), String> {
     let state = BeaconState::from_ssz(case.fork, &case.ssz_bytes("state"))
         .map_err(|err| format!("decoding state: {err:?}"))?;
     let config = case_config(case, &state);
-    let store = build_store(case, &meta, state, &config)?;
+    let mut store = build_store(case, &meta, state, &config)?;
+    if is_builder_market_topic(&meta.topic) {
+        // `on_block` records no head, and the bid and preference rules read
+        // the recorded one.
+        fork_choice::get_head(&mut store, &config)
+            .map_err(|err| format!("computing the head: {err:?}"))?;
+    }
+    let market = BuilderMarket::default();
     let capacity = NonZeroUsize::new(SEEN_CAPACITY).expect("non-zero");
     let mut seen_blocks = SeenBlocks::new(capacity);
     let mut seen_columns = SeenColumns::new(capacity);
@@ -534,6 +588,19 @@ fn run_case(case: &Case) -> Result<(), String> {
                 }
                 outcome
             }
+            // The bid cases mix three message types under one topic (the
+            // preferences and envelope a bid depends on arrive in order, and
+            // share the case's seen state), so the message's own name says
+            // which rule judges it.
+            "execution_payload_bid" | "proposer_preferences" => run_builder_market_message(
+                case,
+                &store,
+                &market,
+                &mut seen_envelopes,
+                message,
+                now_ms,
+                &config,
+            )?,
             other => return Err(format!("topic {other} has no runner")),
         };
         check(message, outcome).map_err(|err| format!("message {index}: {err}"))?;
@@ -541,6 +608,52 @@ fn run_case(case: &Case) -> Result<(), String> {
     Ok(())
 }
 
+/// One message of a `execution_payload_bid` or `proposer_preferences` case.
+fn run_builder_market_message(
+    case: &Case,
+    store: &Store,
+    market: &BuilderMarket,
+    seen_envelopes: &mut SeenEnvelopes,
+    message: &GossipMessage,
+    now_ms: u64,
+    config: &Config,
+) -> Result {
+    let bytes = case.ssz_bytes(&message.message);
+    let decode_err = |err| format!("decoding {}: {err:?}", message.message);
+    if message.message.starts_with("proposer_preferences_") {
+        let preferences =
+            gloas::SignedProposerPreferences::from_ssz_bytes(&bytes).map_err(decode_err)?;
+        let outcome = rules::proposer_preferences::validate(market, store, &preferences, now_ms);
+        if outcome == Outcome::Accept {
+            let wall_slot =
+                now_ms.saturating_sub(config.genesis_time_ms()) / config.slot_duration_ms;
+            market.record_preferences(preferences, wall_slot);
+        }
+        Ok(outcome)
+    } else if message.message.starts_with("execution_payload_envelope_") {
+        let envelope =
+            gloas::SignedExecutionPayloadEnvelope::from_ssz_bytes(&bytes).map_err(decode_err)?;
+        let outcome = rules::envelope::validate(seen_envelopes, store, &envelope);
+        if outcome == Outcome::Accept {
+            seen_envelopes.record(
+                envelope.message.beacon_block_root,
+                envelope.message.builder_index,
+            );
+            market.record_execution_payload(&envelope.message);
+        }
+        Ok(outcome)
+    } else if message.message.starts_with("execution_payload_bid_") {
+        let bid = gloas::SignedExecutionPayloadBid::from_ssz_bytes(&bytes).map_err(decode_err)?;
+        let outcome = rules::execution_payload_bid::validate(market, store, &bid, now_ms);
+        if outcome == Outcome::Accept {
+            market.record_bid(bid);
+        }
+        Ok(outcome)
+    } else {
+        Err(format!("{} is of no known message type", message.message))
+    }
+}
+
 pub fn trials() -> Vec {
     let mut trials = Vec::new();
     for handler in HANDLERS {
diff --git a/crates/common/types/src/beacon/containers/gloas.rs b/crates/common/types/src/beacon/containers/gloas.rs
index 09838fa0f..8284480a6 100644
--- a/crates/common/types/src/beacon/containers/gloas.rs
+++ b/crates/common/types/src/beacon/containers/gloas.rs
@@ -180,23 +180,36 @@ pub type CellsBitList = ProgressiveBitlist;
 /// A registered builder's record in the builder registry (EIP-7732), the
 /// builder-side counterpart of [`super::shared::Validator`].
 #[derive(
-    Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot,
+    Debug,
+    Clone,
+    Default,
+    PartialEq,
+    Eq,
+    serde::Serialize,
+    serde::Deserialize,
+    SszEncode,
+    SszDecode,
+    HashTreeRoot,
 )]
 pub struct Builder {
     pub pubkey: BlsPubkey,
     /// Which shape this record is in. Only [`crate::beacon::constants::PAYLOAD_BUILDER_VERSION`]
     /// exists today; the field exists so a future format change has
     /// somewhere to record it.
+    #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")]
     pub version: u8,
     pub execution_address: ExecutionAddress,
+    #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")]
     pub balance: Gwei,
     /// The epoch this builder's deposit was placed, which
     /// `is_active_builder` compares against the finalized checkpoint before
     /// treating the builder as eligible.
+    #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")]
     pub deposit_epoch: Epoch,
     /// `FAR_FUTURE_EPOCH` until this builder initiates an exit, the same
     /// sentinel convention [`super::shared::Validator::withdrawable_epoch`]
     /// uses.
+    #[serde(with = "crate::beacon::serde_helpers::quoted_or_bare")]
     pub withdrawable_epoch: Epoch,
 }
 
@@ -1115,7 +1128,16 @@ pub struct PartialDataColumnGroupID {
 /// A proposer's broadcast, ahead of its slot, of the fee recipient and gas
 /// limit it wants a builder's bid to target (EIP-7732 p2p-interface.md).
 #[derive(
-    Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot,
+    Debug,
+    Clone,
+    Default,
+    PartialEq,
+    Eq,
+    serde::Serialize,
+    serde::Deserialize,
+    SszEncode,
+    SszDecode,
+    HashTreeRoot,
 )]
 pub struct ProposerPreferences {
     /// The root of the beacon state the proposer duty this message announces
@@ -1132,7 +1154,16 @@ pub struct ProposerPreferences {
 }
 
 #[derive(
-    Debug, Clone, Default, PartialEq, Eq, serde::Serialize, SszEncode, SszDecode, HashTreeRoot,
+    Debug,
+    Clone,
+    Default,
+    PartialEq,
+    Eq,
+    serde::Serialize,
+    serde::Deserialize,
+    SszEncode,
+    SszDecode,
+    HashTreeRoot,
 )]
 pub struct SignedProposerPreferences {
     pub message: ProposerPreferences,
diff --git a/crates/common/types/src/beacon/containers/json_tests.rs b/crates/common/types/src/beacon/containers/json_tests.rs
index 62c6ca3e5..5506897b8 100644
--- a/crates/common/types/src/beacon/containers/json_tests.rs
+++ b/crates/common/types/src/beacon/containers/json_tests.rs
@@ -403,3 +403,84 @@ fn an_oversized_list_is_refused_rather_than_truncated() {
     ]);
     assert!(serde_json::from_value::(json).is_err());
 }
+
+fn bid() -> gloas::SignedExecutionPayloadBid {
+    gloas::SignedExecutionPayloadBid {
+        message: gloas::ExecutionPayloadBid {
+            parent_block_hash: [1; 32].into(),
+            parent_block_root: [2; 32].into(),
+            block_hash: [3; 32].into(),
+            prev_randao: [4; 32].into(),
+            fee_recipient: [5; 20].into(),
+            gas_limit: 30_000_000,
+            builder_index: 7,
+            slot: 33,
+            value: 8,
+            execution_payment: 9,
+            execution_requests_root: [7; 32].into(),
+            blob_kzg_commitments: vec![KzgCommitment([6; 48])].try_into().unwrap(),
+        },
+        signature: signature(3),
+    }
+}
+
+#[test]
+fn a_bid_round_trips_through_json() {
+    round_trip(&bid());
+}
+
+#[test]
+fn proposer_preferences_round_trip_through_json() {
+    round_trip(&gloas::SignedProposerPreferences {
+        message: gloas::ProposerPreferences {
+            dependent_root: [1; 32].into(),
+            proposal_slot: 32,
+            validator_index: 123,
+            fee_recipient: [5; 20].into(),
+            target_gas_limit: 60_000_000,
+        },
+        signature: signature(4),
+    });
+}
+
+#[test]
+fn a_builder_round_trips_and_quotes_its_integers() {
+    let builder = gloas::Builder {
+        pubkey: pubkey(1),
+        version: 3,
+        execution_address: [5; 20].into(),
+        balance: 32_000_000_000,
+        deposit_epoch: 4,
+        withdrawable_epoch: u64::MAX,
+    };
+    round_trip(&builder);
+    let json = serde_json::to_value(&builder).unwrap();
+    for field in ["version", "balance", "deposit_epoch", "withdrawable_epoch"] {
+        assert!(json[field].is_string(), "{field} must be quoted: {json}");
+    }
+}
+
+/// The example `execution_payload_bid` and `proposer_preferences` events in
+/// beacon-APIs' event stream, byte for byte.
+#[test]
+fn the_beacon_apis_example_messages_parse() {
+    let bid = r#"{"message": {"parent_block_hash": "0x9a2fefd2fdb57f74993c7780ea5b9030d2897b615b89f808011ca5aebed54eaf", "parent_block_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2", "block_hash": "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", "prev_randao": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2", "fee_recipient": "0x0000000000000000000000000000000000000000", "gas_limit": "30000000", "builder_index": "42", "slot": "10", "value": "1000000000", "execution_payment": "0", "blob_kzg_commitments": ["0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2"], "execution_requests_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"}, "signature": "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"}"#;
+    let parsed: gloas::SignedExecutionPayloadBid = serde_json::from_str(bid).unwrap();
+    assert_eq!(parsed.message.builder_index, 42);
+    assert_eq!(parsed.message.slot, 10);
+    assert_eq!(parsed.message.value, 1_000_000_000);
+    assert_eq!(parsed.message.blob_kzg_commitments.len(), 1);
+
+    let preferences = r#"{"message": {"dependent_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2", "proposal_slot": "32", "validator_index": "123", "fee_recipient": "0x0000000000000000000000000000000000000000", "target_gas_limit": "60000000"}, "signature": "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"}"#;
+    let parsed: gloas::SignedProposerPreferences = serde_json::from_str(preferences).unwrap();
+    assert_eq!(parsed.message.proposal_slot, 32);
+    assert_eq!(parsed.message.validator_index, 123);
+    assert_eq!(parsed.message.target_gas_limit, 60_000_000);
+}
+
+#[test]
+fn signed_proposer_preferences_are_a_fixed_172_bytes() {
+    use libssz::SszEncode as _;
+    let bytes = gloas::SignedProposerPreferences::default().to_ssz();
+    assert_eq!(bytes.len(), 172);
+}
diff --git a/crates/net/api/src/lib.rs b/crates/net/api/src/lib.rs
index 2f4686961..0f92e2d76 100644
--- a/crates/net/api/src/lib.rs
+++ b/crates/net/api/src/lib.rs
@@ -5,7 +5,10 @@ use ethlambda_types::{
     beacon::containers::{
         DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock,
         electra::SingleAttestation,
-        gloas::{PayloadAttestationMessage, SignedExecutionPayloadEnvelope},
+        gloas::{
+            PayloadAttestationMessage, SignedExecutionPayloadBid, SignedExecutionPayloadEnvelope,
+            SignedProposerPreferences,
+        },
     },
     beacon::primitives::ValidatorIndex,
     block::SignedBlock,
@@ -416,6 +419,19 @@ pub trait RpcToP2P: Send + Sync {
         envelope: Box,
         sidecars: Vec,
     ) -> Result<(), ActorError>;
+    /// Gossip a signed execution payload bid on `execution_payload_bid`. The
+    /// caller ran the gossip rules and recorded it in the shared
+    /// `BuilderMarket` (seen keys and pool), so the p2p actor only publishes.
+    fn publish_execution_payload_bid(
+        &self,
+        bid: SignedExecutionPayloadBid,
+    ) -> Result<(), ActorError>;
+    /// Gossip signed proposer preferences on `proposer_preferences`, under the
+    /// proposal slot's digest. Validated and cached by the caller.
+    fn publish_proposer_preferences(
+        &self,
+        preferences: SignedProposerPreferences,
+    ) -> Result<(), ActorError>;
     /// Gossip a payload timeliness committee member's vote on
     /// `payload_attestation_message` and hand it to the chain actor. Checked by
     /// the caller as above.
diff --git a/crates/net/engine/src/building.rs b/crates/net/engine/src/building.rs
index b407c9326..6ff109248 100644
--- a/crates/net/engine/src/building.rs
+++ b/crates/net/engine/src/building.rs
@@ -130,6 +130,9 @@ pub struct BuiltGloasPayload {
     pub blobs_bundle: BlobsBundle,
     /// The EIP-7685 request list, each entry its type byte then its data.
     pub execution_requests: Vec>,
+    /// The engine's `shouldOverrideBuilder`: it wants this payload built
+    /// locally whatever a builder bid pays. Absent from an answer means false.
+    pub should_override_builder: bool,
 }
 
 /// `BlobsBundleV2`: the payload's blobs, their commitments, and every blob's
@@ -159,6 +162,8 @@ pub(crate) struct GetPayloadV6Response {
     block_value: String,
     blobs_bundle: BlobsBundleJson,
     #[serde(default)]
+    should_override_builder: bool,
+    #[serde(default)]
     execution_requests: Vec,
 }
 
@@ -307,6 +312,7 @@ impl TryFrom for BuiltGloasPayload {
             block_value: parse_uint256(&response.block_value)?,
             blobs_bundle: decode_blobs_bundle(response.blobs_bundle)?,
             execution_requests: decode_requests(&response.execution_requests)?,
+            should_override_builder: response.should_override_builder,
         })
     }
 }
@@ -527,6 +533,30 @@ mod tests {
             built.execution_requests,
             vec![vec![0x00, 0x11], vec![0x02, 0xff]]
         );
+        assert!(!built.should_override_builder);
+    }
+
+    #[test]
+    fn a_v6_answer_carries_should_override_builder_and_defaults_it_to_false() {
+        let mut json = v6_json();
+        json["shouldOverrideBuilder"] = true.into();
+        let response: GetPayloadV6Response = serde_json::from_value(json).unwrap();
+        assert!(
+            BuiltGloasPayload::try_from(response)
+                .unwrap()
+                .should_override_builder
+        );
+
+        let mut json = v6_json();
+        json.as_object_mut()
+            .unwrap()
+            .remove("shouldOverrideBuilder");
+        let response: GetPayloadV6Response = serde_json::from_value(json).unwrap();
+        assert!(
+            !BuiltGloasPayload::try_from(response)
+                .unwrap()
+                .should_override_builder
+        );
     }
 
     #[test]
diff --git a/crates/net/p2p/src/beacon/builder_market.rs b/crates/net/p2p/src/beacon/builder_market.rs
new file mode 100644
index 000000000..f44ce4bbd
--- /dev/null
+++ b/crates/net/p2p/src/beacon/builder_market.rs
@@ -0,0 +1,414 @@
+//! Gloas builder market gossip: the `execution_payload_bid` and
+//! `proposer_preferences` topics.
+//!
+//! Neither message type ever reaches the chain actor: the
+//! rules live in `ethlambda_state_transition::beacon::gossip::{
+//! execution_payload_bid, proposer_preferences}`, and what they accept is
+//! recorded in the node's shared `BuilderMarket` by the verdict.
+
+use ethlambda_state_transition::beacon::gossip::{
+    self, Outcome, RejectReason, execution_payload_bid::MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE,
+};
+use ethlambda_types::beacon::containers::gloas::{
+    SignedExecutionPayloadBid, SignedProposerPreferences,
+};
+use ethlambda_types::beacon::primitives::Slot;
+use ethlambda_types::time::unix_now_ms;
+use libp2p::gossipsub::IdentTopic;
+use libssz::SszEncode;
+use tracing::{debug, error, info, warn};
+
+use crate::beacon::verdict::{Dispatch, Validated};
+use crate::beacon::{decode as beacon_decode, topics as beacon_topics};
+use crate::gossipsub::compress_message;
+use crate::{P2PServer, metrics};
+
+/// Decode a gloas execution payload bid and run its cheap gossip checks.
+///
+/// The size cap is the specification's decompressed bound, applied before any
+/// decode so an oversized payload costs nothing. Same shape as the envelope's
+/// triage, except the seen state lives in the shared
+/// [`BuilderMarket`](ethlambda_state_transition::beacon::builder_market::BuilderMarket),
+/// and the object carries the market on to its stateful checks.
+pub(crate) fn triage_execution_payload_bid(server: &P2PServer, payload: &[u8]) -> Dispatch {
+    const KIND: &str = beacon_topics::EXECUTION_PAYLOAD_BID;
+    if payload.len() > MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE {
+        metrics::inc_beacon_gossip(KIND, "decode_failed");
+        debug!(
+            kind = KIND,
+            bytes = payload.len(),
+            "Beacon gossip payload over the size cap"
+        );
+        return Dispatch::Report(Outcome::Reject(RejectReason::Malformed));
+    }
+    let bid = match beacon_decode::decode_execution_payload_bid(payload) {
+        Ok(bid) => bid,
+        Err(err) => {
+            metrics::inc_beacon_gossip(KIND, "decode_failed");
+            debug!(kind = KIND, %err, bytes = payload.len(), "Beacon gossip decode failed");
+            return Dispatch::Report(Outcome::Reject(RejectReason::Decode));
+        }
+    };
+    metrics::inc_beacon_gossip(KIND, "decoded");
+    debug!(
+        slot = bid.message.slot,
+        builder_index = bid.message.builder_index,
+        value = bid.message.value,
+        bytes = payload.len(),
+        "Beacon execution payload bid decoded"
+    );
+    if let Err(outcome) = gossip::execution_payload_bid::cheap_checks(
+        &server.builder_market,
+        &server.store,
+        &bid,
+        unix_now_ms(),
+    ) {
+        return Dispatch::Report(outcome);
+    }
+    Dispatch::Validate(Validated::ExecutionPayloadBid {
+        bid: Box::new(bid),
+        market: server.builder_market.clone(),
+    })
+}
+
+/// As [`triage_execution_payload_bid`], for `proposer_preferences`. The
+/// container is fixed-size, so decode is its own size check.
+pub(crate) fn triage_proposer_preferences(server: &P2PServer, payload: &[u8]) -> Dispatch {
+    const KIND: &str = beacon_topics::PROPOSER_PREFERENCES;
+    let preferences = match beacon_decode::decode_proposer_preferences(payload) {
+        Ok(preferences) => preferences,
+        Err(err) => {
+            metrics::inc_beacon_gossip(KIND, "decode_failed");
+            debug!(kind = KIND, %err, bytes = payload.len(), "Beacon gossip decode failed");
+            return Dispatch::Report(Outcome::Reject(RejectReason::Decode));
+        }
+    };
+    metrics::inc_beacon_gossip(KIND, "decoded");
+    debug!(
+        proposal_slot = preferences.message.proposal_slot,
+        validator_index = preferences.message.validator_index,
+        bytes = payload.len(),
+        "Beacon proposer preferences decoded"
+    );
+    if let Err(outcome) = gossip::proposer_preferences::cheap_checks(
+        &server.builder_market,
+        &server.store,
+        &preferences,
+        unix_now_ms(),
+    ) {
+        return Dispatch::Report(outcome);
+    }
+    Dispatch::Validate(Validated::ProposerPreferences(Box::new(preferences)))
+}
+
+/// Where and what to publish for one of this module's messages: the topic of
+/// `kind` under the digest `slot` names, and the compressed SSZ. `None` when
+/// this node is not on the beacon wire or holds no digest covering `slot`.
+///
+/// Split from the publish functions so the topic and digest choice can be
+/// checked without a swarm to read the command back from.
+fn publication(
+    server: &P2PServer,
+    kind: &'static str,
+    slot: Slot,
+    ssz: &[u8],
+) -> Option<(IdentTopic, Vec)> {
+    let Some(beacon) = server.wire.beacon() else {
+        error!(
+            kind,
+            slot, "A builder market message reached a lean node; dropping it"
+        );
+        return None;
+    };
+    let Some(digest) = beacon.publish_digest(slot) else {
+        warn!(
+            kind,
+            slot, "No held fork digest covers this message's slot; not publishing"
+        );
+        return None;
+    };
+    let topic = IdentTopic::new(beacon_topics::topic_name(digest, kind));
+    Some((topic, compress_message(ssz)))
+}
+
+/// Publish on `publish_digest(bid.slot)` / `execution_payload_bid`.
+///
+/// Precondition: the caller already recorded it in the market, since
+/// gossipsub never delivers a node its own messages.
+pub(crate) fn publish_execution_payload_bid(
+    server: &mut P2PServer,
+    bid: SignedExecutionPayloadBid,
+) {
+    let slot = bid.message.slot;
+    let Some((topic, data)) = publication(
+        server,
+        beacon_topics::EXECUTION_PAYLOAD_BID,
+        slot,
+        &bid.to_ssz(),
+    ) else {
+        return;
+    };
+    server.swarm_handle.publish(topic, data);
+    info!(
+        slot,
+        builder_index = bid.message.builder_index,
+        value = bid.message.value,
+        "Published execution payload bid to gossipsub"
+    );
+}
+
+/// Publish on `publish_digest(proposal_slot)`: during the epoch before gloas
+/// this is the gloas digest, which is held.
+pub(crate) fn publish_proposer_preferences(
+    server: &mut P2PServer,
+    preferences: SignedProposerPreferences,
+) {
+    let proposal_slot = preferences.message.proposal_slot;
+    let Some((topic, data)) = publication(
+        server,
+        beacon_topics::PROPOSER_PREFERENCES,
+        proposal_slot,
+        &preferences.to_ssz(),
+    ) else {
+        return;
+    };
+    server.swarm_handle.publish(topic, data);
+    info!(
+        proposal_slot,
+        validator_index = preferences.message.validator_index,
+        "Published proposer preferences to gossipsub"
+    );
+}
+
+/// The wall-clock slot, from the store's config.
+pub(crate) fn wall_slot(server: &P2PServer) -> Slot {
+    let config = server.store.config();
+    let genesis_ms = config.genesis_time_ms();
+    unix_now_ms().saturating_sub(genesis_ms) / config.slot_duration_ms.max(1)
+}
+
+#[cfg(test)]
+mod tests {
+    use ethlambda_state_transition::beacon::gossip::{IgnoreReason, Outcome, RejectReason};
+    use ethlambda_types::beacon::config::Config;
+    use ethlambda_types::beacon::fork::ForkName;
+    use ethlambda_types::beacon::preset::SLOTS_PER_EPOCH;
+    use ethlambda_types::beacon::primitives::{Epoch, ExecutionBlockHash};
+    use libssz::SszEncode;
+
+    use super::*;
+    use crate::beacon::transition::apply;
+    use crate::test_support::unconnected_beacon_server;
+
+    const FULU: Epoch = 1_000;
+    const GLOAS: Epoch = 5_000;
+
+    /// Fulu, then gloas at [`GLOAS`], so the rollover window can be opened.
+    fn rollover_config() -> Config {
+        Config::mainnet()
+            .with_fork_epoch(ForkName::Altair, 1)
+            .with_fork_epoch(ForkName::Bellatrix, 2)
+            .with_fork_epoch(ForkName::Capella, 3)
+            .with_fork_epoch(ForkName::Deneb, 4)
+            .with_fork_epoch(ForkName::Electra, 5)
+            .with_fork_epoch(ForkName::Fulu, FULU)
+            .with_fork_epoch(ForkName::Gloas, GLOAS)
+    }
+
+    fn gloas_from_genesis() -> Config {
+        Config::mainnet().with_fork_epoch(ForkName::Gloas, 0)
+    }
+
+    /// A bid the cheap checks have nothing to say against at `slot`.
+    fn bid_at(slot: Slot) -> SignedExecutionPayloadBid {
+        let mut bid = SignedExecutionPayloadBid::default();
+        bid.message.slot = slot;
+        bid.message.builder_index = 3;
+        bid.message.value = 10;
+        bid.message.block_hash = ExecutionBlockHash::repeat_byte(1);
+        bid
+    }
+
+    fn preferences_at(proposal_slot: Slot) -> SignedProposerPreferences {
+        let mut preferences = SignedProposerPreferences::default();
+        preferences.message.proposal_slot = proposal_slot;
+        preferences.message.validator_index = 5;
+        preferences
+    }
+
+    // -- Triage, independent of the gossip rules --
+
+    #[tokio::test]
+    async fn an_oversized_bid_is_malformed_before_it_is_decoded() {
+        let server = unconnected_beacon_server(gloas_from_genesis(), 0).await;
+        let payload = vec![0xff; MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE + 1];
+        assert!(matches!(
+            triage_execution_payload_bid(&server, &payload),
+            Dispatch::Report(Outcome::Reject(RejectReason::Malformed))
+        ));
+    }
+
+    #[tokio::test]
+    async fn garbage_on_either_builder_topic_is_undecodable() {
+        let server = unconnected_beacon_server(gloas_from_genesis(), 0).await;
+        assert!(matches!(
+            triage_execution_payload_bid(&server, &[0xff; 3]),
+            Dispatch::Report(Outcome::Reject(RejectReason::Decode))
+        ));
+        assert!(matches!(
+            triage_proposer_preferences(&server, &[0xff; 3]),
+            Dispatch::Report(Outcome::Reject(RejectReason::Decode))
+        ));
+        // Preferences are fixed-size, so one byte over is not a preference.
+        let mut bytes = preferences_at(1).to_ssz();
+        bytes.push(0);
+        assert!(matches!(
+            triage_proposer_preferences(&server, &bytes),
+            Dispatch::Report(Outcome::Reject(RejectReason::Decode))
+        ));
+    }
+
+    // -- Triage, which needs the real cheap checks (Agent A) --
+
+    #[tokio::test]
+    async fn a_far_slot_bid_is_not_current_or_next() {
+        let server = unconnected_beacon_server(gloas_from_genesis(), 0).await;
+        let bid = bid_at(wall_slot(&server) + 1_000);
+        assert!(matches!(
+            triage_execution_payload_bid(&server, &bid.to_ssz()),
+            Dispatch::Report(Outcome::Ignore(IgnoreReason::NotCurrentOrNextSlot))
+        ));
+    }
+
+    #[tokio::test]
+    async fn a_bid_with_a_nonzero_payment_is_rejected() {
+        let server = unconnected_beacon_server(gloas_from_genesis(), 0).await;
+        let mut bid = bid_at(wall_slot(&server) + 1);
+        bid.message.execution_payment = 1;
+        assert!(matches!(
+            triage_execution_payload_bid(&server, &bid.to_ssz()),
+            Dispatch::Report(Outcome::Reject(RejectReason::ExecutionPaymentNonZero))
+        ));
+    }
+
+    /// A valid-shaped bid goes on to the stateful checks carrying the shared
+    /// market, and once the market holds its builder's key a second is ignored
+    /// before them.
+    #[tokio::test]
+    async fn a_valid_shaped_bid_is_validated_unless_its_builder_key_was_seen() {
+        let server = unconnected_beacon_server(gloas_from_genesis(), 0).await;
+        let bid = bid_at(wall_slot(&server) + 1);
+        let payload = bid.to_ssz();
+        match triage_execution_payload_bid(&server, &payload) {
+            Dispatch::Validate(Validated::ExecutionPayloadBid {
+                bid: decoded,
+                market,
+            }) => {
+                assert_eq!(*decoded, bid);
+                assert!(std::sync::Arc::ptr_eq(&market, &server.builder_market));
+            }
+            _ => panic!("expected the bid to go to its stateful checks"),
+        }
+
+        assert!(server.builder_market.record_bid(bid));
+        assert!(matches!(
+            triage_execution_payload_bid(&server, &payload),
+            Dispatch::Report(Outcome::Ignore(IgnoreReason::AlreadySeen))
+        ));
+    }
+
+    #[tokio::test]
+    async fn preferences_are_judged_on_the_clock() {
+        let server = unconnected_beacon_server(gloas_from_genesis(), 0).await;
+        let wall = wall_slot(&server);
+        let past = preferences_at(wall.saturating_sub(5));
+        assert!(matches!(
+            triage_proposer_preferences(&server, &past.to_ssz()),
+            Dispatch::Report(Outcome::Ignore(IgnoreReason::SlotStarted))
+        ));
+        let two_epochs_ahead = preferences_at((wall / SLOTS_PER_EPOCH + 2) * SLOTS_PER_EPOCH + 1);
+        assert!(matches!(
+            triage_proposer_preferences(&server, &two_epochs_ahead.to_ssz()),
+            Dispatch::Report(Outcome::Ignore(IgnoreReason::BeyondLookahead))
+        ));
+    }
+
+    #[tokio::test]
+    async fn preferences_for_a_cached_key_are_already_seen() {
+        let server = unconnected_beacon_server(gloas_from_genesis(), 0).await;
+        let wall = wall_slot(&server);
+        let preferences = preferences_at(wall + 2);
+        let payload = preferences.to_ssz();
+        assert!(matches!(
+            triage_proposer_preferences(&server, &payload),
+            Dispatch::Validate(Validated::ProposerPreferences(decoded)) if *decoded == preferences
+        ));
+        assert!(server.builder_market.record_preferences(preferences, wall));
+        assert!(matches!(
+            triage_proposer_preferences(&server, &payload),
+            Dispatch::Report(Outcome::Ignore(IgnoreReason::AlreadySeen))
+        ));
+    }
+
+    // -- Publishing --
+
+    fn topic_of(server: &P2PServer, kind: &'static str, slot: Slot) -> Option {
+        publication(server, kind, slot, b"payload").map(|(topic, _)| topic.to_string())
+    }
+
+    #[tokio::test]
+    async fn a_bid_is_published_on_its_slots_digest_and_compressed() {
+        let mut server = unconnected_beacon_server(gloas_from_genesis(), 0).await;
+        // The test server starts on a placeholder digest; this is the startup
+        // call that puts the schedule's own digest in.
+        apply(&mut server, 0);
+        let wire = server.wire.beacon().expect("a beacon wire");
+        let slot = 7;
+        let expected = beacon_topics::topic_name(
+            wire.publish_digest(slot).expect("held"),
+            beacon_topics::EXECUTION_PAYLOAD_BID,
+        );
+        let ssz = bid_at(slot).to_ssz();
+        let (topic, data) =
+            publication(&server, beacon_topics::EXECUTION_PAYLOAD_BID, slot, &ssz).unwrap();
+        assert_eq!(topic.to_string(), expected);
+        assert_eq!(data, compress_message(&ssz));
+        assert!(expected.ends_with("/execution_payload_bid/ssz_snappy"));
+    }
+
+    /// During the epoch before gloas, preferences for a gloas slot go out on
+    /// the gloas digest the node has already joined, and a slot nobody listens
+    /// to is not published at all.
+    #[tokio::test]
+    async fn preferences_for_a_next_fork_slot_use_the_next_forks_digest() {
+        let mut server = unconnected_beacon_server(rollover_config(), 0).await;
+        apply(&mut server, GLOAS - 1);
+        let wire = server.wire.beacon().expect("a beacon wire");
+        let gloas_digest = wire.schedule.digest_at(GLOAS);
+        let fulu_digest = wire.schedule.digest_at(GLOAS - 1);
+        assert_ne!(gloas_digest, fulu_digest);
+
+        let kind = beacon_topics::PROPOSER_PREFERENCES;
+        let first_gloas_slot = GLOAS * SLOTS_PER_EPOCH;
+        assert_eq!(
+            topic_of(&server, kind, first_gloas_slot),
+            Some(beacon_topics::topic_name(gloas_digest, kind))
+        );
+        assert_eq!(
+            topic_of(&server, kind, first_gloas_slot - 1),
+            Some(beacon_topics::topic_name(fulu_digest, kind))
+        );
+        // Long past: its digest is no longer held.
+        assert_eq!(topic_of(&server, kind, 0), None);
+    }
+
+    /// Before the window opens the gloas digest is not held, so there is
+    /// nowhere to publish a bid for a gloas slot.
+    #[tokio::test]
+    async fn nothing_is_published_for_a_digest_that_is_not_held() {
+        let mut server = unconnected_beacon_server(rollover_config(), 0).await;
+        apply(&mut server, GLOAS - 3);
+        let kind = beacon_topics::EXECUTION_PAYLOAD_BID;
+        assert_eq!(topic_of(&server, kind, GLOAS * SLOTS_PER_EPOCH), None);
+    }
+}
diff --git a/crates/net/p2p/src/beacon/decode.rs b/crates/net/p2p/src/beacon/decode.rs
index b4fac1c01..4547d52ac 100644
--- a/crates/net/p2p/src/beacon/decode.rs
+++ b/crates/net/p2p/src/beacon/decode.rs
@@ -24,6 +24,7 @@
 //! | `sync_committee_contribution_and_proof` | No, altair onward |
 //! | `data_column_sidecar_{subnet_id}` | Yes, at gloas, by topic digest |
 //! | `execution_payload`, `payload_attestation_message` | No, gloas onward |
+//! | `execution_payload_bid`, `proposer_preferences` | No, gloas onward |
 
 use ethlambda_types::beacon::config::Config;
 use ethlambda_types::beacon::containers::{
@@ -237,6 +238,22 @@ pub fn decode_payload_attestation_message(
     gloas::PayloadAttestationMessage::from_ssz_bytes(bytes).map_err(|_| DecodeError::Ssz)
 }
 
+/// Decode an `execution_payload_bid` payload. Gloas on, like
+/// [`decode_execution_payload_envelope`].
+pub fn decode_execution_payload_bid(
+    bytes: &[u8],
+) -> Result {
+    gloas::SignedExecutionPayloadBid::from_ssz_bytes(bytes).map_err(|_| DecodeError::Ssz)
+}
+
+/// Decode a `proposer_preferences` payload. Gloas on, like
+/// [`decode_execution_payload_envelope`].
+pub fn decode_proposer_preferences(
+    bytes: &[u8],
+) -> Result {
+    gloas::SignedProposerPreferences::from_ssz_bytes(bytes).map_err(|_| DecodeError::Ssz)
+}
+
 /// Decode a `beacon_aggregate_and_proof` payload, at the fork its slot names.
 pub fn decode_aggregate_and_proof(
     config: &Config,
@@ -653,6 +670,42 @@ mod tests {
         }
     }
 
+    #[test]
+    fn an_execution_payload_bid_round_trips() {
+        let mut bid = gloas::SignedExecutionPayloadBid::default();
+        bid.message.slot = slot_of(10);
+        bid.message.builder_index = 4;
+        bid.message.value = 99;
+        bid.message.parent_block_root = Root::repeat_byte(7);
+        let bytes = bid.to_ssz();
+        assert_eq!(decode_execution_payload_bid(&bytes), Ok(bid));
+        assert!(decode_execution_payload_bid(&bytes[..bytes.len() - 1]).is_err());
+        assert_eq!(
+            decode_execution_payload_bid(&[0xff; 3]),
+            Err(DecodeError::Ssz)
+        );
+    }
+
+    #[test]
+    fn proposer_preferences_round_trip() {
+        let mut preferences = gloas::SignedProposerPreferences::default();
+        preferences.message.proposal_slot = slot_of(10);
+        preferences.message.validator_index = 9;
+        preferences.message.dependent_root = Root::repeat_byte(2);
+        let bytes = preferences.to_ssz();
+        assert_eq!(decode_proposer_preferences(&bytes), Ok(preferences));
+        for length in 0..bytes.len() {
+            assert!(decode_proposer_preferences(&bytes[..length]).is_err());
+        }
+        let mut longer = bytes;
+        longer.push(0);
+        assert!(decode_proposer_preferences(&longer).is_err());
+        assert_eq!(
+            decode_proposer_preferences(&[0xff; 3]),
+            Err(DecodeError::Ssz)
+        );
+    }
+
     #[test]
     fn an_execution_payload_envelope_round_trips() {
         let envelope = crate::test_support::envelope(5, 9);
diff --git a/crates/net/p2p/src/beacon/mod.rs b/crates/net/p2p/src/beacon/mod.rs
index 8ebc3721e..03bd60fa8 100644
--- a/crates/net/p2p/src/beacon/mod.rs
+++ b/crates/net/p2p/src/beacon/mod.rs
@@ -13,6 +13,7 @@
 //! written once and handed the two things the chains disagree about: how wide
 //! the `` field is, and how a chunk body becomes a block.
 
+pub mod builder_market;
 pub mod column_checks;
 pub mod decode;
 pub mod encoding;
diff --git a/crates/net/p2p/src/beacon/topics.rs b/crates/net/p2p/src/beacon/topics.rs
index ff4e6da60..2fe68e720 100644
--- a/crates/net/p2p/src/beacon/topics.rs
+++ b/crates/net/p2p/src/beacon/topics.rs
@@ -1,6 +1,8 @@
 //! The gossipsub topics `ethlambda beacon` subscribes to.
 //!
-//! Seven global topics, plus two families this node's own node id selects a
+//! Seven global topics (four more from gloas: the execution payload envelope,
+//! the payload attestation message, the builder bid and the proposer
+//! preferences), plus two families this node's own node id selects a
 //! narrow slice of: the data column subnets it custodies, and the
 //! `SUBNETS_PER_NODE` attestation subnets it backbones.
 //!
@@ -51,10 +53,19 @@ pub const SYNC_COMMITTEE_CONTRIBUTION_AND_PROOF: &str = "sync_committee_contribu
 pub const EXECUTION_PAYLOAD: &str = "execution_payload";
 /// Topic kind for gloas payload timeliness committee votes.
 pub const PAYLOAD_ATTESTATION_MESSAGE: &str = "payload_attestation_message";
+/// Topic kind for gloas builder bids.
+pub const EXECUTION_PAYLOAD_BID: &str = "execution_payload_bid";
+/// Topic kind for gloas proposer preferences.
+pub const PROPOSER_PREFERENCES: &str = "proposer_preferences";
 
 /// The topic kinds gloas adds to [`SUBSCRIBED_TOPIC_KINDS`], subscribed from
 /// the gloas digest on and never under an earlier one.
-pub const GLOAS_TOPIC_KINDS: [&str; 2] = [EXECUTION_PAYLOAD, PAYLOAD_ATTESTATION_MESSAGE];
+pub const GLOAS_TOPIC_KINDS: [&str; 4] = [
+    EXECUTION_PAYLOAD,
+    PAYLOAD_ATTESTATION_MESSAGE,
+    EXECUTION_PAYLOAD_BID,
+    PROPOSER_PREFERENCES,
+];
 
 /// Every topic kind this node subscribes to at every fork, in the order they
 /// are subscribed. [`GLOAS_TOPIC_KINDS`] follow from gloas.
@@ -344,7 +355,7 @@ mod tests {
     }
 
     #[test]
-    fn gloas_adds_the_envelope_and_payload_attestation_topics() {
+    fn gloas_adds_the_envelope_vote_bid_and_preferences_topics() {
         let fork_topics = |fork| {
             BeaconTopics::for_fork(fork, MAINNET, &[], &[])
                 .topics
@@ -354,8 +365,17 @@ mod tests {
         };
         let gloas = fork_topics(ForkName::Gloas);
         let fulu = fork_topics(ForkName::Fulu);
-        assert_eq!(gloas.len(), SUBSCRIBED_TOPIC_KINDS.len() + 2);
+        assert_eq!(gloas.len(), SUBSCRIBED_TOPIC_KINDS.len() + 4);
         assert_eq!(fulu.len(), SUBSCRIBED_TOPIC_KINDS.len());
+        assert_eq!(
+            GLOAS_TOPIC_KINDS,
+            [
+                "execution_payload",
+                "payload_attestation_message",
+                "execution_payload_bid",
+                "proposer_preferences",
+            ]
+        );
         for kind in GLOAS_TOPIC_KINDS {
             let name = topic_name(MAINNET, kind);
             assert!(gloas.contains(&name), "gloas lacks {name}");
diff --git a/crates/net/p2p/src/beacon/verdict.rs b/crates/net/p2p/src/beacon/verdict.rs
index 7e934d569..0dd83bbe3 100644
--- a/crates/net/p2p/src/beacon/verdict.rs
+++ b/crates/net/p2p/src/beacon/verdict.rs
@@ -15,12 +15,14 @@ use std::panic::{AssertUnwindSafe, catch_unwind};
 use std::time::Instant;
 
 use ethlambda_network_api::{AggregateArrival, BlockAnnouncement, BlockArrival, BlockSource};
+use ethlambda_state_transition::beacon::builder_market::SharedBuilderMarket;
 use ethlambda_state_transition::beacon::gossip::{self, IgnoreReason, Outcome};
 use ethlambda_state_transition::beacon::helpers::accessors::CommitteeCacheExt as _;
 use ethlambda_storage::{CacheKey, Store};
 use ethlambda_types::beacon::containers::electra::{self, SingleAttestation};
 use ethlambda_types::beacon::containers::gloas::{
-    PayloadAttestationMessage, SignedExecutionPayloadEnvelope,
+    PayloadAttestationMessage, SignedExecutionPayloadBid, SignedExecutionPayloadEnvelope,
+    SignedProposerPreferences,
 };
 use ethlambda_types::beacon::containers::{
     DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock,
@@ -32,7 +34,7 @@ use spawned_concurrency::message::Message;
 use spawned_concurrency::tasks::{Context, Handler};
 use tracing::{error, warn};
 
-use crate::beacon::column_checks;
+use crate::beacon::{builder_market, column_checks};
 use crate::{P2PServer, metrics};
 
 /// Which gossip message a verdict is for.
@@ -78,6 +80,14 @@ pub(crate) enum Validated {
     /// A gloas `execution_payload`. Boxed for the reason `Block` is: it
     /// carries a whole execution payload.
     Envelope(Box),
+    /// A gloas `execution_payload_bid`. Carries the market because
+    /// `stateful_checks` receives only the store.
+    ExecutionPayloadBid {
+        bid: Box,
+        market: SharedBuilderMarket,
+    },
+    /// A gloas `proposer_preferences`.
+    ProposerPreferences(Box),
     /// A gloas `payload_attestation_message`.
     PayloadAttestation(PayloadAttestationMessage),
 }
@@ -115,6 +125,12 @@ impl Validated {
                 subnet_id,
             } => gossip::attestation::stateful_checks(store, attestation, *subnet_id),
             Self::Envelope(envelope) => gossip::envelope::stateful_checks(store, envelope),
+            Self::ExecutionPayloadBid { bid, market } => {
+                gossip::execution_payload_bid::stateful_checks(store, market, bid)
+            }
+            Self::ProposerPreferences(preferences) => {
+                gossip::proposer_preferences::stateful_checks(store, preferences)
+            }
             Self::PayloadAttestation(message) => {
                 gossip::payload_attestation::stateful_checks(store, message)
             }
@@ -143,10 +159,26 @@ impl Validated {
             },
             Self::Aggregate { aggregate, .. } => server.seen_aggregates.record(aggregate),
             Self::Attestation { attestation, .. } => server.seen_attestations.record(attestation),
-            Self::Envelope(envelope) => server.seen_envelopes.record(
-                envelope.message.beacon_block_root,
-                envelope.message.builder_index,
-            ),
+            Self::Envelope(envelope) => {
+                let recorded = server.seen_envelopes.record(
+                    envelope.message.beacon_block_root,
+                    envelope.message.builder_index,
+                );
+                if recorded {
+                    // A bid's parent payload is known once its envelope passed
+                    // gossip; the builder market judges bids against this.
+                    server
+                        .builder_market
+                        .record_execution_payload(&envelope.message);
+                }
+                recorded
+            }
+            Self::ExecutionPayloadBid { bid, .. } => {
+                server.builder_market.record_bid((**bid).clone())
+            }
+            Self::ProposerPreferences(preferences) => server
+                .builder_market
+                .record_preferences((**preferences).clone(), builder_market::wall_slot(server)),
             Self::PayloadAttestation(message) => server
                 .seen_payload_attestations
                 .record(message.data.slot, message.validator_index),
@@ -283,6 +315,10 @@ impl Validated {
             Self::Aggregate { .. }
             | Self::Attestation { .. }
             | Self::Envelope(_)
+            // The SSE `execution_payload_bid` and `proposer_preferences` events
+            // hook in here once the events endpoint lands.
+            | Self::ExecutionPayloadBid { .. }
+            | Self::ProposerPreferences(_)
             | Self::PayloadAttestation(_) => {}
         }
     }
@@ -314,7 +350,9 @@ fn record_liveness(server: &P2PServer, object: &Validated) {
         Validated::Block { .. }
         | Validated::Column(_)
         | Validated::Envelope(_)
-        | Validated::PayloadAttestation(_) => {}
+        | Validated::PayloadAttestation(_)
+        | Validated::ExecutionPayloadBid { .. }
+        | Validated::ProposerPreferences(_) => {}
     }
 }
 
@@ -491,6 +529,9 @@ fn permits_for<'a>(
         Validated::Aggregate { .. }
         | Validated::Attestation { .. }
         | Validated::PayloadAttestation(_) => &server.attestation_validation_permits,
+        Validated::ExecutionPayloadBid { .. } | Validated::ProposerPreferences(_) => {
+            &server.builder_validation_permits
+        }
     }
 }
 
@@ -1219,6 +1260,203 @@ mod tests {
         assert!(server.attestation_validation_permits.try_acquire().is_ok());
     }
 
+    fn bid(slot: u64, builder_index: u64, value: u64) -> SignedExecutionPayloadBid {
+        let mut bid = SignedExecutionPayloadBid::default();
+        bid.message.slot = slot;
+        bid.message.builder_index = builder_index;
+        bid.message.value = value;
+        bid
+    }
+
+    fn preferences(proposal_slot: u64, validator: u64) -> SignedProposerPreferences {
+        let mut preferences = SignedProposerPreferences::default();
+        preferences.message.proposal_slot = proposal_slot;
+        preferences.message.validator_index = validator;
+        preferences
+    }
+
+    fn bid_object(server: &P2PServer, bid: SignedExecutionPayloadBid) -> Validated {
+        Validated::ExecutionPayloadBid {
+            bid: Box::new(bid),
+            market: server.builder_market.clone(),
+        }
+    }
+
+    /// Needs the real market (Agent A): its stub never records.
+    #[tokio::test]
+    async fn the_first_accept_for_a_bid_key_stands_and_the_second_is_marked_seen() {
+        let mut server = unconnected_beacon_server(Config::mainnet(), 0).await;
+        let object = bid_object(&server, bid(5, 3, 10));
+
+        assert_eq!(
+            settle(&mut server, Outcome::Accept, &object),
+            Outcome::Accept
+        );
+        assert_eq!(
+            settle(&mut server, Outcome::Accept, &object),
+            Outcome::Ignore(IgnoreReason::AlreadySeen)
+        );
+        // Anything but an accept records nothing. The value must beat the
+        // best one recorded for the same (slot, parent), or the spec's
+        // highest-bid rule would ignore it on its own account.
+        let other = bid_object(&server, bid(5, 4, 11));
+        assert_eq!(
+            settle(
+                &mut server,
+                Outcome::Ignore(IgnoreReason::StateUnavailable),
+                &other
+            ),
+            Outcome::Ignore(IgnoreReason::StateUnavailable)
+        );
+        assert_eq!(
+            settle(&mut server, Outcome::Accept, &other),
+            Outcome::Accept
+        );
+    }
+
+    /// An accepted bid is pooled in the market the API and `produceBlockV4`
+    /// read. Needs the real market (Agent A).
+    #[tokio::test]
+    async fn an_accepted_bid_is_pooled_in_the_shared_market() {
+        let mut server = unconnected_beacon_server(Config::mainnet(), 0).await;
+        let signed = bid(5, 3, 10);
+        let object = bid_object(&server, signed.clone());
+        assert!(!server.builder_market.contains_bid(&signed));
+
+        assert_eq!(
+            settle(&mut server, Outcome::Accept, &object),
+            Outcome::Accept
+        );
+        assert!(server.builder_market.contains_bid(&signed));
+        assert_eq!(
+            server.builder_market.bids_for(
+                5,
+                signed.message.parent_block_root,
+                signed.message.parent_block_hash
+            ),
+            vec![signed]
+        );
+    }
+
+    /// Needs the real market (Agent A).
+    #[tokio::test]
+    async fn the_first_accept_for_a_preferences_key_stands_and_the_second_is_marked_seen() {
+        let mut server = unconnected_beacon_server(Config::mainnet(), 0).await;
+        let object = Validated::ProposerPreferences(Box::new(preferences(40, 5)));
+
+        assert_eq!(
+            settle(&mut server, Outcome::Accept, &object),
+            Outcome::Accept
+        );
+        assert_eq!(
+            settle(&mut server, Outcome::Accept, &object),
+            Outcome::Ignore(IgnoreReason::AlreadySeen)
+        );
+        assert!(server.builder_market.preferences(40, Root::ZERO).is_some());
+    }
+
+    /// Neither builder market type has a consumer on the chain actor, on any
+    /// outcome.
+    #[tokio::test]
+    async fn bids_and_preferences_never_reach_the_chain_actor() {
+        let mut server = unconnected_beacon_server(Config::mainnet(), 0).await;
+        let (sender, mut received) = tokio::sync::mpsc::unbounded_channel();
+        server.blockchain = Some(Arc::new(GloasRecorder(sender)));
+
+        for outcome in [
+            Outcome::Accept,
+            Outcome::Queue(QueueReason::BlockUnknown),
+            Outcome::Ignore(IgnoreReason::Overloaded),
+        ] {
+            bid_object(&server, bid(5, 3, 10)).forward(&server, Instant::now(), outcome);
+            Validated::ProposerPreferences(Box::new(preferences(40, 5))).forward(
+                &server,
+                Instant::now(),
+                outcome,
+            );
+        }
+
+        assert!(received.try_recv().is_err());
+    }
+
+    /// Bids and preferences draw from a pool of their own, so a burst of them
+    /// cannot starve blocks, columns or attestations, and the reverse.
+    #[tokio::test]
+    async fn the_builder_permit_pool_is_independent_of_the_others() {
+        let server = unconnected_beacon_server(Config::mainnet(), 0).await;
+        let bid = bid_object(&server, bid(5, 3, 10));
+        let preferences = Validated::ProposerPreferences(Box::new(preferences(40, 5)));
+        for object in [&bid, &preferences] {
+            assert!(Arc::ptr_eq(
+                permits_for(&server, object),
+                &server.builder_validation_permits
+            ));
+        }
+
+        let gossip_before = server.gossip_validation_permits.available_permits();
+        let attestation_before = server.attestation_validation_permits.available_permits();
+        let mut held = Vec::new();
+        while let Ok(permit) = server
+            .builder_validation_permits
+            .clone()
+            .try_acquire_owned()
+        {
+            held.push(permit);
+        }
+        // Exhausted: a bid's stateful checks would answer `Ignore(Overloaded)`.
+        assert_eq!(server.builder_validation_permits.available_permits(), 0);
+        assert!(
+            permits_for(&server, &bid)
+                .clone()
+                .try_acquire_owned()
+                .is_err()
+        );
+        assert_eq!(
+            server.gossip_validation_permits.available_permits(),
+            gossip_before
+        );
+        assert_eq!(
+            server.attestation_validation_permits.available_permits(),
+            attestation_before
+        );
+    }
+
+    /// An accepted envelope is a known payload for bid validation. Needs the
+    /// real market (Agent A).
+    #[tokio::test]
+    async fn an_accepted_envelope_becomes_a_known_payload() {
+        let mut server = unconnected_beacon_server(Config::mainnet(), 0).await;
+        let signed = envelope(1, 3);
+        let hash = signed.message.payload.block_hash;
+        assert!(server.builder_market.known_payload(hash).is_none());
+        let object = Validated::Envelope(Box::new(signed));
+
+        // A queued envelope has not been judged, so it is not known.
+        settle(
+            &mut server,
+            Outcome::Queue(QueueReason::BlockUnknown),
+            &object,
+        );
+        assert!(server.builder_market.known_payload(hash).is_none());
+
+        settle(&mut server, Outcome::Accept, &object);
+        assert!(server.builder_market.known_payload(hash).is_some());
+    }
+
+    /// The node's own envelope is known too, since gossip never echoes it.
+    /// Needs the real market (Agent A).
+    #[tokio::test]
+    async fn a_published_envelope_becomes_a_known_payload() {
+        let mut server = unconnected_beacon_server(Config::mainnet(), 0).await;
+        let signed = envelope(2, 3);
+        let hash = signed.message.payload.block_hash;
+        assert!(server.builder_market.known_payload(hash).is_none());
+
+        crate::gossipsub::publish_execution_payload_envelope(&mut server, signed, Vec::new()).await;
+
+        assert!(server.builder_market.known_payload(hash).is_some());
+    }
+
     #[test]
     fn only_accept_propagates_and_only_accept_or_queue_reaches_the_chain() {
         assert!(matches!(
diff --git a/crates/net/p2p/src/gossipsub/handler.rs b/crates/net/p2p/src/gossipsub/handler.rs
index 21bba3ef4..b88505104 100644
--- a/crates/net/p2p/src/gossipsub/handler.rs
+++ b/crates/net/p2p/src/gossipsub/handler.rs
@@ -255,6 +255,10 @@ fn handle_beacon_gossip(
         triage_envelope(server, payload)
     } else if kind == beacon_topics::PAYLOAD_ATTESTATION_MESSAGE {
         triage_payload_attestation(server, payload)
+    } else if kind == beacon_topics::EXECUTION_PAYLOAD_BID {
+        crate::beacon::builder_market::triage_execution_payload_bid(server, payload)
+    } else if kind == beacon_topics::PROPOSER_PREFERENCES {
+        crate::beacon::builder_market::triage_proposer_preferences(server, payload)
     } else {
         triage_other(wire, kind, payload)
     };
@@ -834,6 +838,11 @@ pub async fn publish_execution_payload_envelope(
 ) {
     let slot = envelope.message.payload.slot_number;
     let block_root = envelope.message.beacon_block_root;
+    // Gossip never echoes a node's own message, so its own envelope is a
+    // known payload for bid validation only because it is recorded here.
+    server
+        .builder_market
+        .record_execution_payload(&envelope.message);
     let Some(beacon) = server.wire.beacon() else {
         error!(
             slot,
diff --git a/crates/net/p2p/src/gossipsub/mod.rs b/crates/net/p2p/src/gossipsub/mod.rs
index 1701c5f3a..1e2b402fb 100644
--- a/crates/net/p2p/src/gossipsub/mod.rs
+++ b/crates/net/p2p/src/gossipsub/mod.rs
@@ -2,6 +2,7 @@ mod encoding;
 mod handler;
 mod messages;
 
+pub(crate) use encoding::compress_message;
 pub use encoding::decompress_message;
 pub use handler::{
     handle_gossip_message, join_aggregator_subnets, leave_expired_aggregator_subnets,
diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs
index cdeb3e41e..6c44f2c3a 100644
--- a/crates/net/p2p/src/lib.rs
+++ b/crates/net/p2p/src/lib.rs
@@ -45,8 +45,8 @@ use ethlambda_network_api::{
     },
     rpc_to_p2p::{
         PublishBeaconAggregate, PublishBeaconAttestation, PublishBeaconBlock,
-        PublishExecutionPayloadEnvelope, PublishPayloadAttestationMessage,
-        SubscribeAttestationSubnets,
+        PublishExecutionPayloadBid, PublishExecutionPayloadEnvelope,
+        PublishPayloadAttestationMessage, PublishProposerPreferences, SubscribeAttestationSubnets,
     },
 };
 use ethlambda_state_transition::beacon::aggregate::MAX_AGGREGATES_PER_SLOT;
@@ -56,7 +56,8 @@ use ethlambda_state_transition::beacon::gossip::{
     payload_attestation::SeenPayloadAttestations,
 };
 use ethlambda_state_transition::beacon::{
-    attestation_pool::SharedAttestationPool, payload_attestation_pool::SharedPayloadAttestationPool,
+    attestation_pool::SharedAttestationPool, builder_market::SharedBuilderMarket,
+    payload_attestation_pool::SharedPayloadAttestationPool,
 };
 use ethlambda_storage::{Chain, Store};
 use ethlambda_types::beacon::preset::{MAX_VALIDATORS_PER_COMMITTEE, SLOTS_PER_EPOCH};
@@ -244,6 +245,11 @@ const COLUMN_CHECK_PERMITS: usize = 16;
 /// has data from a follower.
 const ATTESTATION_VALIDATION_PERMITS: usize = 128;
 
+/// How many `execution_payload_bid` and `proposer_preferences` stateful checks
+/// may run at once. A pool of its own, so a burst of bids (each costs a BLS
+/// verification) cannot starve blocks, columns or attestations of permits.
+const BUILDER_VALIDATION_PERMITS: usize = 32;
+
 /// Capacity of the first-valid-block cache, keyed by `(slot, proposer)`.
 /// How often to leave aggregator subnets whose slot has passed. One slot's
 /// worth: a subnet outlives its need by at most this, which costs a little
@@ -1098,6 +1104,7 @@ impl P2P {
         discovery: Option,
         attestation_pool: SharedAttestationPool,
         payload_attestation_pool: SharedPayloadAttestationPool,
+        builder_market: SharedBuilderMarket,
     ) -> Result {
         let discovery = match discovery {
             Some(config) => Some(spawn_discovery(config).await?),
@@ -1161,6 +1168,10 @@ impl P2P {
             )),
             attestation_pool,
             payload_attestation_pool,
+            builder_market,
+            builder_validation_permits: Arc::new(tokio::sync::Semaphore::new(
+                BUILDER_VALIDATION_PERMITS,
+            )),
             aggregator_subnets: HashMap::new(),
         };
         let discovery_enabled = server.discovery.is_some();
@@ -1302,6 +1313,15 @@ pub struct P2PServer {
     /// lean never touches it.
     pub(crate) payload_attestation_pool: SharedPayloadAttestationPool,
 
+    /// Bids, proposer preferences and known payloads, shared with the Beacon
+    /// API (which posts bids and preferences, and builds blocks from the pool).
+    /// Gossip's stateful checks read it from blocking threads, so it cannot
+    /// live in the chain actor; lean never touches it.
+    pub(crate) builder_market: SharedBuilderMarket,
+    /// Permits for `execution_payload_bid` and `proposer_preferences` stateful
+    /// checks, see [`BUILDER_VALIDATION_PERMITS`].
+    pub(crate) builder_validation_permits: Arc,
+
     /// The attestation subnets joined for a validator client's aggregators,
     /// each with the last slot it is needed for. Short-lived by design: never
     /// advertised in `attnets`, and left once the slot has passed. The
@@ -1612,6 +1632,18 @@ impl Handler for P2PServer {
     }
 }
 
+impl Handler for P2PServer {
+    async fn handle(&mut self, msg: PublishExecutionPayloadBid, _ctx: &Context) {
+        beacon::builder_market::publish_execution_payload_bid(self, msg.bid);
+    }
+}
+
+impl Handler for P2PServer {
+    async fn handle(&mut self, msg: PublishProposerPreferences, _ctx: &Context) {
+        beacon::builder_market::publish_proposer_preferences(self, msg.preferences);
+    }
+}
+
 impl Handler for P2PServer {
     async fn handle(&mut self, msg: PublishPayloadAttestationMessage, _ctx: &Context) {
         gossipsub::publish_payload_attestation_message(self, msg.message).await;
@@ -2869,6 +2901,10 @@ pub(crate) mod test_support {
             )),
             attestation_pool: Default::default(),
             payload_attestation_pool: Default::default(),
+            builder_market: Default::default(),
+            builder_validation_permits: std::sync::Arc::new(tokio::sync::Semaphore::new(
+                crate::BUILDER_VALIDATION_PERMITS,
+            )),
             aggregator_subnets: HashMap::new(),
         }
     }
diff --git a/crates/net/p2p/src/req_resp/handlers.rs b/crates/net/p2p/src/req_resp/handlers.rs
index 1a3a822e7..18f7c97fb 100644
--- a/crates/net/p2p/src/req_resp/handlers.rs
+++ b/crates/net/p2p/src/req_resp/handlers.rs
@@ -2822,6 +2822,10 @@ pub(crate) mod tests {
             )),
             attestation_pool: Default::default(),
             payload_attestation_pool: Default::default(),
+            builder_market: Default::default(),
+            builder_validation_permits: std::sync::Arc::new(tokio::sync::Semaphore::new(
+                crate::BUILDER_VALIDATION_PERMITS,
+            )),
             aggregator_subnets: HashMap::new(),
         }
     }
diff --git a/crates/net/rpc/src/beacon/bid_selection.rs b/crates/net/rpc/src/beacon/bid_selection.rs
new file mode 100644
index 000000000..82b890c56
--- /dev/null
+++ b/crates/net/rpc/src/beacon/bid_selection.rs
@@ -0,0 +1,230 @@
+//! Choosing between this node's own build and a pooled builder bid for
+//! `produceBlockV4`. Pure: no store, no clock.
+//!
+//! Units differ on the two sides: a bid's value is in Gwei and the execution
+//! client's `blockValue` in Wei, and the specification weights the local value
+//! by 100 against a bid weighted by `builder_boost_factor`. Everything that
+//! compares the two goes through [`choose_payload`], so the conversion lives in
+//! one place.
+
+use ethlambda_types::beacon::containers::gloas::{ExecutionPayloadBid, SignedExecutionPayloadBid};
+use ethlambda_types::beacon::primitives::Uint256;
+
+/// Dividing a wei amount by this yields the amount in Gwei times 100, the
+/// local value's weight: `wei / 1e9 * 100 == wei / 1e7`.
+const WEI_PER_WEIGHTED_GWEI: u128 = 10_000_000;
+
+/// The local build, as the choice sees it.
+pub(crate) struct LocalCandidate {
+    pub(crate) value_wei: u128,
+    /// The execution client's `shouldOverrideBuilder`.
+    pub(crate) should_override_builder: bool,
+}
+
+// The enum is short-lived (one per block production), so boxing the bid buys
+// nothing.
+#[allow(clippy::large_enum_variant)]
+#[derive(Debug, PartialEq, Eq)]
+pub(crate) enum PayloadChoice {
+    Local,
+    Bid(SignedExecutionPayloadBid),
+}
+
+/// `value.saturating_add(execution_payment)`; a p2p bid's payment is zero.
+pub(crate) fn bid_total_gwei(bid: &ExecutionPayloadBid) -> u64 {
+    bid.value.saturating_add(bid.execution_payment)
+}
+
+/// Saturating conversion of a wei amount: the 32 little-endian bytes of a
+/// `uint256` clamped to `u128::MAX`.
+pub(crate) fn wei_u128(value: &Uint256) -> u128 {
+    let (low, high) = value.0.split_at(16);
+    if high.iter().any(|byte| *byte != 0) {
+        return u128::MAX;
+    }
+    u128::from_le_bytes(low.try_into().expect("sixteen bytes"))
+}
+
+/// The bid to build on, or the local build.
+///
+/// 1. The best bid is the first of `bids_desc` (value descending) whose total
+///    is at least `min_bid`.
+/// 2. With no local build, that bid, or `None` if there is none.
+/// 3. A local build that asks to override builders wins.
+/// 4. Otherwise the bid wins iff `builder_boost_factor * bid_gwei` exceeds the
+///    local value in the same weighting: `factor * gwei * 1e9 > 100 * wei`,
+///    which for integers is `factor * gwei > floor(wei / 1e7)`. The left side
+///    is a product of two `u64`s, so it cannot overflow `u128`.
+/// 5. The local build wins a tie, so a factor of `0` prefers it and `u64::MAX`
+///    prefers the bid, each unless step 2 or 3 says otherwise.
+pub(crate) fn choose_payload(
+    local: Option<&LocalCandidate>,
+    bids_desc: &[SignedExecutionPayloadBid],
+    min_bid: u64,
+    builder_boost_factor: u64,
+) -> Option {
+    let best = bids_desc
+        .iter()
+        .find(|signed| bid_total_gwei(&signed.message) >= min_bid);
+    let Some(local) = local else {
+        return best.cloned().map(PayloadChoice::Bid);
+    };
+    let Some(best) = best else {
+        return Some(PayloadChoice::Local);
+    };
+    if local.should_override_builder {
+        return Some(PayloadChoice::Local);
+    }
+    let weighted_bid = u128::from(builder_boost_factor) * u128::from(bid_total_gwei(&best.message));
+    let weighted_local = local.value_wei / WEI_PER_WEIGHTED_GWEI;
+    if weighted_bid > weighted_local {
+        Some(PayloadChoice::Bid(best.clone()))
+    } else {
+        Some(PayloadChoice::Local)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn bid(builder: u64, value: u64) -> SignedExecutionPayloadBid {
+        let mut signed = SignedExecutionPayloadBid::default();
+        signed.message.builder_index = builder;
+        signed.message.value = value;
+        signed
+    }
+
+    fn local(value_wei: u128) -> LocalCandidate {
+        LocalCandidate {
+            value_wei,
+            should_override_builder: false,
+        }
+    }
+
+    fn picks_bid(choice: Option) -> bool {
+        matches!(choice, Some(PayloadChoice::Bid(_)))
+    }
+
+    #[test]
+    fn a_tie_goes_to_the_local_build() {
+        // 1e9 wei is 1 gwei; factor 100 weights both sides to 100.
+        let bids = [bid(1, 1)];
+        assert_eq!(
+            choose_payload(Some(&local(1_000_000_000)), &bids, 0, 100),
+            Some(PayloadChoice::Local)
+        );
+    }
+
+    #[test]
+    fn the_floor_at_ten_million_wei_decides_the_boundary() {
+        let bids = [bid(1, 1)];
+        // One wei short of the tie: the local weighted value floors to 99.
+        assert!(picks_bid(choose_payload(
+            Some(&local(999_999_999)),
+            &bids,
+            0,
+            100
+        )));
+        // Exactly the tie, and above it.
+        for wei in [1_000_000_000, 1_000_000_001, 1_000_000_000_000] {
+            assert_eq!(
+                choose_payload(Some(&local(wei)), &bids, 0, 100),
+                Some(PayloadChoice::Local),
+                "{wei}"
+            );
+        }
+    }
+
+    #[test]
+    fn a_factor_of_zero_prefers_local_and_the_maximum_prefers_the_bid() {
+        let bids = [bid(1, 5)];
+        assert_eq!(
+            choose_payload(Some(&local(1)), &bids, 0, 0),
+            Some(PayloadChoice::Local)
+        );
+        assert!(picks_bid(choose_payload(
+            Some(&local(1_000_000_000_000_000)),
+            &bids,
+            0,
+            u64::MAX
+        )));
+    }
+
+    #[test]
+    fn a_factor_of_zero_still_takes_the_bid_when_the_local_build_failed() {
+        let bids = [bid(1, 5)];
+        assert!(picks_bid(choose_payload(None, &bids, 0, 0)));
+    }
+
+    #[test]
+    fn the_min_bid_floor_skips_bids_below_it() {
+        let bids = [bid(1, 9), bid(2, 5), bid(3, 1)];
+        // The best bid is below the floor, and so are the rest.
+        assert_eq!(
+            choose_payload(Some(&local(0)), &bids, 10, u64::MAX),
+            Some(PayloadChoice::Local)
+        );
+        assert_eq!(choose_payload(None, &bids, 10, u64::MAX), None);
+        // The floor is inclusive.
+        let Some(PayloadChoice::Bid(chosen)) = choose_payload(None, &bids, 9, 0) else {
+            panic!("the bid at the floor is eligible")
+        };
+        assert_eq!(chosen.message.builder_index, 1);
+    }
+
+    #[test]
+    fn the_override_flag_keeps_the_local_build() {
+        let bids = [bid(1, u64::MAX)];
+        let overriding = LocalCandidate {
+            value_wei: 0,
+            should_override_builder: true,
+        };
+        assert_eq!(
+            choose_payload(Some(&overriding), &bids, 0, u64::MAX),
+            Some(PayloadChoice::Local)
+        );
+    }
+
+    #[test]
+    fn no_local_build_and_no_bid_is_nothing() {
+        assert_eq!(choose_payload(None, &[], 0, 100), None);
+        assert_eq!(
+            choose_payload(Some(&local(1)), &[], 0, 100),
+            Some(PayloadChoice::Local)
+        );
+    }
+
+    #[test]
+    fn the_total_counts_execution_payment_and_saturates() {
+        let mut signed = bid(1, u64::MAX);
+        signed.message.execution_payment = 5;
+        assert_eq!(bid_total_gwei(&signed.message), u64::MAX);
+        signed.message.value = 3;
+        assert_eq!(bid_total_gwei(&signed.message), 8);
+    }
+
+    #[test]
+    fn the_weighted_comparison_cannot_overflow() {
+        let bids = [bid(1, u64::MAX)];
+        assert!(picks_bid(choose_payload(
+            Some(&local(u128::MAX)),
+            &bids,
+            0,
+            u64::MAX
+        )));
+    }
+
+    #[test]
+    fn wei_saturates_at_u128() {
+        assert_eq!(wei_u128(&Uint256::from_u128(42)), 42);
+        assert_eq!(wei_u128(&Uint256::from_u128(u128::MAX)), u128::MAX);
+        assert_eq!(wei_u128(&Uint256::MAX), u128::MAX);
+        let mut bytes = [0u8; 32];
+        bytes[16] = 1;
+        assert_eq!(
+            wei_u128(ðlambda_types::beacon::primitives::U256(bytes)),
+            u128::MAX
+        );
+    }
+}
diff --git a/crates/net/rpc/src/beacon/bids.rs b/crates/net/rpc/src/beacon/bids.rs
new file mode 100644
index 000000000..e7da89e78
--- /dev/null
+++ b/crates/net/rpc/src/beacon/bids.rs
@@ -0,0 +1,154 @@
+//! `POST /eth/v1/beacon/execution_payload_bids`: a builder's bid handed to this
+//! node, validated with the gossip rules, pooled in the shared
+//! `BuilderMarket` and gossiped on `execution_payload_bid`.
+//!
+//! The rules are the topic's own (`gossip::execution_payload_bid`), so a bid
+//! that would draw a peer's penalty is refused here instead of being relayed.
+//! A refusal is a 400 whatever the verdict: the specification has no 202 for
+//! "valid but not forwarded", and a bid this node would ignore on gossip is one
+//! it cannot vouch for to the network either.
+
+use axum::{
+    Extension, Router,
+    body::Bytes,
+    extract::State,
+    http::{HeaderMap, StatusCode},
+    response::{IntoResponse, Response},
+    routing::post,
+};
+use ethlambda_network_api::RpcToP2PRef;
+use ethlambda_state_transition::beacon::{
+    builder_market::SharedBuilderMarket,
+    gossip::{
+        IgnoreReason, Outcome,
+        execution_payload_bid::{
+            MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE, cheap_checks, stateful_checks,
+        },
+    },
+};
+use ethlambda_storage::Store;
+use ethlambda_types::beacon::{containers::gloas::SignedExecutionPayloadBid, fork::ForkName};
+use tracing::{debug, warn};
+
+use crate::beacon::{ApiError, BodyEncoding};
+
+pub(crate) fn routes() -> Router {
+    Router::new().route("/eth/v1/beacon/execution_payload_bids", post(post_bid))
+}
+
+/// A 400 whose message names the verdict, in the Beacon API's error shape.
+pub(crate) fn bad_request(message: String) -> Response {
+    let body = serde_json::json!({ "code": 400, "message": message });
+    let mut response = crate::json_response(body);
+    *response.status_mut() = StatusCode::BAD_REQUEST;
+    response
+}
+
+/// `"{outcome}: {reason}"`, the label pair of a verdict.
+pub(crate) fn describe(outcome: &Outcome) -> String {
+    let (outcome, reason) = outcome.labels();
+    format!("{outcome}: {reason}")
+}
+
+pub(crate) fn unix_ms() -> u64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map(|elapsed| elapsed.as_millis() as u64)
+        .unwrap_or(0)
+}
+
+/// `Eth-Consensus-Version` is optional, and must name `allowed` when given.
+pub(crate) fn require_version(headers: &HeaderMap, allowed: &[ForkName]) -> Result<(), ApiError> {
+    let Some(value) = headers.get("eth-consensus-version") else {
+        return Ok(());
+    };
+    match value.to_str().ok().and_then(ForkName::parse) {
+        Some(fork) if allowed.contains(&fork) => Ok(()),
+        _ => Err(ApiError::BadRequest(
+            "Eth-Consensus-Version names a fork this endpoint does not take",
+        )),
+    }
+}
+
+/// `POST /eth/v1/beacon/execution_payload_bids`.
+///
+/// 1. An identical bid already pooled is a success without republishing, so a
+///    builder that retries does not flood the topic.
+/// 2. The cheap rules run inline and the stateful ones (cached states, the
+///    signature) on a blocking thread.
+/// 3. An accepted bid is recorded in the market, which is where block
+///    production reads it and where gossip's own seen rules look, then gossiped.
+async fn post_bid(
+    State(store): State,
+    Extension(p2p): Extension,
+    Extension(market): Extension,
+    headers: HeaderMap,
+    body: Bytes,
+) -> Response {
+    if let Err(err) = require_version(&headers, &[ForkName::Gloas]) {
+        return err.into_response();
+    }
+    let encoding = match BodyEncoding::from_headers(&headers) {
+        Ok(encoding) => encoding,
+        Err(err) => return err.into_response(),
+    };
+    if encoding == BodyEncoding::Ssz && body.len() > MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE {
+        return ApiError::BadRequest("the SignedExecutionPayloadBid exceeds its size bound")
+            .into_response();
+    }
+    let Some(bid) = encoding.decode::(&body) else {
+        return ApiError::BadRequest("the body is not a gloas SignedExecutionPayloadBid")
+            .into_response();
+    };
+
+    if market.contains_bid(&bid) {
+        debug!(
+            slot = bid.message.slot,
+            builder_index = bid.message.builder_index,
+            "Execution payload bid already pooled; not republishing"
+        );
+        return StatusCode::OK.into_response();
+    }
+    let verdict = match cheap_checks(&market, &store, &bid, unix_ms()) {
+        Ok(()) => {
+            let (store, market, bid) = (store.clone(), market.clone(), bid.clone());
+            match tokio::task::spawn_blocking(move || stateful_checks(&store, &market, &bid)).await
+            {
+                Ok(verdict) => verdict,
+                Err(_) => {
+                    return ApiError::Internal("validating the bid failed").into_response();
+                }
+            }
+        }
+        Err(outcome) => outcome,
+    };
+    if verdict != Outcome::Accept {
+        let (outcome, reason) = verdict.labels();
+        warn!(
+            slot = bid.message.slot,
+            builder_index = bid.message.builder_index,
+            outcome,
+            reason,
+            "Refused a submitted execution payload bid"
+        );
+        return bad_request(describe(&verdict));
+    }
+    // The state a stateful check read can have moved on, and another bid can
+    // have taken the key meanwhile: recording re-runs the seen rules under the
+    // market's lock.
+    let slot = bid.message.slot;
+    let builder_index = bid.message.builder_index;
+    if !market.record_bid(bid.clone()) {
+        return bad_request(describe(&Outcome::Ignore(IgnoreReason::AlreadySeen)));
+    }
+    match p2p.publish_execution_payload_bid(bid) {
+        Ok(()) => {
+            debug!(
+                slot,
+                builder_index, "Accepted execution payload bid for gossip"
+            );
+            StatusCode::OK.into_response()
+        }
+        Err(_) => ApiError::Internal("the network actor is not running").into_response(),
+    }
+}
diff --git a/crates/net/rpc/src/beacon/builder_config.rs b/crates/net/rpc/src/beacon/builder_config.rs
new file mode 100644
index 000000000..ebea79e12
--- /dev/null
+++ b/crates/net/rpc/src/beacon/builder_config.rs
@@ -0,0 +1,275 @@
+//! The `BuilderConfig` a validator client sends with `produceBlockV4`.
+//!
+//! Its containers are the Beacon API's own (`types/gloas/builder_entry.yaml`
+//! and `request_auth.yaml`), not consensus containers, so they live with the
+//! API. Each has the SSZ form the specification gives and the JSON form with
+//! quoted integers and hex byte strings.
+//!
+//! Only the top-level `min_bid` and `builder_boost_factor` are used today: they
+//! govern the bids this node sees over p2p. The `builders` entries (bid
+//! requests to a builder's URL) are decoded and left alone.
+
+use axum::http::{HeaderMap, header};
+use ethlambda_types::beacon::primitives::{BlsPubkey, BlsSignature};
+use libssz_derive::{SszDecode, SszEncode};
+use libssz_types::SszList;
+use serde::{Deserialize, Serialize};
+
+use crate::beacon::ApiError;
+
+pub(crate) const MAX_BUILDER_ENTRIES: usize = 64;
+pub(crate) const MAX_BUILDER_URL_SIZE: usize = 2048;
+pub(crate) const MAX_BUILDER_PUBKEYS: usize = 64;
+pub(crate) const MAX_BUILDER_AUTH_DATA_SIZE: usize = 4096;
+
+/// The builder-specs' `BuilderRequestAuth`: opaque authentication bytes and the
+/// slot they authorize.
+#[derive(Debug, Clone, Default, PartialEq, Eq, SszEncode, SszDecode, Serialize, Deserialize)]
+pub(crate) struct BuilderRequestAuth {
+    #[serde(with = "ethlambda_types::beacon::serde_helpers::ssz_hex")]
+    pub(crate) data: SszList,
+    #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")]
+    pub(crate) slot: u64,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq, SszEncode, SszDecode, Serialize, Deserialize)]
+pub(crate) struct SignedBuilderRequestAuth {
+    pub(crate) message: BuilderRequestAuth,
+    pub(crate) signature: BlsSignature,
+}
+
+/// The URL as the SSZ container holds it (UTF-8 bytes) and JSON writes it (a
+/// string).
+mod url_text {
+    use super::{MAX_BUILDER_URL_SIZE, SszList};
+
+    pub fn serialize(
+        value: &SszList,
+        serializer: S,
+    ) -> Result {
+        serializer.serialize_str(&String::from_utf8_lossy(value))
+    }
+
+    pub fn deserialize<'de, D: serde::Deserializer<'de>>(
+        deserializer: D,
+    ) -> Result, D::Error> {
+        let text = ::deserialize(deserializer)?;
+        SszList::try_from(text.into_bytes())
+            .map_err(|_| serde::de::Error::custom("url exceeds MAX_BUILDER_URL_SIZE"))
+    }
+}
+
+/// A per-builder bid request a validator client supplies.
+#[derive(Debug, Clone, Default, PartialEq, Eq, SszEncode, SszDecode, Serialize, Deserialize)]
+pub(crate) struct BuilderEntry {
+    #[serde(with = "url_text")]
+    pub(crate) url: SszList,
+    pub(crate) auth: SignedBuilderRequestAuth,
+    #[serde(with = "ethlambda_types::beacon::serde_helpers::seq")]
+    pub(crate) builder_pubkeys: SszList,
+    #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")]
+    pub(crate) max_execution_payment: u64,
+    #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")]
+    pub(crate) min_bid: u64,
+    #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")]
+    pub(crate) builder_boost_factor: u64,
+}
+
+/// The resolved per-key builder config of one block-production request.
+#[derive(Debug, Clone, Default, PartialEq, Eq, SszEncode, SszDecode, Serialize, Deserialize)]
+pub(crate) struct BuilderConfig {
+    /// Minimum total payment, in Gwei, accepted from a p2p bid.
+    #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")]
+    pub(crate) min_bid: u64,
+    /// Percentage multiplier applied to a p2p bid against the local build.
+    #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")]
+    pub(crate) builder_boost_factor: u64,
+    #[serde(with = "ethlambda_types::beacon::serde_helpers::seq")]
+    pub(crate) builders: SszList,
+}
+
+impl BuilderEntry {
+    /// A non-empty url, non-empty `auth.data` and `auth.message.slot == slot`.
+    /// An unusable entry never fails the request: it yields no bid.
+    pub(crate) fn is_usable_for(&self, slot: u64) -> bool {
+        !self.url.is_empty() && !self.auth.message.data.is_empty() && self.auth.message.slot == slot
+    }
+}
+
+impl BuilderConfig {
+    /// How many entries a builder request could be made for at `slot`.
+    pub(crate) fn usable_entries(&self, slot: u64) -> usize {
+        self.builders
+            .iter()
+            .filter(|entry| entry.is_usable_for(slot))
+            .count()
+    }
+}
+
+/// Decodes the request body as a `BuilderConfig`: SSZ for an
+/// `application/octet-stream` body, JSON otherwise (what a client with no
+/// `Content-Type` sends). A missing or undecodable body is a 400.
+pub(crate) fn decode_builder_config(
+    headers: &HeaderMap,
+    body: &[u8],
+) -> Result {
+    let invalid = || ApiError::BadRequest("the body is not a BuilderConfig");
+    let ssz = headers
+        .get(header::CONTENT_TYPE)
+        .and_then(|value| value.to_str().ok())
+        .is_some_and(|value| value.starts_with(crate::SSZ_CONTENT_TYPE));
+    if ssz {
+        ::from_ssz_bytes(body).map_err(|_| invalid())
+    } else {
+        serde_json::from_slice(body).map_err(|_| invalid())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use libssz::SszEncode as _;
+
+    fn entry(url: &str, data: &[u8], slot: u64) -> BuilderEntry {
+        BuilderEntry {
+            url: SszList::try_from(url.as_bytes().to_vec()).unwrap(),
+            auth: SignedBuilderRequestAuth {
+                message: BuilderRequestAuth {
+                    data: SszList::try_from(data.to_vec()).unwrap(),
+                    slot,
+                },
+                signature: BlsSignature::default(),
+            },
+            builder_pubkeys: vec![BlsPubkey::default()].try_into().unwrap(),
+            max_execution_payment: 7,
+            min_bid: 5,
+            builder_boost_factor: 120,
+        }
+    }
+
+    fn config(entries: Vec) -> BuilderConfig {
+        BuilderConfig {
+            min_bid: 10_000_000,
+            builder_boost_factor: 100,
+            builders: entries.try_into().unwrap(),
+        }
+    }
+
+    fn ssz_headers() -> HeaderMap {
+        let mut headers = HeaderMap::new();
+        headers.insert(
+            header::CONTENT_TYPE,
+            crate::SSZ_CONTENT_TYPE.parse().unwrap(),
+        );
+        headers
+    }
+
+    #[test]
+    fn json_and_ssz_round_trip_with_entries() {
+        let original = config(vec![
+            entry("https://builder.example.com", b"auth", 9),
+            entry("https://other.example.com", b"x", 9),
+        ]);
+        let json = serde_json::to_vec(&original).unwrap();
+        assert_eq!(
+            decode_builder_config(&HeaderMap::new(), &json).unwrap(),
+            original
+        );
+        let ssz = original.to_ssz();
+        assert_eq!(
+            decode_builder_config(&ssz_headers(), &ssz).unwrap(),
+            original
+        );
+    }
+
+    #[test]
+    fn the_json_form_quotes_integers_and_writes_bytes_as_hex() {
+        let json =
+            serde_json::to_value(config(vec![entry("https://b.example", b"\x12\x34", 9)])).unwrap();
+        assert_eq!(json["min_bid"], "10000000");
+        assert_eq!(json["builder_boost_factor"], "100");
+        let builder = &json["builders"][0];
+        assert_eq!(builder["url"], "https://b.example");
+        assert_eq!(builder["auth"]["message"]["data"], "0x1234");
+        assert_eq!(builder["auth"]["message"]["slot"], "9");
+        assert_eq!(builder["max_execution_payment"], "7");
+        assert_eq!(builder["min_bid"], "5");
+        assert_eq!(builder["builder_boost_factor"], "120");
+    }
+
+    #[test]
+    fn an_unusable_entry_still_decodes() {
+        let slot = 9;
+        let unusable = [
+            entry("", b"auth", slot),
+            entry("https://b.example", b"", slot),
+            entry("https://b.example", b"auth", slot + 1),
+        ];
+        for bad in &unusable {
+            assert!(!bad.is_usable_for(slot));
+        }
+        let usable = entry("https://b.example", b"auth", slot);
+        assert!(usable.is_usable_for(slot));
+        let original = config(vec![
+            unusable[0].clone(),
+            unusable[1].clone(),
+            unusable[2].clone(),
+            usable,
+        ]);
+        let json = serde_json::to_vec(&original).unwrap();
+        let decoded = decode_builder_config(&HeaderMap::new(), &json).unwrap();
+        assert_eq!(decoded, original);
+        assert_eq!(decoded.usable_entries(slot), 1);
+        let ssz = decode_builder_config(&ssz_headers(), &original.to_ssz()).unwrap();
+        assert_eq!(ssz, original);
+    }
+
+    #[test]
+    fn an_undecodable_or_oversized_body_is_a_400() {
+        for body in [
+            &b""[..],
+            b"not json",
+            br#"{"min_bid": "1"}"#,
+            br#"{"min_bid": "x", "builder_boost_factor": "1", "builders": []}"#,
+        ] {
+            assert!(matches!(
+                decode_builder_config(&HeaderMap::new(), body),
+                Err(ApiError::BadRequest(_))
+            ));
+        }
+        // SSZ: too short, and an offset that points nowhere.
+        for body in [&[][..], &[0u8; 19][..], &[1u8; 20][..]] {
+            assert!(decode_builder_config(&ssz_headers(), body).is_err());
+        }
+        // More than MAX_BUILDER_ENTRIES entries.
+        let many = serde_json::json!({
+            "min_bid": "0",
+            "builder_boost_factor": "0",
+            "builders": vec![
+                serde_json::to_value(entry("https://b.example", b"a", 1)).unwrap();
+                MAX_BUILDER_ENTRIES + 1
+            ],
+        });
+        assert!(
+            decode_builder_config(&HeaderMap::new(), &serde_json::to_vec(&many).unwrap()).is_err()
+        );
+        // A url above the bound.
+        let long = "a".repeat(MAX_BUILDER_URL_SIZE + 1);
+        let mut json = serde_json::to_value(config(vec![entry("https://b", b"a", 1)])).unwrap();
+        json["builders"][0]["url"] = long.into();
+        assert!(
+            decode_builder_config(&HeaderMap::new(), &serde_json::to_vec(&json).unwrap()).is_err()
+        );
+    }
+
+    #[test]
+    fn the_empty_local_preferred_config_decodes() {
+        let body = br#"{"min_bid":"0","builder_boost_factor":"0","builders":[]}"#;
+        let decoded = decode_builder_config(&HeaderMap::new(), body).unwrap();
+        assert_eq!(decoded, BuilderConfig::default());
+        // Twenty bytes: the two integers and the offset of the empty list.
+        let ssz =
+            decode_builder_config(&ssz_headers(), &BuilderConfig::default().to_ssz()).unwrap();
+        assert_eq!(ssz, decoded);
+    }
+}
diff --git a/crates/net/rpc/src/beacon/builder_market_tests.rs b/crates/net/rpc/src/beacon/builder_market_tests.rs
new file mode 100644
index 000000000..26876e6e8
--- /dev/null
+++ b/crates/net/rpc/src/beacon/builder_market_tests.rs
@@ -0,0 +1,1234 @@
+//! The builder market endpoints and `produceBlockV4`'s bid selection, driven
+//! through the real router with a stand-in execution client.
+//!
+//! Tests that need the gossip rules, `dependent_root_at`, `bid_is_includable`
+//! or `assemble_gloas_block_on_bid` exercise those functions as they are
+//! implemented in `ethlambda-state-transition`; the ones that only check shapes
+//! (headers, status codes, routing, idempotency against a primed market) do not
+//! depend on them.
+
+use std::sync::{Arc, Mutex};
+
+use axum::{
+    Extension, Router,
+    body::Body,
+    http::{HeaderMap, Request, StatusCode},
+};
+use ethlambda_engine::{EngineClient, JwtSecret};
+use ethlambda_network_api::RpcToP2PRef;
+use ethlambda_state_transition::beacon::{
+    attestation_pool::SharedAttestationPool,
+    block_production::advance_to_slot,
+    builder_market::SharedBuilderMarket,
+    gloas_block_production::test_support::{
+        config as chain_config, parent_state, post_state, randao_reveal,
+    },
+    gossip::proposer_preferences::dependent_root_at,
+    helpers::{
+        accessors::get_domain,
+        misc::compute_signing_root,
+        test_state::{secret_key_for, sign_for},
+    },
+    payload_attestation_pool::SharedPayloadAttestationPool,
+};
+use ethlambda_storage::Store;
+use ethlambda_types::{
+    beacon::{
+        constants::{
+            BUILDER_INDEX_SELF_BUILD, DOMAIN_BEACON_BUILDER, DOMAIN_PROPOSER_PREFERENCES,
+            FAR_FUTURE_EPOCH, PAYLOAD_BUILDER_VERSION,
+        },
+        containers::{
+            BeaconState, SignedBeaconBlock,
+            gloas::{
+                BeaconBlock, Builder, ExecutionPayloadBid, ExecutionRequests, ProposerPreferences,
+                SignedExecutionPayloadBid, SignedExecutionPayloadEnvelope,
+                SignedProposerPreferences,
+            },
+        },
+        primitives::{BlsPubkey, BlsSignature, ExecutionAddress, HashTreeRoot as _},
+        signing::compute_epoch_at_slot,
+    },
+    primitives::H256,
+};
+use http_body_util::BodyExt as _;
+use libssz::{SszDecode as _, SszEncode as _};
+use tower::ServiceExt as _;
+
+use super::{Prepared, prepare, routes as produce_routes};
+use crate::{
+    CustodyColumns,
+    beacon::{bids, proposer_preferences, validator::FeeRecipients},
+    test_utils::{RecordingNetwork, beacon_store_with_head_block, gloas_beacon_block},
+};
+
+/// The slot the block is built for. The head block sits one slot earlier, and
+/// the clock is at the head's slot, so this one is the next slot: the one bids
+/// and preferences are accepted for.
+const SLOT: u64 = 33;
+const HEAD_SLOT: u64 = SLOT - 1;
+const GWEI: u64 = 1_000_000_000;
+
+// ---------------------------------------------------------------------------
+// Fake execution client
+// ---------------------------------------------------------------------------
+
+/// A stand-in execution client and what it was asked to build.
+struct FakeEngine {
+    client: EngineClient,
+    /// The payload attributes the last `forkchoiceUpdatedV4` carried.
+    attributes: Arc>>,
+}
+
+/// `forkchoiceUpdated` with attributes answers a payload id, and `getPayloadV6`
+/// a payload extending the requested head with exactly the requested
+/// attributes, worth `block_value_wei`.
+async fn fake_engine(block_value_wei: u64, should_override_builder: bool) -> FakeEngine {
+    use axum::{Json, routing::post};
+
+    let attributes: Arc>> = Arc::default();
+    let recorded = attributes.clone();
+    let handler = move |Json(request): Json| {
+        let recorded = recorded.clone();
+        async move {
+            let result = match request["method"].as_str() {
+                Some("engine_forkchoiceUpdatedV4") => {
+                    let mut attributes = request["params"][1].clone();
+                    attributes["parentHash"] = request["params"][0]["headBlockHash"].clone();
+                    *recorded.lock().unwrap() = Some(attributes);
+                    serde_json::json!({
+                        "payloadStatus": { "status": "VALID", "latestValidHash": null },
+                        "payloadId": "0x0000000000000001",
+                    })
+                }
+                Some("engine_getPayloadV6") => {
+                    let attributes = recorded.lock().unwrap().clone().unwrap();
+                    serde_json::json!({
+                        "executionPayload": {
+                            "parentHash": attributes["parentHash"],
+                            "feeRecipient": attributes["suggestedFeeRecipient"],
+                            "stateRoot": format!("0x{}", "00".repeat(32)),
+                            "receiptsRoot": format!("0x{}", "00".repeat(32)),
+                            "logsBloom": format!("0x{}", "00".repeat(256)),
+                            "prevRandao": attributes["prevRandao"],
+                            "blockNumber": "0x1",
+                            "gasLimit": attributes["targetGasLimit"],
+                            "gasUsed": "0x0",
+                            "timestamp": attributes["timestamp"],
+                            "extraData": "0x",
+                            "baseFeePerGas": "0x7",
+                            "blockHash": format!("0x{}", "ee".repeat(32)),
+                            "transactions": [],
+                            "withdrawals": attributes["withdrawals"],
+                            "blobGasUsed": "0x0",
+                            "excessBlobGas": "0x0",
+                            "blockAccessList": "0xc0",
+                            "slotNumber": attributes["slotNumber"],
+                        },
+                        "blockValue": format!("0x{block_value_wei:x}"),
+                        "blobsBundle": { "commitments": [], "proofs": [], "blobs": [] },
+                        "shouldOverrideBuilder": should_override_builder,
+                        "executionRequests": [],
+                    })
+                }
+                _ => serde_json::Value::Null,
+            };
+            Json(serde_json::json!({ "jsonrpc": "2.0", "id": request["id"], "result": result }))
+        }
+    };
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+    let router = Router::new().route("/", post(handler));
+    tokio::spawn(async move { axum::serve(listener, router).await });
+    FakeEngine {
+        client: EngineClient::new(format!("http://{address}"), JwtSecret::new([0x0f; 32])).unwrap(),
+        attributes,
+    }
+}
+
+/// An execution client nothing listens for, so every build fails.
+fn dead_engine() -> FakeEngine {
+    FakeEngine {
+        client: EngineClient::new("http://127.0.0.1:1".to_string(), JwtSecret::new([0x0f; 32]))
+            .unwrap(),
+        attributes: Arc::default(),
+    }
+}
+
+impl FakeEngine {
+    fn requested(&self) -> serde_json::Value {
+        self.attributes
+            .lock()
+            .unwrap()
+            .clone()
+            .expect("the engine was asked to build")
+    }
+}
+
+// ---------------------------------------------------------------------------
+// The chain
+// ---------------------------------------------------------------------------
+
+fn builder_pubkey() -> BlsPubkey {
+    BlsPubkey(secret_key_for(1000).sk_to_pk().to_bytes())
+}
+
+fn sign_as_builder(root: H256) -> BlsSignature {
+    BlsSignature(
+        secret_key_for(1000)
+            .sign(
+                root.as_slice(),
+                ethlambda_state_transition::beacon::bls::DST,
+                &[],
+            )
+            .to_bytes(),
+    )
+}
+
+/// The block at slot 0 whose root the head state names as the dependent root of
+/// every slot of the first two epochs.
+fn genesis_block() -> SignedBeaconBlock {
+    gloas_beacon_block(0, H256::ZERO, H256::ZERO, H256::repeat_byte(0x0a))
+}
+
+/// A gloas head state at slot 32 whose registry holds one funded, active
+/// builder (index 0, key `secret_key_for(1000)`) and whose block roots name the
+/// genesis block.
+fn chain_state() -> BeaconState {
+    let mut state = parent_state();
+    let genesis_root = genesis_block().message_hash_tree_root();
+    let BeaconState::Gloas(inner) = &mut state else {
+        unreachable!("built as gloas")
+    };
+    // `is_active_builder` wants the deposit epoch below the finalized one.
+    inner.finalized_checkpoint.epoch = 1;
+    inner.builders.push(Builder {
+        pubkey: builder_pubkey(),
+        version: PAYLOAD_BUILDER_VERSION,
+        execution_address: ExecutionAddress::ZERO,
+        balance: 100_000_000_000,
+        deposit_epoch: 0,
+        withdrawable_epoch: FAR_FUTURE_EPOCH,
+    });
+    inner.block_roots[0] = genesis_root;
+    state
+}
+
+struct World {
+    store: Store,
+    state: BeaconState,
+    head_root: H256,
+    market: SharedBuilderMarket,
+    network: Arc,
+    fee_recipients: FeeRecipients,
+}
+
+impl World {
+    /// The head block at slot 32 on a clock where that slot is running.
+    fn new() -> Self {
+        let state = chain_state();
+        let genesis = genesis_block();
+        let genesis_root = genesis.message_hash_tree_root();
+        let block = gloas_beacon_block(
+            state.slot(),
+            genesis_root,
+            H256::repeat_byte(0x01),
+            H256::repeat_byte(0x02),
+        );
+        // The root a block built on this state names as its parent.
+        let mut header = state.latest_block_header().clone();
+        header.state_root = state.hash_tree_root();
+        let head_root = header.hash_tree_root();
+        let mut store = beacon_store_with_head_block(
+            state.clone(),
+            chain_config(),
+            block,
+            head_root,
+            HEAD_SLOT,
+        );
+        store.insert_signed_block(genesis_root, genesis).unwrap();
+        store.insert_state(genesis_root, state.clone()).unwrap();
+        Self {
+            store,
+            state,
+            head_root,
+            market: SharedBuilderMarket::default(),
+            network: Arc::default(),
+            fee_recipients: FeeRecipients::default(),
+        }
+    }
+
+    fn app(&self, engine: Option<&FakeEngine>) -> Router {
+        let p2p: RpcToP2PRef = self.network.clone();
+        produce_routes()
+            .merge(bids::routes())
+            .merge(proposer_preferences::routes())
+            .with_state(self.store.clone())
+            .layer(Extension(p2p))
+            .layer(Extension(engine.map(|engine| engine.client.clone())))
+            .layer(Extension(SharedAttestationPool::default()))
+            .layer(Extension(SharedPayloadAttestationPool::default()))
+            .layer(Extension(self.market.clone()))
+            .layer(Extension(self.fee_recipients.clone()))
+            .layer(Extension(CustodyColumns::default()))
+    }
+
+    fn advanced(&self) -> BeaconState {
+        advance_to_slot(&self.state, SLOT, &chain_config()).unwrap()
+    }
+
+    /// What `produceBlockV4` reads off the chain for [`SLOT`].
+    fn prepared(&self) -> Prepared {
+        prepare(
+            &self.store,
+            &self.market,
+            SLOT,
+            randao_reveal(&self.advanced()),
+        )
+        .unwrap()
+    }
+
+    /// A bid of builder 0 on the parent payload `produceBlockV4` builds on,
+    /// signed under the builder domain.
+    fn bid(&self, value: u64, fee_recipient: ExecutionAddress) -> SignedExecutionPayloadBid {
+        let prepared = self.prepared();
+        let message = ExecutionPayloadBid {
+            parent_block_hash: prepared.inputs.head_block_hash,
+            parent_block_root: prepared.head_root,
+            block_hash: H256::repeat_byte(0xb1),
+            prev_randao: prepared.inputs.prev_randao,
+            fee_recipient,
+            gas_limit: prepared.inputs.target_gas_limit,
+            builder_index: 0,
+            slot: SLOT,
+            value,
+            execution_payment: 0,
+            blob_kzg_commitments: Default::default(),
+            execution_requests_root: ExecutionRequests::default().hash_tree_root(),
+        };
+        let domain = get_domain(&prepared.state, DOMAIN_BEACON_BUILDER, None);
+        let signing_root = compute_signing_root(message.hash_tree_root(), domain);
+        SignedExecutionPayloadBid {
+            message,
+            signature: sign_as_builder(signing_root),
+        }
+    }
+
+    /// The proposer's preferences for [`SLOT`], signed with its key.
+    fn preferences(&self, fee_recipient: ExecutionAddress, gas: u64) -> SignedProposerPreferences {
+        self.preferences_by(self.prepared().proposer, fee_recipient, gas)
+    }
+
+    /// Preferences naming `validator`, signed with that validator's key.
+    fn preferences_by(
+        &self,
+        validator: u64,
+        fee_recipient: ExecutionAddress,
+        gas: u64,
+    ) -> SignedProposerPreferences {
+        let dependent_root = dependent_root_at(&self.state, self.head_root, SLOT)
+            .expect("the head's chain gives the slot a dependent root");
+        let message = ProposerPreferences {
+            dependent_root,
+            proposal_slot: SLOT,
+            validator_index: validator,
+            fee_recipient,
+            target_gas_limit: gas,
+        };
+        let domain = get_domain(
+            &self.state,
+            DOMAIN_PROPOSER_PREFERENCES,
+            Some(compute_epoch_at_slot(SLOT)),
+        );
+        let signing_root = compute_signing_root(message.hash_tree_root(), domain);
+        SignedProposerPreferences {
+            signature: sign_for(validator as usize, signing_root),
+            message,
+        }
+    }
+
+    /// Preferences recorded straight into the market, without the gossip rules.
+    fn prime_preferences(&self, fee_recipient: ExecutionAddress, gas: u64) {
+        let signed = self.preferences(fee_recipient, gas);
+        assert!(self.market.record_preferences(signed, HEAD_SLOT));
+    }
+
+    fn prime_bid(&self, bid: &SignedExecutionPayloadBid) {
+        assert!(self.market.record_bid(bid.clone()));
+    }
+
+    /// The parent payload bids are judged against becomes known to gossip, as
+    /// its envelope arriving would make it.
+    fn reveal_parent_payload(&self) {
+        let prepared = self.prepared();
+        let mut envelope = crate::test_utils::gloas_envelope(self.head_root, HEAD_SLOT);
+        envelope.message.payload.block_hash = prepared.inputs.head_block_hash;
+        envelope.message.payload.gas_limit = 30_000_000;
+        self.market.record_execution_payload(&envelope.message);
+    }
+}
+
+struct Reply {
+    status: StatusCode,
+    headers: HeaderMap,
+    body: Vec,
+}
+
+impl Reply {
+    fn json(&self) -> serde_json::Value {
+        serde_json::from_slice(&self.body).unwrap_or_default()
+    }
+}
+
+async fn send(app: &Router, request: Request) -> Reply {
+    let response = app.clone().oneshot(request).await.unwrap();
+    let status = response.status();
+    let headers = response.headers().clone();
+    let body = response
+        .into_body()
+        .collect()
+        .await
+        .unwrap()
+        .to_bytes()
+        .to_vec();
+    Reply {
+        status,
+        headers,
+        body,
+    }
+}
+
+fn address(byte: u8) -> ExecutionAddress {
+    ExecutionAddress::repeat_byte(byte)
+}
+
+// ---------------------------------------------------------------------------
+// produceBlockV4
+// ---------------------------------------------------------------------------
+
+fn builder_config(min_bid: u64, factor: u64) -> String {
+    format!(r#"{{"min_bid":"{min_bid}","builder_boost_factor":"{factor}","builders":[]}}"#)
+}
+
+fn produce_request(
+    world: &World,
+    include_payload: bool,
+    config: &str,
+    accept_ssz: bool,
+) -> Request {
+    let reveal = randao_reveal(&world.advanced());
+    let mut request = Request::post(format!(
+        "/eth/v4/validator/blocks/{SLOT}?randao_reveal=0x{}&include_payload={include_payload}",
+        hex::encode(reveal.0)
+    ))
+    .header("eth-consensus-version", "gloas")
+    .header("content-type", "application/json");
+    if accept_ssz {
+        request = request.header("accept", "application/octet-stream");
+    }
+    request.body(Body::from(config.to_string())).unwrap()
+}
+
+async fn produce(
+    world: &World,
+    engine: Option<&FakeEngine>,
+    include_payload: bool,
+    config: &str,
+) -> Reply {
+    send(
+        &world.app(engine),
+        produce_request(world, include_payload, config, false),
+    )
+    .await
+}
+
+/// The bid the produced block commits to, from a JSON response either way: a
+/// bare block, or contents carrying it.
+fn produced_block_bid(reply: &Reply) -> ExecutionPayloadBid {
+    let data = &reply.json()["data"];
+    let block = if data.get("block").is_some() {
+        &data["block"]
+    } else {
+        data
+    };
+    serde_json::from_value(block["body"]["signed_execution_payload_bid"]["message"].clone())
+        .expect("a gloas block carries its bid")
+}
+
+fn self_built(reply: &Reply) -> bool {
+    produced_block_bid(reply).builder_index == BUILDER_INDEX_SELF_BUILD
+}
+
+#[tokio::test]
+async fn a_bid_worth_more_than_the_local_payload_is_built_on_and_returned_bare() {
+    let world = World::new();
+    let engine = fake_engine(GWEI, false).await;
+    let bid = world.bid(5, address(0xcc));
+    world.prime_bid(&bid);
+
+    // Factor 100 weights both sides evenly: 5 gwei against 1 gwei.
+    let reply = produce(&world, Some(&engine), true, &builder_config(0, 100)).await;
+
+    assert_eq!(
+        reply.status,
+        StatusCode::OK,
+        "{}",
+        String::from_utf8_lossy(&reply.body)
+    );
+    assert_eq!(produced_block_bid(&reply), bid.message);
+    // `include_payload=true` still comes back bare: the builder reveals.
+    let json = reply.json();
+    assert_eq!(json["execution_payload_included"], false);
+    assert_eq!(json["version"], "gloas");
+    assert!(json["data"].get("execution_payload_envelope").is_none());
+    assert_eq!(reply.headers["eth-execution-payload-included"], "false");
+    assert_eq!(
+        reply.headers["eth-execution-payload-value"],
+        (5 * GWEI).to_string().as_str()
+    );
+    assert_eq!(reply.headers["eth-consensus-block-value"], "0");
+    assert_eq!(reply.headers["eth-consensus-version"], "gloas");
+    assert!(reply.headers.get("eth-builder-url").is_none());
+}
+
+#[tokio::test]
+async fn a_bid_won_block_is_served_as_ssz_too() {
+    let world = World::new();
+    let engine = fake_engine(GWEI, false).await;
+    let bid = world.bid(5, address(0xcc));
+    world.prime_bid(&bid);
+
+    let reply = send(
+        &world.app(Some(&engine)),
+        produce_request(&world, false, &builder_config(0, 100), true),
+    )
+    .await;
+
+    assert_eq!(reply.status, StatusCode::OK);
+    assert_eq!(reply.headers["eth-execution-payload-included"], "false");
+    let block = BeaconBlock::from_ssz_bytes(&reply.body).unwrap();
+    assert_eq!(block.body.signed_execution_payload_bid, bid);
+}
+
+#[tokio::test]
+async fn nothing_is_cached_for_a_bid_block_and_a_self_build_envelope_for_it_is_refused() {
+    let world = World::new();
+    let engine = fake_engine(GWEI, false).await;
+    world.prime_bid(&world.bid(5, address(0xcc)));
+    let app = world.app(Some(&engine));
+    let reply = send(
+        &app,
+        produce_request(&world, false, &builder_config(0, 100), true),
+    )
+    .await;
+    assert_eq!(reply.status, StatusCode::OK);
+    let block = BeaconBlock::from_ssz_bytes(&reply.body).unwrap();
+    let root = block.hash_tree_root();
+
+    // No envelope was cached for the block.
+    let reply = send(
+        &app,
+        Request::get(format!(
+            "/eth/v1/validator/execution_payload_envelopes/{SLOT}/{root:?}"
+        ))
+        .body(Body::empty())
+        .unwrap(),
+    )
+    .await;
+    assert_eq!(reply.status, StatusCode::NOT_FOUND);
+
+    // The block is imported; the proposer's self-build envelope for it names
+    // a different builder than the bid, so it is refused.
+    let mut store = world.store.clone();
+    let post = post_state(&world.advanced(), &block);
+    store
+        .insert_signed_block(
+            root,
+            SignedBeaconBlock::Gloas(
+                ethlambda_types::beacon::containers::gloas::SignedBeaconBlock {
+                    message: block,
+                    signature: Default::default(),
+                },
+            ),
+        )
+        .unwrap();
+    store.insert_state(root, post).unwrap();
+    let mut envelope = crate::test_utils::gloas_envelope(root, SLOT);
+    envelope.message.builder_index = BUILDER_INDEX_SELF_BUILD;
+    let request = Request::post("/eth/v1/beacon/execution_payload_envelopes")
+        .header("eth-consensus-version", "gloas")
+        .header("eth-blob-data-included", "false")
+        .header("content-type", crate::SSZ_CONTENT_TYPE)
+        .body(Body::from(SignedExecutionPayloadEnvelope::to_ssz(
+            &envelope,
+        )))
+        .unwrap();
+    let reply = send(&app, request).await;
+    assert_eq!(reply.status, StatusCode::BAD_REQUEST);
+    assert_eq!(
+        reply.json()["message"],
+        "the envelope does not fulfill the block's bid"
+    );
+    assert!(world.network.envelopes.lock().unwrap().is_empty());
+}
+
+#[tokio::test]
+async fn the_local_payload_wins_ties_a_zero_factor_a_floor_and_an_override() {
+    // (local value in wei, should_override, min_bid in gwei, factor)
+    let cases = [
+        // 5 gwei each side at factor 100: a tie.
+        (5 * GWEI, false, 0, 100),
+        // Factor 0 prefers the local payload.
+        (1, false, 0, 0),
+        // The bid is below the config's floor.
+        (1, false, 6, u64::MAX),
+        // The engine insists on its own payload.
+        (1, true, 0, u64::MAX),
+    ];
+    for (local_wei, should_override, min_bid, factor) in cases {
+        let world = World::new();
+        let engine = fake_engine(local_wei, should_override).await;
+        world.prime_bid(&world.bid(5, address(0xcc)));
+
+        let reply = produce(
+            &world,
+            Some(&engine),
+            true,
+            &builder_config(min_bid, factor),
+        )
+        .await;
+
+        assert_eq!(
+            reply.status,
+            StatusCode::OK,
+            "{local_wei} {min_bid} {factor}"
+        );
+        assert!(
+            self_built(&reply),
+            "{local_wei} {should_override} {min_bid} {factor}"
+        );
+        assert_eq!(reply.json()["execution_payload_included"], true);
+        assert_eq!(reply.headers["eth-execution-payload-included"], "true");
+        assert!(
+            reply.json()["data"]
+                .get("execution_payload_envelope")
+                .is_some()
+        );
+    }
+}
+
+#[tokio::test]
+async fn a_bid_wins_when_the_local_build_is_below_it_by_one_unit_of_weight() {
+    // 1e9 wei is 1 gwei: a tie at factor 100. One wei less and the bid wins.
+    for (local_wei, bid_wins) in [(GWEI, false), (GWEI - 1, true)] {
+        let world = World::new();
+        let engine = fake_engine(local_wei, false).await;
+        world.prime_bid(&world.bid(1, address(0xcc)));
+
+        let reply = produce(&world, Some(&engine), false, &builder_config(0, 100)).await;
+
+        assert_eq!(reply.status, StatusCode::OK);
+        assert_eq!(self_built(&reply), !bid_wins, "{local_wei}");
+    }
+}
+
+#[tokio::test]
+async fn without_an_engine_a_viable_bid_is_built_on_and_none_is_a_503() {
+    let world = World::new();
+    let reply = produce(&world, None, true, &builder_config(0, 0)).await;
+    assert_eq!(reply.status, StatusCode::SERVICE_UNAVAILABLE);
+
+    let bid = world.bid(5, address(0xcc));
+    world.prime_bid(&bid);
+    // Even a factor of 0 takes the bid when there is nothing to prefer.
+    let reply = produce(&world, None, true, &builder_config(0, 0)).await;
+    assert_eq!(reply.status, StatusCode::OK);
+    assert_eq!(produced_block_bid(&reply), bid.message);
+
+    // A bid under the floor is not viable.
+    let reply = produce(&world, None, true, &builder_config(6, 100)).await;
+    assert_eq!(reply.status, StatusCode::SERVICE_UNAVAILABLE);
+}
+
+#[tokio::test]
+async fn a_failed_local_build_falls_back_to_a_viable_bid() {
+    let world = World::new();
+    let engine = dead_engine();
+    let reply = produce(&world, Some(&engine), true, &builder_config(0, 0)).await;
+    assert_eq!(reply.status, StatusCode::SERVICE_UNAVAILABLE);
+
+    let bid = world.bid(5, address(0xcc));
+    world.prime_bid(&bid);
+    let reply = produce(&world, Some(&engine), true, &builder_config(0, 0)).await;
+    assert_eq!(reply.status, StatusCode::OK);
+    assert_eq!(produced_block_bid(&reply), bid.message);
+}
+
+#[tokio::test]
+async fn bids_that_do_not_fit_the_slot_are_left_to_the_local_payload() {
+    // Each would win on value (factor MAX against a 1 wei local build).
+    type Change = fn(&mut ExecutionPayloadBid);
+    let mismatches: [(&str, Change); 3] = [
+        ("randao", |bid| bid.prev_randao = H256::repeat_byte(0x99)),
+        ("parent hash", |bid| {
+            bid.parent_block_hash = H256::repeat_byte(0x98)
+        }),
+        ("parent root", |bid| {
+            bid.parent_block_root = H256::repeat_byte(0x97)
+        }),
+    ];
+    for (name, change) in mismatches {
+        let world = World::new();
+        let engine = fake_engine(1, false).await;
+        let mut bid = world.bid(5, address(0xcc));
+        change(&mut bid.message);
+        // Re-signing is beside the point: the pool is keyed on the fields that
+        // changed, and a bid that does not match never reaches a block.
+        world.market.record_bid(bid);
+
+        let reply = produce(&world, Some(&engine), true, &builder_config(0, u64::MAX)).await;
+
+        assert_eq!(reply.status, StatusCode::OK, "{name}");
+        assert!(self_built(&reply), "{name}");
+    }
+}
+
+#[tokio::test]
+async fn a_bid_paying_someone_other_than_the_preferred_recipient_is_skipped() {
+    let world = World::new();
+    let engine = fake_engine(1, false).await;
+    world.prime_preferences(address(0xaa), 30_000_000);
+    world.prime_bid(&world.bid(5, address(0xcc)));
+
+    let reply = produce(&world, Some(&engine), true, &builder_config(0, u64::MAX)).await;
+
+    assert_eq!(reply.status, StatusCode::OK);
+    assert!(self_built(&reply));
+}
+
+#[tokio::test]
+async fn a_bid_paying_the_preferred_recipient_is_taken() {
+    let world = World::new();
+    let engine = fake_engine(1, false).await;
+    world.prime_preferences(address(0xaa), 30_000_000);
+    let bid = world.bid(5, address(0xaa));
+    world.prime_bid(&bid);
+
+    let reply = produce(&world, Some(&engine), true, &builder_config(0, u64::MAX)).await;
+
+    assert_eq!(reply.status, StatusCode::OK);
+    assert_eq!(produced_block_bid(&reply), bid.message);
+}
+
+#[tokio::test]
+async fn the_self_build_takes_its_fee_recipient_and_gas_target_from_the_preferences() {
+    let world = World::new();
+    let engine = fake_engine(GWEI, false).await;
+    let proposer = world.prepared().proposer;
+    world
+        .fee_recipients
+        .lock()
+        .unwrap()
+        .insert(proposer, address(0xbb));
+    world.prime_preferences(address(0xaa), 31_000_000);
+
+    let reply = produce(&world, Some(&engine), true, &builder_config(0, 0)).await;
+
+    assert_eq!(reply.status, StatusCode::OK);
+    let requested = engine.requested();
+    assert_eq!(
+        requested["suggestedFeeRecipient"],
+        format!("0x{}", "aa".repeat(20))
+    );
+    assert_eq!(
+        requested["targetGasLimit"],
+        format!("0x{:x}", 31_000_000u64)
+    );
+    // The payload the engine built carries them into the block's bid.
+    let bid = produced_block_bid(&reply);
+    assert_eq!(bid.fee_recipient, address(0xaa));
+    assert_eq!(bid.gas_limit, 31_000_000);
+}
+
+#[tokio::test]
+async fn without_preferences_the_self_build_falls_back_to_the_prepared_recipient_and_parent_gas() {
+    let world = World::new();
+    let engine = fake_engine(GWEI, false).await;
+    let proposer = world.prepared().proposer;
+    world
+        .fee_recipients
+        .lock()
+        .unwrap()
+        .insert(proposer, address(0xbb));
+
+    let reply = produce(&world, Some(&engine), true, &builder_config(0, 0)).await;
+
+    assert_eq!(reply.status, StatusCode::OK);
+    let requested = engine.requested();
+    assert_eq!(
+        requested["suggestedFeeRecipient"],
+        format!("0x{}", "bb".repeat(20))
+    );
+    // The parent bid's gas limit, which is what `chain_state` gives it.
+    assert_eq!(
+        requested["targetGasLimit"],
+        format!("0x{:x}", 30_000_000u64)
+    );
+
+    // Neither: the zero address.
+    let world = World::new();
+    let engine = fake_engine(GWEI, false).await;
+    produce(&world, Some(&engine), true, &builder_config(0, 0)).await;
+    assert_eq!(
+        engine.requested()["suggestedFeeRecipient"],
+        format!("0x{}", "00".repeat(20))
+    );
+}
+
+#[tokio::test]
+async fn preferences_of_another_validator_are_not_the_proposers() {
+    let world = World::new();
+    let engine = fake_engine(GWEI, false).await;
+    let proposer = world.prepared().proposer;
+    let other = (proposer + 1) % 64;
+    let stranger = world.preferences_by(other, address(0xaa), 31_000_000);
+    assert!(world.market.record_preferences(stranger, HEAD_SLOT));
+
+    produce(&world, Some(&engine), true, &builder_config(0, 0)).await;
+
+    let requested = engine.requested();
+    assert_eq!(
+        requested["suggestedFeeRecipient"],
+        format!("0x{}", "00".repeat(20))
+    );
+    assert_eq!(
+        requested["targetGasLimit"],
+        format!("0x{:x}", 30_000_000u64)
+    );
+}
+
+#[tokio::test]
+async fn the_existing_error_paths_are_unchanged() {
+    let world = World::new();
+    let app = world.app(None);
+    // A pre-existing client sends the empty local-preferred config.
+    let reply = send(
+        &app,
+        produce_request(&world, true, &builder_config(0, 0), false),
+    )
+    .await;
+    assert_eq!(reply.status, StatusCode::SERVICE_UNAVAILABLE);
+    let reply = send(&app, produce_request(&world, true, "not a config", false)).await;
+    assert_eq!(reply.status, StatusCode::BAD_REQUEST);
+    assert_eq!(reply.json()["message"], "the body is not a BuilderConfig");
+}
+
+// ---------------------------------------------------------------------------
+// POST execution_payload_bids
+// ---------------------------------------------------------------------------
+
+fn bid_request(bid: &SignedExecutionPayloadBid) -> Request {
+    Request::post("/eth/v1/beacon/execution_payload_bids")
+        .header("eth-consensus-version", "gloas")
+        .header("content-type", "application/json")
+        .body(Body::from(serde_json::to_vec(bid).unwrap()))
+        .unwrap()
+}
+
+/// A world where a bid of builder 0 passes every rule: the preferences for the
+/// slot are cached and the parent payload is known.
+fn world_ready_for_bids() -> World {
+    let world = World::new();
+    world.prime_preferences(address(0xaa), 30_000_000);
+    world.reveal_parent_payload();
+    world
+}
+
+#[tokio::test]
+async fn a_valid_bid_is_pooled_and_published_once() {
+    let world = world_ready_for_bids();
+    let app = world.app(None);
+    let bid = world.bid(5, address(0xaa));
+
+    let reply = send(&app, bid_request(&bid)).await;
+
+    assert_eq!(
+        reply.status,
+        StatusCode::OK,
+        "{}",
+        String::from_utf8_lossy(&reply.body)
+    );
+    assert_eq!(*world.network.bids.lock().unwrap(), vec![bid.clone()]);
+    assert!(world.market.contains_bid(&bid));
+
+    // An identical resubmission is a success and is not gossiped again.
+    let reply = send(&app, bid_request(&bid)).await;
+    assert_eq!(reply.status, StatusCode::OK);
+    assert_eq!(world.network.bids.lock().unwrap().len(), 1);
+}
+
+#[tokio::test]
+async fn a_pooled_bid_resubmitted_is_a_200_without_a_publish() {
+    // Needs only the market: the identical bid short-circuits the rules.
+    let world = World::new();
+    let bid = world.bid(5, address(0xaa));
+    world.prime_bid(&bid);
+
+    let reply = send(&world.app(None), bid_request(&bid)).await;
+
+    assert_eq!(reply.status, StatusCode::OK);
+    assert!(world.network.bids.lock().unwrap().is_empty());
+}
+
+#[tokio::test]
+async fn a_bid_is_accepted_as_ssz() {
+    let world = world_ready_for_bids();
+    let bid = world.bid(5, address(0xaa));
+    let request = Request::post("/eth/v1/beacon/execution_payload_bids")
+        .header("eth-consensus-version", "gloas")
+        .header("content-type", crate::SSZ_CONTENT_TYPE)
+        .body(Body::from(bid.to_ssz()))
+        .unwrap();
+
+    let reply = send(&world.app(None), request).await;
+
+    assert_eq!(
+        reply.status,
+        StatusCode::OK,
+        "{}",
+        String::from_utf8_lossy(&reply.body)
+    );
+    assert_eq!(*world.network.bids.lock().unwrap(), vec![bid]);
+}
+
+#[tokio::test]
+async fn a_bid_with_a_bad_signature_is_a_400_naming_the_verdict() {
+    let world = world_ready_for_bids();
+    let mut bid = world.bid(5, address(0xaa));
+    bid.signature = BlsSignature::default();
+
+    let reply = send(&world.app(None), bid_request(&bid)).await;
+
+    assert_eq!(reply.status, StatusCode::BAD_REQUEST);
+    assert_eq!(reply.json()["code"], 400);
+    assert_eq!(reply.json()["message"], "reject: bad_signature");
+    assert!(world.network.bids.lock().unwrap().is_empty());
+    assert!(!world.market.contains_bid(&bid));
+}
+
+#[tokio::test]
+async fn a_bid_for_a_slot_without_preferences_is_a_400() {
+    let world = World::new();
+    world.reveal_parent_payload();
+    let bid = world.bid(5, address(0xaa));
+
+    let reply = send(&world.app(None), bid_request(&bid)).await;
+
+    assert_eq!(reply.status, StatusCode::BAD_REQUEST);
+    assert_eq!(reply.json()["message"], "ignore: preferences_unseen");
+    assert!(world.network.bids.lock().unwrap().is_empty());
+}
+
+#[tokio::test]
+async fn a_lower_bid_after_a_higher_one_is_refused_and_the_verdict_is_a_400() {
+    let world = world_ready_for_bids();
+    let app = world.app(None);
+    let high = world.bid(9, address(0xaa));
+    assert_eq!(send(&app, bid_request(&high)).await.status, StatusCode::OK);
+    let mut low = world.bid(5, address(0xaa));
+    low.message.builder_index = 0;
+
+    let reply = send(&app, bid_request(&low)).await;
+
+    assert_eq!(reply.status, StatusCode::BAD_REQUEST);
+    assert_eq!(world.network.bids.lock().unwrap().len(), 1);
+}
+
+#[tokio::test]
+async fn a_bid_with_a_wrong_header_or_content_type_or_body_is_refused() {
+    let world = world_ready_for_bids();
+    let app = world.app(None);
+    let bid = world.bid(5, address(0xaa));
+    let json = serde_json::to_vec(&bid).unwrap();
+
+    let wrong_fork = Request::post("/eth/v1/beacon/execution_payload_bids")
+        .header("eth-consensus-version", "fulu")
+        .body(Body::from(json.clone()))
+        .unwrap();
+    assert_eq!(send(&app, wrong_fork).await.status, StatusCode::BAD_REQUEST);
+
+    let plain = Request::post("/eth/v1/beacon/execution_payload_bids")
+        .header("content-type", "text/plain")
+        .body(Body::from(json.clone()))
+        .unwrap();
+    assert_eq!(
+        send(&app, plain).await.status,
+        StatusCode::UNSUPPORTED_MEDIA_TYPE
+    );
+
+    let garbage = Request::post("/eth/v1/beacon/execution_payload_bids")
+        .body(Body::from("not a bid"))
+        .unwrap();
+    assert_eq!(send(&app, garbage).await.status, StatusCode::BAD_REQUEST);
+
+    let oversized = Request::post("/eth/v1/beacon/execution_payload_bids")
+        .header("content-type", crate::SSZ_CONTENT_TYPE)
+        .body(Body::from(vec![0u8; 196_933]))
+        .unwrap();
+    assert_eq!(send(&app, oversized).await.status, StatusCode::BAD_REQUEST);
+
+    // No header at all is read leniently.
+    let lenient = Request::post("/eth/v1/beacon/execution_payload_bids")
+        .body(Body::from(json))
+        .unwrap();
+    assert_eq!(send(&app, lenient).await.status, StatusCode::OK);
+}
+
+// ---------------------------------------------------------------------------
+// POST proposer_preferences
+// ---------------------------------------------------------------------------
+
+fn preferences_request(
+    preferences: &[SignedProposerPreferences],
+    version: Option<&str>,
+) -> Request {
+    let mut request = Request::post("/eth/v1/validator/proposer_preferences")
+        .header("content-type", "application/json");
+    if let Some(version) = version {
+        request = request.header("eth-consensus-version", version);
+    }
+    request
+        .body(Body::from(serde_json::to_vec(preferences).unwrap()))
+        .unwrap()
+}
+
+#[tokio::test]
+async fn valid_preferences_are_cached_and_published_once() {
+    let world = World::new();
+    let app = world.app(None);
+    let signed = world.preferences(address(0xaa), 30_000_000);
+
+    let reply = send(
+        &app,
+        preferences_request(std::slice::from_ref(&signed), Some("gloas")),
+    )
+    .await;
+
+    assert_eq!(
+        reply.status,
+        StatusCode::OK,
+        "{}",
+        String::from_utf8_lossy(&reply.body)
+    );
+    assert_eq!(
+        *world.network.proposer_preferences.lock().unwrap(),
+        vec![signed.clone()]
+    );
+    assert_eq!(
+        world
+            .market
+            .preferences(SLOT, signed.message.dependent_root),
+        Some(signed.clone())
+    );
+
+    // The same again is a success without a second publish.
+    let reply = send(&app, preferences_request(&[signed], Some("gloas"))).await;
+    assert_eq!(reply.status, StatusCode::OK);
+    assert_eq!(world.network.proposer_preferences.lock().unwrap().len(), 1);
+}
+
+#[tokio::test]
+async fn cached_preferences_resubmitted_are_a_200_without_a_publish() {
+    // Needs only the market: an identical entry short-circuits the rules.
+    let world = World::new();
+    let signed = world.preferences(address(0xaa), 30_000_000);
+    assert!(world.market.record_preferences(signed.clone(), HEAD_SLOT));
+
+    let reply = send(&world.app(None), preferences_request(&[signed], None)).await;
+
+    assert_eq!(reply.status, StatusCode::OK);
+    assert!(
+        world
+            .network
+            .proposer_preferences
+            .lock()
+            .unwrap()
+            .is_empty()
+    );
+}
+
+#[tokio::test]
+async fn different_preferences_for_a_held_key_fail_with_their_position() {
+    let world = World::new();
+    let first = world.preferences(address(0xaa), 30_000_000);
+    let second = world.preferences(address(0xbb), 30_000_000);
+
+    let reply = send(
+        &world.app(None),
+        preferences_request(&[first.clone(), second], Some("gloas")),
+    )
+    .await;
+
+    assert_eq!(reply.status, StatusCode::BAD_REQUEST);
+    let json = reply.json();
+    assert_eq!(json["code"], 400);
+    assert_eq!(json["failures"].as_array().unwrap().len(), 1);
+    assert_eq!(json["failures"][0]["index"], 1);
+    assert_eq!(json["failures"][0]["message"], "ignore: already_seen");
+    // The first still went out.
+    assert_eq!(
+        *world.network.proposer_preferences.lock().unwrap(),
+        vec![first]
+    );
+}
+
+#[tokio::test]
+async fn preferences_by_the_wrong_proposer_or_signer_fail_and_the_rest_still_go_out() {
+    let world = World::new();
+    let proposer = world.prepared().proposer;
+    let wrong_proposer = world.preferences_by((proposer + 1) % 64, address(0xaa), 30_000_000);
+    let mut bad_signature = world.preferences(address(0xaa), 30_000_000);
+    bad_signature.signature = BlsSignature::default();
+    let good = world.preferences(address(0xcc), 30_000_000);
+
+    let reply = send(
+        &world.app(None),
+        preferences_request(
+            &[wrong_proposer, bad_signature, good.clone()],
+            Some("gloas"),
+        ),
+    )
+    .await;
+
+    assert_eq!(reply.status, StatusCode::BAD_REQUEST);
+    let json = reply.json();
+    let failures: Vec<(u64, String)> = json["failures"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|failure| {
+            (
+                failure["index"].as_u64().unwrap(),
+                failure["message"].as_str().unwrap().to_string(),
+            )
+        })
+        .collect();
+    assert_eq!(
+        failures,
+        vec![
+            (0, "reject: wrong_proposer".to_string()),
+            (1, "reject: bad_signature".to_string()),
+        ]
+    );
+    assert_eq!(
+        *world.network.proposer_preferences.lock().unwrap(),
+        vec![good]
+    );
+}
+
+#[tokio::test]
+async fn preferences_are_accepted_as_an_ssz_list_and_the_header_may_be_fulu() {
+    let world = World::new();
+    let signed = world.preferences(address(0xaa), 30_000_000);
+    let request = Request::post("/eth/v1/validator/proposer_preferences")
+        .header("content-type", crate::SSZ_CONTENT_TYPE)
+        .header("eth-consensus-version", "fulu")
+        .body(Body::from(vec![signed.clone()].to_ssz()))
+        .unwrap();
+
+    let reply = send(&world.app(None), request).await;
+
+    assert_eq!(
+        reply.status,
+        StatusCode::OK,
+        "{}",
+        String::from_utf8_lossy(&reply.body)
+    );
+    assert_eq!(
+        *world.network.proposer_preferences.lock().unwrap(),
+        vec![signed]
+    );
+}
+
+#[tokio::test]
+async fn preferences_with_a_bad_header_type_body_or_count_are_refused() {
+    let world = World::new();
+    let app = world.app(None);
+    let signed = world.preferences(address(0xaa), 30_000_000);
+
+    for version in ["electra", "nonsense"] {
+        let reply = send(
+            &app,
+            preferences_request(std::slice::from_ref(&signed), Some(version)),
+        )
+        .await;
+        assert_eq!(reply.status, StatusCode::BAD_REQUEST, "{version}");
+    }
+    let plain = Request::post("/eth/v1/validator/proposer_preferences")
+        .header("content-type", "text/plain")
+        .body(Body::from("[]"))
+        .unwrap();
+    assert_eq!(
+        send(&app, plain).await.status,
+        StatusCode::UNSUPPORTED_MEDIA_TYPE
+    );
+    let garbage = Request::post("/eth/v1/validator/proposer_preferences")
+        .body(Body::from("not json"))
+        .unwrap();
+    assert_eq!(send(&app, garbage).await.status, StatusCode::BAD_REQUEST);
+
+    // One over the list bound is refused before any entry is looked at.
+    let many =
+        vec![signed; ethlambda_state_transition::beacon::preset::PROPOSER_LOOKAHEAD_LENGTH + 1];
+    let reply = send(&app, preferences_request(&many, Some("gloas"))).await;
+    assert_eq!(reply.status, StatusCode::BAD_REQUEST);
+    assert!(
+        world
+            .network
+            .proposer_preferences
+            .lock()
+            .unwrap()
+            .is_empty()
+    );
+}
+
+// ---------------------------------------------------------------------------
+// Routing
+// ---------------------------------------------------------------------------
+
+/// Every new route answers something other than a 500 through the layer set
+/// the production server applies, and the phase-2 route does not exist.
+#[tokio::test]
+async fn the_new_routes_are_wired_with_the_production_layers() {
+    use ethlambda_blockchain::{SyncStatusController, metrics::SyncStatus};
+
+    let world = World::new();
+    let router =
+        crate::build_beacon_api_router(world.store.clone(), "ethlambda/test", "peer".into())
+            .layer(Extension(SyncStatusController::new(SyncStatus::Synced)))
+            .layer(Extension::(world.network.clone()))
+            .layer(Extension(SharedAttestationPool::default()))
+            .layer(Extension(SharedPayloadAttestationPool::default()))
+            .layer(Extension(world.market.clone()))
+            .layer(Extension(CustodyColumns::default()))
+            .layer(Extension(FeeRecipients::default()))
+            .layer(Extension(None::));
+
+    for (uri, body) in [
+        ("/eth/v1/beacon/execution_payload_bids", "{}"),
+        ("/eth/v1/validator/proposer_preferences", "[]"),
+        ("/eth/v1/beacon/states/head/builders", ""),
+    ] {
+        let reply = send(&router, Request::post(uri).body(Body::from(body)).unwrap()).await;
+        assert_ne!(reply.status, StatusCode::INTERNAL_SERVER_ERROR, "{uri}");
+        assert_ne!(reply.status, StatusCode::NOT_FOUND, "{uri}");
+    }
+    let reply = send(
+        &router,
+        Request::post("/eth/v1/validator/builder_preferences")
+            .body(Body::from("[]"))
+            .unwrap(),
+    )
+    .await;
+    assert_eq!(reply.status, StatusCode::NOT_FOUND);
+}
diff --git a/crates/net/rpc/src/beacon/builders.rs b/crates/net/rpc/src/beacon/builders.rs
new file mode 100644
index 000000000..36a5d8546
--- /dev/null
+++ b/crates/net/rpc/src/beacon/builders.rs
@@ -0,0 +1,336 @@
+//! `POST /eth/v1/beacon/states/{state_id}/builders`: the builder registry of a
+//! gloas state, filtered by id and status.
+//!
+//! The Beacon API defines only the POST form, so a builder list is a request
+//! with a body rather than a query string.
+
+use axum::{
+    Router,
+    body::Bytes,
+    extract::{Path, State},
+    response::{IntoResponse, Response},
+    routing::post,
+};
+use ethlambda_state_transition::beacon::helpers::gloas::is_active_builder;
+use ethlambda_storage::Store;
+use ethlambda_types::beacon::{
+    constants::FAR_FUTURE_EPOCH,
+    containers::{BeaconState, gloas::Builder},
+    primitives::{BlsPubkey, ValidatorIndex},
+};
+use serde::{Deserialize, Serialize};
+
+use crate::beacon::{ApiError, blocks::is_finalized, states::load};
+
+pub(crate) fn routes() -> Router {
+    Router::new().route(
+        "/eth/v1/beacon/states/{state_id}/builders",
+        post(post_builders),
+    )
+}
+
+/// `api.yaml#BuilderStatus`.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum BuilderStatus {
+    Pending,
+    Active,
+    Exited,
+}
+
+impl BuilderStatus {
+    fn name(self) -> &'static str {
+        match self {
+            Self::Pending => "pending",
+            Self::Active => "active",
+            Self::Exited => "exited",
+        }
+    }
+
+    fn parse(text: &str) -> Option {
+        match text {
+            "pending" => Some(Self::Pending),
+            "active" => Some(Self::Active),
+            "exited" => Some(Self::Exited),
+            _ => None,
+        }
+    }
+}
+
+/// A builder id: a registry index or a 48-byte public key.
+enum BuilderId {
+    Index(u64),
+    Pubkey(BlsPubkey),
+}
+
+impl BuilderId {
+    fn parse(text: &str) -> Result {
+        let invalid = || ApiError::BadRequest("invalid builder id");
+        if let Some(digits) = text.strip_prefix("0x") {
+            let bytes: [u8; 48] = hex::decode(digits)
+                .map_err(|_| invalid())?
+                .try_into()
+                .map_err(|_| invalid())?;
+            return Ok(Self::Pubkey(BlsPubkey(bytes)));
+        }
+        text.parse().map(Self::Index).map_err(|_| invalid())
+    }
+
+    fn selects(&self, index: u64, builder: &Builder) -> bool {
+        match self {
+            Self::Index(wanted) => *wanted == index,
+            Self::Pubkey(wanted) => *wanted == builder.pubkey,
+        }
+    }
+}
+
+/// The optional request body. Empty or absent filters select everything.
+#[derive(Debug, Default, Deserialize)]
+struct BuildersRequest {
+    #[serde(default)]
+    ids: Vec,
+    #[serde(default)]
+    statuses: Vec,
+}
+
+#[derive(Debug, Serialize)]
+struct BuilderEntry<'a> {
+    #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")]
+    index: ValidatorIndex,
+    status: &'static str,
+    builder: &'a Builder,
+}
+
+/// `POST /eth/v1/beacon/states/{state_id}/builders`.
+///
+/// An id naming no builder is omitted rather than failing the request. The
+/// answer is in registry order, which the specification leaves unspecified.
+async fn post_builders(
+    Path(state_id): Path,
+    State(store): State,
+    body: Bytes,
+) -> Response {
+    let request = if body.is_empty() {
+        BuildersRequest::default()
+    } else {
+        match serde_json::from_slice(&body) {
+            Ok(request) => request,
+            Err(_) => return ApiError::BadRequest("invalid request body").into_response(),
+        }
+    };
+    let ids = match request
+        .ids
+        .iter()
+        .map(|id| BuilderId::parse(id))
+        .collect::, _>>()
+    {
+        Ok(ids) => ids,
+        Err(err) => return err.into_response(),
+    };
+    let statuses = match request
+        .statuses
+        .iter()
+        .map(|status| BuilderStatus::parse(status).ok_or(ApiError::BadRequest("invalid status")))
+        .collect::, _>>()
+    {
+        Ok(statuses) => statuses,
+        Err(err) => return err.into_response(),
+    };
+    let (root, state) = match load(&store, &state_id) {
+        Ok(found) => found,
+        Err(err) => return err.into_response(),
+    };
+    let BeaconState::Gloas(inner) = state.as_ref() else {
+        return ApiError::BadRequest("the requested state is prior to Gloas").into_response();
+    };
+
+    let mut entries = Vec::new();
+    for (index, builder) in inner.builders.iter().enumerate() {
+        let index = index as u64;
+        if !(ids.is_empty() || ids.iter().any(|id| id.selects(index, builder))) {
+            continue;
+        }
+        let status = if builder.withdrawable_epoch != FAR_FUTURE_EPOCH {
+            BuilderStatus::Exited
+        } else if is_active_builder(inner, index).unwrap_or(false) {
+            BuilderStatus::Active
+        } else {
+            BuilderStatus::Pending
+        };
+        if !(statuses.is_empty() || statuses.contains(&status)) {
+            continue;
+        }
+        entries.push(BuilderEntry {
+            index,
+            status: status.name(),
+            builder,
+        });
+    }
+
+    crate::json_response(serde_json::json!({
+        "execution_optimistic": crate::shared::optimistic::block_is_optimistic(&store, root),
+        "finalized": is_finalized(&store, state.slot()),
+        "data": entries,
+    }))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_utils::beacon_store_with_config;
+    use axum::{
+        body::Body,
+        http::{Request, StatusCode},
+    };
+    use ethlambda_state_transition::beacon::helpers::test_state::with_signing_validators_at;
+    use ethlambda_types::beacon::{config::Config, fork::ForkName};
+    use http_body_util::BodyExt as _;
+    use tower::ServiceExt as _;
+
+    fn builder(seed: u8, deposit_epoch: u64, withdrawable_epoch: u64) -> Builder {
+        Builder {
+            pubkey: BlsPubkey([seed; 48]),
+            version: 3,
+            execution_address: Default::default(),
+            balance: 32_000_000_000 + u64::from(seed),
+            deposit_epoch,
+            withdrawable_epoch,
+        }
+    }
+
+    /// A gloas head state whose registry holds an active builder (0), a
+    /// pending one (1, deposited at the finalized epoch) and an exited one (2).
+    fn gloas_state_with_builders() -> BeaconState {
+        let mut state = with_signing_validators_at(ForkName::Gloas, 64);
+        let BeaconState::Gloas(inner) = &mut state else {
+            unreachable!("built as gloas")
+        };
+        inner.finalized_checkpoint.epoch = 10;
+        for builder in [
+            builder(1, 3, FAR_FUTURE_EPOCH),
+            builder(2, 10, FAR_FUTURE_EPOCH),
+            builder(3, 3, 20),
+        ] {
+            inner.builders.push(builder);
+        }
+        state
+    }
+
+    async fn post_to(store: Store, state_id: &str, body: &str) -> (StatusCode, serde_json::Value) {
+        let request = Request::post(format!("/eth/v1/beacon/states/{state_id}/builders"))
+            .body(Body::from(body.to_string()))
+            .unwrap();
+        let response = routes().with_state(store).oneshot(request).await.unwrap();
+        let status = response.status();
+        let bytes = response.into_body().collect().await.unwrap().to_bytes();
+        (status, serde_json::from_slice(&bytes).unwrap_or_default())
+    }
+
+    fn gloas_store() -> Store {
+        let config = Config::mainnet().with_fork_epoch(ForkName::Gloas, 0);
+        beacon_store_with_config(gloas_state_with_builders(), config).0
+    }
+
+    fn indices(json: &serde_json::Value) -> Vec {
+        json["data"]
+            .as_array()
+            .unwrap()
+            .iter()
+            .map(|entry| entry["index"].as_str().unwrap().to_string())
+            .collect()
+    }
+
+    #[tokio::test]
+    async fn every_builder_is_listed_in_registry_order_with_its_status() {
+        for body in ["", "{}", r#"{"ids":[],"statuses":[]}"#] {
+            let (status, json) = post_to(gloas_store(), "head", body).await;
+            assert_eq!(status, StatusCode::OK, "{body:?}");
+            assert_eq!(indices(&json), ["0", "1", "2"]);
+            let statuses: Vec<_> = json["data"]
+                .as_array()
+                .unwrap()
+                .iter()
+                .map(|entry| entry["status"].as_str().unwrap().to_string())
+                .collect();
+            assert_eq!(statuses, ["active", "pending", "exited"]);
+            assert_eq!(json["execution_optimistic"], false);
+            assert!(json["finalized"].is_boolean());
+        }
+    }
+
+    #[tokio::test]
+    async fn numbers_are_quoted() {
+        let (_, json) = post_to(gloas_store(), "head", "").await;
+        let first = &json["data"][0];
+        assert_eq!(first["index"], "0");
+        assert_eq!(first["builder"]["version"], "3");
+        assert_eq!(first["builder"]["balance"], "32000000001");
+        assert_eq!(first["builder"]["deposit_epoch"], "3");
+        assert_eq!(
+            first["builder"]["withdrawable_epoch"],
+            FAR_FUTURE_EPOCH.to_string()
+        );
+        assert!(
+            first["builder"]["pubkey"]
+                .as_str()
+                .unwrap()
+                .starts_with("0x")
+        );
+    }
+
+    #[tokio::test]
+    async fn builders_are_selected_by_index_or_public_key_and_unknown_ids_are_omitted() {
+        let pubkey = format!("0x{}", "03".repeat(48));
+        let body = format!(r#"{{"ids":["1","{pubkey}","99","0x{}"]}}"#, "ff".repeat(48));
+        let (status, json) = post_to(gloas_store(), "head", &body).await;
+        assert_eq!(status, StatusCode::OK);
+        assert_eq!(indices(&json), ["1", "2"]);
+    }
+
+    #[tokio::test]
+    async fn the_status_filter_narrows_the_list() {
+        let (_, json) = post_to(gloas_store(), "head", r#"{"statuses":["active"]}"#).await;
+        assert_eq!(indices(&json), ["0"]);
+        let (_, json) = post_to(
+            gloas_store(),
+            "head",
+            r#"{"statuses":["pending","exited"]}"#,
+        )
+        .await;
+        assert_eq!(indices(&json), ["1", "2"]);
+        let (_, json) = post_to(
+            gloas_store(),
+            "head",
+            r#"{"ids":["0","1"],"statuses":["exited"]}"#,
+        )
+        .await;
+        assert!(indices(&json).is_empty());
+    }
+
+    #[tokio::test]
+    async fn a_bad_body_id_or_status_is_a_400() {
+        for body in [
+            "not json",
+            r#"{"ids":["zero"]}"#,
+            r#"{"ids":["0x1234"]}"#,
+            r#"{"statuses":["retired"]}"#,
+            r#"{"ids":"0"}"#,
+        ] {
+            let (status, _) = post_to(gloas_store(), "head", body).await;
+            assert_eq!(status, StatusCode::BAD_REQUEST, "{body}");
+        }
+    }
+
+    #[tokio::test]
+    async fn a_pre_gloas_state_is_a_400_and_an_unknown_one_a_404() {
+        let state = with_signing_validators_at(ForkName::Fulu, 64);
+        let (store, _) = beacon_store_with_config(state, Config::mainnet());
+        let (status, json) = post_to(store, "head", "").await;
+        assert_eq!(status, StatusCode::BAD_REQUEST);
+        assert_eq!(json["message"], "the requested state is prior to Gloas");
+
+        let (status, _) = post_to(gloas_store(), "999999", "").await;
+        assert_eq!(status, StatusCode::NOT_FOUND);
+        let (status, _) = post_to(gloas_store(), "nonsense", "").await;
+        assert_eq!(status, StatusCode::BAD_REQUEST);
+    }
+}
diff --git a/crates/net/rpc/src/beacon/gloas_proposal.rs b/crates/net/rpc/src/beacon/gloas_proposal.rs
index 12a3a507d..59d5b7605 100644
--- a/crates/net/rpc/src/beacon/gloas_proposal.rs
+++ b/crates/net/rpc/src/beacon/gloas_proposal.rs
@@ -15,9 +15,15 @@
 //! serves the envelope back for a validator client that asked for the block
 //! with `include_payload=false`.
 //!
-//! This node never takes a builder's bid: the `BuilderConfig` request body is
-//! decoded, as the specification requires of a body that cannot be, and
-//! otherwise ignored.
+//! The node always builds a local payload when it has an execution client, and
+//! may instead build on a bid it saw over p2p (`bid_selection` decides, from the
+//! request's `BuilderConfig`). A bid-won block comes back bare: the builder
+//! reveals the payload, so nothing is cached here and this node never signs or
+//! publishes an envelope for it. The `builders` entries of the config (bid
+//! requests to a builder's URL) are decoded and not consulted.
+//!
+//! A self-built payload's fee recipient and gas target come from the signed
+//! proposer preferences for the slot when this node holds them.
 //!
 //! What `produceBlockV4` builds is kept in a small cache, keyed by slot and
 //! block root and holding the current and the previous slot only, so the
@@ -48,15 +54,18 @@ use ethlambda_state_transition::beacon::{
     attestation_pool::SharedAttestationPool,
     block_production::advance_to_slot,
     bls,
+    builder_market::SharedBuilderMarket,
     fork_choice::{
         get_head_node, gloas_verify_data_column_sidecar,
         gloas_verify_data_column_sidecar_kzg_proofs, should_build_on_full,
     },
     gloas_block_production::{
-        GloasBlockInputs, GloasPayloadInputs, GloasProduced, assemble_gloas_block,
+        GloasBidBlockInputs, GloasBlockInputs, GloasPayloadInputs, GloasProduced,
+        assemble_gloas_block, assemble_gloas_block_on_bid, bid_is_includable,
         gloas_data_column_sidecars, gloas_payload_inputs, pack_gloas_attestations,
         pack_payload_attestations, parse_gloas_execution_requests,
     },
+    gossip::proposer_preferences::dependent_root_at,
     helpers::{
         accessors::{get_beacon_proposer_index, get_domain},
         misc::compute_signing_root,
@@ -74,7 +83,8 @@ use ethlambda_types::{
             deneb::Blob,
             gloas::{
                 BeaconBlock, ExecutionPayloadEnvelope, ExecutionRequests,
-                SignedExecutionPayloadEnvelope,
+                SignedExecutionPayloadBid, SignedExecutionPayloadEnvelope,
+                SignedProposerPreferences,
             },
         },
         fork::ForkName,
@@ -91,6 +101,8 @@ use tracing::{debug, info, warn};
 
 use crate::beacon::{
     ApiError, BodyEncoding,
+    bid_selection::{LocalCandidate, PayloadChoice, bid_total_gwei, choose_payload, wei_u128},
+    builder_config::{BuilderConfig, decode_builder_config},
     proposal::{Blobs, CellKzgProofs, decimal, require_gloas_slot},
     validator::{FeeRecipients, head},
 };
@@ -175,13 +187,6 @@ pub(crate) fn routes() -> Router {
         .layer(Extension(PayloadCache::default()))
 }
 
-fn is_ssz(headers: &HeaderMap) -> bool {
-    headers
-        .get(header::CONTENT_TYPE)
-        .and_then(|value| value.to_str().ok())
-        .is_some_and(|value| value.starts_with(crate::SSZ_CONTENT_TYPE))
-}
-
 fn consensus_version(headers: &HeaderMap) -> Option {
     headers
         .get("eth-consensus-version")
@@ -189,41 +194,6 @@ fn consensus_version(headers: &HeaderMap) -> Option {
         .and_then(ForkName::parse)
 }
 
-/// `BuilderConfig` as it arrives as JSON. Only decoded: this node takes no
-/// builder bids, so nothing past the shape is read.
-#[derive(Debug, Deserialize)]
-struct BuilderConfigJson {
-    #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")]
-    #[allow(dead_code)]
-    min_bid: u64,
-    #[serde(with = "ethlambda_types::beacon::serde_helpers::quoted_or_bare")]
-    #[allow(dead_code)]
-    builder_boost_factor: u64,
-    builders: Vec,
-}
-
-/// Decodes the request body as a `BuilderConfig`, returning how many builder
-/// entries it names. A body that does not decode is invalid per the
-/// specification.
-///
-/// The SSZ form is `min_bid`, `builder_boost_factor` and the offset of the
-/// `builders` list, so a decodable body is at least that long and the offset
-/// points just past those three fields; the entries themselves are not parsed.
-fn decode_builder_config(headers: &HeaderMap, body: &[u8]) -> Result {
-    if is_ssz(headers) {
-        let offset = body
-            .get(16..20)
-            .map(|bytes| u32::from_le_bytes(bytes.try_into().expect("four bytes")));
-        return match offset {
-            Some(20) => Ok(usize::from(body.len() > 20)),
-            _ => Err(ApiError::BadRequest("the body is not a BuilderConfig")),
-        };
-    }
-    serde_json::from_slice::(body)
-        .map(|config| config.builders.len())
-        .map_err(|_| ApiError::BadRequest("the body is not a BuilderConfig"))
-}
-
 #[derive(Debug, Deserialize)]
 struct ProduceQuery {
     randao_reveal: BlsSignature,
@@ -246,6 +216,7 @@ async fn post_produce_block(
     Extension(engine): Extension>,
     Extension(pool): Extension,
     Extension(ptc_pool): Extension,
+    Extension(market): Extension,
     Extension(fee_recipients): Extension,
     Extension(custody): Extension,
     Extension(cache): Extension,
@@ -273,24 +244,29 @@ async fn post_produce_block(
     ) {
         return err.into_response();
     }
-    let builders = match decode_builder_config(&headers, &body) {
-        Ok(builders) => builders,
+    let builder_config = match decode_builder_config(&headers, &body) {
+        Ok(config) => config,
         Err(err) => return err.into_response(),
     };
+    let builders = builder_config.usable_entries(slot);
     if builders > 0 {
-        debug!(%slot, builders, "Ignoring the builders of the block production request");
+        debug!(%slot, builders, "Not consulting the builder entries of the block production request");
     }
-    let Some(engine) = engine else {
+    // Without an execution client a block can only be built on a bid, so a node
+    // holding none for the slot has nothing to offer.
+    if engine.is_none() && !market.has_bids_for_slot(slot) {
         return ApiError::ServiceUnavailable(
             "no execution client configured to build a payload with",
         )
         .into_response();
-    };
+    }
 
     let graffiti = query.graffiti.unwrap_or(Bytes32::ZERO);
     let produced = produce(
         &store,
-        &engine,
+        engine.as_ref(),
+        &market,
+        &builder_config,
         &pool,
         &ptc_pool,
         &fee_recipients,
@@ -304,49 +280,59 @@ async fn post_produce_block(
         Ok(produced) => produced,
         Err(err) => return err.into_response(),
     };
-    let Produced {
-        built,
-        block,
-        envelope,
-        payload_value,
-    } = produced;
-
-    let block_root = block.hash_tree_root();
-    cache.insert(
-        slot,
-        block_root,
-        CachedPayload {
-            envelope: envelope.clone(),
-            blobs: built.blobs_bundle.blobs.clone(),
-            cell_proofs: built.blobs_bundle.proofs.clone(),
-        },
-    );
-
     let accept = headers.get(header::ACCEPT).and_then(|v| v.to_str().ok());
     let encoding = Encoding::from_accept(accept);
-    let included = query.include_payload;
-    let mut response = if included {
-        let (Ok(kzg_proofs), Ok(blobs)) = (
-            CellKzgProofs::try_from(built.blobs_bundle.proofs),
-            blobs_list(built.blobs_bundle.blobs),
-        ) else {
-            return ApiError::Internal("the blobs bundle exceeds the block's bounds")
-                .into_response();
-        };
-        let contents = GloasBlockContents {
-            block,
-            execution_payload_envelope: envelope,
-            kzg_proofs,
-            blobs,
-        };
-        match encoding {
-            Encoding::Ssz => ssz_response(contents.to_ssz()),
-            Encoding::Json => block_json(&payload_value, true, &contents),
+    let (mut response, included, payload_value) = match produced {
+        Production::Bid { block, bid } => {
+            let payload_value = bid_payload_value(&bid);
+            let response = match encoding {
+                Encoding::Ssz => ssz_response(block.to_ssz()),
+                Encoding::Json => block_json(&payload_value, false, &block),
+            };
+            (response, false, payload_value)
         }
-    } else {
-        match encoding {
-            Encoding::Ssz => ssz_response(block.to_ssz()),
-            Encoding::Json => block_json(&payload_value, false, &block),
+        Production::Local(Produced {
+            built,
+            block,
+            envelope,
+            payload_value,
+        }) => {
+            let block_root = block.hash_tree_root();
+            cache.insert(
+                slot,
+                block_root,
+                CachedPayload {
+                    envelope: envelope.clone(),
+                    blobs: built.blobs_bundle.blobs.clone(),
+                    cell_proofs: built.blobs_bundle.proofs.clone(),
+                },
+            );
+            let included = query.include_payload;
+            let response = if included {
+                let (Ok(kzg_proofs), Ok(blobs)) = (
+                    CellKzgProofs::try_from(built.blobs_bundle.proofs),
+                    blobs_list(built.blobs_bundle.blobs),
+                ) else {
+                    return ApiError::Internal("the blobs bundle exceeds the block's bounds")
+                        .into_response();
+                };
+                let contents = GloasBlockContents {
+                    block,
+                    execution_payload_envelope: envelope,
+                    kzg_proofs,
+                    blobs,
+                };
+                match encoding {
+                    Encoding::Ssz => ssz_response(contents.to_ssz()),
+                    Encoding::Json => block_json(&payload_value, true, &contents),
+                }
+            } else {
+                match encoding {
+                    Encoding::Ssz => ssz_response(block.to_ssz()),
+                    Encoding::Json => block_json(&payload_value, false, &block),
+                }
+            };
+            (response, included, payload_value)
         }
     };
     let response_headers = response.headers_mut();
@@ -357,12 +343,17 @@ async fn post_produce_block(
     if let Ok(value) = HeaderValue::from_str(&payload_value) {
         response_headers.insert("eth-execution-payload-value", value);
     }
-    // Not computed: nothing here reads it, and the builder comparison it
-    // exists for does not happen on this node.
+    // Not computed: nothing here reads it, and the bid comparison compares
+    // execution payload values only.
     response_headers.insert("eth-consensus-block-value", HeaderValue::from_static("0"));
     with_consensus_version(response, ForkName::Gloas)
 }
 
+/// A bid's total payment in Wei, the unit `Eth-Execution-Payload-Value` is in.
+fn bid_payload_value(bid: &SignedExecutionPayloadBid) -> String {
+    (u128::from(bid_total_gwei(&bid.message)) * 1_000_000_000).to_string()
+}
+
 fn block_json(payload_value: &str, included: bool, data: &T) -> Response {
     crate::json_response(serde_json::json!({
         "version": ForkName::Gloas.as_str(),
@@ -381,7 +372,21 @@ fn blobs_list(blobs: Vec>) -> Result {
     Blobs::try_from(blobs).map_err(|_| ())
 }
 
-/// Everything `produceBlockV4` built.
+/// What `produceBlockV4` built for one block.
+// Short-lived (one per request), so boxing the larger variant buys nothing.
+#[allow(clippy::large_enum_variant)]
+enum Production {
+    /// A block on this node's own payload, with the envelope that reveals it.
+    Local(Produced),
+    /// A block committing to a builder's bid. The builder reveals the payload,
+    /// so there is no envelope and nothing to cache.
+    Bid {
+        block: BeaconBlock,
+        bid: SignedExecutionPayloadBid,
+    },
+}
+
+/// A self-built block.
 struct Produced {
     built: BuiltGloasPayload,
     block: BeaconBlock,
@@ -389,6 +394,12 @@ struct Produced {
     payload_value: String,
 }
 
+/// This node's own payload for the slot, checked and ready to assemble.
+struct LocalBuild {
+    built: BuiltGloasPayload,
+    execution_requests: ExecutionRequests,
+}
+
 /// What the build needs from the chain, read and advanced off the runtime.
 struct Prepared {
     state: BeaconState,
@@ -399,12 +410,17 @@ struct Prepared {
     /// The parent envelope's requests when building on its full payload and
     /// the parent is gloas; empty otherwise.
     parent_requests: ExecutionRequests,
+    /// The proposer's signed preferences for the slot, when this node holds
+    /// them under the slot's dependent root.
+    preferences: Option,
 }
 
 #[allow(clippy::too_many_arguments)]
 async fn produce(
     store: &Store,
-    engine: &EngineClient,
+    engine: Option<&EngineClient>,
+    market: &SharedBuilderMarket,
+    builder_config: &BuilderConfig,
     pool: &SharedAttestationPool,
     ptc_pool: &SharedPayloadAttestationPool,
     fee_recipients: &FeeRecipients,
@@ -412,27 +428,39 @@ async fn produce(
     slot: Slot,
     randao_reveal: BlsSignature,
     graffiti: Bytes32,
-) -> Result {
+) -> Result {
     let prepare_store = store.clone();
-    let prepared =
-        tokio::task::spawn_blocking(move || prepare(&prepare_store, slot, randao_reveal))
-            .await
-            .map_err(|_| ApiError::Internal("preparing the block failed"))??;
-
-    let built = build_payload(store, engine, fee_recipients, custody, &prepared).await?;
-    let bundle = &built.blobs_bundle;
-    if bundle.commitments.len() != bundle.blobs.len()
-        || bundle.proofs.len()
-            != bundle.blobs.len() * ethlambda_types::beacon::preset::CELLS_PER_EXT_BLOB
-    {
-        warn!(%slot, "The execution client's blobs bundle is inconsistent");
-        return Err(ApiError::ServiceUnavailable(
-            "the execution client returned an inconsistent blobs bundle",
-        ));
-    }
-    let execution_requests = parse_gloas_execution_requests(&built.execution_requests)
-        .map_err(|_| ApiError::Internal("the execution client's request list is malformed"))?;
-    let payload_value = decimal(&built.block_value);
+    let prepare_market = market.clone();
+    let prepared = tokio::task::spawn_blocking(move || {
+        prepare(&prepare_store, &prepare_market, slot, randao_reveal)
+    })
+    .await
+    .map_err(|_| ApiError::Internal("preparing the block failed"))??;
+
+    // The local build first: bids keep arriving while the engine works, and
+    // are read once it is done.
+    let local = match engine {
+        Some(engine) => {
+            Some(build_local(store, engine, fee_recipients, custody, &prepared, slot).await)
+        }
+        None => None,
+    };
+    let (local, local_error) = match local {
+        Some(Ok(local)) => (Some(local), None),
+        Some(Err(err)) => (None, Some(err)),
+        None => (None, None),
+    };
+    let bids = candidate_bids(market, &prepared);
+    let candidate = local.as_ref().map(|local| LocalCandidate {
+        value_wei: wei_u128(&local.built.block_value),
+        should_override_builder: local.built.should_override_builder,
+    });
+    let choice = choose_payload(
+        candidate.as_ref(),
+        &bids,
+        builder_config.min_bid,
+        builder_config.builder_boost_factor,
+    );
 
     let candidates = pool
         .lock()
@@ -443,12 +471,64 @@ async fn produce(
         .expect("payload attestation pool lock poisoned")
         .messages_for(slot.saturating_sub(1), prepared.head_root);
     let config = store.config();
+    let prepared = Arc::new(prepared);
+
+    if let Some(PayloadChoice::Bid(bid)) = choice {
+        let builder_index = bid.message.builder_index;
+        let value_gwei = bid_total_gwei(&bid.message);
+        let assembled = {
+            let (config, prepared, bid) = (config.clone(), prepared.clone(), bid.clone());
+            let (candidates, messages) = (candidates.clone(), messages.clone());
+            tokio::task::spawn_blocking(move || {
+                assemble_on_bid(
+                    &config,
+                    &prepared,
+                    bid,
+                    candidates,
+                    messages,
+                    randao_reveal,
+                    graffiti,
+                )
+            })
+            .await
+        };
+        match assembled {
+            Ok(Ok(block)) => {
+                info!(
+                    %slot,
+                    builder_index,
+                    value_gwei,
+                    "Produced gloas block on a builder bid"
+                );
+                return Ok(Production::Bid { block, bid });
+            }
+            Ok(Err(_)) | Err(_) if local.is_some() => {
+                warn!(%slot, builder_index, "The winning bid failed to build; using the local payload");
+            }
+            Ok(Err(_)) | Err(_) => {
+                return Err(ApiError::ServiceUnavailable(
+                    "the winning bid failed to build",
+                ));
+            }
+        }
+    }
+
+    let Some(local) = local else {
+        return Err(local_error.unwrap_or(ApiError::ServiceUnavailable(
+            "no execution client and no viable builder bid for the slot",
+        )));
+    };
+    let LocalBuild {
+        built,
+        execution_requests,
+    } = local;
+    let payload_value = decimal(&built.block_value);
     let assembled = {
         let built = built.clone();
         tokio::task::spawn_blocking(move || {
             assemble(
                 &config,
-                prepared,
+                &prepared,
                 built,
                 execution_requests,
                 candidates,
@@ -469,17 +549,51 @@ async fn produce(
         blobs = built.blobs_bundle.blobs.len(),
         "Produced gloas block"
     );
-    Ok(Produced {
+    Ok(Production::Local(Produced {
         built,
         block,
         envelope,
         payload_value,
-    })
+    }))
+}
+
+/// The pooled bids this node could build on at the prepared slot: on the same
+/// parent payload the local build extends, with its `prev_randao`, packable
+/// into a block on the advanced state, and, when the proposer's preferences are
+/// held, paying the fee recipient they name. Value descending.
+///
+/// Bids on the head's parent (a proposer reorg) are not considered.
+fn candidate_bids(
+    market: &SharedBuilderMarket,
+    prepared: &Prepared,
+) -> Vec {
+    market
+        .bids_for(
+            prepared.state.slot(),
+            prepared.head_root,
+            prepared.inputs.head_block_hash,
+        )
+        .into_iter()
+        .filter(|signed| {
+            let bid = &signed.message;
+            bid.prev_randao == prepared.inputs.prev_randao
+                && prepared
+                    .preferences
+                    .as_ref()
+                    .is_none_or(|prefs| prefs.message.fee_recipient == bid.fee_recipient)
+                && bid_is_includable(&prepared.state, signed, &prepared.parent_requests)
+        })
+        .collect()
 }
 
 /// The chain-side half of production: pick the parent payload branch, advance
 /// the head state to `slot` and derive the payload inputs from it.
-fn prepare(store: &Store, slot: Slot, randao_reveal: BlsSignature) -> Result {
+fn prepare(
+    store: &Store,
+    market: &SharedBuilderMarket,
+    slot: Slot,
+    randao_reveal: BlsSignature,
+) -> Result {
     let config = store.config();
     let (head_root, head_state) = head(store)?;
     let head_slot = head_state.slot();
@@ -527,8 +641,18 @@ fn prepare(store: &Store, slot: Slot, randao_reveal: BlsSignature) -> Result
 Result
 Result {
+    let built = build_payload(store, engine, fee_recipients, custody, prepared).await?;
+    let bundle = &built.blobs_bundle;
+    if bundle.commitments.len() != bundle.blobs.len()
+        || bundle.proofs.len()
+            != bundle.blobs.len() * ethlambda_types::beacon::preset::CELLS_PER_EXT_BLOB
+    {
+        warn!(%slot, "The execution client's blobs bundle is inconsistent");
+        return Err(ApiError::ServiceUnavailable(
+            "the execution client returned an inconsistent blobs bundle",
+        ));
+    }
+    let execution_requests = parse_gloas_execution_requests(&built.execution_requests)
+        .map_err(|_| ApiError::Internal("the execution client's request list is malformed"))?;
+    Ok(LocalBuild {
+        built,
+        execution_requests,
+    })
+}
+
+/// Where a self-built payload's fee recipient comes from: the proposer's signed
+/// preferences, else `prepare_beacon_proposer`'s, else the zero address.
+fn fee_recipient_for(fee_recipients: &FeeRecipients, prepared: &Prepared) -> ExecutionAddress {
+    if let Some(preferences) = &prepared.preferences {
+        debug!(
+            proposer = prepared.proposer,
+            "Using the signed proposer preferences' fee recipient"
+        );
+        return preferences.message.fee_recipient;
+    }
+    let prepared_recipient = fee_recipients
+        .lock()
+        .expect("fee recipient lock poisoned")
+        .get(&prepared.proposer)
+        .copied();
+    match prepared_recipient {
+        Some(recipient) => {
+            debug!(
+                proposer = prepared.proposer,
+                "Using the fee recipient from prepare_beacon_proposer"
+            );
+            recipient
+        }
+        None => {
+            warn!(
+                proposer = prepared.proposer,
+                "No fee recipient for the proposer; using the zero address"
+            );
+            ExecutionAddress::ZERO
+        }
+    }
+}
+
 /// Ask the execution client to build on the chosen parent payload for the
 /// prepared slot, then collect what it built.
 ///
@@ -575,18 +762,7 @@ async fn build_payload(
     prepared: &Prepared,
 ) -> Result {
     let inputs = &prepared.inputs;
-    let fee_recipient = fee_recipients
-        .lock()
-        .expect("fee recipient lock poisoned")
-        .get(&prepared.proposer)
-        .copied()
-        .unwrap_or_else(|| {
-            warn!(
-                proposer = prepared.proposer,
-                "No fee recipient prepared for the proposer; using the zero address"
-            );
-            ExecutionAddress::ZERO
-        });
+    let fee_recipient = fee_recipient_for(fee_recipients, prepared);
     let forkchoice = ForkchoiceStateV1 {
         head_block_hash: inputs.head_block_hash,
         safe_block_hash: checkpoint_hash(store, store.beacon_justified_checkpoint().root),
@@ -628,7 +804,7 @@ async fn build_payload(
 #[allow(clippy::too_many_arguments)]
 fn assemble(
     config: &Config,
-    prepared: Prepared,
+    prepared: &Prepared,
     built: BuiltGloasPayload,
     execution_requests: ExecutionRequests,
     candidates: Vec,
@@ -643,9 +819,9 @@ fn assemble(
         parent_requests,
         ..
     } = prepared;
-    let attestations = pack_gloas_attestations(&state, candidates);
+    let attestations = pack_gloas_attestations(state, candidates);
     let payload_attestations =
-        pack_payload_attestations(&state, head_root, head_slot, messages, config);
+        pack_payload_attestations(state, *head_root, *head_slot, messages, config);
     let commitments = built.blobs_bundle.commitments;
     let inputs = |attestations, payload_attestations| GloasBlockInputs {
         randao_reveal,
@@ -658,14 +834,55 @@ fn assemble(
         execution_requests: execution_requests.clone(),
     };
     let operations = attestations.len() + payload_attestations.len();
-    match assemble_gloas_block(&state, inputs(attestations, payload_attestations), config) {
+    match assemble_gloas_block(state, inputs(attestations, payload_attestations), config) {
         Ok(produced) => Ok(produced),
         // The packers check every operation's signature against this state, so
         // this should not happen; but a block without them still earns the
         // proposal, and one that fails to build earns nothing.
         Err(err) if operations > 0 => {
             warn!(slot = state.slot(), %err, "Block with operations failed to build; retrying without");
-            assemble_gloas_block(&state, inputs(Vec::new(), Vec::new()), config)
+            assemble_gloas_block(state, inputs(Vec::new(), Vec::new()), config)
+                .map_err(|_| ApiError::Internal("the block failed to build"))
+        }
+        Err(_) => Err(ApiError::Internal("the block failed to build")),
+    }
+}
+
+/// [`assemble`] for a block that commits to a builder's bid: no payload, no
+/// envelope. `process_block` on the block enforces the bid's own rules.
+fn assemble_on_bid(
+    config: &Config,
+    prepared: &Prepared,
+    signed_bid: SignedExecutionPayloadBid,
+    candidates: Vec,
+    messages: Vec,
+    randao_reveal: BlsSignature,
+    graffiti: Bytes32,
+) -> Result {
+    let Prepared {
+        state,
+        head_root,
+        head_slot,
+        parent_requests,
+        ..
+    } = prepared;
+    let attestations = pack_gloas_attestations(state, candidates);
+    let payload_attestations =
+        pack_payload_attestations(state, *head_root, *head_slot, messages, config);
+    let inputs = |attestations, payload_attestations| GloasBidBlockInputs {
+        randao_reveal,
+        graffiti,
+        attestations,
+        payload_attestations,
+        parent_execution_requests: parent_requests.clone(),
+        signed_bid: signed_bid.clone(),
+    };
+    let operations = attestations.len() + payload_attestations.len();
+    match assemble_gloas_block_on_bid(state, inputs(attestations, payload_attestations), config) {
+        Ok(block) => Ok(block),
+        Err(err) if operations > 0 => {
+            warn!(slot = state.slot(), %err, "Bid block with operations failed to build; retrying without");
+            assemble_gloas_block_on_bid(state, inputs(Vec::new(), Vec::new()), config)
                 .map_err(|_| ApiError::Internal("the block failed to build"))
         }
         Err(_) => Err(ApiError::Internal("the block failed to build")),
@@ -922,6 +1139,7 @@ mod tests {
             .layer(Extension(engine))
             .layer(Extension(SharedAttestationPool::default()))
             .layer(Extension(SharedPayloadAttestationPool::default()))
+            .layer(Extension(SharedBuilderMarket::default()))
             .layer(Extension(FeeRecipients::default()))
             .layer(Extension(NodeCustodyColumns::default()))
     }
@@ -1017,7 +1235,10 @@ mod tests {
     fn a_builder_config_decodes_as_json_or_ssz() {
         let mut headers = HeaderMap::new();
         assert_eq!(
-            decode_builder_config(&headers, EMPTY_CONFIG.as_bytes()).unwrap(),
+            decode_builder_config(&headers, EMPTY_CONFIG.as_bytes())
+                .unwrap()
+                .builders
+                .len(),
             0
         );
         assert!(decode_builder_config(&headers, b"{}").is_err());
@@ -1029,7 +1250,13 @@ mod tests {
         // min_bid, boost factor, then the offset of an empty builders list.
         let mut ssz = vec![0u8; 16];
         ssz.extend_from_slice(&20u32.to_le_bytes());
-        assert_eq!(decode_builder_config(&headers, &ssz).unwrap(), 0);
+        assert_eq!(
+            decode_builder_config(&headers, &ssz)
+                .unwrap()
+                .builders
+                .len(),
+            0
+        );
         assert!(decode_builder_config(&headers, &ssz[..19]).is_err());
     }
 
@@ -1614,3 +1841,7 @@ mod tests {
         assert_eq!(rejected.envelopes.lock().unwrap().len(), 1);
     }
 }
+
+#[cfg(test)]
+#[path = "builder_market_tests.rs"]
+mod builder_market_tests;
diff --git a/crates/net/rpc/src/beacon/mod.rs b/crates/net/rpc/src/beacon/mod.rs
index e484cbeef..bf6efc93c 100644
--- a/crates/net/rpc/src/beacon/mod.rs
+++ b/crates/net/rpc/src/beacon/mod.rs
@@ -15,7 +15,11 @@ use serde::Serialize;
 
 use crate::shared::block_id::IdError;
 
+pub(crate) mod bid_selection;
+pub(crate) mod bids;
 pub(crate) mod blocks;
+pub(crate) mod builder_config;
+pub(crate) mod builders;
 pub(crate) mod config;
 pub(crate) mod envelopes;
 pub(crate) mod events;
@@ -25,6 +29,7 @@ pub(crate) mod headers;
 pub(crate) mod node;
 pub(crate) mod pool;
 pub(crate) mod proposal;
+pub(crate) mod proposer_preferences;
 pub(crate) mod ptc;
 pub(crate) mod states;
 pub(crate) mod validator;
@@ -186,6 +191,9 @@ pub(crate) fn routes(version: &'static str, peer_id: String) -> Router {
         .merge(proposal::routes())
         .merge(gloas_proposal::routes())
         .merge(ptc::routes())
+        .merge(bids::routes())
+        .merge(proposer_preferences::routes())
+        .merge(builders::routes())
 }
 
 #[cfg(test)]
diff --git a/crates/net/rpc/src/beacon/proposer_preferences.rs b/crates/net/rpc/src/beacon/proposer_preferences.rs
new file mode 100644
index 000000000..bd217f0b5
--- /dev/null
+++ b/crates/net/rpc/src/beacon/proposer_preferences.rs
@@ -0,0 +1,144 @@
+//! `POST /eth/v1/validator/proposer_preferences`: signed proposer preferences
+//! from a validator client, validated with the gossip rules, cached in the
+//! shared `BuilderMarket` and gossiped on `proposer_preferences`.
+//!
+//! Not gated on the sync status: the specification lists no 503, and a node
+//! that lacks the dependent block simply answers `ignore: unknown_block` per
+//! entry.
+
+use axum::{
+    Extension, Router,
+    body::Bytes,
+    extract::State,
+    http::{HeaderMap, StatusCode},
+    response::{IntoResponse, Response},
+    routing::post,
+};
+use ethlambda_network_api::RpcToP2PRef;
+use ethlambda_state_transition::beacon::{
+    builder_market::SharedBuilderMarket,
+    gossip::{IgnoreReason, Outcome, proposer_preferences::validate},
+    preset,
+};
+use ethlambda_storage::Store;
+use ethlambda_types::beacon::{containers::gloas::SignedProposerPreferences, fork::ForkName};
+use serde::Serialize;
+use tracing::{debug, warn};
+
+use crate::beacon::{
+    ApiError,
+    bids::{describe, require_version, unix_ms},
+    decode_list,
+};
+
+/// The most entries one request may carry: the SSZ list's bound,
+/// `(MIN_SEED_LOOKAHEAD + 1) * SLOTS_PER_EPOCH`.
+const MAX_ENTRIES: usize = preset::PROPOSER_LOOKAHEAD_LENGTH;
+
+pub(crate) fn routes() -> Router {
+    Router::new().route(
+        "/eth/v1/validator/proposer_preferences",
+        post(post_proposer_preferences),
+    )
+}
+
+/// One refused entry, in the Beacon API's `IndexedErrorMessage` shape.
+#[derive(Debug, Serialize)]
+struct Failure {
+    index: usize,
+    message: String,
+}
+
+/// `POST /eth/v1/validator/proposer_preferences`.
+///
+/// Each entry goes through the topic's rules (current slot window, a known
+/// dependent block, the proposer the lookahead names, the signature), since a
+/// message that fails them is one every peer would score this node down for
+/// relaying. Valid ones are cached for block production and for the bids judged
+/// against them, then gossiped; the others are reported by position and the
+/// rest still go out. An entry already cached, identical, is a success that is
+/// not republished.
+async fn post_proposer_preferences(
+    State(store): State,
+    Extension(p2p): Extension,
+    Extension(market): Extension,
+    headers: HeaderMap,
+    body: Bytes,
+) -> Response {
+    // A validator client submits during the epoch before gloas, when the
+    // consensus version it names is still fulu.
+    if let Err(err) = require_version(&headers, &[ForkName::Fulu, ForkName::Gloas]) {
+        return err.into_response();
+    }
+    let preferences = match decode_list::(&headers, &body) {
+        Ok(preferences) => preferences,
+        Err(err) => return err.into_response(),
+    };
+    if preferences.len() > MAX_ENTRIES {
+        return ApiError::BadRequest("too many signed proposer preferences").into_response();
+    }
+
+    let validated =
+        tokio::task::spawn_blocking(move || submit(&store, &p2p, &market, preferences)).await;
+    let failures = match validated {
+        Ok(failures) => failures,
+        Err(_) => return ApiError::Internal("validating the preferences failed").into_response(),
+    };
+    if failures.is_empty() {
+        return StatusCode::OK.into_response();
+    }
+    let body = serde_json::json!({
+        "code": 400,
+        "message": "some signed proposer preferences failed validation and were not published",
+        "failures": failures,
+    });
+    let mut response = crate::json_response(body);
+    *response.status_mut() = StatusCode::BAD_REQUEST;
+    response
+}
+
+/// Validate, record and publish each entry, in order.
+fn submit(
+    store: &Store,
+    p2p: &RpcToP2PRef,
+    market: &SharedBuilderMarket,
+    preferences: Vec,
+) -> Vec {
+    let now_ms = unix_ms();
+    let wall_slot = crate::beacon::node::wall_slot(store);
+    let mut failures = Vec::new();
+    for (index, signed) in preferences.into_iter().enumerate() {
+        let slot = signed.message.proposal_slot;
+        let validator = signed.message.validator_index;
+        let held = market.preferences(slot, signed.message.dependent_root);
+        if held.as_ref() == Some(&signed) {
+            debug!(%slot, validator, "Proposer preferences already cached; not republishing");
+            continue;
+        }
+        let verdict = validate(market, store, &signed, now_ms);
+        if verdict != Outcome::Accept {
+            let (outcome, reason) = verdict.labels();
+            warn!(%slot, validator, outcome, reason, "Refused submitted proposer preferences");
+            failures.push(Failure {
+                index,
+                message: describe(&verdict),
+            });
+            continue;
+        }
+        if !market.record_preferences(signed.clone(), wall_slot) {
+            failures.push(Failure {
+                index,
+                message: describe(&Outcome::Ignore(IgnoreReason::AlreadySeen)),
+            });
+            continue;
+        }
+        match p2p.publish_proposer_preferences(signed) {
+            Ok(()) => debug!(%slot, validator, "Accepted proposer preferences for gossip"),
+            Err(_) => failures.push(Failure {
+                index,
+                message: "the network actor is not running".to_string(),
+            }),
+        }
+    }
+    failures
+}
diff --git a/crates/net/rpc/src/beacon/states.rs b/crates/net/rpc/src/beacon/states.rs
index b4dda41f4..f60f3e047 100644
--- a/crates/net/rpc/src/beacon/states.rs
+++ b/crates/net/rpc/src/beacon/states.rs
@@ -68,7 +68,10 @@ fn resolve_state_id(store: &Store, state_id: &str) -> Result {
 }
 
 /// Load the state a `state_id` names, or the response explaining why not.
-fn load(store: &Store, state_id: &str) -> Result<(H256, std::sync::Arc), ApiError> {
+pub(crate) fn load(
+    store: &Store,
+    state_id: &str,
+) -> Result<(H256, std::sync::Arc), ApiError> {
     let root = resolve_state_id(store, state_id)?;
     let state = store
         .get_state(&root)
diff --git a/crates/net/rpc/src/beacon/validator_client_tests.rs b/crates/net/rpc/src/beacon/validator_client_tests.rs
index 2a6b15dcb..de63f02f1 100644
--- a/crates/net/rpc/src/beacon/validator_client_tests.rs
+++ b/crates/net/rpc/src/beacon/validator_client_tests.rs
@@ -109,6 +109,9 @@ async fn spawn_server(
         .layer(Extension(p2p))
         .layer(Extension(SharedAttestationPool::default()))
         .layer(Extension(payload_pool.clone()))
+        .layer(Extension(
+            ethlambda_state_transition::beacon::builder_market::SharedBuilderMarket::default(),
+        ))
         .layer(Extension(crate::CustodyColumns(Vec::new())))
         .layer(Extension(crate::beacon::validator::FeeRecipients::default()))
         .layer(Extension(engine));
diff --git a/crates/net/rpc/src/lib.rs b/crates/net/rpc/src/lib.rs
index 2ece9a705..595fa4d53 100644
--- a/crates/net/rpc/src/lib.rs
+++ b/crates/net/rpc/src/lib.rs
@@ -4,7 +4,8 @@ use axum::{Extension, Router};
 use ethlambda_blockchain::{EventBus, SyncStatusController};
 use ethlambda_network_api::RpcToP2PRef;
 use ethlambda_state_transition::beacon::{
-    attestation_pool::SharedAttestationPool, payload_attestation_pool::SharedPayloadAttestationPool,
+    attestation_pool::SharedAttestationPool, builder_market::SharedBuilderMarket,
+    payload_attestation_pool::SharedPayloadAttestationPool,
 };
 use ethlambda_storage::Store;
 use ethlambda_types::aggregator::AggregatorController;
@@ -202,6 +203,9 @@ pub struct BeaconApiHandles {
     /// Filled by gossip and the payload attestation pool endpoint, read by
     /// block production and the pool's GET.
     pub payload_attestation_pool: SharedPayloadAttestationPool,
+    /// Bids, proposer preferences and known payloads: filled by the bid and
+    /// preferences endpoints and by gossip, read by block production.
+    pub builder_market: SharedBuilderMarket,
     /// The columns this node custodies: what `payload_attestation_data` checks
     /// a block's blob availability against, and what block production tells
     /// the execution client it samples for when asking it to build a gloas
@@ -234,6 +238,7 @@ pub async fn start_beacon_rpc_server(
         .layer(Extension(handles.p2p))
         .layer(Extension(handles.attestation_pool))
         .layer(Extension(handles.payload_attestation_pool))
+        .layer(Extension(handles.builder_market))
         .layer(Extension(handles.custody_columns))
         .layer(Extension(beacon::validator::FeeRecipients::default()))
         .layer(Extension(handles.engine))
@@ -453,6 +458,12 @@ pub(crate) mod test_utils {
         pub(crate) payload_attestations: std::sync::Mutex<
             Vec,
         >,
+        pub(crate) bids: std::sync::Mutex<
+            Vec,
+        >,
+        pub(crate) proposer_preferences: std::sync::Mutex<
+            Vec,
+        >,
     }
 
     impl ethlambda_network_api::RpcToP2P for RecordingNetwork {
@@ -507,6 +518,22 @@ pub(crate) mod test_utils {
             Ok(())
         }
 
+        fn publish_execution_payload_bid(
+            &self,
+            bid: ethlambda_types::beacon::containers::gloas::SignedExecutionPayloadBid,
+        ) -> Result<(), spawned_concurrency::error::ActorError> {
+            self.bids.lock().unwrap().push(bid);
+            Ok(())
+        }
+
+        fn publish_proposer_preferences(
+            &self,
+            preferences: ethlambda_types::beacon::containers::gloas::SignedProposerPreferences,
+        ) -> Result<(), spawned_concurrency::error::ActorError> {
+            self.proposer_preferences.lock().unwrap().push(preferences);
+            Ok(())
+        }
+
         fn publish_payload_attestation_message(
             &self,
             message: ethlambda_types::beacon::containers::gloas::PayloadAttestationMessage,
diff --git a/docs/beacon_wire.md b/docs/beacon_wire.md
index 83addec59..731777450 100644
--- a/docs/beacon_wire.md
+++ b/docs/beacon_wire.md
@@ -85,14 +85,28 @@ after a boundary read the stale digest and reject the record.
 Seven global topics, `/eth2/{digest}/{name}/ssz_snappy`, plus two subnet
 families this node's own node id selects a narrow slice of: the data column
 subnets it custodies and the attestation subnets it backbones, both described
-below. A gloas digest adds two more, `execution_payload` and
-`payload_attestation_message`, which `BeaconTopics::for_fork` subscribes under
-gloas digests and not under earlier ones. An envelope is validated like a block
+below. A gloas digest adds four more, `execution_payload`,
+`payload_attestation_message`, `execution_payload_bid` and
+`proposer_preferences`, which `BeaconTopics::for_fork` subscribes under gloas
+digests and not under earlier ones (so one epoch before the fork, when the
+digest is joined). An envelope is validated like a block
 (its stateful half on the blocking pool) and goes to the chain actor on `Accept`
 and on `Queue`, since the actor holds an envelope whose block is not imported
 yet. A payload attestation is validated on the attestation permit pool and goes
 to the actor on `Accept` only: a vote for a block not imported here is dropped,
-since it is valid only within its own slot.
+since it is valid only within its own slot. Bids and preferences are the
+builder market: they never reach the chain actor. Their rules live in
+`state_transition::beacon::gossip::{execution_payload_bid, proposer_preferences}`,
+the cheap half runs in p2p's triage (`p2p/src/beacon/builder_market.rs`) and the
+stateful half on a permit pool of their own (`builder_validation_permits`), so
+a bid burst cannot starve blocks, columns or attestations. Whatever is accepted
+goes into the node's one shared `BuilderMarket`, which the Beacon API and block
+production read. Neither is ever queued: a message whose parent or dependent
+block is unknown is ignored. A bid is capped at 196,932 decompressed bytes
+(`Reject(Malformed)` above that). The node publishes both: a bid on the digest of
+its slot, preferences on the digest of the proposal slot, which in the epoch
+before gloas is the gloas digest already joined. Envelopes the node publishes
+itself are recorded as known payloads, since gossip never echoes them.
 
 | Topic | Decoded as |
 | --- | --- |
@@ -106,6 +120,8 @@ since it is valid only within its own slot.
 | `beacon_attestation_{subnet_id}` | `Attestation`, phase0 or electra's `SingleAttestation` (gloas keeps the latter) |
 | `execution_payload` | `SignedExecutionPayloadEnvelope`, gloas digests only |
 | `payload_attestation_message` | `PayloadAttestationMessage`, gloas digests only |
+| `execution_payload_bid` | `SignedExecutionPayloadBid`, gloas digests only |
+| `proposer_preferences` | `SignedProposerPreferences`, gloas digests only |
 
 `beacon_attestation_{0..63}` is no longer wholly unsubscribed. This node holds
 `SUBNETS_PER_NODE` (2 on mainnet) long-lived subscriptions from that family,
diff --git a/docs/metrics.md b/docs/metrics.md
index cbdc72d6d..6d6f7b9e1 100644
--- a/docs/metrics.md
+++ b/docs/metrics.md
@@ -363,15 +363,17 @@ own section. These are ethlambda-specific, not part of the leanMetrics spec.
 
 `kind` is the topic kind, with every `data_column_sidecar_{subnet}` sharing the
 label `data_column_sidecar` and every `beacon_attestation_{subnet_id}` sharing
-`beacon_attestation`. The gloas topics `execution_payload` and
-`payload_attestation_message` are labelled by their own name. `queue` means
+`beacon_attestation`. The gloas topics `execution_payload`,
+`payload_attestation_message`, `execution_payload_bid` and
+`proposer_preferences` are labelled by their own name. `queue` means
 IGNORE to gossipsub while the chain actor still receives the object and parks
 it; of the gloas topics only `execution_payload` answers it, for an envelope
 whose block is not known yet (`block_unknown`), has no post-state yet
 (`block_not_ready`), or has one that is not in the cache the checks read
 (`state_not_cached`, as when the block was imported moments ago; the actor
-verifies the signature itself, so the envelope is forwarded rather than dropped). The aggregate, attestation and
-`payload_attestation_message` topics never answer `queue`, since the vote
+verifies the signature itself, so the envelope is forwarded rather than dropped). The aggregate, attestation,
+`payload_attestation_message`, `execution_payload_bid` and `proposer_preferences`
+topics never answer `queue`, since the vote
 block's post-state is either cached or it is not
 (`IgnoreReason::UnknownBlock`/`StateUnavailable`), with nothing to hold the
 message for. **`verdict_expired_total` should stay at
@@ -390,16 +392,25 @@ only), `not_aggregator` (aggregate only), `not_in_committee`,
 `aggregator_signature` (aggregate only), `aggregate_signature` (aggregate
 only), `target_not_ancestor`, `wrong_subnet` (attestation only), and gloas's
 `data_index_out_of_range`, `same_slot_payload_flag` and `payload_invalid` on the
-reject side. `already_seen`, `overloaded` and `unsupported_fork` are shared with the
+reject side. The builder market topics add, on the ignore side,
+`pre_gloas_slot`, `not_current_or_next_slot`, `not_highest_bid`,
+`beyond_lookahead`, `preferences_unseen`, `fee_recipient_mismatch`,
+`parent_payload_unknown`, `gas_limit_incompatible`, `not_on_head_branch`,
+`builder_cannot_cover`, `builder_may_exit`, `slot_started` and
+`impossible_dependent_root`, and on the reject side `execution_payment_nonzero`,
+`block_hash_equals_parent`, `prev_randao`, `unknown_builder`,
+`not_payload_builder`, `inactive_builder` and `dependent_root_too_late`; a bid
+over its size cap is `malformed`. `already_seen`, `overloaded` and `unsupported_fork` are shared with the
 other topics: `unsupported_fork` is an ignore reason for a message of a fork this
 build has no gossip rules for, so an honest peer past the fork epoch is not
 scored as a bad decoder. Gloas blocks, data columns, aggregates and attestations
 are validated, so they carry their own verdict reasons instead.
 
-Two permit pools bound the blocking-thread half of validation:
-`gossip_validation_permits` for blocks and columns,
-`attestation_validation_permits` for aggregates and subnet attestations. They
-are deliberately separate: a mainnet slot's worth of aggregates and backbone
+Three permit pools bound the blocking-thread half of validation:
+`gossip_validation_permits` for blocks, columns and envelopes,
+`attestation_validation_permits` for aggregates, subnet attestations and
+payload votes, and `builder_validation_permits` for bids and proposer
+preferences. They are deliberately separate: a mainnet slot's worth of aggregates and backbone
 attestations arrives every slot, not only during a range sync, and sharing one
 pool would let that burst answer `Ignore(Overloaded)` for a block or a column
 instead. Neither pool has a metric of its own yet; a permit exhausted on
diff --git a/docs/rpc.md b/docs/rpc.md
index 1e403494c..af119b7be 100644
--- a/docs/rpc.md
+++ b/docs/rpc.md
@@ -258,10 +258,13 @@ surface rather than sitting beside it; a `/lean/v0` path on a beacon node is a
 | `GET` | `/eth/v2/validator/aggregate_attestation` | JSON | The pooled votes for a data root and committee, aggregated |
 | `POST` | `/eth/v2/validator/aggregate_and_proofs` | *(status only)* | Validate and gossip `SignedAggregateAndProof`s (JSON or SSZ body) |
 | `GET` | `/eth/v3/validator/blocks/{slot}` | SSZ or JSON | An unsigned fulu block built on the head (`produceBlockV3`) |
-| `POST` | `/eth/v4/validator/blocks/{slot}` | SSZ or JSON | An unsigned self-built gloas block, with its envelope and blobs when asked (`produceBlockV4`) |
+| `POST` | `/eth/v4/validator/blocks/{slot}` | SSZ or JSON | An unsigned gloas block: self-built, with its envelope and blobs when asked, or on a pooled builder bid (`produceBlockV4`) |
 | `GET` | `/eth/v1/validator/execution_payload_envelopes/{slot}/{beacon_block_root}` | SSZ or JSON | The unsigned envelope `produceBlockV4` built (gloas) |
 | `POST` | `/eth/v2/beacon/blocks` | *(status only)* | Gossip and import a signed fulu or gloas block (`publishBlockV2`; JSON or SSZ body) |
 | `POST` | `/eth/v1/beacon/execution_payload_envelopes` | *(status only)* | Gossip a signed envelope and its data columns (gloas; JSON or SSZ body) |
+| `POST` | `/eth/v1/beacon/execution_payload_bids` | *(status only)* | Validate, pool and gossip a builder's `SignedExecutionPayloadBid` (gloas; JSON or SSZ body) |
+| `POST` | `/eth/v1/validator/proposer_preferences` | *(status only)* | Validate, cache and gossip `SignedProposerPreferences` (gloas; JSON or SSZ body) |
+| `POST` | `/eth/v1/beacon/states/{state_id}/builders` | JSON | The gloas builder registry, filtered by id and status |
 | `POST` | `/eth/v1/validator/prepare_beacon_proposer` | *(status only)* | Acknowledged, not acted on (see below) |
 | `GET` | `/eth/v1/events` | SSE | Live stream of chain events (see below) |
 
@@ -419,10 +422,12 @@ publication below builds and gossips the columns.
 ### Gloas block production
 
 A gloas proposer signs two things on their own, the block and the envelope that
-reveals its payload, and this node serves both halves. It only builds for
-itself: no builder bid is taken from gossip or a builder API, and no
-`SignedProposerPreferences` is read (see
-[Spec Deviations](./spec_deviations.md#self-build-only)).
+reveals its payload, and this node serves both halves for a block it builds
+itself. It may instead build on a builder's bid it holds in the shared builder
+market (see [Builder market](#builder-market)); then the builder, not this node,
+reveals the payload. The builder-API path (a bid requested from a builder's URL
+through the `BuilderConfig`'s `builders` entries) is not implemented: those
+entries are decoded and not consulted.
 
 **`POST /eth/v4/validator/blocks/{slot}`** (`produceBlockV4`):
 
@@ -430,18 +435,21 @@ itself: no builder bid is taken from gossip or a builder API, and no
 |---|---|
 | Query | `randao_reveal` (required), `include_payload` (required, `true` or `false`), `graffiti` (optional, 32-byte hex), `skip_randao_verification` (accepted, ignored) |
 | Request headers | `Eth-Consensus-Version` is optional but must be `gloas` when present. `Accept: application/octet-stream` for SSZ, JSON otherwise |
-| Body | A `BuilderConfig` (`min_bid`, `builder_boost_factor`, `builders`), JSON, or SSZ with `Content-Type: application/octet-stream`. It is decoded and otherwise ignored; a missing or undecodable one is a `400` |
-| `200` SSZ | With `include_payload=true`, `BlockContents` (`block`, `execution_payload_envelope`, `kzg_proofs`, `blobs`); with `false`, the bare `gloas::BeaconBlock` |
+| Body | A `BuilderConfig`, JSON or SSZ with `Content-Type: application/octet-stream`: `min_bid` and `builder_boost_factor` (govern the bids seen over p2p) and `builders` (up to 64 entries, decoded and ignored). Integers are quoted strings, as everywhere in the Beacon API. A missing or undecodable one is a `400` |
+| `200` SSZ | Self-built with `include_payload=true`: `BlockContents` (`block`, `execution_payload_envelope`, `kzg_proofs`, `blobs`). Otherwise, and for every bid-won block, the bare `gloas::BeaconBlock` |
 | `200` JSON | `{version: "gloas", consensus_block_value, execution_payload_value, execution_payload_included, data}`, where `data` is the same container as the SSZ body |
-| Response headers | `Eth-Consensus-Version: gloas`, `Eth-Execution-Payload-Included` (`true` or `false`), `Eth-Execution-Payload-Value` (wei, decimal), `Eth-Consensus-Block-Value` (always `0`: no builder comparison happens on this node) |
+| Response headers | `Eth-Consensus-Version: gloas`, `Eth-Execution-Payload-Included` (`true` or `false`; always `false` for a bid-won block), `Eth-Execution-Payload-Value` (wei, decimal; a bid's value times `10^9` when it won), `Eth-Consensus-Block-Value` (always `0`: the comparison is of execution payload values only). No `Eth-Builder-Url`: that is for builder-API bids |
 | `400` | A slot the schedule does not place at gloas, a missing or malformed query, a wrong `Eth-Consensus-Version`, a bad body, a slot not after the head block, or a `randao_reveal` that does not verify against the slot's proposer. The slot check precedes the execution-client check, so it is a `400` on any node |
-| `503` | No execution client configured, the execution client did not start or return a build, or the node is building on a FULL parent whose envelope it does not hold |
+| `503` | No execution client and no viable pooled bid for the slot, the execution client did not start or return a build (and no bid took its place), the node is building on a FULL parent whose envelope it does not hold, or the winning bid failed to build and there is no local payload to fall back on |
 
 The node advances the head state to the slot, and decides which parent payload
 to build on with `should_build_on_full` over the payload status fork choice
 recorded for the head. It then asks its execution client to build
-(`forkchoiceUpdatedV4` with `PayloadAttributesV4`, then `getPayloadV6`) with the
-proposer's `prepare_beacon_proposer` fee recipient. The body packs the
+(`forkchoiceUpdatedV4` with `PayloadAttributesV4`, then `getPayloadV6`). The
+fee recipient and gas target of that build come from the proposer's signed
+`ProposerPreferences` for the slot when the market holds them under the slot's
+dependent root, else from `prepare_beacon_proposer` (fee recipient, or the zero
+address with a warning) and the parent bid's gas limit. The body packs the
 attestation pool's best aggregates and the payload attestation pool's votes for
 the parent block (an aggregate that does not verify against the advanced state
 is dropped, since one bad operation fails the block), and the state root comes
@@ -449,6 +457,22 @@ from running the block through `process_block`. The bid is a zero-value
 self-build bid read off the built payload. What was built is cached by `(slot,
 block root)`, for the current and previous slot only.
 
+**Choosing between the local payload and a bid.** The bids considered are the
+market's for `(slot, head root, parent payload hash)` with this build's
+`prev_randao`, packable into a block on the advanced state, and paying the
+preferences' fee recipient when those are held. The best bid is the highest
+whose `value + execution_payment` is at least the config's `min_bid`. It wins iff
+`builder_boost_factor * bid_gwei > floor(local_value_wei / 10^7)` (the
+specification's weighting of the local value by 100, in integers), and the local
+payload wins a tie, so a factor of `0` prefers it and `2^64 - 1` prefers the bid.
+An engine that sets `shouldOverrideBuilder` keeps the local payload. Without a
+local payload (no engine, or the build failed) the best bid is taken whatever its
+weight. A bid-won block caches nothing, so the envelope `GET` below answers
+`404` for it, and `POST /eth/v1/beacon/execution_payload_envelopes` refuses a
+proposer-signed (self-build) envelope for it because the builder index differs
+from the bid's. The builder's own envelope goes through that endpoint as for any
+block.
+
 **`GET /eth/v1/validator/execution_payload_envelopes/{slot}/{beacon_block_root}`**
 serves the cached unsigned envelope, for a client that asked for the block with
 `include_payload=false`: `{version: "gloas", data}` as JSON, or the SSZ
@@ -482,6 +506,47 @@ envelope and all `NUMBER_OF_COLUMNS` gloas data column sidecars are handed to
 P2P, which gossips them together. A node that does not subscribe to a column's
 subnet publishes through gossipsub fanout.
 
+### Builder market
+
+Three endpoints serve the gloas builder market. Bids and preferences go through
+the same rules as the `execution_payload_bid` and `proposer_preferences` gossip
+topics and are held in one shared builder market that p2p also fills, so a
+message accepted from either side is seen by both.
+
+- **`POST /eth/v1/beacon/execution_payload_bids`** takes one
+  `SignedExecutionPayloadBid`, as JSON (or no `Content-Type`) or SSZ
+  (`application/octet-stream`, at most 196,932 bytes; any other type is `415`).
+  `Eth-Consensus-Version` is optional and must be `gloas` when present. An
+  identical bid already pooled is `200` and is not republished. Otherwise the
+  gossip rules run (a known parent, the preferences for the slot and their fee
+  recipient and gas limit, a known parent payload, an active funded builder, the
+  signature), and a bid that is anything but accepted is a `400` whose message is
+  `"{outcome}: {reason}"` (for instance `reject: bad_signature` or `ignore:
+  preferences_unseen`), because there is no status for a valid bid this node
+  would not relay. An accepted bid is pooled for block production and gossiped.
+  `500` when the network actor is down.
+- **`POST /eth/v1/validator/proposer_preferences`** takes a list of
+  `SignedProposerPreferences` (at most `(MIN_SEED_LOOKAHEAD + 1) *
+  SLOTS_PER_EPOCH`, 64 on mainnet), as a JSON array or an SSZ list.
+  `Eth-Consensus-Version` is optional and may be `fulu` or `gloas`, since a
+  validator client submits during the epoch before the fork. Each entry runs the
+  gossip rules in order; an entry already cached, identical, succeeds without a
+  republish. Accepted entries are cached (the first per `(proposal slot,
+  dependent root)` wins) and gossiped. If any entry fails the answer is `400`
+  `{code, message, failures: [{index, message}]}` and the others were still
+  published. The endpoint is not gated on the sync status. A node whose
+  dependent block is unknown answers `ignore: unknown_block` for that entry.
+- **`POST /eth/v1/beacon/states/{state_id}/builders`** takes an optional JSON
+  body `{ids?: [index | 0x pubkey], statuses?: [pending | active | exited]}`
+  (empty or absent selects everything) and answers `{execution_optimistic,
+  finalized, data: [{index, status, builder}]}` in registry order, integers
+  quoted. An id naming no builder is omitted. `exited` is a set
+  `withdrawable_epoch`, else `active` per `is_active_builder`, else `pending`.
+  `400` for a malformed body, id or status, or a pre-gloas state; `404` for an
+  unknown state. The Beacon API defines no `GET` form.
+
+`POST /eth/v1/validator/builder_preferences` is not served (`404`).
+
 ### Payload timeliness committee
 
 - **`POST /eth/v1/validator/duties/ptc/{epoch}`** takes a JSON array of quoted
diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md
index e3a781c49..0ffcf6287 100644
--- a/docs/spec_deviations.md
+++ b/docs/spec_deviations.md
@@ -93,47 +93,154 @@ same keys, already signed.
   runs, and treat a restart as an event that needs the same care a manual key
   move would. The client warns about this at startup on every run.
 
-## Self-build only
+## Builder bids: gossip and API only (no builder API)
 
-The gloas validator duties are served for a proposer that builds its own
-payload, and for no one else.
+A gloas proposer served by this node can take a builder's bid from gossip or
+from the Beacon API, and no other way.
 
 - **The specification:** a gloas proposer may take a builder's signed bid
   (`SignedExecutionPayloadBid`, from gossip or a builder API) instead of building
-  its own payload, and publishes a `SignedProposerPreferences` so builders know
-  its fee recipient and gas limit target. `produceBlockV4` takes a
-  `BuilderConfig` (`min_bid`, `builder_boost_factor`, `builders`) to steer that
-  choice.
-- **ethlambda:** the beacon node never takes a bid, from gossip or otherwise, and
-  never builds or reads a `SignedProposerPreferences`. `produceBlockV4` decodes
-  the `BuilderConfig` body (a missing or undecodable one is a `400`, as the
-  specification requires) and ignores it, logging at debug when it names
-  builders. Every block commits to a zero-value self-build bid
-  (`BUILDER_INDEX_SELF_BUILD`, the G2 point at infinity as signature), and
-  `Eth-Consensus-Block-Value` is always `0` since there is nothing to compare.
-  The validator client signs the envelope with the proposer's own key under
-  `DOMAIN_BEACON_BUILDER`, and leaves a block that commits to anyone else's bid
-  alone.
-- **Why:** a scope decision for the first gloas duties: taking bids needs a bid
-  pool, builder payment handling and a builder-facing API, none of which this
-  node has.
-- **Consequence:** a validator run through this node never earns a builder's
-  payment, and an execution client's own block value is what
-  `Eth-Execution-Payload-Value` reports.
-
-## `target_gas_limit` is the parent bid's gas limit
+  its own payload. `produceBlockV4` takes a `BuilderConfig` (`min_bid`,
+  `builder_boost_factor`, `builders`) to steer that choice.
+- **ethlambda:** bids seen on `execution_payload_bid` or posted to
+  `POST /eth/v1/beacon/execution_payload_bids` are pooled, and `produceBlockV4`
+  compares the best one with the local build under the top-level `min_bid` and
+  `builder_boost_factor`: the bid must reach `min_bid`, and wins when
+  `builder_boost_factor * bid_value > local_value / 10^7` (in Gwei against Wei,
+  so a factor of 100 is parity). The local build wins a tie, and
+  `shouldOverrideBuilder` from the execution client is honored. With no
+  execution client, or a failed local build, the best viable bid is taken.
+- **What it does not do:** the `builders` entries are decoded and ignored, so
+  no builder is asked for a bid over HTTP and no `Eth-Builder-Url` is returned.
+  Gossip bids are not filtered by `builder_pubkeys` either, since the Beacon API
+  puts that list on each entry and it governs only that entry's own bid.
+  `Eth-Consensus-Block-Value` stays `0`, because the consensus reward is not
+  computed.
+- **A bid win returns no envelope.** The block commits to the builder's bid, the
+  builder reveals the payload, and nothing is cached for
+  `GET .../execution_payload_envelopes`, so that endpoint answers `404` and
+  `Eth-Execution-Payload-Included` is `false`. The node never signs or
+  publishes an envelope for such a block, and the proposer-signed
+  self-build envelope is refused for it, since its `builder_index` differs from
+  the bid's.
+- **Why:** gossip and API bids need only a pool and the gossip rules. A builder
+  API client is a separate crate with its own failure modes and timeouts, and
+  is left for a later phase.
+
+## Fee recipient and gas target come from proposer preferences, with fallbacks
 
 - **The specification:** the payload is built toward the `target_gas_limit` of
-  the proposer's `SignedProposerPreferences`, and `bid.gas_limit` must be
-  compatible with it (`is_gas_limit_target_compatible`).
-- **ethlambda:** with no preferences to read, `PayloadAttributesV4.targetGasLimit`
-  is the `gas_limit` of `latest_execution_payload_bid` of the state being built
-  on (`gloas_payload_inputs`), so the execution client holds the gas limit where
-  it is.
+  the proposer's `SignedProposerPreferences`, which also names the fee
+  recipient builders must pay, and `bid.gas_limit` must be compatible with the
+  target (`is_gas_limit_target_compatible`).
+- **ethlambda:** the self-build reads the signed preferences for
+  `(slot, dependent_root)` that name the proposer, from gossip or the Beacon API.
+  The fee recipient is theirs, else `prepare_beacon_proposer`'s, else zero with
+  a warning. The target gas limit is theirs, else the `gas_limit` of
+  `latest_execution_payload_bid` of the state being built on, so the execution
+  client holds the gas limit where it is.
 - **Equivalence:** the bid is built from the payload the execution client
   returns, so `bid.gas_limit` is whatever that payload carries and is always
-  consistent with the block. A validator that wants the limit to move cannot say
-  so through this node: it follows the previous block's.
+  consistent with the block. A proposer that wants the limit to move without
+  preferences cannot say so through this node: it follows the previous block's.
+
+## Bid and preference gossip never queues
+
+- **The specification:** several rules say the message "MAY be queued": an
+  unknown parent block, an unimported parent, an unseen dependent block.
+- **ethlambda:** every one of them is IGNORE. A bid or preferences message that
+  names a block or state this node lacks is dropped, never parked, so a burst
+  of them holds no memory and nothing is replayed later.
+- **Why:** both topics are only useful for the next slot or two, and a bid that
+  arrives after its parent was imported would be stale by the time a replay ran.
+  A proposer that missed a message builds locally.
+
+## Bid gossip judges against the recorded head and cached states
+
+- **The specification:** `validate_execution_payload_bid_gossip` reads
+  `get_head(store)`, `store.block_states[parent]` and the parent state advanced
+  with `process_slots` to the bid's slot.
+- **ethlambda:** the head node is the one the chain actor recorded
+  (`Store::head` and `head_payload_status`), with a fresh `get_head_node` walk
+  only when no status is recorded. The dependent root is read from the parent
+  state's `block_roots`, which the lookahead rule keeps in range. The parent
+  state stands in for the advanced one when the bid is in the parent's own
+  epoch, since gloas's `process_slot` touches none of `builders`,
+  `finalized_checkpoint`, `builder_pending_*`, `fork` or
+  `latest_execution_payload_bid`, which are all the later rules read. Across an
+  epoch the cached `CheckpointState` of the bid's epoch is used and filled, the
+  same entry attestation target states use. A state that is not cached is
+  IGNORE and is never rebuilt from disk.
+- **Why:** gossip verdicts are waited on by gossipsub, and the state cache holds
+  32 states, so a spec-literal read of a parent about an epoch old would miss
+  often. The head record and the equivalence above give the same verdict
+  without a replay.
+
+## Known execution payloads are gossip-accepted or self-published envelopes only
+
+- **The specification:** `seen.execution_payloads` holds a payload for every
+  envelope accepted from gossip.
+- **ethlambda:** the known payloads are the market's, a 256-entry LRU filled by
+  envelopes that passed gossip validation and by envelopes this node publishes
+  (gossip never echoes a node's own messages). They are not persisted, and an
+  envelope that was queued and verified later, or fetched by request and
+  response, does not count. After a restart bids on a pre-restart payload are
+  IGNORE until new envelopes arrive.
+- **Exception:** a pre-gloas parent's own payload counts as known, with its
+  execution payload header's gas limit, see the fork boundary entry below.
+
+## Proposer preferences are judged off cached states only
+
+- **The specification:** the lookahead is read from
+  `store.block_states[dependent_root]` advanced to the epoch before the
+  proposal's.
+- **ethlambda:** the cached head state is used when it shares the dependent root
+  and is in the epoch before the proposal's or the proposal's own (the whole
+  canonical case), else the cached `CheckpointState` of the epoch before the
+  proposal's, else IGNORE. Nothing is rebuilt from disk.
+- **Why:** a dependent block about an epoch old is usually out of the 32-state
+  cache, so reading it would IGNORE most honest preferences.
+
+## The fulu-to-gloas boundary for bids and preferences
+
+- **The specification:** says nothing about a parent that is not a gloas block.
+- **ethlambda:** a pre-gloas parent's payload counts as known, with its header's
+  `gas_limit`. A pre-gloas head's payload hashes come from its payload header,
+  and a bid is compatible with it when it builds on that head and its payload.
+  The builder-exit check is skipped for a pre-gloas parent, which carries no
+  envelope.
+- **Preference signatures** are accepted under the lookahead state's own
+  `DOMAIN_PROPOSER_PREFERENCES` domain (the specification's), the fork version of
+  the epoch before the proposal's, or the proposal epoch's. They coincide outside
+  the first epoch of a fork. Lighthouse signs with the proposal epoch's version
+  and the specification gives the earlier one, so accepting both avoids
+  rejecting an honest client's messages around the gloas upgrade.
+- **Consequence:** builders onboarded at the fork are inactive until their
+  deposit epoch is finalized, so gossip bids are rejected for the first epochs of
+  gloas and `produceBlockV4` self-builds.
+
+## Beacon API answers for bids and preferences
+
+- **An IGNORE verdict is a `400`,** the same as a REJECT, because the API has no
+  separate status for it. The message names the verdict and the reason, such as
+  `ignore: preferences_unseen`.
+- **An identical resubmission is a `200`** and is not published again.
+- **The prose and the rules disagree** on a mismatched fee recipient or gas
+  limit: the Beacon API text says the bid is rejected, while consensus-specs
+  IGNOREs it. ethlambda follows consensus-specs.
+- **`Eth-Consensus-Version` may be absent** on the bid and preferences posts. If
+  present it must name a gloas-compatible fork.
+
+## No minimum bid increment or rate limit
+
+- **The specification:** a note says implementations SHOULD guard against
+  builders spamming bids with minimal increments, for example with a minimum
+  threshold or by forwarding only the best bid at intervals.
+- **ethlambda:** neither is implemented. Spam is bounded by one bid per builder
+  per `(slot, parent_hash, parent_root)`, a strictly higher value than the best
+  seen, a funded active registered builder with a valid signature, a cap on keys
+  per slot and on bids pooled per parent, and a separate permit pool for
+  validating them. A configurable minimum increment is a follow-up.
 
 ## `skip_randao_verification` is ignored
 

From 14e24a47f6058c92d719250e4d87e6ae91b83a2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Mon, 26 Oct 2026 17:21:21 -0300
Subject: [PATCH 130/132] style: cargo fmt

---
 crates/net/p2p/src/lib.rs | 7 +++----
 crates/net/rpc/src/lib.rs | 3 +--
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs
index 2859c15ee..b545e39e6 100644
--- a/crates/net/p2p/src/lib.rs
+++ b/crates/net/p2p/src/lib.rs
@@ -47,8 +47,8 @@ use ethlambda_network_api::{
         PublishBeaconAggregate, PublishBeaconAttestation, PublishBeaconBlock,
         PublishExecutionPayloadBid, PublishExecutionPayloadEnvelope,
         PublishPayloadAttestationMessage, PublishProposerPreferences,
-        PublishSyncCommitteeContribution, PublishSyncCommitteeMessage,
-        SubscribeAttestationSubnets, SubscribeSyncCommitteeSubnets,
+        PublishSyncCommitteeContribution, PublishSyncCommitteeMessage, SubscribeAttestationSubnets,
+        SubscribeSyncCommitteeSubnets,
     },
 };
 use ethlambda_state_transition::beacon::aggregate::MAX_AGGREGATES_PER_SLOT;
@@ -61,8 +61,7 @@ use ethlambda_state_transition::beacon::gossip::{
     sync_committee::{SeenSyncCommitteeMessages, SeenSyncContributions},
 };
 use ethlambda_state_transition::beacon::{
-    attestation_pool::SharedAttestationPool,
-    builder_market::SharedBuilderMarket,
+    attestation_pool::SharedAttestationPool, builder_market::SharedBuilderMarket,
     payload_attestation_pool::SharedPayloadAttestationPool,
     sync_committee_pool::SharedSyncCommitteePool,
 };
diff --git a/crates/net/rpc/src/lib.rs b/crates/net/rpc/src/lib.rs
index 032df271c..d28463a4f 100644
--- a/crates/net/rpc/src/lib.rs
+++ b/crates/net/rpc/src/lib.rs
@@ -5,8 +5,7 @@ use axum::{Extension, Router};
 use ethlambda_blockchain::{EventBus, SyncStatusController};
 use ethlambda_network_api::RpcToP2PRef;
 use ethlambda_state_transition::beacon::{
-    attestation_pool::SharedAttestationPool,
-    builder_market::SharedBuilderMarket,
+    attestation_pool::SharedAttestationPool, builder_market::SharedBuilderMarket,
     payload_attestation_pool::SharedPayloadAttestationPool,
     sync_committee_pool::SharedSyncCommitteePool,
 };

From 0d464b8eafd0d132806cc3694759aa2849b1f4ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Mon, 26 Oct 2026 17:21:23 -0300
Subject: [PATCH 131/132] style: cargo fmt

---
 crates/net/p2p/src/lib.rs             | 7 +++----
 crates/net/rpc/src/beacon/proposal.rs | 2 +-
 crates/net/rpc/src/lib.rs             | 3 +--
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs
index 2859c15ee..b545e39e6 100644
--- a/crates/net/p2p/src/lib.rs
+++ b/crates/net/p2p/src/lib.rs
@@ -47,8 +47,8 @@ use ethlambda_network_api::{
         PublishBeaconAggregate, PublishBeaconAttestation, PublishBeaconBlock,
         PublishExecutionPayloadBid, PublishExecutionPayloadEnvelope,
         PublishPayloadAttestationMessage, PublishProposerPreferences,
-        PublishSyncCommitteeContribution, PublishSyncCommitteeMessage,
-        SubscribeAttestationSubnets, SubscribeSyncCommitteeSubnets,
+        PublishSyncCommitteeContribution, PublishSyncCommitteeMessage, SubscribeAttestationSubnets,
+        SubscribeSyncCommitteeSubnets,
     },
 };
 use ethlambda_state_transition::beacon::aggregate::MAX_AGGREGATES_PER_SLOT;
@@ -61,8 +61,7 @@ use ethlambda_state_transition::beacon::gossip::{
     sync_committee::{SeenSyncCommitteeMessages, SeenSyncContributions},
 };
 use ethlambda_state_transition::beacon::{
-    attestation_pool::SharedAttestationPool,
-    builder_market::SharedBuilderMarket,
+    attestation_pool::SharedAttestationPool, builder_market::SharedBuilderMarket,
     payload_attestation_pool::SharedPayloadAttestationPool,
     sync_committee_pool::SharedSyncCommitteePool,
 };
diff --git a/crates/net/rpc/src/beacon/proposal.rs b/crates/net/rpc/src/beacon/proposal.rs
index 1f8cfaf29..33b4626d4 100644
--- a/crates/net/rpc/src/beacon/proposal.rs
+++ b/crates/net/rpc/src/beacon/proposal.rs
@@ -505,7 +505,7 @@ async fn produce(
                 &config,
                 options,
             )
-                .map_err(|_| ApiError::Internal("the block failed to build"))?
+            .map_err(|_| ApiError::Internal("the block failed to build"))?
         }
         Err(_) => return Err(ApiError::Internal("the block failed to build")),
     };
diff --git a/crates/net/rpc/src/lib.rs b/crates/net/rpc/src/lib.rs
index 032df271c..d28463a4f 100644
--- a/crates/net/rpc/src/lib.rs
+++ b/crates/net/rpc/src/lib.rs
@@ -5,8 +5,7 @@ use axum::{Extension, Router};
 use ethlambda_blockchain::{EventBus, SyncStatusController};
 use ethlambda_network_api::RpcToP2PRef;
 use ethlambda_state_transition::beacon::{
-    attestation_pool::SharedAttestationPool,
-    builder_market::SharedBuilderMarket,
+    attestation_pool::SharedAttestationPool, builder_market::SharedBuilderMarket,
     payload_attestation_pool::SharedPayloadAttestationPool,
     sync_committee_pool::SharedSyncCommitteePool,
 };

From 7615a75c739fe2d090411d1dbf067772f841965c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Mon, 26 Oct 2026 19:01:13 -0300
Subject: [PATCH 132/132] test(rpc): give the builder market test routers the
 sync pool and own version

Block production reads the sync committee pool (#668) and the node's own client version (#638) from request extensions; the builder market tests build their own router and had neither, so every produced block answered 500.
---
 .../rpc/src/beacon/builder_market_tests.rs    | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/crates/net/rpc/src/beacon/builder_market_tests.rs b/crates/net/rpc/src/beacon/builder_market_tests.rs
index 26876e6e8..c6751ae76 100644
--- a/crates/net/rpc/src/beacon/builder_market_tests.rs
+++ b/crates/net/rpc/src/beacon/builder_market_tests.rs
@@ -30,6 +30,7 @@ use ethlambda_state_transition::beacon::{
         test_state::{secret_key_for, sign_for},
     },
     payload_attestation_pool::SharedPayloadAttestationPool,
+    sync_committee_pool::SharedSyncCommitteePool,
 };
 use ethlambda_storage::Store;
 use ethlambda_types::{
@@ -268,6 +269,15 @@ impl World {
             .layer(Extension(engine.map(|engine| engine.client.clone())))
             .layer(Extension(SharedAttestationPool::default()))
             .layer(Extension(SharedPayloadAttestationPool::default()))
+            .layer(Extension(SharedSyncCommitteePool::default()))
+            .layer(Extension(crate::beacon::graffiti::OwnVersion(Arc::new(
+                ethlambda_engine::types::ClientVersionV1 {
+                    code: "LA".to_string(),
+                    name: "ethlambda".to_string(),
+                    version: "ethlambda/test".to_string(),
+                    commit: "0x3c4d7e8f".to_string(),
+                },
+            ))))
             .layer(Extension(self.market.clone()))
             .layer(Extension(self.fee_recipients.clone()))
             .layer(Extension(CustodyColumns::default()))
@@ -1209,6 +1219,15 @@ async fn the_new_routes_are_wired_with_the_production_layers() {
             .layer(Extension::(world.network.clone()))
             .layer(Extension(SharedAttestationPool::default()))
             .layer(Extension(SharedPayloadAttestationPool::default()))
+            .layer(Extension(SharedSyncCommitteePool::default()))
+            .layer(Extension(crate::beacon::graffiti::OwnVersion(Arc::new(
+                ethlambda_engine::types::ClientVersionV1 {
+                    code: "LA".to_string(),
+                    name: "ethlambda".to_string(),
+                    version: "ethlambda/test".to_string(),
+                    commit: "0x3c4d7e8f".to_string(),
+                },
+            ))))
             .layer(Extension(world.market.clone()))
             .layer(Extension(CustodyColumns::default()))
             .layer(Extension(FeeRecipients::default()))