From 18d1a47da1114a1c85e109a318035b0eb76d2b18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:08:56 -0300 Subject: [PATCH 1/9] feat(validator): fetch v2 proposer duties and submit proposer preferences Signed proposer preferences must carry the shuffling dependent root, which only v2 of the proposer duties endpoint reports (v1's root is one epoch later). Add both calls to the beacon node trait, with the fallback sending preferences to every node since each one gossips what it accepts. --- crates/validator/src/beacon_node/fallback.rs | 68 ++++++++++++++++ crates/validator/src/beacon_node/http.rs | 85 ++++++++++++++++++++ crates/validator/src/beacon_node/mock.rs | 73 +++++++++++++++++ crates/validator/src/beacon_node/mod.rs | 22 +++++ 4 files changed, 248 insertions(+) diff --git a/crates/validator/src/beacon_node/fallback.rs b/crates/validator/src/beacon_node/fallback.rs index 04b4652e..933ea34b 100644 --- a/crates/validator/src/beacon_node/fallback.rs +++ b/crates/validator/src/beacon_node/fallback.rs @@ -256,6 +256,11 @@ impl BeaconNodeApi for FallbackBeaconNode { .await } + async fn proposer_duties_v2(&self, epoch: Epoch) -> Result { + self.try_each("proposer_duties_v2", |node| node.proposer_duties_v2(epoch)) + .await + } + async fn attestation_data(&self, slot: Slot, fork: ForkName) -> Result { self.try_each("attestation_data", |node| node.attestation_data(slot, fork)) .await @@ -342,6 +347,26 @@ impl BeaconNodeApi for FallbackBeaconNode { .await } + /// To every node, since each one gossips what it accepts: a preference + /// only the first node heard reaches the mesh through that node alone, and + /// builders subscribed elsewhere would ignore the slot. + async fn submit_proposer_preferences( + &self, + fork: ForkName, + preferences: &[gloas::SignedProposerPreferences], + ) -> Result<()> { + self.try_all( + "submit_proposer_preferences", + "it will not gossip these proposer preferences", + Some( + "no node took this client's proposer preferences; builders' bids for those \ + slots are ignored, and from gloas a node may not know the fee recipient", + ), + |node| node.submit_proposer_preferences(fork, preferences), + ) + .await + } + /// Fails over like any query: the envelope the node cached is only on the /// node that produced the block, and one that did not answers 404. async fn execution_payload_envelope( @@ -1061,6 +1086,49 @@ mod tests { .expect_err("the record is gone"); } + #[tokio::test] + async fn proposer_preferences_reach_every_node_past_a_failing_one() { + use ethlambda_types::beacon::containers::gloas::{ + ProposerPreferences, SignedProposerPreferences, + }; + + let preference = SignedProposerPreferences { + message: ProposerPreferences { + proposal_slot: 70, + ..Default::default() + }, + ..Default::default() + }; + let fallback = FallbackBeaconNode::new(vec![ + MockBeaconNode::new(), + MockBeaconNode::failing("middle down"), + MockBeaconNode::new(), + ]); + + fallback + .submit_proposer_preferences(ForkName::Fulu, std::slice::from_ref(&preference)) + .await + .expect("two of three accepted"); + + for position in [0, 2] { + assert_eq!( + *fallback.nodes[position] + .submitted_preferences + .lock() + .expect("lock"), + vec![(ForkName::Fulu, vec![preference.clone()])], + "node {position}" + ); + } + assert!( + fallback.nodes[1] + .submitted_preferences + .lock() + .expect("lock") + .is_empty() + ); + } + #[tokio::test] async fn registrations_reach_every_node() { use ethlambda_types::beacon::builder::{ diff --git a/crates/validator/src/beacon_node/http.rs b/crates/validator/src/beacon_node/http.rs index 59911989..f127acf3 100644 --- a/crates/validator/src/beacon_node/http.rs +++ b/crates/validator/src/beacon_node/http.rs @@ -627,6 +627,18 @@ impl BeaconNodeApi for HttpBeaconNode { }) } + /// [`Self::proposer_duties`] against the v2 path, whose `dependent_root` + /// is the shuffling dependent root proposer preferences must carry. + async fn proposer_duties_v2(&self, epoch: Epoch) -> Result { + let response: DutiesResponse> = self + .get(&format!("/eth/v2/validator/duties/proposer/{epoch}")) + .await?; + Ok(ProposerDuties { + dependent_root: parse_root(&response.dependent_root)?, + duties: response.data, + }) + } + async fn attestation_data(&self, slot: Slot, fork: ForkName) -> Result { let response: DataResponse = self.get(&Self::attestation_data_path(slot, fork)).await?; @@ -807,6 +819,24 @@ impl BeaconNodeApi for HttpBeaconNode { .await } + /// JSON, which every beacon node must accept, an empty list never reaching + /// the wire. The gloas container serialises as the endpoint's JSON itself. + async fn submit_proposer_preferences( + &self, + fork: ForkName, + preferences: &[gloas::SignedProposerPreferences], + ) -> Result<()> { + if preferences.is_empty() { + return Ok(()); + } + self.post_no_content( + "/eth/v1/validator/proposer_preferences", + &preferences, + Some(fork.as_str()), + ) + .await + } + async fn execution_payload_envelope( &self, slot: Slot, @@ -1469,6 +1499,61 @@ mod tests { assert!(served.seen.lock().expect("lock").is_empty()); } + #[tokio::test] + async fn proposer_preferences_are_posted_as_json_with_the_fork_header() { + use ethlambda_types::beacon::primitives::{BlsSignature, H160}; + + let preferences = vec![gloas::SignedProposerPreferences { + message: gloas::ProposerPreferences { + dependent_root: Root::from([9u8; 32]), + proposal_slot: 70, + validator_index: 4, + fee_recipient: H160([0xcd; 20]), + target_gas_limit: 36_000_000, + }, + signature: BlsSignature([5; 96]), + }]; + let served = serve(200, vec![], vec![]).await; + served + .node + .submit_proposer_preferences(ForkName::Fulu, &preferences) + .await + .expect("accepted"); + + let seen = served.seen.lock().expect("lock"); + assert_eq!(seen.len(), 1); + assert!( + seen[0] + .0 + .ends_with("/eth/v1/validator/proposer_preferences") + ); + assert_eq!(seen[0].2["eth-consensus-version"], "fulu"); + let decoded: Vec = + serde_json::from_slice(&seen[0].1).expect("the body is the endpoint's JSON"); + assert_eq!(decoded, preferences); + } + + #[tokio::test] + async fn an_empty_preference_list_never_reaches_the_wire() { + let served = serve(200, vec![], vec![]).await; + served + .node + .submit_proposer_preferences(ForkName::Fulu, &[]) + .await + .expect("a no-op"); + assert!(served.seen.lock().expect("lock").is_empty()); + } + + #[tokio::test] + async fn v2_proposer_duties_use_the_v2_path() { + let reply = br#"{"dependent_root":"0x0909090909090909090909090909090909090909090909090909090909090909","execution_optimistic":false,"data":[]}"#.to_vec(); + let served = serve(200, vec![("content-type", "application/json")], reply).await; + let duties = served.node.proposer_duties_v2(3).await.expect("decodes"); + assert_eq!(duties.dependent_root, Root::from([9u8; 32])); + let seen = served.seen.lock().expect("lock"); + assert!(seen[0].0.ends_with("/eth/v2/validator/duties/proposer/3")); + } + #[tokio::test] async fn a_blinded_publish_maps_each_status() { for (status, expected) in [ diff --git a/crates/validator/src/beacon_node/mock.rs b/crates/validator/src/beacon_node/mock.rs index 409e3aed..c5628770 100644 --- a/crates/validator/src/beacon_node/mock.rs +++ b/crates/validator/src/beacon_node/mock.rs @@ -64,6 +64,11 @@ pub struct MockBeaconNode { pub validators: Vec, pub duties: Mutex>, pub proposers: Mutex>, + /// What `proposer_duties_v2` answers with, by epoch, like `proposers`. + pub proposers_v2: Mutex>, + /// Every proposer preferences batch `submit_proposer_preferences` + /// received, with the fork it was sent under, in call order. + pub submitted_preferences: Mutex)>>, pub attestation_data: Option, /// The slot and proposer the mock will produce a block for. `None` makes /// `produce_block` fail, the way an unset `attestation_data` does. @@ -136,6 +141,7 @@ pub struct MockBeaconNode { pub duties_calls: Mutex, pub validator_indices_calls: Mutex, pub proposer_duties_calls: Mutex, + pub proposer_duties_v2_calls: Mutex, pub attestation_data_calls: Mutex, /// Sync duties by the epoch they are asked at. An epoch never set answers /// no duties, so tests that do not care about sync committees are @@ -253,6 +259,44 @@ impl MockBeaconNode { } } + /// Set the v2 proposer duties for `epoch`, replacing any earlier ones: a + /// reorg is the same epoch answering with another dependent root. + pub fn set_proposers_v2( + &self, + epoch: Epoch, + dependent_root: Root, + duties: Vec, + ) { + let mut stored = self.proposers_v2.lock().expect("lock"); + let entry = ProposerDuties { + dependent_root, + duties, + }; + match stored + .iter_mut() + .find(|(stored_epoch, _)| *stored_epoch == epoch) + { + Some((_, existing)) => *existing = entry, + None => stored.push((epoch, entry)), + } + } + + pub fn with_proposers_v2( + self, + epoch: Epoch, + dependent_root: Root, + duties: Vec, + ) -> Self { + self.set_proposers_v2(epoch, dependent_root, duties); + self + } + + /// How many times `proposer_duties_v2` has been called, successes and + /// failures alike. + pub fn proposer_duties_v2_call_count(&self) -> usize { + *self.proposer_duties_v2_calls.lock().expect("lock") + } + pub fn with_attestation_data(mut self, slot: Slot) -> Self { self.attestation_data = Some(AttestationData { slot, @@ -634,6 +678,22 @@ impl BeaconNodeApi for MockBeaconNode { }) } + async fn proposer_duties_v2(&self, epoch: Epoch) -> Result { + self.guard("proposer_duties_v2")?; + *self.proposer_duties_v2_calls.lock().expect("lock") += 1; + self.proposers_v2 + .lock() + .expect("lock") + .iter() + .find(|(stored, _)| *stored == epoch) + .map(|(_, duties)| duties.clone()) + .ok_or_else(|| Error::BeaconNode { + url: "mock".to_string(), + failure: BeaconNodeFailure::Request, + detail: format!("no v2 proposer duties for epoch {epoch}"), + }) + } + async fn attestation_data(&self, slot: Slot, fork: ForkName) -> Result { self.guard("attestation_data")?; *self.attestation_data_calls.lock().expect("lock") += 1; @@ -777,6 +837,19 @@ impl BeaconNodeApi for MockBeaconNode { Ok(()) } + async fn submit_proposer_preferences( + &self, + fork: ForkName, + preferences: &[gloas::SignedProposerPreferences], + ) -> Result<()> { + self.guard("submit_proposer_preferences")?; + self.submitted_preferences + .lock() + .expect("lock") + .push((fork, preferences.to_vec())); + Ok(()) + } + async fn execution_payload_envelope( &self, slot: Slot, diff --git a/crates/validator/src/beacon_node/mod.rs b/crates/validator/src/beacon_node/mod.rs index ed292c5f..0f6ac0ab 100644 --- a/crates/validator/src/beacon_node/mod.rs +++ b/crates/validator/src/beacon_node/mod.rs @@ -322,6 +322,13 @@ pub trait BeaconNodeApi: Send + Sync { /// treats the two the same way will act on a stale proposer schedule. async fn proposer_duties(&self, epoch: Epoch) -> Result; + /// Every proposer duty in `epoch` with v2's `dependent_root`: the block at + /// `compute_start_slot_at_epoch(epoch - 1) - 1`, the shuffling dependent + /// root fulu's proposer lookahead fixes the epoch's proposers from. Signed + /// proposer preferences must carry exactly this root, which v1's + /// definition (one epoch later) is not. + async fn proposer_duties_v2(&self, epoch: Epoch) -> Result; + /// Produce the attestation data for `slot`, which is in `fork`. /// /// No committee index is ever a parameter here. Before gloas the query @@ -416,6 +423,21 @@ pub trait BeaconNodeApi: Send + Sync { registrations: &[SignedValidatorRegistrationV1], ) -> Result<()>; + /// Submit signed proposer preferences, which the node validates with the + /// `proposer_preferences` gossip rules and publishes. `fork` is the + /// `Eth-Consensus-Version`: the fork of the current epoch. + /// + /// A 400 naming per-index failures is an `Err`: the whole call failed for + /// the caller's purposes. The caller re-submits the entries next slot, and + /// the node answers per entry, so the ones that already went out are + /// deduplicated by the node. State on the node's gossip, so failover sends + /// it to every node. + async fn submit_proposer_preferences( + &self, + fork: ForkName, + preferences: &[gloas::SignedProposerPreferences], + ) -> Result<()>; + /// The envelope the node cached for the self-built payload of the block /// whose root is `block_root`, proposed at `slot`. /// From 38d4f5e62e3555e97535d339955759ecf268098c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:10:11 -0300 Subject: [PATCH 2/9] feat(validator): sign proposer preferences under the gossip rule's domain The gossip validator verifies a preference at the fork version of the epoch before its proposal, while the validator guide signs with the head state. The two differ only in the first epoch of a fork, where the gossip rule is what peers accept, so sign for that one. --- crates/validator/src/signing.rs | 116 +++++++++++++++++++++++++++++++- 1 file changed, 113 insertions(+), 3 deletions(-) diff --git a/crates/validator/src/signing.rs b/crates/validator/src/signing.rs index cafa4099..d43a3602 100644 --- a/crates/validator/src/signing.rs +++ b/crates/validator/src/signing.rs @@ -9,14 +9,15 @@ use ethlambda_types::beacon::builder::ValidatorRegistrationV1; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::constants::{ DOMAIN_AGGREGATE_AND_PROOF, DOMAIN_BEACON_ATTESTER, DOMAIN_BEACON_BUILDER, - DOMAIN_BEACON_PROPOSER, DOMAIN_CONTRIBUTION_AND_PROOF, DOMAIN_PTC_ATTESTER, DOMAIN_RANDAO, - DOMAIN_SELECTION_PROOF, DOMAIN_SYNC_COMMITTEE, DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF, + DOMAIN_BEACON_PROPOSER, DOMAIN_CONTRIBUTION_AND_PROOF, DOMAIN_PROPOSER_PREFERENCES, + DOMAIN_PTC_ATTESTER, DOMAIN_RANDAO, DOMAIN_SELECTION_PROOF, DOMAIN_SYNC_COMMITTEE, + DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF, }; use ethlambda_types::beacon::containers::altair::{ ContributionAndProof, SyncAggregatorSelectionData, }; use ethlambda_types::beacon::containers::gloas::{ - ExecutionPayloadEnvelope, PayloadAttestationData, + ExecutionPayloadEnvelope, PayloadAttestationData, ProposerPreferences, }; use ethlambda_types::beacon::containers::shared::AttestationData; use ethlambda_types::beacon::primitives::{ @@ -25,6 +26,7 @@ use ethlambda_types::beacon::primitives::{ }; // `AttestationData`, `Checkpoint`, `Fork` and `SigningData` all live in // `containers::shared`, verified against `containers/shared.rs:158,120,97,220`. +use ethlambda_types::beacon::preset; use ethlambda_types::beacon::signing::{ compute_builder_domain, compute_domain, compute_epoch_at_slot, compute_signing_root, }; @@ -316,6 +318,31 @@ impl SigningContext { self.sign_root(store, ®istration.pubkey, root) } + /// A proposer preferences message's signature, under + /// `DOMAIN_PROPOSER_PREFERENCES` at the fork version of the epoch before + /// the proposal's. + /// + /// That is the domain the `proposer_preferences` gossip rule verifies + /// against: `get_domain(state, .., proposal_epoch)` on the dependent + /// block's state advanced to the lookahead start, whose fork is the one + /// in force an epoch before the proposal. The validator guide signs with + /// the head state instead; the two differ only in the first epoch of a + /// fork, where this one is the one peers accept. Not slashable. + pub fn sign_proposer_preferences( + &self, + store: &ValidatorStore, + pubkey: &BlsPubkey, + preferences: &ProposerPreferences, + ) -> Result { + let proposal_epoch = compute_epoch_at_slot(preferences.proposal_slot); + let domain = self.domain( + DOMAIN_PROPOSER_PREFERENCES, + proposal_epoch.saturating_sub(preset::MIN_SEED_LOOKAHEAD), + ); + let root = compute_signing_root(preferences.hash_tree_root(), domain); + self.sign_root(store, pubkey, root) + } + /// Sign a payload timeliness committee vote on behalf of `pubkey`. /// /// Not slashable by the specification, and deduplicated per validator and @@ -590,6 +617,89 @@ mod tests { assert!(!verify(&pubkey, &signature, envelope_root)); } + /// A config whose gloas fork starts at epoch 500000, fulu being active + /// well before it, with a non-zero validators root. + fn gloas_scheduled_context() -> (SigningContext, Epoch) { + const GLOAS_EPOCH: Epoch = 500_000; + let mut config = Config::mainnet(); + config.gloas_fork_epoch = GLOAS_EPOCH; + ( + SigningContext { + config, + genesis_validators_root: Root::repeat_byte(0x42), + }, + GLOAS_EPOCH, + ) + } + + fn preferences_for(slot: Slot) -> ProposerPreferences { + ProposerPreferences { + dependent_root: Root::repeat_byte(0x11), + proposal_slot: slot, + validator_index: 3, + fee_recipient: ethlambda_types::beacon::primitives::H160([0xab; 20]), + target_gas_limit: 36_000_000, + } + } + + /// The first gloas epoch's preference is signed at the fork of the epoch + /// before it (fulu), which is what the gossip rule verifies against, and + /// not under the gloas version the validator guide's head-state signer + /// would use. + #[test] + fn a_preference_for_the_first_gloas_epoch_is_signed_under_the_fulu_version() { + let (store, pubkey) = store_with_key(); + let (context, gloas_epoch) = gloas_scheduled_context(); + let preferences = preferences_for(gloas_epoch * preset::SLOTS_PER_EPOCH + 2); + + let signature = context + .sign_proposer_preferences(&store, &pubkey, &preferences) + .expect("signs"); + + let under = |version| { + compute_signing_root( + preferences.hash_tree_root(), + compute_domain( + DOMAIN_PROPOSER_PREFERENCES, + version, + context.genesis_validators_root, + ), + ) + }; + assert!(verify( + &pubkey, + &signature, + under(context.config.fulu_fork_version) + )); + assert!(!verify( + &pubkey, + &signature, + under(context.config.gloas_fork_version) + )); + } + + /// One epoch later the epoch before the proposal's is itself gloas. + #[test] + fn a_preference_for_the_second_gloas_epoch_is_signed_under_the_gloas_version() { + let (store, pubkey) = store_with_key(); + let (context, gloas_epoch) = gloas_scheduled_context(); + let preferences = preferences_for((gloas_epoch + 1) * preset::SLOTS_PER_EPOCH); + + let signature = context + .sign_proposer_preferences(&store, &pubkey, &preferences) + .expect("signs"); + + let root = compute_signing_root( + preferences.hash_tree_root(), + compute_domain( + DOMAIN_PROPOSER_PREFERENCES, + context.config.gloas_fork_version, + context.genesis_validators_root, + ), + ); + assert!(verify(&pubkey, &signature, root)); + } + #[test] fn a_randao_reveal_verifies_under_the_randao_signing_root() { let (store, pubkey) = store_with_key(); From 231c602f9b1df047b7d4abb22b0455f82dc065d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:12:16 -0300 Subject: [PATCH 3/9] feat(validator): publish proposer preferences for gloas proposals From gloas a proposer's fee recipient and gas limit travel in signed proposer preferences, and a node ignores trustless bids for a slot whose proposer published none. A per-slot service signs one per owned proposal in the current and next gloas epoch, keyed by (slot, dependent root) so a reorg re-publishes, and sends it to every node. --- crates/validator/src/lib.rs | 1 + crates/validator/src/metrics.rs | 31 ++ crates/validator/src/preferences.rs | 610 ++++++++++++++++++++++++++++ 3 files changed, 642 insertions(+) create mode 100644 crates/validator/src/preferences.rs diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index 558991d3..658d3792 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -40,6 +40,7 @@ pub mod http_api; pub mod keys; pub mod metrics; pub mod payload_attestation; +pub mod preferences; pub mod proposal; pub mod proposal_guard; pub mod proposer_settings; diff --git a/crates/validator/src/metrics.rs b/crates/validator/src/metrics.rs index 64d7ca0c..92fc44b2 100644 --- a/crates/validator/src/metrics.rs +++ b/crates/validator/src/metrics.rs @@ -134,6 +134,27 @@ static BUILDER_REGISTRATION_FAILURES_TOTAL: LazyLock = LazyLock::new .unwrap() }); +/// Proposer preferences a beacon node accepted. Counted per preference, so a +/// full epoch's worth reads as that many. +static PROPOSER_PREFERENCES_PUBLISHED_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter!( + "ethlambda_validator_proposer_preferences_published_total", + "Proposer preferences a beacon node accepted" + ) + .unwrap() +}); + +/// Proposer preference batches no beacon node accepted. The next slot retries, +/// so this is a warning sign: until one is accepted, builders' bids for the +/// slot are ignored. +static PROPOSER_PREFERENCES_FAILURES_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter!( + "ethlambda_validator_proposer_preferences_failures_total", + "Proposer preference batches no beacon node accepted" + ) + .unwrap() +}); + /// Blinded blocks a beacon node handed to its builder. static BLINDED_BLOCKS_PROPOSED_TOTAL: LazyLock = LazyLock::new(|| { register_int_counter!( @@ -358,6 +379,8 @@ pub fn init() { LazyLock::force(&BLOCKS_BROADCAST_NOT_IMPORTED_TOTAL); LazyLock::force(&BUILDER_REGISTRATIONS_SENT_TOTAL); LazyLock::force(&BUILDER_REGISTRATION_FAILURES_TOTAL); + LazyLock::force(&PROPOSER_PREFERENCES_PUBLISHED_TOTAL); + LazyLock::force(&PROPOSER_PREFERENCES_FAILURES_TOTAL); LazyLock::force(&BLINDED_BLOCKS_PROPOSED_TOTAL); LazyLock::force(&BLINDED_BLOCKS_UNCONFIRMED_TOTAL); LazyLock::force(&BLOCKS_REFUSED_TOTAL); @@ -428,6 +451,14 @@ pub fn inc_builder_registration_failures() { BUILDER_REGISTRATION_FAILURES_TOTAL.inc(); } +pub fn inc_proposer_preferences_published(count: usize) { + PROPOSER_PREFERENCES_PUBLISHED_TOTAL.inc_by(count as u64); +} + +pub fn inc_proposer_preferences_failures() { + PROPOSER_PREFERENCES_FAILURES_TOTAL.inc(); +} + pub fn inc_blinded_blocks_proposed() { BLINDED_BLOCKS_PROPOSED_TOTAL.inc(); } diff --git a/crates/validator/src/preferences.rs b/crates/validator/src/preferences.rs new file mode 100644 index 00000000..667a8887 --- /dev/null +++ b/crates/validator/src/preferences.rs @@ -0,0 +1,610 @@ +//! Signed proposer preferences: how a validator tells builders, and from +//! gloas its own beacon node, where its proposals should pay and which gas +//! limit to target. +//! +//! # Why it exists +//! +//! From gloas, beacon-APIs makes a signed `ProposerPreferences` the source of a +//! proposer's fee recipient and gas limit, superseding `prepare_beacon_proposer`. +//! It is also the trustless-bid channel: a node ignores p2p builder bids for a +//! slot whose proposer published no preferences, so a validator that stays +//! silent self-builds every gloas proposal. +//! +//! # Why v2 proposer duties +//! +//! A preference names the `dependent_root` its duty was computed from, and the +//! gossip rule compares it with the node's own shuffling dependent root. Fulu's +//! proposer lookahead fixes an epoch's proposers one epoch earlier than v1's +//! definition says, so only v2's `dependent_root` is the one a preference must +//! carry. The duty loop keeps its own v1 schedule; this service fetches v2 for +//! itself. +//! +//! # The signing domain +//! +//! See [`SigningContext::sign_proposer_preferences`]: the gossip rule's domain, +//! at the fork of the epoch before the proposal's, which is not the validator +//! guide's head-state one in the first epoch of a fork. +//! +//! # Why every slot +//! +//! [`PreferencesService::poll`] runs each slot rather than each epoch. A reorg +//! changes a dependent root, which makes the old preference unusable and needs +//! a new one; a validator can become active, or be given a fee recipient, +//! mid-epoch; and a node that refused a send must be tried again. What was +//! already accepted is remembered per `(slot, dependent_root)`, so a quiet +//! slot costs the two duty fetches and no signing. +//! +//! # Why every node gets them +//! +//! Each node gossips what it accepts, so the fallback client sends to all of +//! them: a preference only one node heard reaches builders through that node's +//! mesh alone. +//! +//! # No replacement +//! +//! A node deduplicates on `(slot, dependent_root)`, so a preference cannot be +//! replaced for the same slot and root. A fee recipient changed after a +//! preference went out applies from the next dependent root onward only. +//! +//! # Failures are never propagated +//! +//! A missed preference costs a builder bid and, from gloas, possibly the +//! node's knowledge of the fee recipient, never a duty. A failed send is logged +//! and counted, and the next slot retries. + +use std::collections::HashSet; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use ethlambda_types::beacon::containers::gloas::{ProposerPreferences, SignedProposerPreferences}; +use ethlambda_types::beacon::fork::ForkName; +use ethlambda_types::beacon::preset; +use ethlambda_types::beacon::primitives::{BlsPubkey, Epoch, Root, Slot}; +use ethlambda_types::beacon::signing::compute_epoch_at_slot; +use tokio::sync::RwLock; +use tracing::{debug, info, warn}; + +use crate::beacon_node::BeaconNodeApi; +use crate::beacon_node::dto::{encode_hex, parse_pubkey}; +use crate::keys::ValidatorStore; +use crate::proposer_settings::ProposerSettings; +use crate::signing::SigningContext; + +/// Clears [`PreferencesService`]'s `in_flight` flag on every exit from a poll, +/// a panic included, so one failed poll cannot stop every later one. +struct InFlightGuard<'a>(&'a AtomicBool); + +impl Drop for InFlightGuard<'_> { + fn drop(&mut self) { + self.0.store(false, Ordering::Release); + } +} + +/// Signs and publishes proposer preferences. +pub struct PreferencesService { + beacon_node: Arc, + context: Arc, + settings: Arc, + /// `(proposal_slot, dependent_root)` pairs a node accepted, so a + /// preference is signed and sent once per dependent root. A reorg that + /// moves the root sends the slot again; pruned below the current epoch. + published: std::sync::Mutex>, + /// Epochs whose missing fee recipients were already warned about, so the + /// warning is once per epoch rather than once per slot. Pruned like + /// `published`. + warned_epochs: std::sync::Mutex>, + /// One poll at a time: a slow node must not stack polls up. + in_flight: AtomicBool, +} + +impl PreferencesService { + pub fn new( + beacon_node: Arc, + context: Arc, + settings: Arc, + ) -> Self { + Self { + beacon_node, + context, + settings, + published: std::sync::Mutex::new(HashSet::new()), + warned_epochs: std::sync::Mutex::new(HashSet::new()), + in_flight: AtomicBool::new(false), + } + } + + /// Publish what this client's validators still owe for the current and the + /// next epoch, as of `current_slot`. Called once per slot, off the slot + /// loop, so a slow node cannot delay the next slot's duties. + /// + /// Returns at once when the previous poll is still running. + pub async fn poll(self: Arc, current_slot: Slot, store: Arc>) { + if self.in_flight.swap(true, Ordering::AcqRel) { + return; + } + let _guard = InFlightGuard(&self.in_flight); + + let epoch = compute_epoch_at_slot(current_slot); + // Snapshotted with the guard dropped at once: nothing below awaits a + // node while holding the store. + let pubkeys: HashSet = store.read().await.pubkeys().into_iter().collect(); + let fork = self.context.config.fork_at_epoch(epoch); + + for target in [epoch, epoch + 1] { + if self.context.config.fork_at_epoch(target) < ForkName::Gloas { + continue; + } + self.publish_epoch(fork, current_slot, target, &pubkeys, &store) + .await; + } + + let epoch_start = epoch * preset::SLOTS_PER_EPOCH; + self.published + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .retain(|(slot, _)| *slot >= epoch_start); + self.warned_epochs + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .retain(|warned| *warned >= epoch); + } + + /// One target epoch: fetch its v2 duties, sign what is still owed and send it. + async fn publish_epoch( + &self, + fork: ForkName, + current_slot: Slot, + target: Epoch, + pubkeys: &HashSet, + store: &Arc>, + ) { + let duties = match self.beacon_node.proposer_duties_v2(target).await { + Ok(duties) => duties, + Err(err) => { + debug!(epoch = target, %err, "Could not fetch proposer duties for preferences"); + return; + } + }; + + let mut owed = Vec::new(); + let mut no_fee_recipient = 0usize; + for duty in &duties.duties { + // Started slots are skipped: gossip would IGNORE them. + if duty.slot <= current_slot { + continue; + } + let Ok(pubkey) = parse_pubkey(&duty.pubkey) else { + continue; + }; + if !pubkeys.contains(&pubkey) { + continue; + } + let already = self + .published + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .contains(&(duty.slot, duties.dependent_root)); + if already { + continue; + } + let Some(fee_recipient) = self.settings.fee_recipient(&pubkey) else { + no_fee_recipient += 1; + continue; + }; + owed.push(( + pubkey, + ProposerPreferences { + dependent_root: duties.dependent_root, + proposal_slot: duty.slot, + validator_index: duty.validator_index, + fee_recipient, + target_gas_limit: self.settings.gas_limit(&pubkey), + }, + )); + } + + if no_fee_recipient > 0 { + let first_warning = self + .warned_epochs + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(target); + if first_warning { + warn!( + epoch = target, + skipped = no_fee_recipient, + "Validators without a fee recipient publish no proposer preferences" + ); + } + } + if owed.is_empty() { + return; + } + + // At most a few per epoch, so signing inline is cheap; the read guard + // is released before the submit. + let mut signed = Vec::with_capacity(owed.len()); + { + let store = store.read().await; + for (pubkey, message) in owed { + match self + .context + .sign_proposer_preferences(&store, &pubkey, &message) + { + Ok(signature) => signed.push(SignedProposerPreferences { message, signature }), + Err(err) => { + warn!( + pubkey = %encode_hex(&pubkey.0), + slot = message.proposal_slot, + %err, + "Could not sign proposer preferences; skipping them" + ); + } + } + } + } + if signed.is_empty() { + return; + } + + match self + .beacon_node + .submit_proposer_preferences(fork, &signed) + .await + { + Ok(()) => { + let count = signed.len(); + self.published + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .extend( + signed.iter().map(|entry| { + (entry.message.proposal_slot, entry.message.dependent_root) + }), + ); + crate::metrics::inc_proposer_preferences_published(count); + info!(epoch = target, count, "Proposer preferences published"); + } + Err(err) => { + warn!(epoch = target, count = signed.len(), %err, "No beacon node accepted the proposer preferences"); + crate::metrics::inc_proposer_preferences_failures(); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::beacon_node::dto::ProposerDutyDto; + use crate::beacon_node::mock::MockBeaconNode; + use ethlambda_types::beacon::config::Config; + use ethlambda_types::beacon::constants::DOMAIN_PROPOSER_PREFERENCES; + use ethlambda_types::beacon::primitives::{Bytes32, H160, HashTreeRoot as _}; + use ethlambda_types::beacon::signing::{compute_domain, compute_signing_root}; + + const GLOAS_EPOCH: Epoch = 500_000; + const FEE_RECIPIENT: H160 = H160([0xab; 20]); + + fn context() -> Arc { + let mut config = Config::mainnet(); + config.gloas_fork_epoch = GLOAS_EPOCH; + Arc::new(SigningContext { + config, + genesis_validators_root: Root::repeat_byte(0x42), + }) + } + + /// `count` distinct keys. The secrets are small integers, which are valid + /// BLS scalars, so no keystore is needed. + fn store_with_keys(count: usize) -> (ValidatorStore, Vec) { + let mut store = ValidatorStore::new(); + let pubkeys = (0..count) + .map(|index| { + let mut secret = [0u8; 32]; + secret[24..].copy_from_slice(&(index as u64 + 1).to_be_bytes()); + store.insert_secret("test", &secret).expect("inserts") + }) + .collect(); + (store, pubkeys) + } + + fn duty(pubkey: &BlsPubkey, validator_index: u64, slot: Slot) -> ProposerDutyDto { + ProposerDutyDto { + pubkey: encode_hex(&pubkey.0), + validator_index, + slot, + } + } + + fn settings(fee_recipient: Option) -> Arc { + Arc::new(ProposerSettings::new(Bytes32::ZERO, fee_recipient)) + } + + fn service( + node: MockBeaconNode, + settings: Arc, + ) -> (Arc>, Arc) { + let node = Arc::new(node); + ( + Arc::new(PreferencesService::new(node.clone(), context(), settings)), + node, + ) + } + + fn slot_in(epoch: Epoch, offset: Slot) -> Slot { + epoch * preset::SLOTS_PER_EPOCH + offset + } + + fn submitted(node: &MockBeaconNode) -> Vec<(ForkName, Vec)> { + node.submitted_preferences.lock().expect("lock").clone() + } + + fn verifies(pubkey: &BlsPubkey, entry: &SignedProposerPreferences) -> bool { + use blst::min_pk::{PublicKey, Signature}; + + let context = context(); + let proposal_epoch = compute_epoch_at_slot(entry.message.proposal_slot); + let domain = context.domain(DOMAIN_PROPOSER_PREFERENCES, proposal_epoch - 1); + assert_eq!( + domain, + compute_domain( + DOMAIN_PROPOSER_PREFERENCES, + context + .config + .fork_version(context.config.fork_at_epoch(proposal_epoch - 1)), + context.genesis_validators_root, + ) + ); + let root = compute_signing_root(entry.message.hash_tree_root(), domain); + let pk = PublicKey::from_bytes(&pubkey.0).expect("valid pubkey"); + let sig = Signature::from_bytes(&entry.signature.0).expect("valid signature"); + sig.verify( + true, + root.as_slice(), + b"BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_", + &[], + &pk, + true, + ) == blst::BLST_ERROR::BLST_SUCCESS + } + + #[tokio::test] + async fn publishes_for_this_clients_gloas_duties_in_the_current_and_next_epoch() { + let (store, pubkeys) = store_with_keys(1); + let store = Arc::new(RwLock::new(store)); + let settings = settings(Some(FEE_RECIPIENT)); + settings.set_gas_limit(&pubkeys[0], 31_000_000); + let current = slot_in(GLOAS_EPOCH, 1); + let node = MockBeaconNode::new() + .with_proposers_v2( + GLOAS_EPOCH, + Root::repeat_byte(0x11), + vec![duty(&pubkeys[0], 3, slot_in(GLOAS_EPOCH, 10))], + ) + .with_proposers_v2( + GLOAS_EPOCH + 1, + Root::repeat_byte(0x22), + vec![duty(&pubkeys[0], 3, slot_in(GLOAS_EPOCH + 1, 4))], + ); + let (service, node) = service(node, settings); + + service.poll(current, store).await; + + let sent = submitted(&node); + assert_eq!(sent.len(), 2, "one send per target epoch"); + let mut entries: Vec<_> = sent + .iter() + .inspect(|(fork, _)| assert_eq!(*fork, ForkName::Gloas)) + .flat_map(|(_, entries)| entries.clone()) + .collect(); + entries.sort_by_key(|entry| entry.message.proposal_slot); + assert_eq!(entries.len(), 2); + let expected = [ + (slot_in(GLOAS_EPOCH, 10), Root::repeat_byte(0x11)), + (slot_in(GLOAS_EPOCH + 1, 4), Root::repeat_byte(0x22)), + ]; + for (entry, (slot, root)) in entries.iter().zip(expected) { + assert_eq!(entry.message.proposal_slot, slot); + assert_eq!(entry.message.dependent_root, root); + assert_eq!(entry.message.validator_index, 3); + assert_eq!(entry.message.fee_recipient, FEE_RECIPIENT); + assert_eq!(entry.message.target_gas_limit, 31_000_000); + assert!(verifies(&pubkeys[0], entry), "slot {slot}"); + } + } + + #[tokio::test] + async fn a_second_poll_does_not_resubmit() { + let (store, pubkeys) = store_with_keys(1); + let store = Arc::new(RwLock::new(store)); + let node = MockBeaconNode::new().with_proposers_v2( + GLOAS_EPOCH, + Root::repeat_byte(0x11), + vec![duty(&pubkeys[0], 3, slot_in(GLOAS_EPOCH, 10))], + ); + let (service, node) = service(node, settings(Some(FEE_RECIPIENT))); + let current = slot_in(GLOAS_EPOCH, 1); + + service.clone().poll(current, store.clone()).await; + service.poll(current + 1, store).await; + + assert_eq!(submitted(&node).len(), 1); + } + + #[tokio::test] + async fn a_changed_dependent_root_resubmits() { + let (store, pubkeys) = store_with_keys(1); + let store = Arc::new(RwLock::new(store)); + let slot = slot_in(GLOAS_EPOCH, 10); + let node = MockBeaconNode::new().with_proposers_v2( + GLOAS_EPOCH, + Root::repeat_byte(0x11), + vec![duty(&pubkeys[0], 3, slot)], + ); + let (service, node) = service(node, settings(Some(FEE_RECIPIENT))); + let current = slot_in(GLOAS_EPOCH, 1); + + service.clone().poll(current, store.clone()).await; + node.set_proposers_v2( + GLOAS_EPOCH, + Root::repeat_byte(0x33), + vec![duty(&pubkeys[0], 3, slot)], + ); + service.poll(current + 1, store).await; + + let sent = submitted(&node); + assert_eq!(sent.len(), 2); + assert_eq!(sent[1].1[0].message.dependent_root, Root::repeat_byte(0x33)); + assert!(verifies(&pubkeys[0], &sent[1].1[0])); + } + + #[tokio::test] + async fn pre_gloas_epochs_are_skipped() { + let (store, pubkeys) = store_with_keys(1); + let store = Arc::new(RwLock::new(store)); + let node = MockBeaconNode::new().with_proposers_v2( + GLOAS_EPOCH, + Root::repeat_byte(0x11), + vec![duty(&pubkeys[0], 3, slot_in(GLOAS_EPOCH, 10))], + ); + let (service, node) = service(node, settings(Some(FEE_RECIPIENT))); + + // The epoch before gloas: only the next, gloas, epoch is owed. + service + .clone() + .poll(slot_in(GLOAS_EPOCH - 1, 1), store.clone()) + .await; + let sent = submitted(&node); + assert_eq!(sent.len(), 1); + assert_eq!(sent[0].0, ForkName::Fulu, "the current epoch's fork"); + assert_eq!(sent[0].1[0].message.proposal_slot, slot_in(GLOAS_EPOCH, 10)); + assert_eq!(node.proposer_duties_v2_call_count(), 1); + + // Two epochs before: nothing is submitted and no v2 call is made. + let (service, node) = service_for_two_epochs_before(&pubkeys); + service.poll(slot_in(GLOAS_EPOCH - 2, 1), store).await; + assert!(submitted(&node).is_empty()); + assert_eq!(node.proposer_duties_v2_call_count(), 0); + } + + fn service_for_two_epochs_before( + pubkeys: &[BlsPubkey], + ) -> (Arc>, Arc) { + let node = MockBeaconNode::new().with_proposers_v2( + GLOAS_EPOCH - 1, + Root::repeat_byte(0x11), + vec![duty(&pubkeys[0], 3, slot_in(GLOAS_EPOCH - 1, 10))], + ); + service(node, settings(Some(FEE_RECIPIENT))) + } + + #[tokio::test] + async fn started_slots_and_other_validators_are_skipped() { + let (store, pubkeys) = store_with_keys(2); + // The store holds only the first key; the second is another client's. + let (held, _) = store_with_keys(1); + let held = Arc::new(RwLock::new(held)); + let current = slot_in(GLOAS_EPOCH, 5); + let node = MockBeaconNode::new().with_proposers_v2( + GLOAS_EPOCH, + Root::repeat_byte(0x11), + vec![ + duty(&pubkeys[0], 3, current), + duty(&pubkeys[0], 3, current - 1), + duty(&pubkeys[1], 4, slot_in(GLOAS_EPOCH, 10)), + duty(&pubkeys[0], 3, slot_in(GLOAS_EPOCH, 12)), + ], + ); + drop(store); + let (service, node) = service(node, settings(Some(FEE_RECIPIENT))); + + service.poll(current, held).await; + + let sent = submitted(&node); + assert_eq!(sent.len(), 1); + assert_eq!(sent[0].1.len(), 1); + assert_eq!(sent[0].1[0].message.proposal_slot, slot_in(GLOAS_EPOCH, 12)); + assert_eq!(sent[0].1[0].message.validator_index, 3); + } + + #[tokio::test] + async fn a_validator_without_a_fee_recipient_is_skipped() { + let (store, pubkeys) = store_with_keys(2); + let store = Arc::new(RwLock::new(store)); + let settings = settings(None); + settings.set_fee_recipient(&pubkeys[1], FEE_RECIPIENT); + let node = MockBeaconNode::new().with_proposers_v2( + GLOAS_EPOCH, + Root::repeat_byte(0x11), + vec![ + duty(&pubkeys[0], 3, slot_in(GLOAS_EPOCH, 10)), + duty(&pubkeys[1], 4, slot_in(GLOAS_EPOCH, 11)), + ], + ); + let (service, node) = service(node, settings); + + service.poll(slot_in(GLOAS_EPOCH, 1), store).await; + + let sent = submitted(&node); + assert_eq!(sent.len(), 1); + assert_eq!(sent[0].1.len(), 1); + assert_eq!(sent[0].1[0].message.validator_index, 4); + } + + #[tokio::test] + async fn a_failed_submit_is_retried_next_poll() { + let (store, pubkeys) = store_with_keys(1); + let store = Arc::new(RwLock::new(store)); + let node = MockBeaconNode::new().with_proposers_v2( + GLOAS_EPOCH, + Root::repeat_byte(0x11), + vec![duty(&pubkeys[0], 3, slot_in(GLOAS_EPOCH, 10))], + ); + node.failures + .lock() + .expect("lock") + .insert("submit_proposer_preferences", "node refused".to_string()); + let (service, node) = service(node, settings(Some(FEE_RECIPIENT))); + let current = slot_in(GLOAS_EPOCH, 1); + + service.clone().poll(current, store.clone()).await; + assert!(submitted(&node).is_empty()); + + node.failures.lock().expect("lock").clear(); + service.poll(current + 1, store).await; + assert_eq!(submitted(&node).len(), 1, "sent once the node recovered"); + } + + #[tokio::test] + async fn overlapping_polls_do_not_run_twice() { + let (store, pubkeys) = store_with_keys(1); + let store = Arc::new(RwLock::new(store)); + let node = MockBeaconNode::new().with_proposers_v2( + GLOAS_EPOCH, + Root::repeat_byte(0x11), + vec![duty(&pubkeys[0], 3, slot_in(GLOAS_EPOCH, 10))], + ); + let (service, node) = service(node, settings(Some(FEE_RECIPIENT))); + + service.in_flight.store(true, Ordering::Release); + service + .clone() + .poll(slot_in(GLOAS_EPOCH, 1), store.clone()) + .await; + assert!(submitted(&node).is_empty()); + assert_eq!(node.proposer_duties_v2_call_count(), 0); + assert!( + service.in_flight.load(Ordering::Acquire), + "the running poll's flag is not cleared by the one that bounced" + ); + + service.in_flight.store(false, Ordering::Release); + service.clone().poll(slot_in(GLOAS_EPOCH, 1), store).await; + assert_eq!(submitted(&node).len(), 1); + assert!( + !service.in_flight.load(Ordering::Acquire), + "cleared after a poll" + ); + } +} From b3562ff81b4f1755a61c631ed45424b588fd95ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:14:15 -0300 Subject: [PATCH 4/9] feat(validator): publish proposer preferences every slot and weigh gloas bids by the boost factor Wire the preferences service into the slot loop, unconditionally since from gloas they carry every proposal's fee recipient. produceBlockV4 now sends the configured builder_boost_factor in its body, so with --builder-proposals a pooled p2p bid can win on value; without it the factor stays 0 as before. --- crates/validator/src/beacon_node/http.rs | 57 +++++++++++++++++++----- crates/validator/src/beacon_node/mod.rs | 5 ++- crates/validator/src/lib.rs | 13 ++++++ 3 files changed, 61 insertions(+), 14 deletions(-) diff --git a/crates/validator/src/beacon_node/http.rs b/crates/validator/src/beacon_node/http.rs index f127acf3..453248c8 100644 --- a/crates/validator/src/beacon_node/http.rs +++ b/crates/validator/src/beacon_node/http.rs @@ -79,10 +79,17 @@ struct SszResponse { body: Vec, } -/// The body `produceBlockV4` requires. Self-build only, so no minimum bid, no -/// boost and no builders: the node decodes it, as the specification demands, -/// and builds locally. -const BUILDER_CONFIG: &str = r#"{"min_bid":"0","builder_boost_factor":"0","builders":[]}"#; +/// The body `produceBlockV4` requires: the node decodes it, as the +/// specification demands, even when it builds locally. +/// +/// `min_bid` stays `0` and `builders` stays empty, since the builder API +/// (relay or `builder_pubkeys` bids) is a later phase. The factor is the +/// configured `builder_boost_factor` with `--builder-proposals`, so a pooled +/// p2p bid can win on value, and `0` without it: self-build only, as before. +fn builder_config(boost_factor: Option) -> String { + let factor = boost_factor.unwrap_or(0); + format!(r#"{{"min_bid":"0","builder_boost_factor":"{factor}","builders":[]}}"#) +} /// A [`BeaconNodeApi`] backed by one beacon node's standard REST Beacon API. pub struct HttpBeaconNode { @@ -256,7 +263,7 @@ impl HttpBeaconNode { async fn post_json_for_ssz( &self, path: &str, - json_body: &'static str, + json_body: String, consensus_version: &str, ) -> Result { let url = format!("{}{path}", self.base_url); @@ -393,7 +400,11 @@ impl HttpBeaconNode { encode_hex(&request.graffiti.0), ); let response = self - .post_json_for_ssz(&path, BUILDER_CONFIG, request.fork.as_str()) + .post_json_for_ssz( + &path, + builder_config(request.builder_boost_factor), + request.fork.as_str(), + ) .await?; if response.fork != ForkName::Gloas { return Err(Error::InconsistentResponse(format!( @@ -1330,13 +1341,35 @@ mod tests { ); } - /// The body `produceBlockV4` requires: self-build only, so no builders. + /// The body `produceBlockV4` requires: no minimum bid and no builders, + /// the boost factor being the configured one or `0`. #[test] - fn the_builder_config_is_the_documented_self_build_body() { - let value: serde_json::Value = serde_json::from_str(BUILDER_CONFIG).expect("valid json"); - assert_eq!(value["min_bid"], "0"); - assert_eq!(value["builder_boost_factor"], "0"); - assert_eq!(value["builders"], serde_json::json!([])); + fn the_builder_config_carries_the_factor_or_zero() { + for (factor, expected) in [(None, "0"), (Some(250), "250"), (Some(0), "0")] { + let value: serde_json::Value = + serde_json::from_str(&builder_config(factor)).expect("valid json"); + assert_eq!(value["min_bid"], "0"); + assert_eq!(value["builder_boost_factor"], expected, "{factor:?}"); + assert_eq!(value["builders"], serde_json::json!([])); + } + } + + /// The body reaches the wire: a gloas production request posts it. + #[tokio::test] + async fn a_gloas_production_request_posts_the_factor_in_its_body() { + for (factor, expected) in [(None, "0"), (Some(250), "250")] { + let served = serve(500, vec![], vec![]).await; + let request = BlockRequest { + fork: ForkName::Gloas, + ..block_request(factor) + }; + let _ = served.node.produce_block(&request).await; + let seen = served.seen.lock().expect("lock"); + assert_eq!(seen.len(), 1); + assert!(seen[0].0.contains("/eth/v4/validator/blocks/5")); + let body: serde_json::Value = serde_json::from_slice(&seen[0].1).expect("json body"); + assert_eq!(body["builder_boost_factor"], expected); + } } /// A Beacon API double that answers every route the same way: whatever diff --git a/crates/validator/src/beacon_node/mod.rs b/crates/validator/src/beacon_node/mod.rs index 0f6ac0ab..5c8c189d 100644 --- a/crates/validator/src/beacon_node/mod.rs +++ b/crates/validator/src/beacon_node/mod.rs @@ -133,8 +133,9 @@ pub struct BlockRequest { pub graffiti: Bytes32, /// The `builder_boost_factor` to send, or `None` when builder proposals are /// off. `None` sends `0` and refuses a blinded answer; `Some` sends the - /// factor and accepts one. Ignored from gloas, where `produceBlockV4` takes - /// the builder configuration in its body. + /// factor and accepts one. From gloas the factor travels in + /// `produceBlockV4`'s body instead of the query, so a pooled p2p bid can + /// win on value; a gloas answer is never blinded. pub builder_boost_factor: Option, } diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index 658d3792..05d832b9 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -70,6 +70,7 @@ use crate::beacon_node::{BeaconNodeApi, Published}; use crate::duties::DutiesService; use crate::keys::ValidatorStore; use crate::payload_attestation::PayloadAttestationService; +use crate::preferences::PreferencesService; use crate::proposal::ProposalService; use crate::proposer_settings::ProposerSettings; use crate::registration::RegistrationService; @@ -225,6 +226,14 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { )) }); + // Unconditional, as Lighthouse does: from gloas the preferences carry the + // fee recipient of every proposal, builders or not. See the module doc. + let preferences = Arc::new(PreferencesService::new( + beacon_node.clone(), + context.clone(), + settings.clone(), + )); + if let Some(address) = config.keymanager { let token = http_api::load_or_create_token(&config.validators_dir)?; let context = http_api::KeymanagerContext { @@ -371,6 +380,10 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { if let Some(registration) = ®istration { tokio::spawn(registration.clone().register_changed(epoch, store.clone())); } + + // Likewise off the slot loop: the poll is single-flight, and a slow + // node must not delay the next slot's duties. + tokio::spawn(preferences.clone().poll(slot, store.clone())); } } From 1960d86214f63123f2669e928ff946a9173b81ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:15:45 -0300 Subject: [PATCH 5/9] test(rpc): the validator client's proposer preferences pass this node's gossip rules The signing domain is the part of the preferences flow most likely to drift between the signer and the verifier, so drive the validator client's own signer and HTTP client against the real route and gossip rules. --- .../rpc/src/beacon/builder_market_tests.rs | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/crates/net/rpc/src/beacon/builder_market_tests.rs b/crates/net/rpc/src/beacon/builder_market_tests.rs index c2876290..638e6a94 100644 --- a/crates/net/rpc/src/beacon/builder_market_tests.rs +++ b/crates/net/rpc/src/beacon/builder_market_tests.rs @@ -1061,6 +1061,74 @@ async fn valid_preferences_are_cached_and_published_once() { assert_eq!(world.network.proposer_preferences.lock().unwrap().len(), 1); } +/// The validator client's own signer and HTTP client against this node's +/// gossip rules: the domain `sign_proposer_preferences` picks (the fork of the +/// epoch before the proposal's) is the one `validate_proposer_preferences_gossip` +/// verifies under, and the JSON the client posts is what the route decodes. +/// +/// The fork-boundary case (a preference for the first gloas epoch, signed +/// under the fulu version) is not driven here: this world's schedule has gloas +/// at epoch 0 and a second world in the epoch before the fork would be a new +/// chain fixture. `signing.rs` pins that domain in a unit test. +#[tokio::test] +async fn preferences_signed_by_the_validator_client_pass_this_nodes_gossip_rules() { + use ethlambda_validator::{ + beacon_node::{BeaconNodeApi as _, http::HttpBeaconNode}, + keys::ValidatorStore, + signing::SigningContext, + }; + + let world = World::new(); + let proposer = world.prepared().proposer; + let dependent_root = dependent_root_at(&world.state, world.head_root, SLOT) + .expect("the head's chain gives the slot a dependent root"); + let message = ProposerPreferences { + dependent_root, + proposal_slot: SLOT, + validator_index: proposer, + fee_recipient: address(0xaa), + target_gas_limit: 30_000_000, + }; + + let mut keys = ValidatorStore::new(); + let pubkey = keys + .insert_secret("test", &secret_key_for(proposer as usize).to_bytes()) + .expect("the test secret is a valid key"); + assert_eq!( + pubkey, + world.state.validator(proposer).unwrap().pubkey, + "the test registry's key for the proposer" + ); + let context = SigningContext { + config: chain_config(), + genesis_validators_root: world.state.genesis_validators_root(), + }; + let signature = context + .sign_proposer_preferences(&keys, &pubkey, &message) + .expect("signs"); + let signed = SignedProposerPreferences { message, signature }; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let router = world.app(None); + tokio::spawn(async move { axum::serve(listener, router).await }); + let client = HttpBeaconNode::new(format!("http://{address}")).unwrap(); + + client + .submit_proposer_preferences( + ethlambda_types::beacon::fork::ForkName::Gloas, + std::slice::from_ref(&signed), + ) + .await + .expect("the node accepts what the validator client signed"); + + assert_eq!( + *world.network.proposer_preferences.lock().unwrap(), + vec![signed.clone()] + ); + assert_eq!(world.market.preferences(SLOT, dependent_root), Some(signed)); +} + #[tokio::test] async fn cached_preferences_resubmitted_are_a_200_without_a_publish() { // Needs only the market: an identical entry short-circuits the rules. From 1e0e2f371754fbb60736c8e4bce56709d5dbd1d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:17:04 -0300 Subject: [PATCH 6/9] docs: describe the validator client's proposer preferences Document the per-slot preferences, their two counters, the signing domain departure from the validator guide, and that the boost factor now also weighs gloas p2p bids. --- CLAUDE.md | 7 +++++-- docs/cli.md | 15 ++++++++++++++- docs/metrics.md | 2 ++ docs/spec_deviations.md | 23 ++++++++++++++++++++++- 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 632a3d0e..cbe26c10 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -853,8 +853,11 @@ transitions are in `ethlambda-types`, per the section above. Nothing above `SlotClock` picks the offsets by each slot's own fork, and `/eth/v1/config/spec` supplies them. With `--builder-proposals`, pre-gloas slots register validators with the node's builder - and may sign a blinded block, published to the producing node only. It - still keeps no slashing-protection record; PTC votes have an in-memory + and may sign a blinded block, published to the producing node only. From the + epoch before gloas it signs proposer preferences every slot for its proposals in + the current and next epoch (`PreferencesService`, v2 duties, the gossip rule's + signing domain), unconditionally, and `produceBlockV4` carries the boost factor. + It still keeps no slashing-protection record; PTC votes have an in-memory `(validator, slot)` dedup only, and envelopes are unguarded. `get_head_node` is one bottom-up walk for every fork, with the current slot's fork selecting the payload rules and the boost gate, so a diff --git a/docs/cli.md b/docs/cli.md index a8cd8c65..e65ed84f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -291,7 +291,7 @@ implementation's, so none of the [common flags](#common-flags) apply to it. | `--graffiti` | empty | Text for the graffiti field of proposed blocks, for every validator the keymanager API gives no graffiti of its own. At most 32 bytes as UTF-8, right-padded with zeros. Refused rather than truncated if longer. An ethlambda beacon node appends both clients' versions to it (see [`rpc.md`](rpc.md#validator-endpoints)) | | `--enable-keymanager` | off | Serve the keymanager API: keystores, plus each validator's own fee recipient, graffiti and gas limit. Off by default because it mutates key material | | `--builder-proposals` | off | Propose builder blocks: register each validator with the beacon node's builder (an ethlambda node: `--builder`) and accept a blinded block when the builder's bid wins. Fulu and earlier only; from gloas the builder market works through proposer preferences instead | -| `--builder-boost-factor` | `100` | How a builder's bid is weighed against the local payload: the local payload wins iff `local >= factor * bid / 100`. `0` prefers the local payload unless it cannot be built. Requires `--builder-proposals`; conflicts with `--prefer-builder-proposals` | +| `--builder-boost-factor` | `100` | How a builder's bid is weighed against the local payload: the local payload wins iff `local >= factor * bid / 100`. `0` prefers the local payload unless it cannot be built. From gloas it is also sent in `produceBlockV4`'s body, so a pooled p2p bid can win on value; without `--builder-proposals` the body carries `0`. Requires `--builder-proposals`; conflicts with `--prefer-builder-proposals` | | `--prefer-builder-proposals` | off | Take the builder's block whenever there is a valid bid, unless the execution client asks to override the builder (sends `builder_boost_factor = u64::MAX`). Requires `--builder-proposals`; conflicts with `--builder-boost-factor` | ### What it does today @@ -317,6 +317,19 @@ produced it, with no failover. A publish that node does not confirm is logged at error and counted as unconfirmed, not as a missed proposal, since the relay may still publish the block. +From the epoch before gloas on, it also signs a proposer preferences message +(fee recipient and target gas limit) for each of its proposal slots in the +current and next gloas epoch, and sends them to every node. This happens every +slot, whether or not `--builder-proposals` is set: from gloas the preferences +are what carry a proposal's fee recipient, and a node ignores builders' bids +for a slot with none. It reads those duties from the v2 proposer duties +endpoint, whose dependent root the preferences must carry, and publishes each +`(slot, dependent root)` once, again after a reorg changes the root. A +validator with no fee recipient is skipped, with one warning per epoch naming +the count. A preference cannot be replaced for the same slot and root, so a +fee recipient changed through the keymanager API applies from the next +dependent root. + Each slot it wakes at the boundary. If one of its validators proposes that slot, it signs the RANDAO reveal, asks the beacon node for a block, checks that the block is for the slot and proposer it asked about, signs it and publishes diff --git a/docs/metrics.md b/docs/metrics.md index 2fefa779..0c6338c4 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -743,6 +743,8 @@ and absent is not the same as zero. | `ethlambda_validator_fee_recipient_mismatches_total` | Counter | Blocks paying execution rewards to an address this client did not request. Should read zero forever; a non-zero value means every proposal is paying somewhere else | | `ethlambda_validator_builder_registrations_sent_total` | Counter | Builder registrations a beacon node accepted (`--builder-proposals`) | | `ethlambda_validator_builder_registration_failures_total` | Counter | Registration batches no beacon node accepted. A missed registration costs at most a builder block, never a duty | +| `ethlambda_validator_proposer_preferences_published_total` | Counter | Signed proposer preferences a beacon node accepted, counted per preference. Published for every proposal from the epoch before gloas, whether or not `--builder-proposals` is set | +| `ethlambda_validator_proposer_preferences_failures_total` | Counter | Proposer preference batches no beacon node accepted. The next slot retries; until one is accepted, builders' bids for those slots are ignored | | `ethlambda_validator_blinded_blocks_proposed_total` | Counter | Blinded blocks a beacon node handed to its builder. A subset of `blocks_proposed_total` | | `ethlambda_validator_blinded_blocks_unconfirmed_total` | Counter | Blinded blocks the beacon node did not confirm (a 5xx or no answer), which the relay may still have published. Not retried, since a second block for the slot would be an equivocation | | `ethlambda_validator_block_publication_delay_seconds` | Histogram | Slot start to block accepted. Its buckets are tighter than the attestation histogram's, because a block is due at the slot boundary rather than a third of the way in | diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 8a42579b..84ebe397 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -698,6 +698,27 @@ the beacon node's builder and may sign a blinded block (builder-specs and relay may still publish the block. - **Registrations stop at gloas.** The client sends no `register_validator` from the gloas fork epoch on: the builder market works through proposer - preferences there, and `produceBlockV4` keeps its self-build configuration. + preferences there (see the next section), and `produceBlockV4` sends the + configured boost factor with no minimum bid and no builders. Validators with no fee recipient are not registered either (the field is required), and only validators the beacon node resolved are signed for. + +## The validator client signs proposer preferences under the gossip rule's domain + +`validator.md` ("Broadcasting `SignedProposerPreferences`") signs with +`get_domain(state, DOMAIN_PROPOSER_PREFERENCES, compute_epoch_at_slot(proposal_slot))` +on the head state. `p2p-interface.md`'s `validate_proposer_preferences_gossip` +verifies against the dependent block's state advanced to the lookahead start, +so the fork version it uses is the one in force an epoch before the proposal's. + +- **ethlambda:** the validator client signs at the epoch before the proposal's + (`SigningContext::sign_proposer_preferences`), which is the domain the gossip + rule verifies. +- **Why:** the two agree except in the first epoch of a fork, where the head + state still carries the old fork version and a proposal's epoch resolves to + the new one, so the guide's signature would fail the gossip rule that peers + apply. This node's own `proposer_preferences` endpoint accepts both. +- **Published unconditionally,** not gated on `--builder-proposals`, since from + gloas the preferences carry every proposal's fee recipient. Validators with no + fee recipient are skipped (the field is required), and a preference cannot be + replaced for the same slot and dependent root. From 6d57e2197fb48746c7ce226b5c4a4f39eb6febf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:35:08 -0300 Subject: [PATCH 7/9] fix(validator): send proposer preferences one per request and tighten their reporting A node answers 400 for a whole call when any entry fails, and one it holds for the same slot and root with other content is such a failure, so a batch kept every entry unpublished and re-sent until that slot passed (a restart after a fee recipient change triggers it). Send each preference alone and remember each success. Also: a failed v2 duties fetch no longer warns "duty likely missed" twice a slot (the service warns once per epoch and counts it), the fork header is the target epoch's, the poll is spawned ahead of the proposal, the signing-domain explanation is corrected (get_domain reads state.fork, so the guide and the gossip rule differ only for a proposal in a fork's first epoch signed during it), v2 dependent slot reuses the state transition's definition, and the end-to-end test now reads its duty and dependent root through the client. --- bin/ethlambda/src/validator.rs | 7 +- .../rpc/src/beacon/builder_market_tests.rs | 60 ++-- crates/net/rpc/src/beacon/validator.rs | 14 +- crates/validator/src/beacon_node/fallback.rs | 40 ++- crates/validator/src/beacon_node/mock.rs | 27 +- crates/validator/src/beacon_node/mod.rs | 13 +- crates/validator/src/lib.rs | 20 +- crates/validator/src/preferences.rs | 271 ++++++++++++------ crates/validator/src/registration.rs | 21 +- crates/validator/src/signing.rs | 15 +- crates/validator/src/test_support.rs | 27 ++ docs/cli.md | 2 +- docs/spec_deviations.md | 11 +- 13 files changed, 353 insertions(+), 175 deletions(-) create mode 100644 crates/validator/src/test_support.rs diff --git a/bin/ethlambda/src/validator.rs b/bin/ethlambda/src/validator.rs index 5929d07f..2dd24bdf 100644 --- a/bin/ethlambda/src/validator.rs +++ b/bin/ethlambda/src/validator.rs @@ -78,9 +78,10 @@ pub(crate) struct ValidatorOptions { /// Propose builder blocks: register each validator with the beacon node's /// builder and accept a blinded block when the builder's bid wins. Requires - /// a beacon node started with a builder (ethlambda: `--builder`). Fulu and - /// earlier only; from gloas the builder market works through proposer - /// preferences instead. + /// a beacon node started with a builder (ethlambda: `--builder`). The + /// registrations and blinded blocks are fulu and earlier only; from gloas + /// the builder market works through proposer preferences, and this flag's + /// boost factor weighs p2p bids in `produceBlockV4`. #[arg(long, default_value = "false")] pub(crate) builder_proposals: bool, diff --git a/crates/net/rpc/src/beacon/builder_market_tests.rs b/crates/net/rpc/src/beacon/builder_market_tests.rs index 638e6a94..d536f2c7 100644 --- a/crates/net/rpc/src/beacon/builder_market_tests.rs +++ b/crates/net/rpc/src/beacon/builder_market_tests.rs @@ -1061,29 +1061,48 @@ async fn valid_preferences_are_cached_and_published_once() { assert_eq!(world.network.proposer_preferences.lock().unwrap().len(), 1); } -/// The validator client's own signer and HTTP client against this node's -/// gossip rules: the domain `sign_proposer_preferences` picks (the fork of the -/// epoch before the proposal's) is the one `validate_proposer_preferences_gossip` -/// verifies under, and the JSON the client posts is what the route decodes. +/// The validator client's own path end to end against this node: it reads the +/// proposer duty and its v2 `dependent_root` from the served duties route, +/// signs with `SigningContext::sign_proposer_preferences`, and posts through +/// `HttpBeaconNode`, and the node's gossip rules accept it. That pins the +/// duties route's dependent root as the one the preference must carry, and the +/// JSON the client posts as the one the route decodes. /// -/// The fork-boundary case (a preference for the first gloas epoch, signed -/// under the fulu version) is not driven here: this world's schedule has gloas -/// at epoch 0 and a second world in the epoch before the fork would be a new -/// chain fixture. `signing.rs` pins that domain in a unit test. +/// It does not pin the fork-version choice: this world has gloas at epoch 0 +/// and the node accepts either version, so `signing.rs`'s unit test is the pin +/// for the domain at a fork boundary. #[tokio::test] async fn preferences_signed_by_the_validator_client_pass_this_nodes_gossip_rules() { + use ethlambda_blockchain::{SyncStatusController, metrics::SyncStatus}; use ethlambda_validator::{ - beacon_node::{BeaconNodeApi as _, http::HttpBeaconNode}, + beacon_node::{BeaconNodeApi as _, dto::parse_pubkey, http::HttpBeaconNode}, keys::ValidatorStore, signing::SigningContext, }; let world = World::new(); - let proposer = world.prepared().proposer; - let dependent_root = dependent_root_at(&world.state, world.head_root, SLOT) - .expect("the head's chain gives the slot a dependent root"); + let duties_routes = crate::beacon::validator::routes() + .with_state(world.store.clone()) + .layer(Extension(SyncStatusController::new(SyncStatus::Synced))); + let router = world.app(None).merge(duties_routes); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let bound = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, router).await }); + let client = HttpBeaconNode::new(format!("http://{bound}")).unwrap(); + + // The duty and the dependent root the way the validator client reads them. + let duties = client + .proposer_duties_v2(compute_epoch_at_slot(SLOT)) + .await + .expect("the node serves v2 proposer duties"); + let duty = duties + .duties + .iter() + .find(|duty| duty.slot == SLOT) + .expect("the slot has a proposer"); + let proposer = duty.validator_index; let message = ProposerPreferences { - dependent_root, + dependent_root: duties.dependent_root, proposal_slot: SLOT, validator_index: proposer, fee_recipient: address(0xaa), @@ -1096,7 +1115,7 @@ async fn preferences_signed_by_the_validator_client_pass_this_nodes_gossip_rules .expect("the test secret is a valid key"); assert_eq!( pubkey, - world.state.validator(proposer).unwrap().pubkey, + parse_pubkey(&duty.pubkey).expect("the duty names a key"), "the test registry's key for the proposer" ); let context = SigningContext { @@ -1108,12 +1127,6 @@ async fn preferences_signed_by_the_validator_client_pass_this_nodes_gossip_rules .expect("signs"); let signed = SignedProposerPreferences { message, signature }; - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let router = world.app(None); - tokio::spawn(async move { axum::serve(listener, router).await }); - let client = HttpBeaconNode::new(format!("http://{address}")).unwrap(); - client .submit_proposer_preferences( ethlambda_types::beacon::fork::ForkName::Gloas, @@ -1126,7 +1139,12 @@ async fn preferences_signed_by_the_validator_client_pass_this_nodes_gossip_rules *world.network.proposer_preferences.lock().unwrap(), vec![signed.clone()] ); - assert_eq!(world.market.preferences(SLOT, dependent_root), Some(signed)); + assert_eq!( + world + .market + .preferences(SLOT, signed.message.dependent_root), + Some(signed) + ); } #[tokio::test] diff --git a/crates/net/rpc/src/beacon/validator.rs b/crates/net/rpc/src/beacon/validator.rs index 58a7f34b..28a2e310 100644 --- a/crates/net/rpc/src/beacon/validator.rs +++ b/crates/net/rpc/src/beacon/validator.rs @@ -35,7 +35,9 @@ use serde::{Deserialize, Serialize}; use ethlambda_network_api::RpcToP2PRef; use ethlambda_state_transition::beacon::{ - fork_choice::{checkpoint_state, get_current_store_epoch, get_head_node}, + fork_choice::{ + checkpoint_state, compute_shuffling_dependent_slot, get_current_store_epoch, get_head_node, + }, gossip::attestation::compute_subnet_for_attestation, helpers::accessors::{CommitteeCacheExt as _, get_block_root_at_slot}, helpers::altair::compute_sync_committee_period, @@ -535,11 +537,11 @@ impl DependentRoot { /// definition names the genesis block where its subtraction would /// underflow, which is slot 0. fn slot(self, epoch: Epoch) -> Slot { - let epoch = match self { - Self::V1 => epoch, - Self::V2 => epoch.saturating_sub(1), - }; - compute_start_slot_at_epoch(epoch).saturating_sub(1) + match self { + Self::V1 => compute_start_slot_at_epoch(epoch).saturating_sub(1), + // The state transition's own definition, so the two cannot drift. + Self::V2 => compute_shuffling_dependent_slot(epoch), + } } } diff --git a/crates/validator/src/beacon_node/fallback.rs b/crates/validator/src/beacon_node/fallback.rs index 933ea34b..32cca315 100644 --- a/crates/validator/src/beacon_node/fallback.rs +++ b/crates/validator/src/beacon_node/fallback.rs @@ -86,6 +86,32 @@ impl FallbackBeaconNode { /// answered, for the calls whose follow-up must go to the same node. The /// logging and the final `warn!` live here so the two cannot drift. async fn try_each_at<'p, T, F>(&'p self, what: &'static str, call: F) -> Result<(usize, T)> + where + F: Fn(&'p B) -> BoxFuture<'p, Result>, + { + self.try_each_at_inner(what, true, call).await + } + + /// [`Self::try_each`] without the final `warn!`, for a caller whose own + /// duty did not depend on the answer and which reports the failure itself + /// (the counterpart of [`Self::try_all`]'s `all_failed: None`). A node that + /// is syncing, or lacks the endpoint, would otherwise make this warn on + /// every call for as long as it lasts. + async fn try_each_quiet<'p, T, F>(&'p self, what: &'static str, call: F) -> Result + where + F: Fn(&'p B) -> BoxFuture<'p, Result>, + { + self.try_each_at_inner(what, false, call) + .await + .map(|(_, value)| value) + } + + async fn try_each_at_inner<'p, T, F>( + &'p self, + what: &'static str, + warn_when_all_fail: bool, + call: F, + ) -> Result<(usize, T)> where F: Fn(&'p B) -> BoxFuture<'p, Result>, { @@ -106,7 +132,9 @@ impl FallbackBeaconNode { let detail = last .map(|err| err.to_string()) .unwrap_or_else(|| "no nodes configured".to_string()); - warn!(%what, %detail, "All configured beacon nodes failed; duty likely missed"); + if warn_when_all_fail { + warn!(%what, %detail, "All configured beacon nodes failed; duty likely missed"); + } Err(Error::AllBeaconNodesFailed(detail)) } @@ -257,7 +285,9 @@ impl BeaconNodeApi for FallbackBeaconNode { } async fn proposer_duties_v2(&self, epoch: Epoch) -> Result { - self.try_each("proposer_duties_v2", |node| node.proposer_duties_v2(epoch)) + // Quiet: the preferences service reports its own failure once per + // epoch, and no duty depends on this answer. + self.try_each_quiet("proposer_duties_v2", |node| node.proposer_duties_v2(epoch)) .await } @@ -358,10 +388,8 @@ impl BeaconNodeApi for FallbackBeaconNode { self.try_all( "submit_proposer_preferences", "it will not gossip these proposer preferences", - Some( - "no node took this client's proposer preferences; builders' bids for those \ - slots are ignored, and from gloas a node may not know the fee recipient", - ), + // The service reports it, per preference, with what it costs. + None, |node| node.submit_proposer_preferences(fork, preferences), ) .await diff --git a/crates/validator/src/beacon_node/mock.rs b/crates/validator/src/beacon_node/mock.rs index c5628770..bd3747f1 100644 --- a/crates/validator/src/beacon_node/mock.rs +++ b/crates/validator/src/beacon_node/mock.rs @@ -69,6 +69,10 @@ pub struct MockBeaconNode { /// Every proposer preferences batch `submit_proposer_preferences` /// received, with the fork it was sent under, in call order. pub submitted_preferences: Mutex)>>, + /// Proposal slots whose preference the mock refuses, as a node does one it + /// holds with other content: a call carrying any of them is an `Err` and + /// records nothing, like the node's 400 for the whole call. + pub refused_preference_slots: Mutex>, pub attestation_data: Option, /// The slot and proposer the mock will produce a block for. `None` makes /// `produce_block` fail, the way an unset `attestation_data` does. @@ -291,8 +295,9 @@ impl MockBeaconNode { self } - /// How many times `proposer_duties_v2` has been called, successes and - /// failures alike. + /// How many times `proposer_duties_v2` has been called and got past the + /// configured failures, answers and misses alike. A call `failures` or + /// `fail_everything` rejects is not counted. pub fn proposer_duties_v2_call_count(&self) -> usize { *self.proposer_duties_v2_calls.lock().expect("lock") } @@ -579,8 +584,9 @@ impl MockBeaconNode { *self.validator_indices_calls.lock().expect("lock") } - /// How many times `proposer_duties` has been called, successes and - /// failures alike. + /// How many times `proposer_duties` has been called and got past the + /// configured failures, answers and misses alike. A call `failures` or + /// `fail_everything` rejects is not counted. pub fn proposer_duties_call_count(&self) -> usize { *self.proposer_duties_calls.lock().expect("lock") } @@ -843,6 +849,19 @@ impl BeaconNodeApi for MockBeaconNode { preferences: &[gloas::SignedProposerPreferences], ) -> Result<()> { self.guard("submit_proposer_preferences")?; + let refused = self.refused_preference_slots.lock().expect("lock").clone(); + if let Some(entry) = preferences + .iter() + .find(|entry| refused.contains(&entry.message.proposal_slot)) + { + return Err(Error::BeaconNodeStatus { + status: 400, + body: format!( + "preference for slot {} refused", + entry.message.proposal_slot + ), + }); + } self.submitted_preferences .lock() .expect("lock") diff --git a/crates/validator/src/beacon_node/mod.rs b/crates/validator/src/beacon_node/mod.rs index 5c8c189d..4eb8d802 100644 --- a/crates/validator/src/beacon_node/mod.rs +++ b/crates/validator/src/beacon_node/mod.rs @@ -426,13 +426,16 @@ pub trait BeaconNodeApi: Send + Sync { /// Submit signed proposer preferences, which the node validates with the /// `proposer_preferences` gossip rules and publishes. `fork` is the - /// `Eth-Consensus-Version`: the fork of the current epoch. + /// `Eth-Consensus-Version`: the fork the preferences belong to, that of + /// their proposal epoch. /// /// A 400 naming per-index failures is an `Err`: the whole call failed for - /// the caller's purposes. The caller re-submits the entries next slot, and - /// the node answers per entry, so the ones that already went out are - /// deduplicated by the node. State on the node's gossip, so failover sends - /// it to every node. + /// the caller's purposes, although the node may have published the entries + /// that passed. A node only short-circuits an identical entry, so one it + /// holds for the same `(slot, dependent_root)` with other content fails + /// every call that carries it. The caller therefore sends one preference + /// per call. State on the node's gossip, so failover sends it to every + /// node. async fn submit_proposer_preferences( &self, fork: ForkName, diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index 05d832b9..3cc62fda 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -50,6 +50,8 @@ pub mod signing; pub mod slot_clock; pub mod subscriptions; pub mod sync_committee; +#[cfg(test)] +mod test_support; pub use error::{Error, Result}; @@ -107,9 +109,10 @@ pub struct ValidatorConfig { /// and accept a blinded block when the builder's bid wins. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct BuilderProposals { - /// The `builder_boost_factor` sent with every pre-gloas block request: - /// 100 compares the bid and the local payload as they are; `u64::MAX` - /// takes the builder whenever there is a bid. + /// The `builder_boost_factor` sent with every block request (in the query + /// before gloas, in `produceBlockV4`'s body from gloas): 100 compares the + /// bid and the local payload as they are; `u64::MAX` takes the builder + /// whenever there is a bid. pub boost_factor: u64, } @@ -341,6 +344,13 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { } } + // First in the slot, ahead of the proposal: the preferences for the + // slots after this one must be out before those slots start (gossip + // IGNOREs a started slot), and a poll queued behind a duty could land + // late. Off the slot loop, since it is single-flight and a slow node + // must not delay this slot's duties. + tokio::spawn(preferences.clone().poll(slot, store.clone())); + // A block is due now, at the boundary this iteration woke on. // // Cloned out of `duties` so the borrow ends before the await: the duty @@ -380,10 +390,6 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { if let Some(registration) = ®istration { tokio::spawn(registration.clone().register_changed(epoch, store.clone())); } - - // Likewise off the slot loop: the poll is single-flight, and a slow - // node must not delay the next slot's duties. - tokio::spawn(preferences.clone().poll(slot, store.clone())); } } diff --git a/crates/validator/src/preferences.rs b/crates/validator/src/preferences.rs index 667a8887..ec87a925 100644 --- a/crates/validator/src/preferences.rs +++ b/crates/validator/src/preferences.rs @@ -22,8 +22,9 @@ //! # The signing domain //! //! See [`SigningContext::sign_proposer_preferences`]: the gossip rule's domain, -//! at the fork of the epoch before the proposal's, which is not the validator -//! guide's head-state one in the first epoch of a fork. +//! at the fork of the epoch before the proposal's. The validator guide's +//! head-state signer differs from it only for a proposal in the first epoch of +//! a fork, signed during that epoch. //! //! # Why every slot //! @@ -46,15 +47,25 @@ //! replaced for the same slot and root. A fee recipient changed after a //! preference went out applies from the next dependent root onward only. //! +//! # One preference per request +//! +//! A node answers 400 for a whole call when any entry in it fails, and one it +//! holds for the same `(slot, dependent_root)` with other content is such a +//! failure (a restart after a fee recipient change, say). Batching would then +//! fail every entry every slot until that slot passed, so each preference is +//! sent alone, and each success is remembered on its own. There are only a +//! handful per epoch. +//! //! # Failures are never propagated //! //! A missed preference costs a builder bid and, from gloas, possibly the -//! node's knowledge of the fee recipient, never a duty. A failed send is logged -//! and counted, and the next slot retries. +//! node's knowledge of the fee recipient, never a duty. A failed send, or a +//! failed duties fetch, is logged (the fetch once per epoch) and counted, and +//! the next slot retries. use std::collections::HashSet; -use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; use ethlambda_types::beacon::containers::gloas::{ProposerPreferences, SignedProposerPreferences}; use ethlambda_types::beacon::fork::ForkName; @@ -88,15 +99,26 @@ pub struct PreferencesService { /// `(proposal_slot, dependent_root)` pairs a node accepted, so a /// preference is signed and sent once per dependent root. A reorg that /// moves the root sends the slot again; pruned below the current epoch. - published: std::sync::Mutex>, + published: Mutex>, /// Epochs whose missing fee recipients were already warned about, so the /// warning is once per epoch rather than once per slot. Pruned like /// `published`. - warned_epochs: std::sync::Mutex>, + no_fee_warned: Mutex>, + /// Epochs whose failed duties fetch was already warned about, for the same + /// reason: a syncing node, or one without v2, fails every slot. + fetch_warned: Mutex>, /// One poll at a time: a slow node must not stack polls up. in_flight: AtomicBool, } +/// Lock `mutex`, recovering a poisoned one: every write here is one set +/// operation, so the data behind it is never half-changed. +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + impl PreferencesService { pub fn new( beacon_node: Arc, @@ -107,8 +129,9 @@ impl PreferencesService { beacon_node, context, settings, - published: std::sync::Mutex::new(HashSet::new()), - warned_epochs: std::sync::Mutex::new(HashSet::new()), + published: Mutex::new(HashSet::new()), + no_fee_warned: Mutex::new(HashSet::new()), + fetch_warned: Mutex::new(HashSet::new()), in_flight: AtomicBool::new(false), } } @@ -128,31 +151,24 @@ impl PreferencesService { // Snapshotted with the guard dropped at once: nothing below awaits a // node while holding the store. let pubkeys: HashSet = store.read().await.pubkeys().into_iter().collect(); - let fork = self.context.config.fork_at_epoch(epoch); for target in [epoch, epoch + 1] { if self.context.config.fork_at_epoch(target) < ForkName::Gloas { continue; } - self.publish_epoch(fork, current_slot, target, &pubkeys, &store) + self.publish_epoch(current_slot, target, &pubkeys, &store) .await; } let epoch_start = epoch * preset::SLOTS_PER_EPOCH; - self.published - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .retain(|(slot, _)| *slot >= epoch_start); - self.warned_epochs - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .retain(|warned| *warned >= epoch); + lock(&self.published).retain(|(slot, _)| *slot >= epoch_start); + lock(&self.no_fee_warned).retain(|warned| *warned >= epoch); + lock(&self.fetch_warned).retain(|warned| *warned >= epoch); } /// One target epoch: fetch its v2 duties, sign what is still owed and send it. async fn publish_epoch( &self, - fork: ForkName, current_slot: Slot, target: Epoch, pubkeys: &HashSet, @@ -162,6 +178,15 @@ impl PreferencesService { Ok(duties) => duties, Err(err) => { debug!(epoch = target, %err, "Could not fetch proposer duties for preferences"); + crate::metrics::inc_proposer_preferences_failures(); + if lock(&self.fetch_warned).insert(target) { + warn!( + epoch = target, + %err, + "No beacon node answered the v2 proposer duties; proposer preferences \ + for this epoch wait for a node that does" + ); + } return; } }; @@ -179,12 +204,7 @@ impl PreferencesService { if !pubkeys.contains(&pubkey) { continue; } - let already = self - .published - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .contains(&(duty.slot, duties.dependent_root)); - if already { + if lock(&self.published).contains(&(duty.slot, duties.dependent_root)) { continue; } let Some(fee_recipient) = self.settings.fee_recipient(&pubkey) else { @@ -203,26 +223,19 @@ impl PreferencesService { )); } - if no_fee_recipient > 0 { - let first_warning = self - .warned_epochs - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert(target); - if first_warning { - warn!( - epoch = target, - skipped = no_fee_recipient, - "Validators without a fee recipient publish no proposer preferences" - ); - } + if no_fee_recipient > 0 && lock(&self.no_fee_warned).insert(target) { + warn!( + epoch = target, + skipped = no_fee_recipient, + "Validators without a fee recipient publish no proposer preferences" + ); } if owed.is_empty() { return; } // At most a few per epoch, so signing inline is cheap; the read guard - // is released before the submit. + // is released before the submits. let mut signed = Vec::with_capacity(owed.len()); { let store = store.read().await; @@ -243,33 +256,47 @@ impl PreferencesService { } } } - if signed.is_empty() { - return; - } - match self - .beacon_node - .submit_proposer_preferences(fork, &signed) - .await - { - Ok(()) => { - let count = signed.len(); - self.published - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .extend( - signed.iter().map(|entry| { - (entry.message.proposal_slot, entry.message.dependent_root) - }), + // The target epoch's fork, which is what the submitted objects belong + // to (always gloas here). + let fork = self.context.config.fork_at_epoch(target); + let mut published = 0usize; + for entry in signed { + // One request per preference: a node answers 400 for the whole + // call when any entry fails, and the one it refuses (for a + // `(slot, root)` it holds with other content, say) would + // otherwise keep the others from ever being marked published. + let key = (entry.message.proposal_slot, entry.message.dependent_root); + match self + .beacon_node + .submit_proposer_preferences(fork, std::slice::from_ref(&entry)) + .await + { + Ok(()) => { + lock(&self.published).insert(key); + published += 1; + } + Err(err) => { + warn!( + epoch = target, + slot = key.0, + validator = entry.message.validator_index, + %err, + "No beacon node accepted a proposer preference; builders' bids for that \ + slot are ignored, and from gloas a node may not know the fee recipient" ); - crate::metrics::inc_proposer_preferences_published(count); - info!(epoch = target, count, "Proposer preferences published"); - } - Err(err) => { - warn!(epoch = target, count = signed.len(), %err, "No beacon node accepted the proposer preferences"); - crate::metrics::inc_proposer_preferences_failures(); + crate::metrics::inc_proposer_preferences_failures(); + } } } + if published > 0 { + crate::metrics::inc_proposer_preferences_published(published); + info!( + epoch = target, + count = published, + "Proposer preferences published" + ); + } } } @@ -278,9 +305,10 @@ mod tests { use super::*; use crate::beacon_node::dto::ProposerDutyDto; use crate::beacon_node::mock::MockBeaconNode; + use crate::test_support::{settings, store_with_keys}; use ethlambda_types::beacon::config::Config; use ethlambda_types::beacon::constants::DOMAIN_PROPOSER_PREFERENCES; - use ethlambda_types::beacon::primitives::{Bytes32, H160, HashTreeRoot as _}; + use ethlambda_types::beacon::primitives::{H160, HashTreeRoot as _}; use ethlambda_types::beacon::signing::{compute_domain, compute_signing_root}; const GLOAS_EPOCH: Epoch = 500_000; @@ -295,20 +323,6 @@ mod tests { }) } - /// `count` distinct keys. The secrets are small integers, which are valid - /// BLS scalars, so no keystore is needed. - fn store_with_keys(count: usize) -> (ValidatorStore, Vec) { - let mut store = ValidatorStore::new(); - let pubkeys = (0..count) - .map(|index| { - let mut secret = [0u8; 32]; - secret[24..].copy_from_slice(&(index as u64 + 1).to_be_bytes()); - store.insert_secret("test", &secret).expect("inserts") - }) - .collect(); - (store, pubkeys) - } - fn duty(pubkey: &BlsPubkey, validator_index: u64, slot: Slot) -> ProposerDutyDto { ProposerDutyDto { pubkey: encode_hex(&pubkey.0), @@ -317,10 +331,6 @@ mod tests { } } - fn settings(fee_recipient: Option) -> Arc { - Arc::new(ProposerSettings::new(Bytes32::ZERO, fee_recipient)) - } - fn service( node: MockBeaconNode, settings: Arc, @@ -345,16 +355,19 @@ mod tests { let context = context(); let proposal_epoch = compute_epoch_at_slot(entry.message.proposal_slot); - let domain = context.domain(DOMAIN_PROPOSER_PREFERENCES, proposal_epoch - 1); - assert_eq!( - domain, - compute_domain( - DOMAIN_PROPOSER_PREFERENCES, - context - .config - .fork_version(context.config.fork_at_epoch(proposal_epoch - 1)), - context.genesis_validators_root, - ) + // Spelled out rather than asked of `SigningContext::domain`: the + // first gloas epoch's preference verifies under fulu's version (the + // fork of the epoch before it), every later one under gloas's. + let version = if proposal_epoch == GLOAS_EPOCH { + context.config.fulu_fork_version + } else { + assert!(proposal_epoch > GLOAS_EPOCH); + context.config.gloas_fork_version + }; + let domain = compute_domain( + DOMAIN_PROPOSER_PREFERENCES, + version, + context.genesis_validators_root, ); let root = compute_signing_root(entry.message.hash_tree_root(), domain); let pk = PublicKey::from_bytes(&pubkey.0).expect("valid pubkey"); @@ -392,7 +405,7 @@ mod tests { service.poll(current, store).await; let sent = submitted(&node); - assert_eq!(sent.len(), 2, "one send per target epoch"); + assert_eq!(sent.len(), 2, "one request per preference"); let mut entries: Vec<_> = sent .iter() .inspect(|(fork, _)| assert_eq!(*fork, ForkName::Gloas)) @@ -477,7 +490,7 @@ mod tests { .await; let sent = submitted(&node); assert_eq!(sent.len(), 1); - assert_eq!(sent[0].0, ForkName::Fulu, "the current epoch's fork"); + assert_eq!(sent[0].0, ForkName::Gloas, "the target epoch's fork"); assert_eq!(sent[0].1[0].message.proposal_slot, slot_in(GLOAS_EPOCH, 10)); assert_eq!(node.proposer_duties_v2_call_count(), 1); @@ -552,6 +565,76 @@ mod tests { assert_eq!(sent[0].1[0].message.validator_index, 4); } + /// A node that refuses one preference (it holds another for that slot and + /// root) must not keep the others from going out and being remembered, nor + /// make them re-sent every slot. + #[tokio::test] + async fn a_refused_preference_does_not_block_the_others() { + let (store, pubkeys) = store_with_keys(2); + let store = Arc::new(RwLock::new(store)); + let refused = slot_in(GLOAS_EPOCH, 10); + let accepted = slot_in(GLOAS_EPOCH, 11); + let node = MockBeaconNode::new().with_proposers_v2( + GLOAS_EPOCH, + Root::repeat_byte(0x11), + vec![ + duty(&pubkeys[0], 3, refused), + duty(&pubkeys[1], 4, accepted), + ], + ); + node.refused_preference_slots + .lock() + .expect("lock") + .push(refused); + let (service, node) = service(node, settings(Some(FEE_RECIPIENT))); + let current = slot_in(GLOAS_EPOCH, 1); + + service.clone().poll(current, store.clone()).await; + let sent = submitted(&node); + assert_eq!(sent.len(), 1, "only the accepted one was recorded"); + assert_eq!(sent[0].1[0].message.proposal_slot, accepted); + + // Next poll: the accepted one is not re-sent, the refused one is + // tried again (and refused again). + service.clone().poll(current + 1, store.clone()).await; + assert_eq!(submitted(&node).len(), 1); + + node.refused_preference_slots.lock().expect("lock").clear(); + service.poll(current + 2, store).await; + let sent = submitted(&node); + assert_eq!(sent.len(), 2); + assert_eq!(sent[1].1[0].message.proposal_slot, refused); + } + + /// A failed duties fetch is warned about once per target epoch, and the + /// next poll that gets an answer publishes. + #[tokio::test] + async fn a_failed_duties_fetch_is_retried_and_warned_once_per_epoch() { + let (store, pubkeys) = store_with_keys(1); + let store = Arc::new(RwLock::new(store)); + let node = MockBeaconNode::new().with_proposers_v2( + GLOAS_EPOCH, + Root::repeat_byte(0x11), + vec![duty(&pubkeys[0], 3, slot_in(GLOAS_EPOCH, 10))], + ); + node.failures + .lock() + .expect("lock") + .insert("proposer_duties_v2", "syncing".to_string()); + let (service, node) = service(node, settings(Some(FEE_RECIPIENT))); + let current = slot_in(GLOAS_EPOCH, 1); + + service.clone().poll(current, store.clone()).await; + service.clone().poll(current + 1, store.clone()).await; + assert!(submitted(&node).is_empty()); + let warned: Vec = lock(&service.fetch_warned).iter().copied().collect(); + assert_eq!(warned.len(), 2, "one entry per target epoch, not per poll"); + + node.failures.lock().expect("lock").clear(); + service.poll(current + 2, store).await; + assert_eq!(submitted(&node).len(), 1); + } + #[tokio::test] async fn a_failed_submit_is_retried_next_poll() { let (store, pubkeys) = store_with_keys(1); diff --git a/crates/validator/src/registration.rs b/crates/validator/src/registration.rs index c9d5a717..0918096f 100644 --- a/crates/validator/src/registration.rs +++ b/crates/validator/src/registration.rs @@ -318,8 +318,9 @@ impl RegistrationService { mod tests { use super::*; use crate::beacon_node::mock::MockBeaconNode; + use crate::test_support::{settings, store_with_keys}; use ethlambda_types::beacon::config::Config; - use ethlambda_types::beacon::primitives::{Bytes32, H160, Root}; + use ethlambda_types::beacon::primitives::{H160, Root}; fn context(config: Config) -> Arc { Arc::new(SigningContext { @@ -328,24 +329,6 @@ mod tests { }) } - /// `count` distinct keys. The secrets are small integers, which are valid - /// BLS scalars, so no keystore is needed. - fn store_with_keys(count: usize) -> (ValidatorStore, Vec) { - let mut store = ValidatorStore::new(); - let pubkeys = (0..count) - .map(|index| { - let mut secret = [0u8; 32]; - secret[24..].copy_from_slice(&(index as u64 + 1).to_be_bytes()); - store.insert_secret("test", &secret).expect("inserts") - }) - .collect(); - (store, pubkeys) - } - - fn settings(fee_recipient: Option) -> Arc { - Arc::new(ProposerSettings::new(Bytes32::ZERO, fee_recipient)) - } - fn service( config: Config, settings: Arc, diff --git a/crates/validator/src/signing.rs b/crates/validator/src/signing.rs index d43a3602..2ece1aca 100644 --- a/crates/validator/src/signing.rs +++ b/crates/validator/src/signing.rs @@ -326,8 +326,12 @@ impl SigningContext { /// against: `get_domain(state, .., proposal_epoch)` on the dependent /// block's state advanced to the lookahead start, whose fork is the one /// in force an epoch before the proposal. The validator guide signs with - /// the head state instead; the two differ only in the first epoch of a - /// fork, where this one is the one peers accept. Not slashable. + /// the head state (`get_domain` reads `state.fork`, not the schedule); + /// the two agree everywhere except for a proposal in the first epoch of a + /// fork, signed during that epoch: the head state already carries the new + /// version, while the gossip rule's state, at the start of the previous + /// epoch, still carries the old one. This signer takes the gossip rule's, + /// which is the one peers verify. Not slashable. pub fn sign_proposer_preferences( &self, store: &ValidatorStore, @@ -643,9 +647,10 @@ mod tests { } /// The first gloas epoch's preference is signed at the fork of the epoch - /// before it (fulu), which is what the gossip rule verifies against, and - /// not under the gloas version the validator guide's head-state signer - /// would use. + /// before it (fulu), which is what the gossip rule verifies against. The + /// validator guide's head-state signer would give gloas for a preference + /// signed during that epoch, which the gossip rule's state (still fulu) + /// would reject. #[test] fn a_preference_for_the_first_gloas_epoch_is_signed_under_the_fulu_version() { let (store, pubkey) = store_with_key(); diff --git a/crates/validator/src/test_support.rs b/crates/validator/src/test_support.rs new file mode 100644 index 00000000..d9216b9a --- /dev/null +++ b/crates/validator/src/test_support.rs @@ -0,0 +1,27 @@ +//! Helpers the service tests share. + +use std::sync::Arc; + +use ethlambda_types::beacon::primitives::{BlsPubkey, Bytes32, H160}; + +use crate::keys::ValidatorStore; +use crate::proposer_settings::ProposerSettings; + +/// `count` distinct keys. The secrets are small integers, which are valid BLS +/// scalars, so no keystore is needed. +pub(crate) fn store_with_keys(count: usize) -> (ValidatorStore, Vec) { + let mut store = ValidatorStore::new(); + let pubkeys = (0..count) + .map(|index| { + let mut secret = [0u8; 32]; + secret[24..].copy_from_slice(&(index as u64 + 1).to_be_bytes()); + store.insert_secret("test", &secret).expect("inserts") + }) + .collect(); + (store, pubkeys) +} + +/// Settings with `fee_recipient` as the default one and the default graffiti. +pub(crate) fn settings(fee_recipient: Option) -> Arc { + Arc::new(ProposerSettings::new(Bytes32::ZERO, fee_recipient)) +} diff --git a/docs/cli.md b/docs/cli.md index e65ed84f..2d90ab2d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -290,7 +290,7 @@ implementation's, so none of the [common flags](#common-flags) apply to it. | `--suggested-fee-recipient` | none | Execution address to receive block rewards, `0x`-prefixed, for every validator the keymanager API gives no address of its own. Optional, and startup warns when it is absent: without it the beacon node picks an address, and it will not be yours | | `--graffiti` | empty | Text for the graffiti field of proposed blocks, for every validator the keymanager API gives no graffiti of its own. At most 32 bytes as UTF-8, right-padded with zeros. Refused rather than truncated if longer. An ethlambda beacon node appends both clients' versions to it (see [`rpc.md`](rpc.md#validator-endpoints)) | | `--enable-keymanager` | off | Serve the keymanager API: keystores, plus each validator's own fee recipient, graffiti and gas limit. Off by default because it mutates key material | -| `--builder-proposals` | off | Propose builder blocks: register each validator with the beacon node's builder (an ethlambda node: `--builder`) and accept a blinded block when the builder's bid wins. Fulu and earlier only; from gloas the builder market works through proposer preferences instead | +| `--builder-proposals` | off | Propose builder blocks: register each validator with the beacon node's builder (an ethlambda node: `--builder`) and accept a blinded block when the builder's bid wins. Registrations and blinded blocks are fulu and earlier only; from gloas the builder market works through proposer preferences, and the boost factor weighs p2p bids in `produceBlockV4` | | `--builder-boost-factor` | `100` | How a builder's bid is weighed against the local payload: the local payload wins iff `local >= factor * bid / 100`. `0` prefers the local payload unless it cannot be built. From gloas it is also sent in `produceBlockV4`'s body, so a pooled p2p bid can win on value; without `--builder-proposals` the body carries `0`. Requires `--builder-proposals`; conflicts with `--prefer-builder-proposals` | | `--prefer-builder-proposals` | off | Take the builder's block whenever there is a valid bid, unless the execution client asks to override the builder (sends `builder_boost_factor = u64::MAX`). Requires `--builder-proposals`; conflicts with `--builder-boost-factor` | diff --git a/docs/spec_deviations.md b/docs/spec_deviations.md index 84ebe397..a9b0459d 100644 --- a/docs/spec_deviations.md +++ b/docs/spec_deviations.md @@ -714,10 +714,13 @@ so the fork version it uses is the one in force an epoch before the proposal's. - **ethlambda:** the validator client signs at the epoch before the proposal's (`SigningContext::sign_proposer_preferences`), which is the domain the gossip rule verifies. -- **Why:** the two agree except in the first epoch of a fork, where the head - state still carries the old fork version and a proposal's epoch resolves to - the new one, so the guide's signature would fail the gossip rule that peers - apply. This node's own `proposer_preferences` endpoint accepts both. +- **Why:** `get_domain` reads `state.fork`, not the schedule, so the two agree + everywhere except for a proposal in the first epoch of a fork, signed during + that epoch. There the head state already carries the new fork version, while + the gossip rule's state (at the start of the previous epoch) still carries the + old one, so the guide's signature would fail the rule peers apply. During the + epoch before the fork both give the old version. This node's own + `proposer_preferences` endpoint accepts both. - **Published unconditionally,** not gated on `--builder-proposals`, since from gloas the preferences carry every proposal's fee recipient. Validators with no fee recipient are skipped (the field is required), and a preference cannot be From e44c0741c30032d83d90565285509c63f194e2fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:36:13 -0300 Subject: [PATCH 8/9] fix(validator): warn once per refused proposer preference A preference a node keeps refusing is retried every slot until its slot passes, which warned up to two epochs of slots per entry. Warn on the first refusal of each (slot, dependent root), log repeats at debug, and keep counting every failure. Reword the failures metric to what it now counts. --- crates/validator/src/metrics.rs | 9 ++++--- crates/validator/src/preferences.rs | 40 +++++++++++++++++++++++------ docs/metrics.md | 2 +- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/crates/validator/src/metrics.rs b/crates/validator/src/metrics.rs index 92fc44b2..4367cedb 100644 --- a/crates/validator/src/metrics.rs +++ b/crates/validator/src/metrics.rs @@ -144,13 +144,14 @@ static PROPOSER_PREFERENCES_PUBLISHED_TOTAL: LazyLock = LazyLock::ne .unwrap() }); -/// Proposer preference batches no beacon node accepted. The next slot retries, -/// so this is a warning sign: until one is accepted, builders' bids for the -/// slot are ignored. +/// Proposer preferences no beacon node accepted, one count per preference per +/// attempt, plus failed v2 proposer-duty fetches. The next slot retries, so +/// this is a warning sign: until one is accepted, builders' bids for the slot +/// are ignored. static PROPOSER_PREFERENCES_FAILURES_TOTAL: LazyLock = LazyLock::new(|| { register_int_counter!( "ethlambda_validator_proposer_preferences_failures_total", - "Proposer preference batches no beacon node accepted" + "Proposer preferences no beacon node accepted, plus failed v2 duty fetches" ) .unwrap() }); diff --git a/crates/validator/src/preferences.rs b/crates/validator/src/preferences.rs index ec87a925..72ec3b50 100644 --- a/crates/validator/src/preferences.rs +++ b/crates/validator/src/preferences.rs @@ -104,6 +104,10 @@ pub struct PreferencesService { /// warning is once per epoch rather than once per slot. Pruned like /// `published`. no_fee_warned: Mutex>, + /// `(proposal_slot, dependent_root)` entries whose refusal was already + /// warned about, so a persistently refused one warns once rather than every + /// slot until its slot passes. Pruned with `published`. + refusal_warned: Mutex>, /// Epochs whose failed duties fetch was already warned about, for the same /// reason: a syncing node, or one without v2, fails every slot. fetch_warned: Mutex>, @@ -131,6 +135,7 @@ impl PreferencesService { settings, published: Mutex::new(HashSet::new()), no_fee_warned: Mutex::new(HashSet::new()), + refusal_warned: Mutex::new(HashSet::new()), fetch_warned: Mutex::new(HashSet::new()), in_flight: AtomicBool::new(false), } @@ -162,6 +167,7 @@ impl PreferencesService { let epoch_start = epoch * preset::SLOTS_PER_EPOCH; lock(&self.published).retain(|(slot, _)| *slot >= epoch_start); + lock(&self.refusal_warned).retain(|(slot, _)| *slot >= epoch_start); lock(&self.no_fee_warned).retain(|warned| *warned >= epoch); lock(&self.fetch_warned).retain(|warned| *warned >= epoch); } @@ -277,14 +283,27 @@ impl PreferencesService { published += 1; } Err(err) => { - warn!( - epoch = target, - slot = key.0, - validator = entry.message.validator_index, - %err, - "No beacon node accepted a proposer preference; builders' bids for that \ - slot are ignored, and from gloas a node may not know the fee recipient" - ); + // A refused entry is tried again every slot until its slot + // passes, so only its first refusal is a warning. + if lock(&self.refusal_warned).insert(key) { + warn!( + epoch = target, + slot = key.0, + validator = entry.message.validator_index, + %err, + "No beacon node accepted a proposer preference; builders' bids for \ + that slot are ignored, and from gloas a node may not know the fee \ + recipient" + ); + } else { + debug!( + epoch = target, + slot = key.0, + validator = entry.message.validator_index, + %err, + "A proposer preference was refused again" + ); + } crate::metrics::inc_proposer_preferences_failures(); } } @@ -598,6 +617,11 @@ mod tests { // tried again (and refused again). service.clone().poll(current + 1, store.clone()).await; assert_eq!(submitted(&node).len(), 1); + assert_eq!( + *lock(&service.refusal_warned), + HashSet::from([(refused, Root::repeat_byte(0x11))]), + "two refusals of one entry leave one warn-path record" + ); node.refused_preference_slots.lock().expect("lock").clear(); service.poll(current + 2, store).await; diff --git a/docs/metrics.md b/docs/metrics.md index 0c6338c4..c11ed81a 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -744,7 +744,7 @@ and absent is not the same as zero. | `ethlambda_validator_builder_registrations_sent_total` | Counter | Builder registrations a beacon node accepted (`--builder-proposals`) | | `ethlambda_validator_builder_registration_failures_total` | Counter | Registration batches no beacon node accepted. A missed registration costs at most a builder block, never a duty | | `ethlambda_validator_proposer_preferences_published_total` | Counter | Signed proposer preferences a beacon node accepted, counted per preference. Published for every proposal from the epoch before gloas, whether or not `--builder-proposals` is set | -| `ethlambda_validator_proposer_preferences_failures_total` | Counter | Proposer preference batches no beacon node accepted. The next slot retries; until one is accepted, builders' bids for those slots are ignored | +| `ethlambda_validator_proposer_preferences_failures_total` | Counter | Proposer preferences a beacon node refused or failed to take (one count per preference per attempt), plus failed v2 proposer-duty fetches. The next slot retries; until a preference is accepted, builders' bids for its slot are ignored | | `ethlambda_validator_blinded_blocks_proposed_total` | Counter | Blinded blocks a beacon node handed to its builder. A subset of `blocks_proposed_total` | | `ethlambda_validator_blinded_blocks_unconfirmed_total` | Counter | Blinded blocks the beacon node did not confirm (a 5xx or no answer), which the relay may still have published. Not retried, since a second block for the slot would be an equivocation | | `ethlambda_validator_block_publication_delay_seconds` | Histogram | Slot start to block accepted. Its buckets are tighter than the attestation histogram's, because a block is due at the slot boundary rather than a third of the way in | From ae1a3d2f3bf4629dd2ba69f446ec7e1df1a030e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:39:56 -0300 Subject: [PATCH 9/9] fix(validator): keep refused proposer preferences quiet per node and split the fetch counter try_all warned per node on every refusal, so a preference a node kept refusing still warned every slot despite the per-entry dedup; add a quiet variant for it. Poll after the proposal rather than before, so the next epoch duties fetch does not compete with block production. Count failed v2 duty fetches on their own counter, since a syncing node is not a refused preference. --- crates/validator/src/beacon_node/fallback.rs | 45 ++++++++++++++++++-- crates/validator/src/lib.rs | 16 ++++--- crates/validator/src/metrics.rs | 24 +++++++++-- crates/validator/src/preferences.rs | 26 +++++++++-- docs/metrics.md | 3 +- 5 files changed, 94 insertions(+), 20 deletions(-) diff --git a/crates/validator/src/beacon_node/fallback.rs b/crates/validator/src/beacon_node/fallback.rs index 32cca315..90359e9b 100644 --- a/crates/validator/src/beacon_node/fallback.rs +++ b/crates/validator/src/beacon_node/fallback.rs @@ -170,6 +170,39 @@ impl FallbackBeaconNode { all_failed: Option<&'static str>, call: F, ) -> Result<()> + where + F: Fn(&'p B) -> BoxFuture<'p, Result<()>>, + { + self.try_all_inner(what, node_effect, all_failed, true, call) + .await + } + + /// [`Self::try_all`] logging each node's refusal at `debug!` instead of + /// `warn!`, for a call the caller retries every slot and reports itself + /// (once per entry): a per-node warning here would repeat on every retry + /// for as long as the refusal lasts. Always without an `all_failed` + /// message, for the same reason. + async fn try_all_quiet<'p, F>( + &'p self, + what: &'static str, + node_effect: &'static str, + call: F, + ) -> Result<()> + where + F: Fn(&'p B) -> BoxFuture<'p, Result<()>>, + { + self.try_all_inner(what, node_effect, None, false, call) + .await + } + + async fn try_all_inner<'p, F>( + &'p self, + what: &'static str, + node_effect: &'static str, + all_failed: Option<&'static str>, + warn_per_node: bool, + call: F, + ) -> Result<()> where F: Fn(&'p B) -> BoxFuture<'p, Result<()>>, { @@ -179,7 +212,11 @@ impl FallbackBeaconNode { match call(node).await { Ok(()) => accepted += 1, Err(err) => { - warn!(%what, position, %err, node_effect, "Beacon node did not accept"); + if warn_per_node { + warn!(%what, position, %err, node_effect, "Beacon node did not accept"); + } else { + debug!(%what, position, %err, node_effect, "Beacon node did not accept"); + } last = Some(err); } } @@ -385,11 +422,11 @@ impl BeaconNodeApi for FallbackBeaconNode { fork: ForkName, preferences: &[gloas::SignedProposerPreferences], ) -> Result<()> { - self.try_all( + // Quiet: the service retries a refused preference every slot and warns + // once per entry, with what it costs. + self.try_all_quiet( "submit_proposer_preferences", "it will not gossip these proposer preferences", - // The service reports it, per preference, with what it costs. - None, |node| node.submit_proposer_preferences(fork, preferences), ) .await diff --git a/crates/validator/src/lib.rs b/crates/validator/src/lib.rs index 3cc62fda..98447c94 100644 --- a/crates/validator/src/lib.rs +++ b/crates/validator/src/lib.rs @@ -344,13 +344,6 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { } } - // First in the slot, ahead of the proposal: the preferences for the - // slots after this one must be out before those slots start (gossip - // IGNOREs a started slot), and a poll queued behind a duty could land - // late. Off the slot loop, since it is single-flight and a slow node - // must not delay this slot's duties. - tokio::spawn(preferences.clone().poll(slot, store.clone())); - // A block is due now, at the boundary this iteration woke on. // // Cloned out of `duties` so the borrow ends before the await: the duty @@ -360,6 +353,15 @@ pub async fn run(config: ValidatorConfig) -> Result<()> { propose(&proposal, &clock, slot, &duty, &store).await; } + // Right after the proposal and ahead of the slot's other duties: the + // preferences for the slots after this one must be out before those + // slots start (gossip IGNOREs a started slot), and the poll's v2 duties + // fetch for the next epoch can make the node advance a state, which + // should not compete with it producing this slot's block. Off the slot + // loop, since it is single-flight and a slow node must not delay this + // slot's duties. + tokio::spawn(preferences.clone().poll(slot, store.clone())); + // The rest of the way to the first of the slot's remaining duties: // the attester offset, or the sync committee offset when the network // puts it earlier. Zero if the refresh or the proposal above already diff --git a/crates/validator/src/metrics.rs b/crates/validator/src/metrics.rs index 4367cedb..16c8539b 100644 --- a/crates/validator/src/metrics.rs +++ b/crates/validator/src/metrics.rs @@ -145,13 +145,24 @@ static PROPOSER_PREFERENCES_PUBLISHED_TOTAL: LazyLock = LazyLock::ne }); /// Proposer preferences no beacon node accepted, one count per preference per -/// attempt, plus failed v2 proposer-duty fetches. The next slot retries, so -/// this is a warning sign: until one is accepted, builders' bids for the slot -/// are ignored. +/// attempt. The next slot retries, so this is a warning sign: until one is +/// accepted, builders' bids for the slot are ignored. static PROPOSER_PREFERENCES_FAILURES_TOTAL: LazyLock = LazyLock::new(|| { register_int_counter!( "ethlambda_validator_proposer_preferences_failures_total", - "Proposer preferences no beacon node accepted, plus failed v2 duty fetches" + "Proposer preferences no beacon node accepted" + ) + .unwrap() +}); + +/// v2 proposer-duty fetches no beacon node answered, one count per target epoch +/// per attempt. Without the duties no preference is signed for that epoch, so a +/// node that is syncing or lacks the endpoint shows up here and not as refused +/// preferences. +static PROPOSER_PREFERENCES_DUTY_FETCH_FAILURES_TOTAL: LazyLock = LazyLock::new(|| { + register_int_counter!( + "ethlambda_validator_proposer_preferences_duty_fetch_failures_total", + "v2 proposer-duty fetches no beacon node answered" ) .unwrap() }); @@ -382,6 +393,7 @@ pub fn init() { LazyLock::force(&BUILDER_REGISTRATION_FAILURES_TOTAL); LazyLock::force(&PROPOSER_PREFERENCES_PUBLISHED_TOTAL); LazyLock::force(&PROPOSER_PREFERENCES_FAILURES_TOTAL); + LazyLock::force(&PROPOSER_PREFERENCES_DUTY_FETCH_FAILURES_TOTAL); LazyLock::force(&BLINDED_BLOCKS_PROPOSED_TOTAL); LazyLock::force(&BLINDED_BLOCKS_UNCONFIRMED_TOTAL); LazyLock::force(&BLOCKS_REFUSED_TOTAL); @@ -460,6 +472,10 @@ pub fn inc_proposer_preferences_failures() { PROPOSER_PREFERENCES_FAILURES_TOTAL.inc(); } +pub fn inc_proposer_preferences_duty_fetch_failures() { + PROPOSER_PREFERENCES_DUTY_FETCH_FAILURES_TOTAL.inc(); +} + pub fn inc_blinded_blocks_proposed() { BLINDED_BLOCKS_PROPOSED_TOTAL.inc(); } diff --git a/crates/validator/src/preferences.rs b/crates/validator/src/preferences.rs index 72ec3b50..8cb049a0 100644 --- a/crates/validator/src/preferences.rs +++ b/crates/validator/src/preferences.rs @@ -184,7 +184,7 @@ impl PreferencesService { Ok(duties) => duties, Err(err) => { debug!(epoch = target, %err, "Could not fetch proposer duties for preferences"); - crate::metrics::inc_proposer_preferences_failures(); + crate::metrics::inc_proposer_preferences_duty_fetch_failures(); if lock(&self.fetch_warned).insert(target) { warn!( epoch = target, @@ -641,6 +641,13 @@ mod tests { Root::repeat_byte(0x11), vec![duty(&pubkeys[0], 3, slot_in(GLOAS_EPOCH, 10))], ); + // Both target epochs have duties, and the failure is injected on the + // call itself, so neither fetch fails for want of data. + node.set_proposers_v2( + GLOAS_EPOCH + 1, + Root::repeat_byte(0x22), + vec![duty(&pubkeys[0], 3, slot_in(GLOAS_EPOCH + 1, 4))], + ); node.failures .lock() .expect("lock") @@ -651,12 +658,23 @@ mod tests { service.clone().poll(current, store.clone()).await; service.clone().poll(current + 1, store.clone()).await; assert!(submitted(&node).is_empty()); - let warned: Vec = lock(&service.fetch_warned).iter().copied().collect(); - assert_eq!(warned.len(), 2, "one entry per target epoch, not per poll"); + assert_eq!( + *lock(&service.fetch_warned), + HashSet::from([GLOAS_EPOCH, GLOAS_EPOCH + 1]), + "one entry per target epoch, not per poll" + ); node.failures.lock().expect("lock").clear(); service.poll(current + 2, store).await; - assert_eq!(submitted(&node).len(), 1); + let mut slots: Vec = submitted(&node) + .iter() + .flat_map(|(_, entries)| entries.iter().map(|entry| entry.message.proposal_slot)) + .collect(); + slots.sort(); + assert_eq!( + slots, + vec![slot_in(GLOAS_EPOCH, 10), slot_in(GLOAS_EPOCH + 1, 4)] + ); } #[tokio::test] diff --git a/docs/metrics.md b/docs/metrics.md index c11ed81a..4de578d9 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -744,7 +744,8 @@ and absent is not the same as zero. | `ethlambda_validator_builder_registrations_sent_total` | Counter | Builder registrations a beacon node accepted (`--builder-proposals`) | | `ethlambda_validator_builder_registration_failures_total` | Counter | Registration batches no beacon node accepted. A missed registration costs at most a builder block, never a duty | | `ethlambda_validator_proposer_preferences_published_total` | Counter | Signed proposer preferences a beacon node accepted, counted per preference. Published for every proposal from the epoch before gloas, whether or not `--builder-proposals` is set | -| `ethlambda_validator_proposer_preferences_failures_total` | Counter | Proposer preferences a beacon node refused or failed to take (one count per preference per attempt), plus failed v2 proposer-duty fetches. The next slot retries; until a preference is accepted, builders' bids for its slot are ignored | +| `ethlambda_validator_proposer_preferences_failures_total` | Counter | Proposer preferences a beacon node refused or failed to take (one count per preference per attempt). The next slot retries; until a preference is accepted, builders' bids for its slot are ignored | +| `ethlambda_validator_proposer_preferences_duty_fetch_failures_total` | Counter | v2 proposer-duty fetches no beacon node answered (one count per target epoch per attempt). Without the duties no preference is signed for that epoch; a syncing node, or one without the v2 endpoint, shows up here | | `ethlambda_validator_blinded_blocks_proposed_total` | Counter | Blinded blocks a beacon node handed to its builder. A subset of `blocks_proposed_total` | | `ethlambda_validator_blinded_blocks_unconfirmed_total` | Counter | Blinded blocks the beacon node did not confirm (a 5xx or no answer), which the relay may still have published. Not retried, since a second block for the slot would be an equivocation | | `ethlambda_validator_block_publication_delay_seconds` | Histogram | Slot start to block accepted. Its buckets are tighter than the attestation histogram's, because a block is due at the slot boundary rather than a third of the way in |