From 8529fe0a7d665f0d63ca08188f92cd1be13b6943 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:48:57 -0300 Subject: [PATCH 1/6] perf(beacon): back the participation lists and pending deposits with persistent trees On mainnet (about 2.4M validators) the post-block state_root check costs about 27 ms per block, and perf attributes most of BeaconState::hash_tree_root to plain SszList fields re-merkleized from scratch on every call: the two participation lists (2.4 MB each) and pending_deposits. Lighthouse keeps every state list in milhouse trees with cached node hashes. EpochParticipation (altair through fulu), PendingDeposits (electra, fulu) and their gloas progressive counterparts now use ethlambda_ssz_tree lists. Only the current epoch's participation list is written by a block, so the previous epoch's list and an untouched deposit queue reuse their cached roots. The SSZ encoding and roots are unchanged (tests compare against the flat lists), so DB_VERSION is not bumped. Participation writes are buffered densely, not in a BTreeMap: at one slot's worth of scattered attesters the BTreeMap inserts cost several times the hashing they save. A quick ignored benchmark at 2.4M entries and 75k writes measures about 13 ms for the flat lists against about 5 ms for the trees. Reads go through ParticipationRef and ParticipationMut, since a tree has no slice view. The participation scan walks the list in lockstep with the registry instead of a tree descent per validator. Both lists are also rebased onto a resident relative of a state decoded from storage. --- CLAUDE.md | 1 + .../src/beacon/block_production.rs | 5 +- .../src/beacon/helpers/electra.rs | 26 +- .../src/beacon/helpers/participation.rs | 13 +- .../src/beacon/stf/epoch/altair.rs | 3 +- .../src/beacon/stf/epoch/gloas.rs | 12 +- .../state_transition/src/beacon/stf/gloas.rs | 2 +- .../state_transition/src/beacon/upgrade.rs | 2 +- .../types/src/beacon/containers/electra.rs | 47 +++- .../types/src/beacon/containers/gloas.rs | 10 +- .../common/types/src/beacon/containers/mod.rs | 258 ++++++++++++++++-- .../types/src/beacon/containers/shared.rs | 150 +++++++++- crates/common/types/tests/beacon_json.rs | 30 +- crates/storage/src/beacon_state_delta.rs | 4 +- docs/beacon_stf.md | 29 +- 15 files changed, 513 insertions(+), 79 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a171704a8..dddf2dba8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -646,6 +646,7 @@ existing once per chain. `[patch.crates-io]` override because nothing outside this workspace depends on libssz, so there is no second copy to unify; that also keeps the manifest free of a `[patch]` table, which `shadow/cargo-patch.toml` would collide with. +- **The participation lists and `pending_deposits` are tree-backed too** (`EpochParticipation`, `PendingDeposits`; progressive from gloas), reached through `altair_validator_lists` (`ParticipationRef`, no slice) and `epoch_participation_mut`; scan with `ParticipationRef::iter`, not `get(i)`. - **`Validators` and `Balances` are `ethlambda_ssz_tree::List`s** (gloas: `ethlambda_ssz_tree::ProgressiveList`, same method set), persistent Merkle trees that cache node hashes and share unchanged subtrees between diff --git a/crates/blockchain/state_transition/src/beacon/block_production.rs b/crates/blockchain/state_transition/src/beacon/block_production.rs index 9c3d0f8ce..2c02e240b 100644 --- a/crates/blockchain/state_transition/src/beacon/block_production.rs +++ b/crates/blockchain/state_transition/src/beacon/block_production.rs @@ -905,9 +905,8 @@ mod tests { let BeaconState::Fulu(inner) = &mut state else { unreachable!("built as fulu") }; - for flags in inner.current_epoch_participation.iter_mut() { - *flags = 0b111; - } + let count = inner.current_epoch_participation.len(); + inner.current_epoch_participation = vec![0b111; count].try_into().unwrap(); assert!(pack_attestations(&state, vec![candidate]).is_empty()); } diff --git a/crates/blockchain/state_transition/src/beacon/helpers/electra.rs b/crates/blockchain/state_transition/src/beacon/helpers/electra.rs index 94fa7d20b..ce9c9bc6b 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/electra.rs @@ -1070,14 +1070,14 @@ impl<'a> PendingQueueFields<'a> { /// Queues one more [`electra::PendingDeposit`], pushed by /// [`queue_excess_active_balance`], [`queue_entire_balance_and_reset_validator`], - /// and the deposit-request processors. Infallible on gloas's unbounded - /// queue; electra's and fulu's bounded one can still reject a length over - /// its own SSZ limit. + /// and the deposit-request processors. Electra's and fulu's bounded queue + /// can reject a length over its own SSZ limit; gloas's progressive one + /// never does. pub(crate) fn push_pending_deposit(&mut self, deposit: electra::PendingDeposit) -> Result<()> { match self { PendingQueueFields::Electra(state) => state.pending_deposits.push(deposit)?, PendingQueueFields::Fulu(state) => state.pending_deposits.push(deposit)?, - PendingQueueFields::Gloas(state) => state.pending_deposits.push(deposit), + PendingQueueFields::Gloas(state) => state.pending_deposits.push(deposit)?, } Ok(()) } @@ -1093,19 +1093,19 @@ impl<'a> PendingQueueFields<'a> { pub(crate) fn take_pending_deposits(&mut self) -> Vec { match self { PendingQueueFields::Electra(state) => { - core::mem::take(&mut state.pending_deposits).into_inner() + core::mem::take(&mut state.pending_deposits).to_vec() } PendingQueueFields::Fulu(state) => { - core::mem::take(&mut state.pending_deposits).into_inner() + core::mem::take(&mut state.pending_deposits).to_vec() } PendingQueueFields::Gloas(state) => { - core::mem::take(&mut state.pending_deposits).into_inner() + core::mem::take(&mut state.pending_deposits).to_vec() } } } /// See [`Self::take_pending_deposits`]. Fallible only for electra's and - /// fulu's bounded `SszList`; gloas's progressive one never rejects a + /// fulu's bounded list; gloas's progressive one never rejects a /// length. pub(crate) fn set_pending_deposits( &mut self, @@ -1230,9 +1230,9 @@ impl<'a> PendingQueueFields<'a> { /// and `inactivity_scores` by one all-zero entry each, keeping them /// exactly as long as the registry after a new validator is appended. /// - /// Gloas's own three lists are progressive (`libssz_types::ProgressiveList`), - /// so their `push` is infallible, unlike electra's and fulu's bounded - /// `SszList`. + /// Gloas's own lists are progressive, so their pushes cannot hit a length + /// limit; its inactivity scores are still a flat + /// `libssz_types::ProgressiveList`, whose `push` is infallible outright. pub(crate) fn push_empty_participation_and_inactivity(&mut self) -> Result<()> { match self { PendingQueueFields::Electra(state) => { @@ -1246,8 +1246,8 @@ impl<'a> PendingQueueFields<'a> { state.inactivity_scores.push(0)?; } PendingQueueFields::Gloas(state) => { - state.previous_epoch_participation.push(0); - state.current_epoch_participation.push(0); + state.previous_epoch_participation.push(0)?; + state.current_epoch_participation.push(0)?; state.inactivity_scores.push(0); } } diff --git a/crates/blockchain/state_transition/src/beacon/helpers/participation.rs b/crates/blockchain/state_transition/src/beacon/helpers/participation.rs index 10266925f..6a627801b 100644 --- a/crates/blockchain/state_transition/src/beacon/helpers/participation.rs +++ b/crates/blockchain/state_transition/src/beacon/helpers/participation.rs @@ -134,7 +134,16 @@ fn scan( let mut previous_error = None; let mut current_error = None; + // Walked in lockstep with the registry, not indexed per validator: each + // `get` on a tree-backed list is a descent, which at mainnet size was the + // largest cost of this pass. A list shorter than the registry yields + // `None` from the index it runs out at, the same as `get` did. + let mut previous_flags_iter = previous_list.iter(); + let mut current_flags_iter = current_list.iter(); + for (index, validator) in state.iter_validators().enumerate() { + let previous_entry = previous_flags_iter.next(); + let current_entry = current_flags_iter.next(); let effective_balance = validator.effective_balance; let active_now = is_active_validator(validator, current_epoch); let active_before = is_active_validator(validator, previous_epoch); @@ -145,7 +154,7 @@ fn scan( let mut bits = 0u8; if active_before { - match previous_list.get(index) { + match previous_entry { Some(&participation) => { if !validator.slashed { for (flag_index, total) in previous_flags.iter_mut().enumerate() { @@ -172,7 +181,7 @@ fn scan( } if current_target && active_now { - match current_list.get(index) { + match current_entry { Some(&participation) => { if has_flag(participation, constants::TIMELY_TARGET_FLAG_INDEX) && !validator.slashed diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/altair.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/altair.rs index f33f7641a..92223be4e 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/altair.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/altair.rs @@ -498,7 +498,8 @@ mod tests { // none has yet this epoch: only the previous epoch should justify. { let (previous_epoch_participation, _, _) = state.altair_validator_lists_mut().unwrap(); - for flags in previous_epoch_participation.iter_mut() { + for index in 0..previous_epoch_participation.len() { + let flags = &mut previous_epoch_participation[index]; *flags = add_flag(*flags, constants::TIMELY_TARGET_FLAG_INDEX); } } diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs index c07dddf8b..9daa541cf 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs @@ -326,7 +326,10 @@ mod tests { signature: Default::default(), slot: constants::GENESIS_SLOT, }; - gloas_inner_mut(&mut state).pending_deposits.push(deposit); + gloas_inner_mut(&mut state) + .pending_deposits + .push(deposit) + .unwrap(); let balance_before = state.balance(0).unwrap(); process_pending_deposits(&mut state, &config).unwrap(); @@ -360,7 +363,8 @@ mod tests { }; gloas_inner_mut(&mut state) .pending_deposits - .push(deposit.clone()); + .push(deposit.clone()) + .unwrap(); let balance_before = state.balance(0).unwrap(); process_pending_deposits(&mut state, &config).unwrap(); @@ -439,8 +443,8 @@ mod tests { }; { let inner = gloas_inner_mut(&mut state); - inner.pending_deposits.push(first); - inner.pending_deposits.push(second.clone()); + inner.pending_deposits.push(first).unwrap(); + inner.pending_deposits.push(second.clone()).unwrap(); } let balance_0_before = state.balance(0).unwrap(); diff --git a/crates/blockchain/state_transition/src/beacon/stf/gloas.rs b/crates/blockchain/state_transition/src/beacon/stf/gloas.rs index 0ef4c1c8c..82642934e 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/gloas.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/gloas.rs @@ -1495,7 +1495,7 @@ pub fn process_attestation( // Write phase: apply exactly the flags the read phase decided on. { - let participation_mut = state.epoch_participation_mut(current_epoch_target)?; + let mut participation_mut = state.epoch_participation_mut(current_epoch_target)?; let participation_len = participation_mut.len(); for (index, new_flags) in flag_updates { let flags = diff --git a/crates/blockchain/state_transition/src/beacon/upgrade.rs b/crates/blockchain/state_transition/src/beacon/upgrade.rs index 311d3637a..102bc528c 100644 --- a/crates/blockchain/state_transition/src/beacon/upgrade.rs +++ b/crates/blockchain/state_transition/src/beacon/upgrade.rs @@ -1020,7 +1020,7 @@ fn onboard_builders_from_pending_deposits( // and taking ownership here is what lets the loop body hold `state` // mutably (`add_builder_to_registry`, the balance top-up) without a // lingering borrow of one of its own fields. - for deposit in std::mem::take(&mut state.pending_deposits) { + for deposit in std::mem::take(&mut state.pending_deposits).to_vec() { if validator_pubkeys.contains(&deposit.pubkey) { requeue(&mut pending_deposits, &mut pending_index, deposit); continue; diff --git a/crates/common/types/src/beacon/containers/electra.rs b/crates/common/types/src/beacon/containers/electra.rs index f22a60e01..896e0482c 100644 --- a/crates/common/types/src/beacon/containers/electra.rs +++ b/crates/common/types/src/beacon/containers/electra.rs @@ -108,7 +108,16 @@ pub type Attestations = SszList; /// Deposits queued in the state, not yet credited to the validator registry. -pub type PendingDeposits = SszList; +/// +/// Tree-backed: a block only appends (deposit requests) and the epoch drops a +/// prefix, so every other block leaves the list untouched and the state root +/// reuses its cached hash instead of re-merkleizing every queued deposit. +/// Writes are buffered in a `BTreeMap`, since they are a handful of pushes. +pub type PendingDeposits = ethlambda_ssz_tree::List< + PendingDeposit, + { preset::PENDING_DEPOSITS_LIMIT }, + std::collections::BTreeMap, +>; /// Partial withdrawals queued in the state, not yet paid out. pub type PendingPartialWithdrawals = @@ -874,4 +883,40 @@ mod tests { assert!(!::is_fixed_size()); assert!(!::is_fixed_size()); } + + fn pending_deposit(i: u64) -> PendingDeposit { + PendingDeposit { + pubkey: BlsPubkey([i as u8; 48]), + withdrawal_credentials: [(i >> 8) as u8; 32].into(), + amount: i, + signature: BlsSignature([i as u8; 96]), + slot: i, + } + } + + #[test] + fn tree_backed_pending_deposits_hash_like_the_flat_list() { + use crate::primitives::HashTreeRoot as _; + type Flat = SszList; + + for len in [0usize, 1, 2, 3, 17, 1000] { + let values: Vec<_> = (0..len as u64).map(pending_deposit).collect(); + let flat = Flat::try_from(values.clone()).unwrap(); + let mut tree = PendingDeposits::try_from(values).unwrap(); + assert_eq!(tree.hash_tree_root(), flat.hash_tree_root(), "len {len}"); + assert_eq!(tree.to_ssz(), flat.to_ssz(), "len {len}"); + + // An append, the one per-block write, rehashes like a rebuild. + tree.push(pending_deposit(9_999)).unwrap(); + tree.apply_updates(); + let mut grown: Vec<_> = flat.iter().cloned().collect(); + grown.push(pending_deposit(9_999)); + let flat = Flat::try_from(grown).unwrap(); + assert_eq!( + tree.hash_tree_root(), + flat.hash_tree_root(), + "len {len} + 1" + ); + } + } } diff --git a/crates/common/types/src/beacon/containers/gloas.rs b/crates/common/types/src/beacon/containers/gloas.rs index 2e7964bf4..8b60f51c1 100644 --- a/crates/common/types/src/beacon/containers/gloas.rs +++ b/crates/common/types/src/beacon/containers/gloas.rs @@ -67,8 +67,7 @@ use super::shared::{ use crate::beacon::preset; use crate::beacon::primitives::{ BlsPubkey, BlsSignature, Bytes32, ColumnIndex, Epoch, ExecutionAddress, ExecutionBlockHash, - Gwei, KzgCommitment, KzgProof, ParticipationFlags, Root, Slot, Uint256, ValidatorIndex, - WithdrawalIndex, + Gwei, KzgCommitment, KzgProof, Root, Slot, Uint256, ValidatorIndex, WithdrawalIndex, }; // --------------------------------------------------------------------------- @@ -123,11 +122,14 @@ pub type BuilderDepositRequests = ProgressiveList; pub type BuilderExitRequests = ProgressiveList; /// Per-validator participation flags for one epoch, positionally parallel to /// the registry. -pub type EpochParticipation = ProgressiveList; +pub type EpochParticipation = super::shared::ProgressiveEpochParticipation; /// Per-validator inactivity scores, positionally parallel to the registry. pub type InactivityScores = ProgressiveList; /// Deposits known but not yet credited to the validator registry. -pub type PendingDeposits = ProgressiveList; +pub type PendingDeposits = ethlambda_ssz_tree::ProgressiveList< + PendingDeposit, + std::collections::BTreeMap, +>; /// Partial withdrawals known but not yet paid out. pub type PendingPartialWithdrawals = ProgressiveList; /// Validator consolidations known but not yet applied. diff --git a/crates/common/types/src/beacon/containers/mod.rs b/crates/common/types/src/beacon/containers/mod.rs index a7859de2e..e2a4b43ed 100644 --- a/crates/common/types/src/beacon/containers/mod.rs +++ b/crates/common/types/src/beacon/containers/mod.rs @@ -435,33 +435,35 @@ tree_fields!( phase0::BeaconState => validators, balances, block_roots, state_roots, historical_roots, eth1_data_votes, randao_mixes, slashings ); tree_fields!( - altair::BeaconState => validators, balances, block_roots, state_roots, historical_roots, eth1_data_votes, randao_mixes, slashings, inactivity_scores + altair::BeaconState => validators, balances, block_roots, state_roots, historical_roots, eth1_data_votes, randao_mixes, slashings, inactivity_scores, + previous_epoch_participation, current_epoch_participation ); tree_fields!( - bellatrix::BeaconState => validators, balances, block_roots, state_roots, historical_roots, eth1_data_votes, randao_mixes, slashings, inactivity_scores + bellatrix::BeaconState => validators, balances, block_roots, state_roots, historical_roots, eth1_data_votes, randao_mixes, slashings, inactivity_scores, + previous_epoch_participation, current_epoch_participation ); tree_fields!( capella::BeaconState => validators, balances, block_roots, state_roots, historical_roots, eth1_data_votes, randao_mixes, slashings, inactivity_scores, - historical_summaries + historical_summaries, previous_epoch_participation, current_epoch_participation ); tree_fields!( deneb::BeaconState => validators, balances, block_roots, state_roots, historical_roots, eth1_data_votes, randao_mixes, slashings, inactivity_scores, - historical_summaries + historical_summaries, previous_epoch_participation, current_epoch_participation ); tree_fields!( electra::BeaconState => validators, balances, block_roots, state_roots, historical_roots, eth1_data_votes, randao_mixes, slashings, inactivity_scores, - historical_summaries + historical_summaries, previous_epoch_participation, current_epoch_participation, pending_deposits ); tree_fields!( fulu::BeaconState => validators, balances, block_roots, state_roots, historical_roots, eth1_data_votes, randao_mixes, slashings, inactivity_scores, - historical_summaries + historical_summaries, previous_epoch_participation, current_epoch_participation, pending_deposits ); // Gloas keeps `validators` and `balances` in progressive trees, and its // `inactivity_scores` in a flat progressive list that buffers nothing, so the // scores are not listed. Every other field is the shared tree type. tree_fields!( gloas::BeaconState => validators, balances, block_roots, state_roots, historical_roots, eth1_data_votes, randao_mixes, slashings, - historical_summaries + historical_summaries, previous_epoch_participation, current_epoch_participation, pending_deposits ); /// Generates read and write accessors for state fields that every fork shares. @@ -613,6 +615,154 @@ enum RegistryMut<'a> { Progressive(&'a mut ProgressiveValidators, &'a mut ProgressiveBalances), } +/// One of a state's participation lists, whichever list kind its fork uses. +/// +/// Every fork from altair on keeps them in a tree-backed list, a bounded one +/// before gloas and a progressive one from gloas on. Neither has a slice +/// view, and a read needs only length, element and in-order access, which +/// this offers over both. Prefer [`Self::iter`] to [`Self::get`] when +/// visiting many: each `get` on a tree is a descent. +#[derive(Clone, Copy)] +pub enum ParticipationRef<'a> { + /// Altair through fulu. + Bounded(&'a EpochParticipation), + /// Gloas. + Progressive(&'a gloas::EpochParticipation), +} + +impl<'a> From<&'a EpochParticipation> for ParticipationRef<'a> { + fn from(list: &'a EpochParticipation) -> Self { + Self::Bounded(list) + } +} + +impl<'a> From<&'a gloas::EpochParticipation> for ParticipationRef<'a> { + fn from(list: &'a gloas::EpochParticipation) -> Self { + Self::Progressive(list) + } +} + +impl<'a> ParticipationRef<'a> { + /// The number of entries. + pub fn len(&self) -> usize { + match self { + Self::Bounded(list) => list.len(), + Self::Progressive(list) => list.len(), + } + } + + /// Whether there are no entries. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// The flags at `index`, or `None` past the end. A tree descent. + pub fn get(&self, index: usize) -> Option<&'a ParticipationFlags> { + match self { + Self::Bounded(list) => list.get(index), + Self::Progressive(list) => list.get(index), + } + } + + /// Every entry in order. + pub fn iter(&self) -> impl ExactSizeIterator + use<'a> { + match self { + Self::Bounded(list) => RegistryIter::Bounded(list.iter()), + Self::Progressive(list) => RegistryIter::Progressive(list.iter()), + } + } + + /// Whether a write is buffered and not yet folded into the tree. + pub fn has_pending_updates(&self) -> bool { + match self { + Self::Bounded(list) => list.has_pending_updates(), + Self::Progressive(list) => list.has_pending_updates(), + } + } + + /// Whether both lists share one allocation. Never true across list kinds. + pub fn ptr_eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Bounded(a), Self::Bounded(b)) => a.ptr_eq(b), + (Self::Progressive(a), Self::Progressive(b)) => a.ptr_eq(b), + _ => false, + } + } + + /// The entries, copied out. + pub fn to_vec(&self) -> Vec { + self.iter().copied().collect() + } +} + +impl PartialEq for ParticipationRef<'_> { + fn eq(&self, other: &Self) -> bool { + self.iter().eq(other.iter()) + } +} + +impl std::fmt::Debug for ParticipationRef<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_list().entries(self.iter()).finish() + } +} + +impl std::ops::Index for ParticipationRef<'_> { + type Output = ParticipationFlags; + + fn index(&self, index: usize) -> &ParticipationFlags { + self.get(index) + .unwrap_or_else(|| panic!("index {index} out of bounds for {} entries", self.len())) + } +} + +/// One of a state's participation lists, mutably, whichever list kind its +/// fork uses: element writes only, so it cannot change the list's length. +/// See [`BeaconState::epoch_participation_mut`]. +pub enum ParticipationMut<'a> { + /// Altair through fulu. + Bounded(&'a mut EpochParticipation), + /// Gloas. + Progressive(&'a mut gloas::EpochParticipation), +} + +impl<'a> From<&'a mut EpochParticipation> for ParticipationMut<'a> { + fn from(list: &'a mut EpochParticipation) -> Self { + Self::Bounded(list) + } +} + +impl<'a> From<&'a mut gloas::EpochParticipation> for ParticipationMut<'a> { + fn from(list: &'a mut gloas::EpochParticipation) -> Self { + Self::Progressive(list) + } +} + +impl ParticipationMut<'_> { + /// The number of entries. + pub fn len(&self) -> usize { + match self { + Self::Bounded(list) => list.len(), + Self::Progressive(list) => list.len(), + } + } + + /// Whether there are no entries. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// The flags at `index`, mutably, or `None` past the end. The entry is + /// copied into the list's write buffer on first touch whether or not the + /// caller changes it, so write only when the value differs. + pub fn get_mut(&mut self, index: usize) -> Option<&mut ParticipationFlags> { + match self { + Self::Bounded(list) => list.get_mut(index), + Self::Progressive(list) => list.get_mut(index), + } + } +} + /// The inactivity scores of a state, whichever list kind its fork uses. /// /// Every fork before gloas keeps them in a tree-backed list, which has no @@ -1016,6 +1166,8 @@ impl BeaconState { self.randao_mixes_mut().rebase_on(base.randao_mixes()); self.slashings_mut().rebase_on(base.slashings()); self.rebase_inactivity_scores_on(base); + self.rebase_participation_on(base); + self.rebase_pending_deposits_on(base); self.rebase_historical_summaries_on(base); } @@ -1047,6 +1199,55 @@ impl BeaconState { } } + /// Shares both participation lists with `base`'s where both states keep + /// the same list kind (altair through fulu, or both gloas). Positional, + /// so across an epoch boundary (the child's previous list is the base's + /// current one) it finds little to share; the lists are rebuilt wholesale + /// there anyway. + fn rebase_participation_on(&mut self, base: &BeaconState) { + if let (Ok((previous, current, _)), Ok((base_previous, base_current, _))) = ( + self.altair_validator_lists_mut(), + base.altair_validator_lists(), + ) { + if let ( + ParticipationRef::Bounded(base_previous), + ParticipationRef::Bounded(base_current), + ) = (base_previous, base_current) + { + previous.rebase_on(base_previous); + current.rebase_on(base_current); + } + } else if let (BeaconState::Gloas(state), BeaconState::Gloas(base)) = (self, base) { + state + .previous_epoch_participation + .rebase_on(&base.previous_epoch_participation); + state + .current_epoch_participation + .rebase_on(&base.current_epoch_participation); + } + } + + /// Shares `pending_deposits` with `base`'s where both keep the same list + /// kind (electra and fulu, or both gloas). + fn rebase_pending_deposits_on(&mut self, base: &BeaconState) { + let base_deposits = match base { + BeaconState::Electra(state) => &state.pending_deposits, + BeaconState::Fulu(state) => &state.pending_deposits, + BeaconState::Gloas(base) => { + if let BeaconState::Gloas(state) = self { + state.pending_deposits.rebase_on(&base.pending_deposits); + } + return; + } + _ => return, + }; + match self { + BeaconState::Electra(state) => state.pending_deposits.rebase_on(base_deposits), + BeaconState::Fulu(state) => state.pending_deposits.rebase_on(base_deposits), + _ => {} + } + } + /// Shares `historical_summaries` with `base`'s where both carry it, which /// is every fork from capella on. Earlier forks have no summaries, so any /// pairing with one of those does nothing. @@ -1175,9 +1376,9 @@ impl BeaconState { } /// The three per-validator lists that exist from altair on, by reference and - /// all at once: the two participation lists as plain slices, the inactivity - /// scores as an [`InactivityScoresRef`] since a tree-backed list has no - /// slice to hand out. + /// all at once: the two participation lists as [`ParticipationRef`]s and + /// the inactivity scores as an [`InactivityScoresRef`], since a tree-backed + /// list has no slice to hand out. /// /// These cannot join `shared_state_accessors`' lists, since phase0 has /// none of them, and a per-fork projection to a concrete state struct (the @@ -1190,11 +1391,10 @@ impl BeaconState { /// Gloas joins `carried_by` here, unlike [`Self::altair_validator_lists_mut`]: /// EIP-7688 makes all three progressive lists on a gloas state /// (`ProgressiveList` rather than `SszList`), a different concrete Rust - /// type from every earlier fork's, but both list kinds `Deref` to a plain - /// `[T]`, and a slice is all a *read* ever needs (`.get`, `.len`, - /// `.binary_search`, ...). Returning slices rather than the list types - /// themselves is what lets one `dispatch_state_from!` body serve both - /// kinds: the write side still needs the concrete container type, to grow + /// type from every earlier fork's. The `Ref` types are what let one + /// `dispatch_state_from!` body serve both kinds, since a read needs only + /// length, element and in-order access. The write side still needs the + /// concrete container type, to grow /// or replace the whole list, which is why [`Self::altair_validator_lists_mut`] /// stays bounded-only. On gloas, element writes go through /// [`Self::inactivity_score_mut`], and growing or replacing a list goes @@ -1208,16 +1408,16 @@ impl BeaconState { pub fn altair_validator_lists( &self, ) -> Result<( - &[ParticipationFlags], - &[ParticipationFlags], + ParticipationRef<'_>, + ParticipationRef<'_>, InactivityScoresRef<'_>, )> { dispatch_state_from!( self, "BeaconState::altair_validator_lists", |state| ( - &state.previous_epoch_participation[..], - &state.current_epoch_participation[..], + ParticipationRef::from(&state.previous_epoch_participation), + ParticipationRef::from(&state.current_epoch_participation), InactivityScoresRef::from(&state.inactivity_scores), ), carried_by: [Altair, Bellatrix, Capella, Deneb, Electra, Fulu, Gloas], @@ -1246,9 +1446,11 @@ impl BeaconState { /// accessor calls would each borrow the whole enum. /// /// Gloas is in `absent_from` here, not `carried_by`: the container type - /// this returns (`EpochParticipation`/`InactivityScores`, both `SszList`) - /// is fixed to the pre-gloas one, and `gloas::BeaconState`'s own three - /// lists are the progressive `ProgressiveList` instead, a different Rust + /// this returns (`EpochParticipation`/`InactivityScores`, both bounded + /// `ethlambda_ssz_tree::List`s) is fixed to the pre-gloas one, and + /// `gloas::BeaconState`'s own three lists are progressive instead + /// (`ethlambda_ssz_tree::ProgressiveList` for participation, a flat + /// `libssz_types::ProgressiveList` for the scores), a different Rust /// type a shared return type cannot name. A gloas caller either grows the /// registry through a per-fork projection /// (`PendingQueueFields::push_empty_participation_and_inactivity`), writes @@ -1310,7 +1512,7 @@ impl BeaconState { } /// `previous_epoch_participation` or `current_epoch_participation`, - /// mutably and as a plain slice, whichever `current` selects: element + /// mutably and as a [`ParticipationMut`], whichever `current` selects: element /// writes only, the same contract [`Self::inactivity_score_mut`]'s own /// doc gives for why that is enough to include gloas where /// [`Self::altair_validator_lists_mut`] cannot. @@ -1322,15 +1524,15 @@ impl BeaconState { /// [`Self::altair_validator_lists_mut`], on a fork that carries it, or /// through a gloas caller's own per-fork projection, the way /// [`Self::altair_validator_lists_mut`]'s own doc describes). - pub fn epoch_participation_mut(&mut self, current: bool) -> Result<&mut [ParticipationFlags]> { + pub fn epoch_participation_mut(&mut self, current: bool) -> Result> { dispatch_state_from!( self, "BeaconState::epoch_participation_mut", - |state| if current { - &mut state.current_epoch_participation[..] + |state| ParticipationMut::from(if current { + &mut state.current_epoch_participation } else { - &mut state.previous_epoch_participation[..] - }, + &mut state.previous_epoch_participation + }), carried_by: [Altair, Bellatrix, Capella, Deneb, Electra, Fulu, Gloas], absent_from: [Phase0], ) diff --git a/crates/common/types/src/beacon/containers/shared.rs b/crates/common/types/src/beacon/containers/shared.rs index 0b024f9f1..868b8265a 100644 --- a/crates/common/types/src/beacon/containers/shared.rs +++ b/crates/common/types/src/beacon/containers/shared.rs @@ -11,7 +11,7 @@ use std::collections::BTreeMap; use ethlambda_ssz_tree::{List, ProgressiveList as TreeProgressiveList, Vector}; use libssz_derive::{HashTreeRoot, SszDecode, SszEncode}; -use libssz_types::{SszBitvector, SszList, SszVector}; +use libssz_types::{SszBitvector, SszVector}; use crate::beacon::constants; use crate::beacon::preset; @@ -102,7 +102,22 @@ pub type DepositProof = SszVector; +/// +/// Tree-backed: the state root otherwise re-merkleized both multi-megabyte +/// lists from scratch on every block, while a block flips flags for a few +/// thousand validators at most. With the tree, only the leaves holding those +/// validators are rehashed; a flag is one byte, so 32 share a chunk and a +/// block's writes dirty far fewer leaves than it has attesters. +/// +/// Writes are buffered densely (the default `VecMap`), not in a `BTreeMap` +/// like the registry's. A block credits a thirty-second of the registry at +/// scattered positions, and measured at mainnet size the `BTreeMap`'s inserts +/// cost several times what the hashing it saves does (a few dozen +/// milliseconds against a few); the dense buffer is an indexed write, and its +/// slot-per-validator allocation is small next to that. The pass that writes +/// nearly every entry, the epoch rotation, does not buffer either: it replaces +/// the list with a freshly built one (see `process_participation_flag_updates`). +pub type EpochParticipation = List; /// Per-validator inactivity scores, positionally parallel to [`Validators`] /// (altair and later). @@ -132,6 +147,10 @@ pub type ProgressiveValidators = TreeProgressiveList; +/// The gloas participation flags, positionally parallel to +/// [`ProgressiveValidators`]. Tree-backed and buffered like [`EpochParticipation`]. +pub type ProgressiveEpochParticipation = TreeProgressiveList; + // --------------------------------------------------------------------------- // Misc // --------------------------------------------------------------------------- @@ -568,4 +587,131 @@ mod tests { assert!(::is_fixed_size()); assert!(::is_fixed_size()); } + + /// Flags of mixed values, so a packing or ordering mistake changes the root. + fn flags(len: usize) -> Vec { + (0..len) + .map(|i| (i * 7 % 8) as ParticipationFlags) + .collect() + } + + /// Lengths that hit an empty list, part of one chunk, exactly one chunk, a + /// partial last chunk and several leaf pages. + const LENGTHS: [usize; 8] = [0, 1, 31, 32, 33, 1000, 65_536 + 5, 300_001]; + + #[test] + fn tree_backed_participation_hashes_like_the_flat_list() { + use crate::primitives::HashTreeRoot as _; + type Flat = libssz_types::SszList; + + for len in LENGTHS { + let values = flags(len); + let flat = Flat::try_from(values.clone()).unwrap(); + let tree = EpochParticipation::try_from(values).unwrap(); + assert_eq!(tree.hash_tree_root(), flat.hash_tree_root(), "len {len}"); + assert_eq!(tree.to_ssz(), flat.to_ssz(), "len {len}"); + } + } + + #[test] + fn participation_writes_rehash_like_a_rebuilt_list() { + use crate::primitives::HashTreeRoot as _; + type Flat = libssz_types::SszList; + + let mut values = flags(70_000); + let mut tree = EpochParticipation::try_from(values.clone()).unwrap(); + // Warm every cached hash, so the writes below must invalidate the paths + // they touch rather than recompute the whole list. + tree.apply_updates(); + let _ = tree.hash_tree_root(); + + for index in [0usize, 31, 32, 4_095, 4_096, 69_999] { + values[index] |= 0b100; + *tree.get_mut(index).unwrap() |= 0b100; + } + tree.push(0b11).unwrap(); + values.push(0b11); + tree.apply_updates(); + + let flat = Flat::try_from(values).unwrap(); + assert_eq!(tree.hash_tree_root(), flat.hash_tree_root()); + assert_eq!(tree.to_ssz(), flat.to_ssz()); + } + + #[test] + fn progressive_participation_hashes_like_the_flat_list() { + use crate::primitives::HashTreeRoot as _; + + for len in LENGTHS { + let values = flags(len); + let flat = libssz_types::ProgressiveList::::from(values.clone()); + let tree = ProgressiveEpochParticipation::from(values); + assert_eq!(tree.hash_tree_root(), flat.hash_tree_root(), "len {len}"); + assert_eq!(tree.to_ssz(), flat.to_ssz(), "len {len}"); + } + } + + /// Run with `--ignored --nocapture`. Compares what a state root pays for + /// the two participation lists after one block, before (flat lists, + /// rehashed whole) and after (trees) at mainnet registry size. The block + /// credits one slot's worth of validators (a thirty-second of the + /// registry, at pseudo-random positions as committee shuffling places + /// them) in the current epoch's list; the previous one is untouched. + #[test] + #[ignore = "benchmark"] + fn participation_root_after_a_block_of_writes_bench() { + use crate::primitives::HashTreeRoot as _; + use std::time::Instant; + type Flat = libssz_types::SszList; + + const COUNT: usize = 2_400_000; + const TOUCHED: usize = COUNT / 32; + let values = flags(COUNT); + let mut state = 12345u64; + let touched: Vec = (0..TOUCHED) + .map(|_| { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + (state >> 33) as usize % COUNT + }) + .collect(); + + let flat_previous = Flat::try_from(values.clone()).unwrap(); + let mut flat_current = Flat::try_from(values.clone()).unwrap(); + let mut tree_previous = EpochParticipation::try_from(values.clone()).unwrap(); + let mut tree_current = EpochParticipation::try_from(values).unwrap(); + for tree in [&mut tree_previous, &mut tree_current] { + tree.apply_updates(); + let _ = tree.hash_tree_root(); + } + + let start = Instant::now(); + for &index in &touched { + flat_current[index] |= 0b100; + } + let flat_roots = ( + flat_previous.hash_tree_root(), + flat_current.hash_tree_root(), + ); + let flat_time = start.elapsed(); + + let start = Instant::now(); + for &index in &touched { + *tree_current.get_mut(index).unwrap() |= 0b100; + } + let write_time = start.elapsed(); + tree_previous.apply_updates(); + tree_current.apply_updates(); + let apply_time = start.elapsed(); + let tree_roots = ( + tree_previous.hash_tree_root(), + tree_current.hash_tree_root(), + ); + let tree_time = start.elapsed(); + println!("tree phases: writes {write_time:?}, apply {apply_time:?}, hash {tree_time:?}"); + + assert_eq!(flat_roots, tree_roots); + println!("flat: {flat_time:?}, tree: {tree_time:?} ({TOUCHED} of {COUNT} entries written)"); + } } diff --git a/crates/common/types/tests/beacon_json.rs b/crates/common/types/tests/beacon_json.rs index 5f50bbd60..8508a810e 100644 --- a/crates/common/types/tests/beacon_json.rs +++ b/crates/common/types/tests/beacon_json.rs @@ -492,8 +492,8 @@ fn altair_state() -> altair::BeaconState { balances: shared::Balances::try_from(vec![32_000_000_001u64]).expect("within capacity"), randao_mixes: fixtures::tree_vector(H256::repeat_byte(0xaa)), slashings: fixtures::tree_vector(1_000_000_000u64), - previous_epoch_participation: SszList::try_from(vec![7u8]).expect("within capacity"), - current_epoch_participation: SszList::try_from(vec![9u8]).expect("within capacity"), + previous_epoch_participation: vec![7u8].try_into().expect("within capacity"), + current_epoch_participation: vec![9u8].try_into().expect("within capacity"), justification_bits: { let mut bits = shared::JustificationBits::new(); bits.set(0, true).expect("in bounds"); @@ -639,8 +639,8 @@ fn bellatrix_state() -> bellatrix::BeaconState { balances: shared::Balances::try_from(vec![32_000_000_001u64]).expect("within capacity"), randao_mixes: fixtures::tree_vector(H256::repeat_byte(0xaa)), slashings: fixtures::tree_vector(1_000_000_000u64), - previous_epoch_participation: SszList::try_from(vec![7u8]).expect("within capacity"), - current_epoch_participation: SszList::try_from(vec![9u8]).expect("within capacity"), + previous_epoch_participation: vec![7u8].try_into().expect("within capacity"), + current_epoch_participation: vec![9u8].try_into().expect("within capacity"), justification_bits: { let mut bits = shared::JustificationBits::new(); bits.set(0, true).expect("in bounds"); @@ -796,8 +796,8 @@ fn capella_state() -> capella::BeaconState { balances: shared::Balances::try_from(vec![32_000_000_001u64]).expect("within capacity"), randao_mixes: fixtures::tree_vector(H256::repeat_byte(0xaa)), slashings: fixtures::tree_vector(1_000_000_000u64), - previous_epoch_participation: SszList::try_from(vec![7u8]).expect("within capacity"), - current_epoch_participation: SszList::try_from(vec![9u8]).expect("within capacity"), + previous_epoch_participation: vec![7u8].try_into().expect("within capacity"), + current_epoch_participation: vec![9u8].try_into().expect("within capacity"), justification_bits: { let mut bits = shared::JustificationBits::new(); bits.set(0, true).expect("in bounds"); @@ -965,8 +965,8 @@ fn deneb_state() -> deneb::BeaconState { balances: shared::Balances::try_from(vec![32_000_000_001u64]).expect("within capacity"), randao_mixes: fixtures::tree_vector(H256::repeat_byte(0xaa)), slashings: fixtures::tree_vector(1_000_000_000u64), - previous_epoch_participation: SszList::try_from(vec![7u8]).expect("within capacity"), - current_epoch_participation: SszList::try_from(vec![9u8]).expect("within capacity"), + previous_epoch_participation: vec![7u8].try_into().expect("within capacity"), + current_epoch_participation: vec![9u8].try_into().expect("within capacity"), justification_bits: { let mut bits = shared::JustificationBits::new(); bits.set(0, true).expect("in bounds"); @@ -1160,8 +1160,8 @@ fn electra_state() -> electra::BeaconState { balances: shared::Balances::try_from(vec![32_000_000_001u64]).expect("within capacity"), randao_mixes: fixtures::tree_vector(H256::repeat_byte(0xaa)), slashings: fixtures::tree_vector(1_000_000_000u64), - previous_epoch_participation: SszList::try_from(vec![7u8]).expect("within capacity"), - current_epoch_participation: SszList::try_from(vec![9u8]).expect("within capacity"), + previous_epoch_participation: vec![7u8].try_into().expect("within capacity"), + current_epoch_participation: vec![9u8].try_into().expect("within capacity"), justification_bits: { let mut bits = shared::JustificationBits::new(); bits.set(0, true).expect("in bounds"); @@ -1187,7 +1187,8 @@ fn electra_state() -> electra::BeaconState { earliest_exit_epoch: 53, consolidation_balance_to_consume: 54, earliest_consolidation_epoch: 55, - pending_deposits: SszList::try_from(vec![fixtures::pending_deposit(60)]) + pending_deposits: vec![fixtures::pending_deposit(60)] + .try_into() .expect("within capacity"), pending_partial_withdrawals: SszList::try_from(vec![fixtures::pending_partial_withdrawal( 70, @@ -1251,8 +1252,8 @@ fn fulu_state() -> fulu::BeaconState { balances: shared::Balances::try_from(vec![32_000_000_001u64]).expect("within capacity"), randao_mixes: fixtures::tree_vector(H256::repeat_byte(0xaa)), slashings: fixtures::tree_vector(1_000_000_000u64), - previous_epoch_participation: SszList::try_from(vec![7u8]).expect("within capacity"), - current_epoch_participation: SszList::try_from(vec![9u8]).expect("within capacity"), + previous_epoch_participation: vec![7u8].try_into().expect("within capacity"), + current_epoch_participation: vec![9u8].try_into().expect("within capacity"), justification_bits: { let mut bits = shared::JustificationBits::new(); bits.set(0, true).expect("in bounds"); @@ -1278,7 +1279,8 @@ fn fulu_state() -> fulu::BeaconState { earliest_exit_epoch: 53, consolidation_balance_to_consume: 54, earliest_consolidation_epoch: 55, - pending_deposits: SszList::try_from(vec![fixtures::pending_deposit(60)]) + pending_deposits: vec![fixtures::pending_deposit(60)] + .try_into() .expect("within capacity"), pending_partial_withdrawals: SszList::try_from(vec![fixtures::pending_partial_withdrawal( 70, diff --git a/crates/storage/src/beacon_state_delta.rs b/crates/storage/src/beacon_state_delta.rs index d03d8a88d..2f009bb7c 100644 --- a/crates/storage/src/beacon_state_delta.rs +++ b/crates/storage/src/beacon_state_delta.rs @@ -408,8 +408,8 @@ pub(crate) mod tests { // One slot's attesters: one epoch's worth of committees split across // SLOTS_PER_EPOCH slots. let attesters = VALIDATOR_COUNT / preset::SLOTS_PER_EPOCH as usize; - for participation in state.current_epoch_participation[..attesters].iter_mut() { - *participation = 0b0000_0111; + for index in 0..attesters { + state.current_epoch_participation[index] = 0b0000_0111; } state.latest_execution_payload_header = execution_payload_header_for_slot(state.slot); diff --git a/docs/beacon_stf.md b/docs/beacon_stf.md index b2df5a0f2..fe6c208b9 100644 --- a/docs/beacon_stf.md +++ b/docs/beacon_stf.md @@ -361,6 +361,28 @@ whose registry-sized lists differ in length fall back to the step-by-step path. The debug oracle runs the specification-shaped steps on a clone and asserts the two full states hash equal. +### Participation lists and pending deposits + +The two participation lists (one byte per validator, 2.4 MB each on mainnet) +and `pending_deposits` are tree-backed too. As flat `SszList`s they were +re-merkleized from scratch for every state root, which profiled at most of +`BeaconState::hash_tree_root`; a block only flips flags for the validators it +attests for, so the tree rehashes the leaves holding those. Writes to the +participation lists are buffered densely (the default `VecMap`), unlike the +registry's `BTreeMap`: a block writes tens of thousands of scattered entries, and +measured at mainnet size the `BTreeMap`'s inserts cost several times the hashing +they save, while the dense buffer is an indexed write. The epoch +rotation does not buffer: it moves `current` into `previous` and builds a fresh +zeroed list. `pending_deposits` only grows at the end during a block and is +rebuilt from a `Vec` once per epoch, so a `BTreeMap` buffer fits it too. + +Reads go through `BeaconState::altair_validator_lists`, which returns +`ParticipationRef`s (there is no slice to hand out); a full scan walks +`ParticipationRef::iter` in lockstep with `iter_validators`, since each `get` is +a tree descent. Element writes go through `epoch_participation_mut`, which +returns a `ParticipationMut`. The SSZ encoding is unchanged, so `DB_VERSION` is +too. + ## Gloas Five of gloas's EIPs shape this module: ePBS (EIP-7732), progressive lists and @@ -456,11 +478,12 @@ of keeping copies. EIP-7688 makes most lists in the gloas state and body unbounded, merkleized as a chain of growing subtrees (EIP-7916), and makes the big containers progressive (EIP-7495); a progressive container has no fixed depth. The other progressive -lists are libssz's `ProgressiveList`. The registry is the exception: -`validators` and `balances` stay tree-backed, through +lists are libssz's `ProgressiveList`. The exceptions stay tree-backed, through `ethlambda_ssz_tree::ProgressiveList`, which has the same persistent nodes, update buffer and rebase as the bounded `List` and the same method set, so a -gloas state shares subtrees with its parent the way earlier ones do. +gloas state shares subtrees with its parent the way earlier ones do: `validators` +and `balances`, the two participation lists and `pending_deposits`. (The +inactivity scores are still a flat progressive list.) The registry therefore has two list types across the forks, and an accessor cannot return either one. The whole-registry accessors were replaced by From 6de47281237e40f3effba51eda5b2fa85c519183 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 3 Nov 2026 12:00:29 -0300 Subject: [PATCH 2/6] feat(ssz-tree): add a blind set, a sorted reader and a shared-node repeat Three primitives the participation-list fixes that follow need, kept in their own commit so the fixes read as pure call-site changes. - `List::set` / `ProgressiveList::set` buffer a value without first descending the tree to copy the old one into the write buffer, which `get_mut` does even when the caller already knows the old value. - `sorted_reader()` reads non-decreasing indices keeping the current leaf's slice, descending only when an index leaves it. Pending writes are consulted first, exactly as `get` does, so the answers are identical. - `List::repeat` / `ProgressiveList::repeat` build a list of identical elements by sharing one full leaf and one full inner node per level, each caching its hash once, so building and hashing cost the tree height plus the partial last leaf instead of the whole length. Nodes are immutable and every write copies the path it touches, so a shared leaf can never be mutated; tests write into a repeated list and check its clone and the untouched leaves, and compare roots against the Vec-built list at lengths around every leaf and node boundary. --- crates/common/ssz-tree/src/interface.rs | 101 +++++++++ crates/common/ssz-tree/src/lib.rs | 5 +- crates/common/ssz-tree/src/list.rs | 195 +++++++++++++++++- .../common/ssz-tree/src/progressive_list.rs | 130 +++++++++++- crates/common/ssz-tree/src/tree.rs | 86 ++++++++ 5 files changed, 514 insertions(+), 3 deletions(-) diff --git a/crates/common/ssz-tree/src/interface.rs b/crates/common/ssz-tree/src/interface.rs index 19f3b8fff..9612fd63a 100644 --- a/crates/common/ssz-tree/src/interface.rs +++ b/crates/common/ssz-tree/src/interface.rs @@ -69,6 +69,43 @@ impl> Interface { self.updates.get_mut(index) } + /// Buffers `value` as the new value of the element at `index`, without + /// reading the old one. Whether `index` is in range is returned; nothing is + /// written past the end. + pub(crate) fn set(&mut self, index: usize, value: T) -> bool { + if index >= self.len { + return false; + } + self.updates.insert(index, value); + true + } + + /// `len` copies of `value` in a new tree of height `depth`, sharing every + /// identical subtree. The caller checks the type's limit. + pub(crate) fn repeat(value: &T, len: usize, depth: usize) -> Self { + Self { + tree: Tree::repeat(value, len, depth), + depth, + committed_len: len, + len, + updates: U::default(), + } + } + + /// A reader for ascending indices: see [`SortedReader`]. + pub(crate) fn sorted_reader(&self) -> SortedReader<'_, T, U> { + SortedReader { + tree: &self.tree, + depth: self.depth, + updates: &self.updates, + no_updates: self.updates.is_empty(), + len: self.len, + first: 0, + run: &[], + last: 0, + } + } + /// Buffers `value` as the new last element. The caller checks the limit. pub(crate) fn push(&mut self, value: T) { self.updates.insert(self.len, value); @@ -225,3 +262,67 @@ impl> Interface { } } } + +/// Reads elements at non-decreasing indices, descending the tree only when an +/// index leaves the leaf it last read. +/// +/// What [`Interface::get`] does per index is a full descent; a pass that +/// visits many indices in order, such as the attesters of one attestation, +/// pays that for every one although consecutive indices mostly share a leaf. +/// Pending writes are consulted first, exactly as `get` does, so the answers +/// are identical to `get`'s. +pub struct SortedReader<'a, T, U> { + tree: &'a Tree, + depth: usize, + updates: &'a U, + no_updates: bool, + len: usize, + /// Index of the first element of `run`. + first: usize, + /// The elements of the leaf last read from. + run: &'a [T], + /// The previous index, to check the order in debug builds. + last: usize, +} + +impl<'a, T: Value, U: UpdateMap> SortedReader<'a, T, U> { + /// The element at `index`, or `None` past the end. `index` must not be + /// below the previous call's (checked in debug builds); the answer would + /// still be right, but each step back would cost a descent. + pub fn get(&mut self, index: usize) -> Option<&'a T> { + debug_assert!( + index >= self.last, + "sorted reader read {index} after {}", + self.last + ); + self.last = index; + if index >= self.len { + return None; + } + if !self.no_updates + && let Some(value) = self.updates.get(index) + { + return Some(value); + } + if let Some(value) = index + .checked_sub(self.first) + .and_then(|offset| self.run.get(offset)) + { + return Some(value); + } + let (first, run) = self.tree.leaf_run(index, self.depth)?; + self.first = first; + self.run = run; + run.get(index - first) + } + + /// The number of elements, pending pushes included. + pub fn len(&self) -> usize { + self.len + } + + /// Whether the list holds no elements. + pub fn is_empty(&self) -> bool { + self.len == 0 + } +} diff --git a/crates/common/ssz-tree/src/lib.rs b/crates/common/ssz-tree/src/lib.rs index fbac1f77b..039c1a7fb 100644 --- a/crates/common/ssz-tree/src/lib.rs +++ b/crates/common/ssz-tree/src/lib.rs @@ -82,9 +82,12 @@ mod update_map; mod vector; pub use cursor::{ElemCow, IterCow}; +pub use interface::SortedReader; pub use iter::Iter; pub use list::List; -pub use progressive_list::{ProgressiveIter, ProgressiveIterCow, ProgressiveList}; +pub use progressive_list::{ + ProgressiveIter, ProgressiveIterCow, ProgressiveList, ProgressiveSortedReader, +}; pub use update_map::{UpdateMap, VecMap}; pub use vector::Vector; diff --git a/crates/common/ssz-tree/src/list.rs b/crates/common/ssz-tree/src/list.rs index a36fbb57a..be7fc2faf 100644 --- a/crates/common/ssz-tree/src/list.rs +++ b/crates/common/ssz-tree/src/list.rs @@ -8,7 +8,7 @@ use libssz_merkle::{HashTreeRoot, Sha2Hasher, Sha256Hasher, mix_in_length}; use libssz_types::TypeError; use crate::cursor::{ElemCow, IterCow}; -use crate::interface::Interface; +use crate::interface::{Interface, SortedReader}; use crate::iter::Iter; use crate::update_map::{UpdateMap, VecMap}; use crate::{Hash256, Value, tree_depth}; @@ -65,6 +65,34 @@ impl> List { self.interface.get_mut(index) } + /// Buffers `value` as the new value of the element at `index`, without + /// reading the old one (unlike [`List::get_mut`], which copies it into the + /// buffer first). Returns `false`, writing nothing, past the end. + #[must_use = "a write past the end is refused"] + pub fn set(&mut self, index: usize, value: T) -> bool { + self.interface.set(index, value) + } + + /// A reader for non-decreasing indices that descends the tree only when + /// an index leaves the leaf it last read. See [`SortedReader`]. + pub fn sorted_reader(&self) -> SortedReader<'_, T, U> { + self.interface.sorted_reader() + } + + /// A list of `len` copies of `value`, in which every identical subtree is + /// shared rather than built again, so building it and hashing it cost + /// about the tree's height plus a partial last leaf, not `len`. It behaves + /// as the list built from the equivalent `Vec`: writes copy the path they + /// touch and never change a shared node. + pub fn repeat(value: T, len: usize) -> Result { + if len > N { + return Err(TypeError::OverCapacity { max: N, got: len }); + } + Ok(Self { + interface: Interface::repeat(&value, len, Self::depth()), + }) + } + /// Appends `value`, buffered until [`List::apply_updates`]. pub fn push(&mut self, value: T) -> Result<(), TypeError> { if self.len() >= N { @@ -575,6 +603,171 @@ mod tests { assert_eq!(list_err, ssz_err); } + /// A registry-sized limit, so the tree has inner nodes above its leaves. + type Flags = List; + + #[test] + fn set_writes_without_reading_and_refuses_past_the_end() { + let mut list = Balances::try_from(vec![1, 2, 3]).unwrap(); + assert!(list.set(1, 20)); + assert!(!list.set(3, 40)); + assert_eq!(list.to_vec(), vec![1, 20, 3]); + list.push(4).unwrap(); + // A pushed element is in range too. + assert!(list.set(3, 40)); + let mut expected = Balances::try_from(vec![1, 2, 3, 4]).unwrap(); + *expected.get_mut(1).unwrap() = 20; + *expected.get_mut(3).unwrap() = 40; + assert_eq!(list, expected); + list.apply_updates(); + assert_eq!(root(&list), root(&expected)); + } + + #[test] + fn a_sorted_reader_agrees_with_get_at_every_index() { + for len in [0usize, 1, 31, 32, 4_095, 4_096, 4_097, 10_000, 70_000] { + let values: Vec = (0..len).map(|i| (i % 251) as u8).collect(); + let mut list = Flags::try_from(values).unwrap(); + for index in [0usize, 5, 4_096, 9_999, 69_999] { + if index < len { + assert!(list.set(index, 255)); + } + } + list.push(9).unwrap(); + for step in [1usize, 7, 5_000] { + let mut reader = list.sorted_reader(); + for index in (0..list.len() + 2).step_by(step) { + assert_eq!( + reader.get(index), + list.get(index), + "len {len} index {index}" + ); + } + } + // Applied, the reader reads the tree alone. + list.apply_updates(); + let mut reader = list.sorted_reader(); + for index in 0..list.len() + 2 { + assert_eq!( + reader.get(index), + list.get(index), + "len {len} index {index}" + ); + } + } + } + + #[test] + fn repeat_hashes_like_the_list_built_from_a_vec_at_every_length() { + const LEAF: usize = 4096; + const NODE: usize = LEAF * 512; + let lengths = [ + 0, + 1, + 31, + 32, + 33, + LEAF - 1, + LEAF, + LEAF + 1, + 3 * LEAF + 17, + NODE - 1, + NODE, + NODE + 1, + 2 * NODE, + 2 * NODE + LEAF, + 2_400_000, + ]; + for len in lengths { + for value in [0u8, 5] { + let repeated = Flags::repeat(value, len).unwrap(); + let built = Flags::try_from(vec![value; len]).unwrap(); + assert_eq!(repeated.len(), len, "len {len}"); + assert_eq!(root(&repeated), root(&built), "len {len} value {value}"); + assert_eq!(repeated.to_ssz(), built.to_ssz(), "len {len}"); + assert_eq!(repeated.iter().count(), len, "len {len}"); + } + } + } + + #[test] + fn repeat_fits_trees_without_inner_nodes_and_composite_elements() { + // The whole tree is one leaf. + for len in [0usize, 1, 100, 1024] { + let repeated = List::::repeat(9, len).unwrap(); + let built = List::::try_from(vec![9u64; len]).unwrap(); + assert_eq!(root(&repeated), root(&built), "len {len}"); + } + // One element per leaf. + for len in [0usize, 1, 2, 7, 100] { + let repeated = Items::repeat(item(3), len).unwrap(); + let built = Items::try_from(vec![item(3); len]).unwrap(); + assert_eq!(root(&repeated), root(&built), "len {len}"); + assert_eq!(repeated, built); + } + assert_eq!( + Items::repeat(item(3), 101), + Err(TypeError::OverCapacity { max: 100, got: 101 }) + ); + } + + #[test] + fn a_write_to_a_repeated_list_never_touches_a_shared_leaf() { + // Three leaves, the last partial, all but that one the same allocation. + const LEN: usize = 4096 * 2 + 1808; + let mut list = Flags::repeat(0, LEN).unwrap(); + let clone = list.clone(); + let hashed_before = root(&list); + + assert!(list.set(5_000, 3)); + *list.get_mut(0).unwrap() = 1; + list.apply_updates(); + + let mut expected = vec![0u8; LEN]; + expected[5_000] = 3; + expected[0] = 1; + assert_eq!(list.to_vec(), expected); + assert_eq!( + root(&list), + root(&Flags::try_from(expected.clone()).unwrap()) + ); + // The clone, and the leaves the writes did not land in, are unchanged. + assert_eq!(clone.to_vec(), vec![0u8; LEN]); + assert_eq!(root(&clone), hashed_before); + assert_eq!(list.get(4_096), Some(&0)); + assert_eq!(list.get(8_192), Some(&0)); + + // The other write paths behave as on any list. + let mut cow = Flags::repeat(0, LEN).unwrap(); + cow.try_update_each(|element| { + if element.index() % 1000 == 0 { + *element.make_mut() = 7; + } + Ok::<_, ()>(()) + }) + .unwrap(); + let cow_expected: Vec = (0..LEN) + .map(|i| if i % 1000 == 0 { 7 } else { 0 }) + .collect(); + assert_eq!(cow.to_vec(), cow_expected); + assert_eq!(root(&cow), root(&Flags::try_from(cow_expected).unwrap())); + + let mut rebased = Flags::try_from(vec![0u8; LEN]).unwrap(); + rebased.rebase_on(&Flags::repeat(0, LEN).unwrap()); + assert_eq!(rebased.to_vec(), vec![0u8; LEN]); + + // Growing a repeated list past a full leaf. + let mut grown = Flags::repeat(0, 4096).unwrap(); + grown.push(1).unwrap(); + grown.apply_updates(); + let mut grown_expected = vec![0u8; 4096]; + grown_expected.push(1); + assert_eq!( + root(&grown), + root(&Flags::try_from(grown_expected).unwrap()) + ); + } + #[test] fn rebasing_a_rebuilt_list_shares_the_base_and_keeps_its_writes() { let values: Vec = (0..1000).collect(); diff --git a/crates/common/ssz-tree/src/progressive_list.rs b/crates/common/ssz-tree/src/progressive_list.rs index 63faaf4a2..cc68a71ca 100644 --- a/crates/common/ssz-tree/src/progressive_list.rs +++ b/crates/common/ssz-tree/src/progressive_list.rs @@ -10,7 +10,7 @@ use libssz_merkle::{HashTreeRoot, Sha2Hasher, Sha256Hasher, hash_nodes, mix_in_l use libssz_types::TypeError; use crate::cursor::{ElemCow, IterCow}; -use crate::interface::Interface; +use crate::interface::{Interface, SortedReader}; use crate::iter::Iter; use crate::update_map::{UpdateMap, VecMap}; use crate::{Hash256, Value, packing_factor}; @@ -97,6 +97,50 @@ impl> ProgressiveList { self.subtrees[k].get_mut(offset) } + /// Buffers `value` as the new value of the element at `index`, without + /// reading the old one. Returns `false`, writing nothing, past the end. + #[must_use = "a write past the end is refused"] + pub fn set(&mut self, index: usize, value: T) -> bool { + match self.locate(index) { + Some((k, offset)) => self.subtrees[k].set(offset, value), + None => false, + } + } + + /// A reader for non-decreasing indices. See [`ProgressiveSortedReader`]. + pub fn sorted_reader(&self) -> ProgressiveSortedReader<'_, T, U> { + let mut start = 0; + let subtrees = self + .subtrees + .iter() + .map(|subtree| { + let entry = (start, subtree.sorted_reader()); + start += subtree.len(); + entry + }) + .collect(); + ProgressiveSortedReader { + subtrees, + current: 0, + last: 0, + } + } + + /// A list of `len` copies of `value`, sharing identical subtrees as + /// [`List::repeat`](crate::List::repeat) does, within each of its + /// subtrees. + pub fn repeat(value: T, len: usize) -> Self { + let mut subtrees = Vec::new(); + let mut remaining = len; + while remaining > 0 { + let k = subtrees.len(); + let count = remaining.min(Self::capacity(k)); + subtrees.push(Interface::repeat(&value, count, Self::height(k))); + remaining -= count; + } + Self { subtrees } + } + /// Buffers `value` as the new last element. /// /// Returns a `Result` only to mirror [`List::push`](crate::List::push); @@ -204,6 +248,37 @@ impl> ProgressiveList { } } +/// Reads a [`ProgressiveList`] at non-decreasing indices, one +/// [`SortedReader`] per subtree: see [`ProgressiveList::sorted_reader`]. +pub struct ProgressiveSortedReader<'a, T, U> { + /// Each subtree's first index and its reader. + subtrees: Vec<(usize, SortedReader<'a, T, U>)>, + /// The subtree the previous read was in. + current: usize, + /// The previous index, to check the order in debug builds. + last: usize, +} + +impl<'a, T: Value, U: UpdateMap> ProgressiveSortedReader<'a, T, U> { + /// The element at `index`, or `None` past the end. `index` must not be + /// below the previous call's (checked in debug builds). + pub fn get(&mut self, index: usize) -> Option<&'a T> { + debug_assert!( + index >= self.last, + "sorted reader read {index} after {}", + self.last + ); + self.last = index; + while let Some((start, reader)) = self.subtrees.get_mut(self.current) { + if index < *start + reader.len() { + return reader.get(index - *start); + } + self.current += 1; + } + None + } +} + /// An in-order rewriting pass over a [`ProgressiveList`], made by /// [`ProgressiveList::iter_cow`]: one [`IterCow`] per subtree, in turn. pub struct ProgressiveIterCow<'a, T: Value, U> { @@ -430,3 +505,56 @@ impl<'a, T: Value, U: UpdateMap> Iterator for ProgressiveIter<'a, T, U> { impl> ExactSizeIterator for ProgressiveIter<'_, T, U> {} impl> FusedIterator for ProgressiveIter<'_, T, U> {} + +#[cfg(test)] +mod tests { + use super::*; + + fn root(list: &L) -> Hash256 { + list.hash_tree_root(&Sha2Hasher) + } + + #[test] + fn repeat_hashes_and_reads_like_the_list_built_from_a_vec() { + // Around each subtree's capacity (32, 128, 512, ... entries of u8). + for len in [ + 0usize, 1, 31, 32, 33, 159, 160, 161, 700, 2_720, 2_721, 50_000, + ] { + let repeated = ProgressiveList::::repeat(7, len); + let built = ProgressiveList::::from(vec![7u8; len]); + assert_eq!(repeated.len(), len, "len {len}"); + assert_eq!(root(&repeated), root(&built), "len {len}"); + assert_eq!(repeated.to_ssz(), built.to_ssz(), "len {len}"); + assert_eq!(repeated, built, "len {len}"); + } + } + + #[test] + fn a_write_to_a_repeated_list_leaves_its_clone_and_siblings_alone() { + let mut list = ProgressiveList::::repeat(0, 9_000); + let snapshot = list.clone(); + assert!(list.set(5_000, 3)); + assert!(!list.set(9_000, 3)); + list.apply_updates(); + let mut expected = vec![0u8; 9_000]; + expected[5_000] = 3; + assert_eq!(list.to_vec(), expected); + assert_eq!(snapshot.to_vec(), vec![0u8; 9_000]); + assert_eq!(root(&list), root(&ProgressiveList::::from(expected))); + } + + #[test] + fn a_sorted_reader_agrees_with_get_across_subtrees_and_pending_writes() { + let values: Vec = (0..10_000u32).map(|i| (i % 251) as u8).collect(); + let mut list = ProgressiveList::::from(values); + assert!(list.set(40, 200)); + assert!(list.set(9_999, 201)); + list.push(202).unwrap(); + let mut reader = list.sorted_reader(); + for index in (0..list.len() + 3).step_by(3) { + assert_eq!(reader.get(index), list.get(index), "index {index}"); + } + assert_eq!(list.sorted_reader().get(10_000), Some(&202)); + assert_eq!(list.sorted_reader().get(10_001), None); + } +} diff --git a/crates/common/ssz-tree/src/tree.rs b/crates/common/ssz-tree/src/tree.rs index 9339e722c..194cabb54 100644 --- a/crates/common/ssz-tree/src/tree.rs +++ b/crates/common/ssz-tree/src/tree.rs @@ -162,6 +162,66 @@ impl Tree { (Self::from_leaves(leaves, leaf_height, depth), len) } + /// Builds a tree of height `depth` holding `len` copies of `value`, + /// sharing every identical subtree instead of building it again. + /// + /// All full leaves are one allocation, and so are all the full inner nodes + /// at each level, each caching its hash once for every place it appears. + /// Building and hashing is therefore proportional to the tree's height + /// plus the partial leaf at the right edge, not to `len`. The nodes are + /// immutable and every write copies the path it touches, so the sharing is + /// never observable. + /// + /// # Panics + /// + /// If `len` elements do not fit in a tree of height `depth`. Callers check + /// the type's limit first. + pub(crate) fn repeat(value: &T, len: usize, depth: usize) -> Arc { + if len == 0 { + return Arc::new(Tree::Zero(depth)); + } + let leaf_height = leaf_height::(depth); + let per_leaf = packing_factor::() << leaf_height; + let mut full_count = len / per_leaf; + let tail_len = len % per_leaf; + let levels = depth - leaf_height; + assert!( + levels >= usize::BITS as usize || full_count + usize::from(tail_len > 0) <= 1 << levels, + "{len} values do not fit in a tree of depth {depth}" + ); + // The shared full node of the level being built, if it has any. + let mut full = + (full_count > 0).then(|| Arc::new(Self::leaf(vec![value.clone(); per_leaf], None))); + // The one partial node at the right edge of the level, if any. + let mut tail = + (tail_len > 0).then(|| Arc::new(Self::leaf(vec![value.clone(); tail_len], None))); + let mut height = leaf_height; + while height < depth { + let parent = (height + NODE_LEVELS).min(depth); + debug_assert_eq!(child_height::(parent), height); + let fan_out = 1 << (parent - height); + let rest = full_count % fan_out; + let tail_children: Vec> = full + .iter() + .flat_map(|full| std::iter::repeat_n(Arc::clone(full), rest)) + .chain(tail.take()) + .collect(); + let next_full = full + .as_ref() + .filter(|_| full_count >= fan_out) + .map(|full| Arc::new(Self::node(vec![Arc::clone(full); fan_out]))); + tail = (!tail_children.is_empty()).then(|| Arc::new(Self::node(tail_children))); + full_count /= fan_out; + full = next_full; + height = parent; + } + match (full, tail) { + (Some(root), None) if full_count == 1 => root, + (None, Some(root)) => root, + _ => unreachable!("a tree that fits groups up into one root"), + } + } + /// Groups `level`, left to right, from `leaf_height` up into a tree of /// height `depth`, [`NODE_LEVELS`] binary levels per inner node. fn from_leaves(mut level: Vec>, leaf_height: usize, depth: usize) -> Arc { @@ -217,6 +277,32 @@ impl Tree { } } + /// The leaf holding element `index` of a tree of height `depth`, as the + /// index of its first element and its run of elements, or `None` past the + /// end of the data. The element itself is `run.get(index - first)`: a + /// partial last leaf is shorter than the span it covers. + pub(crate) fn leaf_run(&self, index: usize, depth: usize) -> Option<(usize, &[T])> { + let packing = packing_factor::(); + let chunk_index = index / packing; + let mut node = self; + let mut height = depth; + loop { + match node { + Tree::Node(inner) => { + let below = child_height::(height); + let slot = (chunk_index >> below) & ((1 << (height - below)) - 1); + node = inner.children.get(slot)?; + height = below; + } + Tree::Leaf(leaf) => { + let first = index & !((packing << height) - 1); + return Some((first, &leaf.values)); + } + Tree::Zero(_) => return None, + } + } + } + /// This subtree's root, computing and caching whatever is not cached yet. /// /// `height` is this node's height. An inner node folds its children's From 099d5687b902208f71379d26e0f6c4afc7c5bec6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 3 Nov 2026 12:01:05 -0300 Subject: [PATCH 3/6] test(types): add ignored benchmarks for the attestation pass and the rotation Two micro-benchmarks next to `participation_root_after_a_block_of_writes_bench`, at 2.4M validators: the participation part of one `process_attestation` (about 75k ascending reads, then the writes) in each combination of per-index `get`/sorted reader and `get_mut`/`set`, and the epoch rotation's new list built from a zeroed `Vec` against `List::repeat`, with the root that follows. Run with `cargo test --profile release-fast -p ethlambda-types --lib -- --ignored --nocapture attestation_pass_bench participation_rotation_bench`. --- .../types/src/beacon/containers/shared.rs | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/crates/common/types/src/beacon/containers/shared.rs b/crates/common/types/src/beacon/containers/shared.rs index 868b8265a..8850ed4b2 100644 --- a/crates/common/types/src/beacon/containers/shared.rs +++ b/crates/common/types/src/beacon/containers/shared.rs @@ -714,4 +714,118 @@ mod tests { assert_eq!(flat_roots, tree_roots); println!("flat: {flat_time:?}, tree: {tree_time:?} ({TOUCHED} of {COUNT} entries written)"); } + + /// Run with `--ignored --nocapture`. Times the participation part of one + /// `process_attestation` at mainnet registry size: read the current flags + /// of ~75k ascending attesters, then write the new ones. "per-index" is + /// the first tree-backed version (a descent per `get`, then a `get_mut` + /// that copies the old value into the write buffer); the others swap in + /// the sorted reader and the blind `set`. + #[test] + #[ignore = "benchmark"] + fn attestation_pass_bench() { + use crate::primitives::HashTreeRoot as _; + use std::time::Instant; + + const COUNT: usize = 2_400_000; + const ATTESTERS: usize = COUNT / 32; + let mut state = 99u64; + let mut attesters: Vec = (0..ATTESTERS) + .map(|_| { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + (state >> 33) as usize % COUNT + }) + .collect(); + attesters.sort_unstable(); + attesters.dedup(); + + let fresh = || { + let mut list = EpochParticipation::try_from(flags(COUNT)).unwrap(); + list.apply_updates(); + let _ = list.hash_tree_root(); + list + }; + + for (name, sorted_reads, blind_writes) in [ + ("per-index get + get_mut", false, false), + ("per-index get + set", false, true), + ("sorted reader + get_mut", true, false), + ("sorted reader + set", true, true), + ] { + // Best of several runs: the first is cold. + let mut best = (std::time::Duration::MAX, std::time::Duration::MAX); + for _ in 0..5 { + let mut list = fresh(); + let start = Instant::now(); + let mut updates: Vec<(usize, ParticipationFlags)> = Vec::new(); + if sorted_reads { + let mut reader = list.sorted_reader(); + for &index in &attesters { + let current = *reader.get(index).unwrap(); + updates.push((index, current)); + } + } else { + for &index in &attesters { + let current = *list.get(index).unwrap(); + updates.push((index, current)); + } + } + let read = start.elapsed(); + let start = Instant::now(); + for &(index, current) in &updates { + if blind_writes { + assert!(list.set(index, current | 0b100)); + } else { + *list.get_mut(index).unwrap() |= 0b100; + } + } + let write = start.elapsed(); + best = (best.0.min(read), best.1.min(write)); + } + println!( + "{name}: read {:?}, write {:?}, total {:?} ({} attesters)", + best.0, + best.1, + best.0 + best.1, + attesters.len() + ); + } + } + + /// Run with `--ignored --nocapture`. Times the epoch rotation's new + /// current-epoch participation list at mainnet registry size, then the + /// state root that follows: built from a zeroed `Vec` (what the rotation + /// did) against `List::repeat`. + #[test] + #[ignore = "benchmark"] + fn participation_rotation_bench() { + use crate::primitives::HashTreeRoot as _; + use std::time::Instant; + + const COUNT: usize = 2_400_000; + for _ in 0..3 { + let start = Instant::now(); + let from_vec = EpochParticipation::try_from(vec![0u8; COUNT]).unwrap(); + let build_vec = start.elapsed(); + let start = Instant::now(); + let root_vec = from_vec.hash_tree_root(); + let hash_vec = start.elapsed(); + + let start = Instant::now(); + let repeated = EpochParticipation::repeat(0, COUNT).unwrap(); + let build_repeat = start.elapsed(); + let start = Instant::now(); + let root_repeat = repeated.hash_tree_root(); + let hash_repeat = start.elapsed(); + + assert_eq!(root_vec, root_repeat); + println!( + "vec: build {build_vec:?} + hash {hash_vec:?} = {:?}; repeat: build {build_repeat:?} + hash {hash_repeat:?} = {:?}", + build_vec + hash_vec, + build_repeat + hash_repeat + ); + } + } } From d08f73137b197bb5cfe50a446cc1854f92f0b5e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 3 Nov 2026 12:01:58 -0300 Subject: [PATCH 4/6] perf(beacon): write participation flags blindly in process_attestation The write phase did `get_mut(index)` and `*flags |= new_flags` for every attester. On a tree-backed list `get_mut` descends the tree to copy the old value into the write buffer, although the read phase had already read it as `current_flags`. Micro-benchmark at 2.4M validators and about 74k ascending attesters (best of five, release-fast): the write phase takes 2.16 ms with `get_mut` and 0.94 ms with the blind `set`. The read phase now records `current_flags | new_flags` and the write phase stores it with the new `ParticipationMut::set`. That is exactly what `|=` would leave: the attesting indices are distinct (members of disjoint committees, and the signature set built before the read phase refuses a repeated index), so each index is written once, and nothing writes the list between the two phases, so the stored value is still `current_flags`. Applies to the altair, deneb, electra (electra and fulu) and gloas versions of `process_attestation`. --- .../state_transition/src/beacon/stf/altair.rs | 20 +++++++++------ .../state_transition/src/beacon/stf/deneb.rs | 20 +++++++++------ .../src/beacon/stf/electra.rs | 25 +++++++++++-------- .../state_transition/src/beacon/stf/gloas.rs | 25 +++++++++++-------- .../common/types/src/beacon/containers/mod.rs | 11 ++++++++ 5 files changed, 67 insertions(+), 34 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/stf/altair.rs b/crates/blockchain/state_transition/src/beacon/stf/altair.rs index 2b33e79be..9b18f2d28 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/altair.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/altair.rs @@ -212,7 +212,7 @@ pub fn process_attestation( .ok_or(Error::ArithmeticOverflow("proposer_reward_numerator"))?; } if new_flags != 0 { - updates.push((index, new_flags)); + updates.push((index, current_flags | new_flags)); } } @@ -227,14 +227,20 @@ pub fn process_attestation( previous_epoch_participation }; let epoch_participation_len = epoch_participation.len(); - for (index, new_flags) in updates { - let flags = epoch_participation - .get_mut(index as usize) - .ok_or(Error::IndexOutOfBounds { + // A blind write of `current_flags | new_flags`, which the read phase + // stored, rather than `|=` on the stored value. That is exact: the + // attesting indices are distinct (members of disjoint committees, and the + // signature set built above refuses a repeated index), so each index is + // written once, and nothing writes this list between the read phase and + // here, so the stored value is still `current_flags`. It saves the tree + // descent `get_mut` makes to copy the old value into the write buffer. + for (index, flags) in updates { + if !epoch_participation.set(index as usize, flags) { + return Err(Error::IndexOutOfBounds { index: index as usize, len: epoch_participation_len, - })?; - *flags |= new_flags; + }); + } } // The weights partition `WEIGHT_DENOMINATOR` between the three flags, the diff --git a/crates/blockchain/state_transition/src/beacon/stf/deneb.rs b/crates/blockchain/state_transition/src/beacon/stf/deneb.rs index 72b5ac946..69c3f29f1 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/deneb.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/deneb.rs @@ -412,7 +412,7 @@ pub fn process_attestation( .ok_or(Error::ArithmeticOverflow("proposer_reward_numerator"))?; } if new_flags != 0 { - updates.push((index, new_flags)); + updates.push((index, current_flags | new_flags)); } } @@ -424,14 +424,20 @@ pub fn process_attestation( previous_participation }; let epoch_participation_len = epoch_participation.len(); - for (index, new_flags) in updates { - let flags = epoch_participation - .get_mut(index as usize) - .ok_or(Error::IndexOutOfBounds { + // A blind write of `current_flags | new_flags`, which the read phase + // stored, rather than `|=` on the stored value. That is exact: the + // attesting indices are distinct (members of disjoint committees, and the + // signature set built above refuses a repeated index), so each index is + // written once, and nothing writes this list between the read phase and + // here, so the stored value is still `current_flags`. It saves the tree + // descent `get_mut` makes to copy the old value into the write buffer. + for (index, flags) in updates { + if !epoch_participation.set(index as usize, flags) { + return Err(Error::IndexOutOfBounds { index: index as usize, len: epoch_participation_len, - })?; - *flags |= new_flags; + }); + } } const NON_PROPOSER_WEIGHT: u64 = constants::WEIGHT_DENOMINATOR - constants::PROPOSER_WEIGHT; diff --git a/crates/blockchain/state_transition/src/beacon/stf/electra.rs b/crates/blockchain/state_transition/src/beacon/stf/electra.rs index 9b3519caf..05fa1e536 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/electra.rs @@ -793,7 +793,7 @@ pub fn process_attestation( .ok_or(Error::ArithmeticOverflow("proposer_reward_numerator"))?; } if new_flags != 0 { - updates.push((index, new_flags)); + updates.push((index, current_flags | new_flags)); } } @@ -804,15 +804,20 @@ pub fn process_attestation( let mut fields = block_mut(state, "process_attestation")?; let participation_mut = fields.epoch_participation_mut(current_epoch_target); let participation_len = participation_mut.len(); - for (index, new_flags) in updates { - let flags = - participation_mut - .get_mut(index as usize) - .ok_or(Error::IndexOutOfBounds { - index: index as usize, - len: participation_len, - })?; - *flags |= new_flags; + // A blind write of `current_flags | new_flags`, which the read phase + // stored, rather than `|=` on the stored value. That is exact: the + // attesting indices are distinct (members of disjoint committees, and the + // signature set built above refuses a repeated index), so each index is + // written once, and nothing writes this list between the read phase and + // here, so the stored value is still `current_flags`. It saves the tree + // descent `get_mut` makes to copy the old value into the write buffer. + for (index, flags) in updates { + if !participation_mut.set(index as usize, flags) { + return Err(Error::IndexOutOfBounds { + index: index as usize, + len: participation_len, + }); + } } } diff --git a/crates/blockchain/state_transition/src/beacon/stf/gloas.rs b/crates/blockchain/state_transition/src/beacon/stf/gloas.rs index 82642934e..0d56641a9 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/gloas.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/gloas.rs @@ -1488,7 +1488,7 @@ pub fn process_attestation( )?; } if will_set_new_flag { - flag_updates.push((index, new_flags)); + flag_updates.push((index, current_flags | new_flags)); } } } @@ -1497,15 +1497,20 @@ pub fn process_attestation( { let mut participation_mut = state.epoch_participation_mut(current_epoch_target)?; let participation_len = participation_mut.len(); - for (index, new_flags) in flag_updates { - let flags = - participation_mut - .get_mut(index as usize) - .ok_or(Error::IndexOutOfBounds { - index: index as usize, - len: participation_len, - })?; - *flags |= new_flags; + // A blind write of `current_flags | new_flags`, which the read phase + // stored, rather than `|=` on the stored value. That is exact: the + // attesting indices are distinct (members of disjoint committees, and the + // signature set built above refuses a repeated index), so each index is + // written once, and nothing writes this list between the read phase and + // here, so the stored value is still `current_flags`. It saves the tree + // descent `get_mut` makes to copy the old value into the write buffer. + for (index, flags) in flag_updates { + if !participation_mut.set(index as usize, flags) { + return Err(Error::IndexOutOfBounds { + index: index as usize, + len: participation_len, + }); + } } } diff --git a/crates/common/types/src/beacon/containers/mod.rs b/crates/common/types/src/beacon/containers/mod.rs index e2a4b43ed..362127187 100644 --- a/crates/common/types/src/beacon/containers/mod.rs +++ b/crates/common/types/src/beacon/containers/mod.rs @@ -761,6 +761,17 @@ impl ParticipationMut<'_> { Self::Progressive(list) => list.get_mut(index), } } + + /// Buffers `flags` as the entry at `index` without reading the old one, + /// which [`Self::get_mut`] does (a tree descent) even when the caller + /// already holds it. Returns `false`, writing nothing, past the end. + #[must_use = "a write past the end is refused"] + pub fn set(&mut self, index: usize, flags: ParticipationFlags) -> bool { + match self { + Self::Bounded(list) => list.set(index, flags), + Self::Progressive(list) => list.set(index, flags), + } + } } /// The inactivity scores of a state, whichever list kind its fork uses. From c67f22443e8570c9f3ba2af11bae0263b0f23bdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 3 Nov 2026 12:02:29 -0300 Subject: [PATCH 5/6] perf(beacon): read an attestation's participation flags with a sorted reader Once the participation lists became trees, the read phase of `process_attestation` paid a full tree descent per attester (`get` was a slice index before). An attestation carries tens of thousands of attesters. Its indices are sorted ascending and consecutive ones almost always share a leaf, which holds thousands of flags. `sorted_reader()` keeps the current leaf's slice and descends only when an index leaves it, consulting pending writes first exactly as `get` does, so the answers are identical. Micro-benchmark at 2.4M validators and about 74k ascending attesters (best of five, release-fast): the read phase takes 0.70 ms per-index and 0.10 ms with the reader; with the blind write from the previous commit the whole participation pass goes from 2.86 ms to 1.03 ms. Every `process_attestation` (altair, deneb, electra and fulu, gloas) now uses it; the reader debug-asserts that indices do not decrease, which `get_attesting_indices` guarantees by sorting. --- .../state_transition/src/beacon/stf/altair.rs | 18 ++++---- .../state_transition/src/beacon/stf/deneb.rs | 18 ++++---- .../src/beacon/stf/electra.rs | 18 ++++---- .../state_transition/src/beacon/stf/gloas.rs | 17 +++++--- .../common/types/src/beacon/containers/mod.rs | 42 +++++++++++++++++++ 5 files changed, 83 insertions(+), 30 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/stf/altair.rs b/crates/blockchain/state_transition/src/beacon/stf/altair.rs index 9b18f2d28..bbeb35250 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/altair.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/altair.rs @@ -167,15 +167,17 @@ pub fn process_attestation( let mut proposer_reward_numerator: Gwei = 0; let mut updates: Vec<(ValidatorIndex, ParticipationFlags)> = Vec::new(); + // `attesting_indices` is sorted ascending, so the reader walks each leaf + // once rather than descending the tree for every attester. + let mut reader = epoch_participation.sorted_reader(); for index in attesting_indices { - let current_flags = - epoch_participation - .get(index as usize) - .copied() - .ok_or(Error::IndexOutOfBounds { - index: index as usize, - len: epoch_participation.len(), - })?; + let current_flags = reader + .get(index as usize) + .copied() + .ok_or(Error::IndexOutOfBounds { + index: index as usize, + len: epoch_participation.len(), + })?; let mut new_flags: ParticipationFlags = 0; // The attester's validator is read at most once, on the first flag diff --git a/crates/blockchain/state_transition/src/beacon/stf/deneb.rs b/crates/blockchain/state_transition/src/beacon/stf/deneb.rs index 69c3f29f1..2b01516ab 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/deneb.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/deneb.rs @@ -367,15 +367,17 @@ pub fn process_attestation( let mut proposer_reward_numerator: Gwei = 0; let mut updates: Vec<(ValidatorIndex, ParticipationFlags)> = Vec::new(); + // `attesting_indices` is sorted ascending, so the reader walks each leaf + // once rather than descending the tree for every attester. + let mut reader = epoch_participation.sorted_reader(); for index in attesting_indices { - let current_flags = - epoch_participation - .get(index as usize) - .copied() - .ok_or(Error::IndexOutOfBounds { - index: index as usize, - len: epoch_participation.len(), - })?; + let current_flags = reader + .get(index as usize) + .copied() + .ok_or(Error::IndexOutOfBounds { + index: index as usize, + len: epoch_participation.len(), + })?; let mut new_flags: ParticipationFlags = 0; // The attester's validator is read at most once, on the first flag diff --git a/crates/blockchain/state_transition/src/beacon/stf/electra.rs b/crates/blockchain/state_transition/src/beacon/stf/electra.rs index 05fa1e536..7c2836a31 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/electra.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/electra.rs @@ -750,15 +750,17 @@ pub fn process_attestation( let mut proposer_reward_numerator: Gwei = 0; let mut updates: Vec<(ValidatorIndex, ParticipationFlags)> = Vec::new(); + // `attesting_indices` is sorted ascending, so the reader walks each leaf + // once rather than descending the tree for every attester. + let mut reader = participation.sorted_reader(); for index in attesting_indices { - let current_flags = - participation - .get(index as usize) - .copied() - .ok_or(Error::IndexOutOfBounds { - index: index as usize, - len: participation.len(), - })?; + let current_flags = reader + .get(index as usize) + .copied() + .ok_or(Error::IndexOutOfBounds { + index: index as usize, + len: participation.len(), + })?; let mut new_flags: ParticipationFlags = 0; // The attester's validator is read at most once, on the first flag diff --git a/crates/blockchain/state_transition/src/beacon/stf/gloas.rs b/crates/blockchain/state_transition/src/beacon/stf/gloas.rs index 0d56641a9..8e8e1744c 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/gloas.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/gloas.rs @@ -1428,13 +1428,18 @@ pub fn process_attestation( } else { previous_participation }; + // `attesting_indices` is sorted ascending, so the reader walks each leaf + // once rather than descending the tree for every attester. + let mut reader = epoch_participation.sorted_reader(); for index in attesting_indices { - let current_flags = epoch_participation.get(index as usize).copied().ok_or( - Error::IndexOutOfBounds { - index: index as usize, - len: epoch_participation.len(), - }, - )?; + let current_flags = + reader + .get(index as usize) + .copied() + .ok_or(Error::IndexOutOfBounds { + index: index as usize, + len: epoch_participation.len(), + })?; let had_no_participation = current_flags == 0; // The attester's validator is read at most once, on the first diff --git a/crates/common/types/src/beacon/containers/mod.rs b/crates/common/types/src/beacon/containers/mod.rs index 362127187..2cb81438d 100644 --- a/crates/common/types/src/beacon/containers/mod.rs +++ b/crates/common/types/src/beacon/containers/mod.rs @@ -672,6 +672,16 @@ impl<'a> ParticipationRef<'a> { } } + /// A reader for ascending indices, which descends the tree only when an + /// index leaves the leaf it last read. Prefer it to [`Self::get`] when + /// visiting many entries in order, such as one attestation's attesters. + pub fn sorted_reader(&self) -> ParticipationSortedReader<'a> { + match self { + Self::Bounded(list) => ParticipationSortedReader::Bounded(list.sorted_reader()), + Self::Progressive(list) => ParticipationSortedReader::Progressive(list.sorted_reader()), + } + } + /// Whether a write is buffered and not yet folded into the tree. pub fn has_pending_updates(&self) -> bool { match self { @@ -695,6 +705,38 @@ impl<'a> ParticipationRef<'a> { } } +/// Reads a [`ParticipationRef`] at non-decreasing indices: see +/// [`ParticipationRef::sorted_reader`]. +pub enum ParticipationSortedReader<'a> { + /// Altair through fulu. + Bounded( + ethlambda_ssz_tree::SortedReader< + 'a, + ParticipationFlags, + ethlambda_ssz_tree::VecMap, + >, + ), + /// Gloas. + Progressive( + ethlambda_ssz_tree::ProgressiveSortedReader< + 'a, + ParticipationFlags, + ethlambda_ssz_tree::VecMap, + >, + ), +} + +impl<'a> ParticipationSortedReader<'a> { + /// The flags at `index`, or `None` past the end. `index` must not be + /// below the previous call's (checked in debug builds). + pub fn get(&mut self, index: usize) -> Option<&'a ParticipationFlags> { + match self { + Self::Bounded(reader) => reader.get(index), + Self::Progressive(reader) => reader.get(index), + } + } +} + impl PartialEq for ParticipationRef<'_> { fn eq(&self, other: &Self) -> bool { self.iter().eq(other.iter()) From fb7b208ab348e0523abd9e0d5b4255d183d401e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 3 Nov 2026 12:02:54 -0300 Subject: [PATCH 6/6] perf(beacon): build the rotated participation list by sharing one zero leaf `process_participation_flag_updates` installed the new current list as `vec![0; validator_count]` converted into a tree: about 2.4M bytes allocated and filled into hundreds of separate leaves, every one of which the next state root then hashed. At epoch boundaries `process_epoch` went from 96.3 ms to 105.5 ms in the offline import replay once the lists became trees. `List::repeat` / `ProgressiveList::repeat` (added with the other ssz-tree primitives) build the list from one full leaf and one full inner node per level, each shared through its `Arc` with its cached hash, so building and hashing cost the tree height plus the partial last leaf. Micro-benchmark at 2.4M validators (release-fast, steady state over three runs): the old build plus the root takes 3.0 to 6.0 ms, the repeated list 0.10 to 0.13 ms. Nodes are immutable and every write copies its path, so sharing is invisible to later writes; the roots and SSZ bytes are tested equal to the flat list's at lengths around every leaf and node boundary, and a write to a repeated list is tested to leave its clone and the other leaves alone. Used by the altair-through-fulu rotation, gloas's own copy of it, and the altair upgrade's two fresh lists. PendingDeposits is rebuilt from a `Vec` once per epoch by `process_pending_deposits`, which only drops a prefix and appends postponed deposits. The tree cannot keep the suffix's cached hashes: leaves are aligned runs, so dropping a prefix that is not a multiple of a leaf shifts every later element into a different leaf, and the API has no operation that re-chunks while carrying cached element roots. An ignored benchmark in `ethlambda-types` puts the cost at about 7 ms for the copy and rebuild plus about 24 ms to hash, for a queue of 200k deposits (about 1 ms plus 3 ms at 20k), once per epoch. The hash was paid on every block before the trees, so this is left as it is. --- .../src/beacon/stf/epoch/altair.rs | 7 +++-- .../src/beacon/stf/epoch/gloas.rs | 11 +++---- .../state_transition/src/beacon/upgrade.rs | 4 +-- .../types/src/beacon/containers/electra.rs | 30 +++++++++++++++++++ .../types/src/beacon/containers/shared.rs | 22 ++++++++++++++ 5 files changed, 65 insertions(+), 9 deletions(-) diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/altair.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/altair.rs index 92223be4e..2e3af0b09 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/altair.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/altair.rs @@ -18,7 +18,7 @@ use crate::beacon::config::Config; use crate::beacon::constants; -use crate::beacon::containers::BeaconState; +use crate::beacon::containers::{BeaconState, EpochParticipation}; use crate::beacon::error::{Error, Result}; use crate::beacon::helpers::accessors::get_current_epoch; use crate::beacon::helpers::altair::get_next_sync_committee; @@ -400,7 +400,10 @@ pub fn process_participation_flag_updates(state: &mut BeaconState) -> Result<()> state.altair_validator_lists_mut()?; *previous_epoch_participation = core::mem::take(current_epoch_participation); - *current_epoch_participation = vec![0; validator_count].try_into()?; + // `repeat` rather than a zeroed `Vec`: the `Vec` builds hundreds of + // separate leaves that the next state root then hashes one by one, while + // `repeat` shares one zero leaf and its cached hash among all of them. + *current_epoch_participation = EpochParticipation::repeat(0, validator_count)?; Ok(()) } diff --git a/crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs b/crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs index 9daa541cf..778be7ab9 100644 --- a/crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs +++ b/crates/blockchain/state_transition/src/beacon/stf/epoch/gloas.rs @@ -57,8 +57,8 @@ //! It is unmodified in the specification, //! but replaces the whole list (a new length, not an element write), which //! only the fork's own concrete container type can do: electra's and -//! fulu's `EpochParticipation::try_from` is fallible where gloas's -//! `ProgressiveList::from` is not, so there is no shared return type a +//! fulu's `EpochParticipation::repeat` is fallible where gloas's +//! `ProgressiveList::repeat` is not, so there is no shared return type a //! `dispatch_state_from!` body could give both. use crate::beacon::config::Config; @@ -110,8 +110,8 @@ pub fn process_epoch(state: &mut BeaconState, config: &Config) -> Result<()> { /// installs a fresh, all-zero current list, gloas's own copy of /// [`crate::beacon::stf::epoch::altair::process_participation_flag_updates`]: /// identical logic, writing gloas's own progressive lists through -/// [`gloas_state`] (an infallible `From>` conversion, unlike the -/// bounded `EpochParticipation`'s fallible `TryFrom` altair's version needs). +/// [`gloas_state`] (an infallible `repeat`, unlike the bounded +/// `EpochParticipation`'s fallible one altair's version needs). /// See this module's own doc for why that difference is what keeps this a /// separate copy rather than a shared, `Vec`-level function the way /// [`super::electra::process_pending_consolidations`] is. @@ -120,7 +120,8 @@ pub fn process_participation_flag_updates(state: &mut BeaconState) -> Result<()> let inner = gloas_state(state, "process_participation_flag_updates")?; inner.previous_epoch_participation = core::mem::take(&mut inner.current_epoch_participation); - inner.current_epoch_participation = vec![0; validator_count].into(); + // `repeat` rather than a zeroed `Vec`, for the reason given in altair's. + inner.current_epoch_participation = gloas::EpochParticipation::repeat(0, validator_count); Ok(()) } diff --git a/crates/blockchain/state_transition/src/beacon/upgrade.rs b/crates/blockchain/state_transition/src/beacon/upgrade.rs index 102bc528c..e17eadafb 100644 --- a/crates/blockchain/state_transition/src/beacon/upgrade.rs +++ b/crates/blockchain/state_transition/src/beacon/upgrade.rs @@ -172,8 +172,8 @@ pub fn upgrade_to_altair( balances: pre.balances.clone(), randao_mixes: pre.randao_mixes.clone(), slashings: pre.slashings.clone(), - previous_epoch_participation: EpochParticipation::try_from(vec![0u8; validator_count])?, - current_epoch_participation: EpochParticipation::try_from(vec![0u8; validator_count])?, + previous_epoch_participation: EpochParticipation::repeat(0, validator_count)?, + current_epoch_participation: EpochParticipation::repeat(0, validator_count)?, justification_bits: pre.justification_bits.clone(), previous_justified_checkpoint: pre.previous_justified_checkpoint, current_justified_checkpoint: pre.current_justified_checkpoint, diff --git a/crates/common/types/src/beacon/containers/electra.rs b/crates/common/types/src/beacon/containers/electra.rs index 896e0482c..35b5e85d2 100644 --- a/crates/common/types/src/beacon/containers/electra.rs +++ b/crates/common/types/src/beacon/containers/electra.rs @@ -894,6 +894,36 @@ mod tests { } } + /// Run with `--ignored --nocapture`. What `process_pending_deposits` costs + /// once per epoch on a queue of `len` deposits when it takes the queue out + /// as a `Vec`, drops a prefix and rebuilds the list from the rest: the + /// copy out, the rebuild, and the root that follows. + #[test] + #[ignore = "benchmark"] + fn pending_deposits_epoch_rebuild_bench() { + use crate::primitives::HashTreeRoot as _; + use std::time::Instant; + + for len in [1_000usize, 20_000, 200_000] { + let values: Vec<_> = (0..len as u64).map(pending_deposit).collect(); + let mut queue = PendingDeposits::try_from(values).unwrap(); + queue.apply_updates(); + let _ = queue.hash_tree_root(); + + let start = Instant::now(); + let all = queue.to_vec(); + let copy = start.elapsed(); + let start = Instant::now(); + let remaining: Vec<_> = all.into_iter().skip(16).collect(); + let rebuilt = PendingDeposits::try_from(remaining).unwrap(); + let build = start.elapsed(); + let start = Instant::now(); + let _ = rebuilt.hash_tree_root(); + let hash = start.elapsed(); + println!("{len} deposits: copy out {copy:?}, rebuild {build:?}, hash {hash:?}"); + } + } + #[test] fn tree_backed_pending_deposits_hash_like_the_flat_list() { use crate::primitives::HashTreeRoot as _; diff --git a/crates/common/types/src/beacon/containers/shared.rs b/crates/common/types/src/beacon/containers/shared.rs index 8850ed4b2..a2dd2c6bb 100644 --- a/crates/common/types/src/beacon/containers/shared.rs +++ b/crates/common/types/src/beacon/containers/shared.rs @@ -613,6 +613,28 @@ mod tests { } } + #[test] + fn repeated_participation_hashes_like_the_flat_list() { + use crate::primitives::HashTreeRoot as _; + type Flat = libssz_types::SszList; + + for len in LENGTHS { + let flat = Flat::try_from(vec![0; len]).unwrap(); + let bounded = EpochParticipation::repeat(0, len).unwrap(); + assert_eq!(bounded.hash_tree_root(), flat.hash_tree_root(), "len {len}"); + assert_eq!(bounded.to_ssz(), flat.to_ssz(), "len {len}"); + + let flat = libssz_types::ProgressiveList::::from(vec![0; len]); + let progressive = ProgressiveEpochParticipation::repeat(0, len); + assert_eq!( + progressive.hash_tree_root(), + flat.hash_tree_root(), + "len {len}" + ); + assert_eq!(progressive.to_ssz(), flat.to_ssz(), "len {len}"); + } + } + #[test] fn participation_writes_rehash_like_a_rebuilt_list() { use crate::primitives::HashTreeRoot as _;