perf(beacon): back the participation lists and pending deposits with persistent trees

CLAUDE.md

@@ -646,6 +646,7 @@ existing once per chain.
   `[patch.crates-io]` override because nothing outside this workspace depends on
   libssz, so there is no second copy to unify; that also keeps the manifest free
   of a `[patch]` table, which `shadow/cargo-patch.toml` would collide with.
+- **The participation lists and `pending_deposits` are tree-backed too** (`EpochParticipation`, `PendingDeposits`; progressive from gloas), reached through `altair_validator_lists` (`ParticipationRef`, no slice) and `epoch_participation_mut`; scan with `ParticipationRef::iter`, not `get(i)`.
 - **`Validators` and `Balances` are `ethlambda_ssz_tree::List`s** (gloas:
   `ethlambda_ssz_tree::ProgressiveList`, same method set), persistent
   Merkle trees that cache node hashes and share unchanged subtrees between

crates/blockchain/state_transition/src/beacon/block_production.rs

@@ -905,9 +905,8 @@ mod tests {
         let BeaconState::Fulu(inner) = &mut state else {
             unreachable!("built as fulu")
         };
-        for flags in inner.current_epoch_participation.iter_mut() {
-            *flags = 0b111;
-        }
+        let count = inner.current_epoch_participation.len();
+        inner.current_epoch_participation = vec![0b111; count].try_into().unwrap();
         assert!(pack_attestations(&state, vec![candidate]).is_empty());
     }
 

crates/blockchain/state_transition/src/beacon/helpers/electra.rs

@@ -1070,14 +1070,14 @@ impl<'a> PendingQueueFields<'a> {
 
     /// Queues one more [`electra::PendingDeposit`], pushed by
     /// [`queue_excess_active_balance`], [`queue_entire_balance_and_reset_validator`],
-    /// and the deposit-request processors. Infallible on gloas's unbounded
-    /// queue; electra's and fulu's bounded one can still reject a length over
-    /// its own SSZ limit.
+    /// and the deposit-request processors. Electra's and fulu's bounded queue
+    /// can reject a length over its own SSZ limit; gloas's progressive one
+    /// never does.
     pub(crate) fn push_pending_deposit(&mut self, deposit: electra::PendingDeposit) -> Result<()> {
         match self {
             PendingQueueFields::Electra(state) => state.pending_deposits.push(deposit)?,
             PendingQueueFields::Fulu(state) => state.pending_deposits.push(deposit)?,
-            PendingQueueFields::Gloas(state) => state.pending_deposits.push(deposit),
+            PendingQueueFields::Gloas(state) => state.pending_deposits.push(deposit)?,
         }
         Ok(())
     }
@@ -1093,19 +1093,19 @@ impl<'a> PendingQueueFields<'a> {
     pub(crate) fn take_pending_deposits(&mut self) -> Vec<electra::PendingDeposit> {
         match self {
             PendingQueueFields::Electra(state) => {
-                core::mem::take(&mut state.pending_deposits).into_inner()
+                core::mem::take(&mut state.pending_deposits).to_vec()
             }
             PendingQueueFields::Fulu(state) => {
-                core::mem::take(&mut state.pending_deposits).into_inner()
+                core::mem::take(&mut state.pending_deposits).to_vec()
             }
             PendingQueueFields::Gloas(state) => {
-                core::mem::take(&mut state.pending_deposits).into_inner()
+                core::mem::take(&mut state.pending_deposits).to_vec()
             }
         }
     }
 
     /// See [`Self::take_pending_deposits`]. Fallible only for electra's and
-    /// fulu's bounded `SszList`; gloas's progressive one never rejects a
+    /// fulu's bounded list; gloas's progressive one never rejects a
     /// length.
     pub(crate) fn set_pending_deposits(
         &mut self,
@@ -1230,9 +1230,9 @@ impl<'a> PendingQueueFields<'a> {
     /// and `inactivity_scores` by one all-zero entry each, keeping them
     /// exactly as long as the registry after a new validator is appended.
     ///
-    /// Gloas's own three lists are progressive (`libssz_types::ProgressiveList`),
-    /// so their `push` is infallible, unlike electra's and fulu's bounded
-    /// `SszList`.
+    /// Gloas's own lists are progressive, so their pushes cannot hit a length
+    /// limit; its inactivity scores are still a flat
+    /// `libssz_types::ProgressiveList`, whose `push` is infallible outright.
     pub(crate) fn push_empty_participation_and_inactivity(&mut self) -> Result<()> {
         match self {
             PendingQueueFields::Electra(state) => {
@@ -1246,8 +1246,8 @@ impl<'a> PendingQueueFields<'a> {
                 state.inactivity_scores.push(0)?;
             }
             PendingQueueFields::Gloas(state) => {
-                state.previous_epoch_participation.push(0);
-                state.current_epoch_participation.push(0);
+                state.previous_epoch_participation.push(0)?;
+                state.current_epoch_participation.push(0)?;
                 state.inactivity_scores.push(0);
             }
         }

crates/blockchain/state_transition/src/beacon/helpers/participation.rs

@@ -134,7 +134,16 @@ fn scan(
     let mut previous_error = None;
     let mut current_error = None;
 
+    // Walked in lockstep with the registry, not indexed per validator: each
+    // `get` on a tree-backed list is a descent, which at mainnet size was the
+    // largest cost of this pass. A list shorter than the registry yields
+    // `None` from the index it runs out at, the same as `get` did.
+    let mut previous_flags_iter = previous_list.iter();
+    let mut current_flags_iter = current_list.iter();
+
     for (index, validator) in state.iter_validators().enumerate() {
+        let previous_entry = previous_flags_iter.next();
+        let current_entry = current_flags_iter.next();
         let effective_balance = validator.effective_balance;
         let active_now = is_active_validator(validator, current_epoch);
         let active_before = is_active_validator(validator, previous_epoch);
@@ -145,7 +154,7 @@ fn scan(
 
         let mut bits = 0u8;
         if active_before {
-            match previous_list.get(index) {
+            match previous_entry {
                 Some(&participation) => {
                     if !validator.slashed {
                         for (flag_index, total) in previous_flags.iter_mut().enumerate() {
@@ -172,7 +181,7 @@ fn scan(
         }
 
         if current_target && active_now {
-            match current_list.get(index) {
+            match current_entry {
                 Some(&participation) => {
                     if has_flag(participation, constants::TIMELY_TARGET_FLAG_INDEX)
                         && !validator.slashed

crates/blockchain/state_transition/src/beacon/stf/altair.rs

@@ -167,15 +167,17 @@ pub fn process_attestation(
 
     let mut proposer_reward_numerator: Gwei = 0;
     let mut updates: Vec<(ValidatorIndex, ParticipationFlags)> = Vec::new();
+    // `attesting_indices` is sorted ascending, so the reader walks each leaf
+    // once rather than descending the tree for every attester.
+    let mut reader = epoch_participation.sorted_reader();
     for index in attesting_indices {
-        let current_flags =
-            epoch_participation
-                .get(index as usize)
-                .copied()
-                .ok_or(Error::IndexOutOfBounds {
-                    index: index as usize,
-                    len: epoch_participation.len(),
-                })?;
+        let current_flags = reader
+            .get(index as usize)
+            .copied()
+            .ok_or(Error::IndexOutOfBounds {
+                index: index as usize,
+                len: epoch_participation.len(),
+            })?;
 
         let mut new_flags: ParticipationFlags = 0;
         // The attester's validator is read at most once, on the first flag
@@ -212,7 +214,7 @@ pub fn process_attestation(
                 .ok_or(Error::ArithmeticOverflow("proposer_reward_numerator"))?;
         }
         if new_flags != 0 {
-            updates.push((index, new_flags));
+            updates.push((index, current_flags | new_flags));
         }
     }
 
@@ -227,14 +229,20 @@ pub fn process_attestation(
         previous_epoch_participation
     };
     let epoch_participation_len = epoch_participation.len();
-    for (index, new_flags) in updates {
-        let flags = epoch_participation
-            .get_mut(index as usize)
-            .ok_or(Error::IndexOutOfBounds {
+    // A blind write of `current_flags | new_flags`, which the read phase
+    // stored, rather than `|=` on the stored value. That is exact: the
+    // attesting indices are distinct (members of disjoint committees, and the
+    // signature set built above refuses a repeated index), so each index is
+    // written once, and nothing writes this list between the read phase and
+    // here, so the stored value is still `current_flags`. It saves the tree
+    // descent `get_mut` makes to copy the old value into the write buffer.
+    for (index, flags) in updates {
+        if !epoch_participation.set(index as usize, flags) {
+            return Err(Error::IndexOutOfBounds {
                 index: index as usize,
                 len: epoch_participation_len,
-            })?;
-        *flags |= new_flags;
+            });
+        }
     }
 
     // The weights partition `WEIGHT_DENOMINATOR` between the three flags, the

crates/blockchain/state_transition/src/beacon/stf/deneb.rs

@@ -367,15 +367,17 @@ pub fn process_attestation(
 
     let mut proposer_reward_numerator: Gwei = 0;
     let mut updates: Vec<(ValidatorIndex, ParticipationFlags)> = Vec::new();
+    // `attesting_indices` is sorted ascending, so the reader walks each leaf
+    // once rather than descending the tree for every attester.
+    let mut reader = epoch_participation.sorted_reader();
     for index in attesting_indices {
-        let current_flags =
-            epoch_participation
-                .get(index as usize)
-                .copied()
-                .ok_or(Error::IndexOutOfBounds {
-                    index: index as usize,
-                    len: epoch_participation.len(),
-                })?;
+        let current_flags = reader
+            .get(index as usize)
+            .copied()
+            .ok_or(Error::IndexOutOfBounds {
+                index: index as usize,
+                len: epoch_participation.len(),
+            })?;
 
         let mut new_flags: ParticipationFlags = 0;
         // The attester's validator is read at most once, on the first flag
@@ -412,7 +414,7 @@ pub fn process_attestation(
                 .ok_or(Error::ArithmeticOverflow("proposer_reward_numerator"))?;
         }
         if new_flags != 0 {
-            updates.push((index, new_flags));
+            updates.push((index, current_flags | new_flags));
         }
     }
 
@@ -424,14 +426,20 @@ pub fn process_attestation(
         previous_participation
     };
     let epoch_participation_len = epoch_participation.len();
-    for (index, new_flags) in updates {
-        let flags = epoch_participation
-            .get_mut(index as usize)
-            .ok_or(Error::IndexOutOfBounds {
+    // A blind write of `current_flags | new_flags`, which the read phase
+    // stored, rather than `|=` on the stored value. That is exact: the
+    // attesting indices are distinct (members of disjoint committees, and the
+    // signature set built above refuses a repeated index), so each index is
+    // written once, and nothing writes this list between the read phase and
+    // here, so the stored value is still `current_flags`. It saves the tree
+    // descent `get_mut` makes to copy the old value into the write buffer.
+    for (index, flags) in updates {
+        if !epoch_participation.set(index as usize, flags) {
+            return Err(Error::IndexOutOfBounds {
                 index: index as usize,
                 len: epoch_participation_len,
-            })?;
-        *flags |= new_flags;
+            });
+        }
     }
 
     const NON_PROPOSER_WEIGHT: u64 = constants::WEIGHT_DENOMINATOR - constants::PROPOSER_WEIGHT;

crates/blockchain/state_transition/src/beacon/stf/electra.rs

@@ -750,15 +750,17 @@ pub fn process_attestation(
 
     let mut proposer_reward_numerator: Gwei = 0;
     let mut updates: Vec<(ValidatorIndex, ParticipationFlags)> = Vec::new();
+    // `attesting_indices` is sorted ascending, so the reader walks each leaf
+    // once rather than descending the tree for every attester.
+    let mut reader = participation.sorted_reader();
     for index in attesting_indices {
-        let current_flags =
-            participation
-                .get(index as usize)
-                .copied()
-                .ok_or(Error::IndexOutOfBounds {
-                    index: index as usize,
-                    len: participation.len(),
-                })?;
+        let current_flags = reader
+            .get(index as usize)
+            .copied()
+            .ok_or(Error::IndexOutOfBounds {
+                index: index as usize,
+                len: participation.len(),
+            })?;
 
         let mut new_flags: ParticipationFlags = 0;
         // The attester's validator is read at most once, on the first flag
@@ -793,7 +795,7 @@ pub fn process_attestation(
                 .ok_or(Error::ArithmeticOverflow("proposer_reward_numerator"))?;
         }
         if new_flags != 0 {
-            updates.push((index, new_flags));
+            updates.push((index, current_flags | new_flags));
         }
     }
 
@@ -804,15 +806,20 @@ pub fn process_attestation(
         let mut fields = block_mut(state, "process_attestation")?;
         let participation_mut = fields.epoch_participation_mut(current_epoch_target);
         let participation_len = participation_mut.len();
-        for (index, new_flags) in updates {
-            let flags =
-                participation_mut
-                    .get_mut(index as usize)
-                    .ok_or(Error::IndexOutOfBounds {
-                        index: index as usize,
-                        len: participation_len,
-                    })?;
-            *flags |= new_flags;
+        // A blind write of `current_flags | new_flags`, which the read phase
+        // stored, rather than `|=` on the stored value. That is exact: the
+        // attesting indices are distinct (members of disjoint committees, and the
+        // signature set built above refuses a repeated index), so each index is
+        // written once, and nothing writes this list between the read phase and
+        // here, so the stored value is still `current_flags`. It saves the tree
+        // descent `get_mut` makes to copy the old value into the write buffer.
+        for (index, flags) in updates {
+            if !participation_mut.set(index as usize, flags) {
+                return Err(Error::IndexOutOfBounds {
+                    index: index as usize,
+                    len: participation_len,
+                });
+            }
         }
     }
 

crates/blockchain/state_transition/src/beacon/stf/epoch/altair.rs

@@ -18,7 +18,7 @@
 
 use crate::beacon::config::Config;
 use crate::beacon::constants;
-use crate::beacon::containers::BeaconState;
+use crate::beacon::containers::{BeaconState, EpochParticipation};
 use crate::beacon::error::{Error, Result};
 use crate::beacon::helpers::accessors::get_current_epoch;
 use crate::beacon::helpers::altair::get_next_sync_committee;
@@ -400,7 +400,10 @@ pub fn process_participation_flag_updates(state: &mut BeaconState) -> Result<()>
         state.altair_validator_lists_mut()?;
 
     *previous_epoch_participation = core::mem::take(current_epoch_participation);
-    *current_epoch_participation = vec![0; validator_count].try_into()?;
+    // `repeat` rather than a zeroed `Vec`: the `Vec` builds hundreds of
+    // separate leaves that the next state root then hashes one by one, while
+    // `repeat` shares one zero leaf and its cached hash among all of them.
+    *current_epoch_participation = EpochParticipation::repeat(0, validator_count)?;
 
     Ok(())
 }
@@ -498,7 +501,8 @@ mod tests {
         // none has yet this epoch: only the previous epoch should justify.
         {
             let (previous_epoch_participation, _, _) = state.altair_validator_lists_mut().unwrap();
-            for flags in previous_epoch_participation.iter_mut() {
+            for index in 0..previous_epoch_participation.len() {
+                let flags = &mut previous_epoch_participation[index];
                 *flags = add_flag(*flags, constants::TIMELY_TARGET_FLAG_INDEX);
             }
         }