Merge branch 'master' into rr-squash-branch

.cargo/audit.toml

@@ -27,4 +27,8 @@ ignore = [
     # older versions of parking-lot are vulnerable, but used by wasmer0, which we need to keep alive for replayability reasons.
     # We should remove it, as well as this ignore, as soon as we get limited replayability.
     "RUSTSEC-2020-0070",
+
+    # proc-macro-error is unmaintained, but hard to replace right now.
+    # Follow https://github.com/Kyuuhachi/syn_derive/issues/4
+    "RUSTSEC-2024-0370",
 ]

Justfile

@@ -165,12 +165,21 @@ check-protocol-schema:
     rustup toolchain install nightly
     rustup target add wasm32-unknown-unknown --toolchain nightly
 
+    # Below, we *should* have been used `cargo +nightly ...` instead of
+    # `RUSTC_BOOTSTRAP=1`. However, the env var appears to be more stable.
+    # `nightly` builds are updated daily and may be broken sometimes, e.g.
+    # https://github.com/rust-lang/rust/issues/130769.
+    #
+    # If there is an issue with the env var, fall back to `cargo +nightly ...`.
+
     # Test that checker is not broken
-    RUSTFLAGS="--cfg enable_const_type_id" cargo +nightly test -p protocol-schema-check --profile dev-artifacts
+    RUSTC_BOOTSTRAP=1 RUSTFLAGS="--cfg enable_const_type_id" \
+        cargo test -p protocol-schema-check --profile dev-artifacts
+
     # Run the checker
-    RUSTFLAGS="--cfg enable_const_type_id" \
+    RUSTC_BOOTSTRAP=1 RUSTFLAGS="--cfg enable_const_type_id" \
         {{ with_macos_incremental }} \
-        cargo +nightly run -p protocol-schema-check --profile dev-artifacts
+        cargo run -p protocol-schema-check --profile dev-artifacts
 
 check_build_public_libraries:
     cargo check {{public_libraries}}

chain/chain/src/stateless_validation/chunk_endorsement.rs

@@ -1,4 +1,4 @@
-use std::collections::HashSet;
+use std::collections::HashMap;
 
 use itertools::Itertools;
 use near_chain_primitives::Error;
@@ -79,7 +79,7 @@ pub fn validate_chunk_endorsements_in_block(
         // Verify that the signature in block body are valid for given chunk_validator.
         // Signature can be either None, or Some(signature).
         // We calculate the stake of the chunk_validators for who we have the signature present.
-        let mut endorsed_chunk_validators = HashSet::new();
+        let mut endorsed_chunk_validators = HashMap::new();
         for (account_id, signature) in ordered_chunk_validators.iter().zip(signatures) {
             let Some(signature) = signature else { continue };
             let (validator, _) = epoch_manager.get_validator_by_account_id(
@@ -104,13 +104,13 @@ pub fn validate_chunk_endorsements_in_block(
             }
 
             // Add validators with signature in endorsed_chunk_validators. We later use this to check stake.
-            endorsed_chunk_validators.insert(account_id);
+            endorsed_chunk_validators.insert(account_id, *signature.clone());
         }
 
-        let endorsement_stats =
-            chunk_validator_assignments.compute_endorsement_stats(&endorsed_chunk_validators);
-        if !endorsement_stats.has_enough_stake() {
-            tracing::error!(target: "chain", ?endorsement_stats, "Chunk does not have enough stake to be endorsed");
+        let endorsement_state =
+            chunk_validator_assignments.compute_endorsement_state(endorsed_chunk_validators);
+        if !endorsement_state.is_endorsed {
+            tracing::error!(target: "chain", ?endorsement_state, "Chunk does not have enough stake to be endorsed");
             return Err(Error::InvalidChunkEndorsement);
         }
 

chain/client/src/chunk_inclusion_tracker.rs

@@ -6,14 +6,13 @@ use near_o11y::log_assert_fail;
 use near_primitives::block_body::ChunkEndorsementSignatures;
 use near_primitives::hash::CryptoHash;
 use near_primitives::sharding::{ChunkHash, ShardChunkHeader};
+use near_primitives::stateless_validation::validator_assignment::ChunkEndorsementsState;
 use near_primitives::types::{AccountId, EpochId, ShardId};
 use std::collections::HashMap;
 use std::num::NonZeroUsize;
 
 use crate::metrics;
-use crate::stateless_validation::chunk_endorsement::{
-    ChunkEndorsementTracker, ChunkEndorsementsState,
-};
+use crate::stateless_validation::chunk_endorsement::ChunkEndorsementTracker;
 
 const CHUNK_HEADERS_FOR_INCLUSION_CACHE_SIZE: usize = 2048;
 const NUM_EPOCH_CHUNK_PRODUCERS_TO_KEEP_IN_BLOCKLIST: usize = 1000;
@@ -27,12 +26,6 @@ struct ChunkInfo {
     pub endorsements: ChunkEndorsementsState,
 }
 
-impl ChunkInfo {
-    fn is_endorsed(&self) -> bool {
-        matches!(self.endorsements, ChunkEndorsementsState::Endorsed(_, _))
-    }
-}
-
 pub struct ChunkInclusionTracker {
     // Track chunks that are ready to be included in a block.
     // Key is the previous_block_hash as the chunk is created based on this block. It's possible that
@@ -88,7 +81,7 @@ impl ChunkInclusionTracker {
             chunk_header,
             received_time: Utc::now_utc(),
             chunk_producer,
-            endorsements: ChunkEndorsementsState::NotEnoughStake(None),
+            endorsements: ChunkEndorsementsState::default(),
         };
         self.chunk_hash_to_chunk_info.insert(chunk_hash, chunk_info);
     }
@@ -155,16 +148,16 @@ impl ChunkInclusionTracker {
         for (shard_id, chunk_hash) in entry {
             let chunk_info = self.chunk_hash_to_chunk_info.get(chunk_hash).unwrap();
             let banned = self.is_banned(epoch_id, &chunk_info);
-            let has_chunk_endorsements = chunk_info.is_endorsed();
-            if !has_chunk_endorsements {
+            let is_endorsed = chunk_info.endorsements.is_endorsed;
+            if !is_endorsed {
                 tracing::debug!(
                     target: "client",
                     chunk_hash = ?chunk_info.chunk_header.chunk_hash(),
                     chunk_producer = ?chunk_info.chunk_producer,
                     "Not including chunk because of insufficient chunk endorsements"
                 );
             }
-            if !banned && has_chunk_endorsements {
+            if !banned && is_endorsed {
                 // only add to chunk_headers_ready_for_inclusion if chunk is not from a banned chunk producer
                 // and chunk has sufficient chunk endorsements.
                 // Chunk endorsements are got as part of call to prepare_chunk_headers_ready_for_inclusion
@@ -203,10 +196,7 @@ impl ChunkInclusionTracker {
     ) -> Result<(ShardChunkHeader, ChunkEndorsementSignatures), Error> {
         let chunk_info = self.get_chunk_info(chunk_hash)?;
         let chunk_header = chunk_info.chunk_header.clone();
-        let signatures = match &chunk_info.endorsements {
-            ChunkEndorsementsState::Endorsed(_, signatures) => signatures.clone(),
-            ChunkEndorsementsState::NotEnoughStake(_) => vec![],
-        };
+        let signatures = chunk_info.endorsements.signatures.clone();
         Ok((chunk_header, signatures))
     }
 
@@ -228,9 +218,10 @@ impl ChunkInclusionTracker {
                 log_assert_fail!("Chunk info is missing for shard {shard_id} chunk {chunk_hash:?}");
                 continue;
             };
-            let Some(stats) = chunk_info.endorsements.stats() else {
+            let stats = &chunk_info.endorsements;
+            if stats.total_stake == 0 {
                 continue;
-            };
+            }
             let shard_label = shard_id.to_string();
             let label_values = &[shard_label.as_ref()];
             metrics::BLOCK_PRODUCER_ENDORSED_STAKE_RATIO

chain/client/src/debug.rs

@@ -714,7 +714,7 @@ impl ClientActorInner {
                 continue;
             }
             // Compute total stake and endorsed stake.
-            let mut endorsed_chunk_validators = HashSet::new();
+            let mut endorsed_chunk_validators = HashMap::new();
             for (account_id, signature) in ordered_chunk_validators.iter().zip(signatures) {
                 let Some(signature) = signature else { continue };
                 let Ok((validator, _)) = self.client.epoch_manager.get_validator_by_account_id(
@@ -731,13 +731,13 @@ impl ClientActorInner {
                 ) {
                     continue;
                 }
-                endorsed_chunk_validators.insert(account_id);
+                endorsed_chunk_validators.insert(account_id, *signature.clone());
             }
-            let endorsement_stats =
-                chunk_validator_assignments.compute_endorsement_stats(&endorsed_chunk_validators);
+            let endorsement_state =
+                chunk_validator_assignments.compute_endorsement_state(endorsed_chunk_validators);
             chunk_endorsements.insert(
                 chunk_header.chunk_hash(),
-                endorsement_stats.endorsed_stake as f64 / endorsement_stats.total_stake as f64,
+                endorsement_state.endorsed_stake as f64 / endorsement_state.total_stake as f64,
             );
         }
         Some(chunk_endorsements)

chain/client/src/stateless_validation/chunk_endorsement/mod.rs

@@ -3,10 +3,9 @@ use std::sync::Arc;
 use near_chain::ChainStoreAccess;
 use near_chain_primitives::Error;
 use near_epoch_manager::EpochManagerAdapter;
-use near_primitives::block_body::ChunkEndorsementSignatures;
 use near_primitives::sharding::ShardChunkHeader;
 use near_primitives::stateless_validation::chunk_endorsement::ChunkEndorsement;
-use near_primitives::stateless_validation::validator_assignment::EndorsementStats;
+use near_primitives::stateless_validation::validator_assignment::ChunkEndorsementsState;
 use near_primitives::version::ProtocolFeature;
 use near_store::Store;
 
@@ -15,20 +14,6 @@ use crate::Client;
 mod tracker_v1;
 mod tracker_v2;
 
-pub enum ChunkEndorsementsState {
-    Endorsed(Option<EndorsementStats>, ChunkEndorsementSignatures),
-    NotEnoughStake(Option<EndorsementStats>),
-}
-
-impl ChunkEndorsementsState {
-    pub fn stats(&self) -> Option<&EndorsementStats> {
-        match self {
-            Self::Endorsed(stats, _) => stats.as_ref(),
-            Self::NotEnoughStake(stats) => stats.as_ref(),
-        }
-    }
-}
-
 /// Module to track chunk endorsements received from chunk validators.
 pub struct ChunkEndorsementTracker {
     epoch_manager: Arc<dyn EpochManagerAdapter>,

chain/client/src/stateless_validation/chunk_endorsement/tracker_v1.rs

@@ -1,5 +1,6 @@
 use lru::LruCache;
 use near_primitives::stateless_validation::chunk_endorsement::ChunkEndorsementV1;
+use near_primitives::stateless_validation::validator_assignment::ChunkEndorsementsState;
 use std::collections::HashMap;
 use std::num::NonZeroUsize;
 use std::sync::{Arc, Mutex};
@@ -10,8 +11,6 @@ use near_primitives::checked_feature;
 use near_primitives::sharding::{ChunkHash, ShardChunkHeader};
 use near_primitives::types::AccountId;
 
-use super::ChunkEndorsementsState;
-
 // This is the number of unique chunks for which we would track the chunk endorsements.
 // Ideally, we should not be processing more than num_shards chunks at a time.
 const NUM_CHUNKS_IN_CHUNK_ENDORSEMENTS_CACHE: usize = 100;
@@ -104,13 +103,8 @@ impl ChunkEndorsementTracker {
         )
     }
 
-    /// Called by block producer.
-    /// Returns ChunkEndorsementsState::Endorsed if node has enough signed stake for the chunk
-    /// represented by chunk_header.
-    /// Signatures have the same order as ordered_chunk_validators, thus ready to be included in a block as is.
-    /// Returns ChunkEndorsementsState::NotEnoughStake if chunk doesn't have enough stake.
-    /// For older protocol version, we return ChunkEndorsementsState::Endorsed with an empty array of
-    /// chunk endorsements.
+    /// This function is called by block producer potentially multiple times if there's not enough stake.
+    /// For older protocol version, we return an empty array of chunk endorsements.
     pub fn collect_chunk_endorsements(
         &self,
         chunk_header: &ShardChunkHeader,
@@ -209,7 +203,10 @@ impl ChunkEndorsementTrackerInner {
         let protocol_version = self.epoch_manager.get_epoch_protocol_version(&epoch_id)?;
         if !checked_feature!("stable", StatelessValidation, protocol_version) {
             // Return an empty array of chunk endorsements for older protocol versions.
-            return Ok(ChunkEndorsementsState::Endorsed(None, vec![]));
+            return Ok(ChunkEndorsementsState {
+                is_endorsed: true,
+                ..ChunkEndorsementsState::default()
+            });
         }
 
         let chunk_validator_assignments = self.epoch_manager.get_chunk_validator_assignments(
@@ -226,29 +223,14 @@ impl ChunkEndorsementTrackerInner {
             self.chunk_endorsements.get(&chunk_header.chunk_hash())
         else {
             // Early return if no chunk_endorsements found in our cache.
-            return Ok(ChunkEndorsementsState::NotEnoughStake(None));
+            return Ok(ChunkEndorsementsState::default());
         };
 
-        let endorsement_stats = chunk_validator_assignments
-            .compute_endorsement_stats(&chunk_endorsements.keys().collect());
-
-        // Check whether the current set of chunk_validators have enough stake to include chunk in block.
-        if !endorsement_stats.has_enough_stake() {
-            return Ok(ChunkEndorsementsState::NotEnoughStake(Some(endorsement_stats)));
-        }
-
-        // We've already verified the chunk_endorsements are valid, collect signatures.
-        let signatures = chunk_validator_assignments
-            .ordered_chunk_validators()
-            .iter()
-            .map(|account_id| {
-                // map Option<ChunkEndorsement> to Option<Box<Signature>>
-                chunk_endorsements
-                    .get(account_id)
-                    .map(|endorsement| Box::new(endorsement.signature.clone()))
-            })
+        let validator_signatures = chunk_endorsements
+            .into_iter()
+            .map(|(account_id, endorsement)| (account_id, endorsement.signature.clone()))
             .collect();
 
-        Ok(ChunkEndorsementsState::Endorsed(Some(endorsement_stats), signatures))
+        Ok(chunk_validator_assignments.compute_endorsement_state(validator_signatures))
     }
 }

chain/client/src/stateless_validation/chunk_endorsement/tracker_v2.rs

@@ -7,15 +7,14 @@ use near_chain_primitives::Error;
 use near_epoch_manager::EpochManagerAdapter;
 use near_primitives::sharding::ShardChunkHeader;
 use near_primitives::stateless_validation::chunk_endorsement::ChunkEndorsementV2;
+use near_primitives::stateless_validation::validator_assignment::ChunkEndorsementsState;
 use near_primitives::stateless_validation::ChunkProductionKey;
 use near_primitives::types::AccountId;
 use near_primitives::version::ProtocolFeature;
 use near_store::Store;
 
 use crate::stateless_validation::validate::validate_chunk_endorsement;
 
-use super::ChunkEndorsementsState;
-
 // This is the number of unique chunks for which we would track the chunk endorsements.
 // Ideally, we should not be processing more than num_shards chunks at a time.
 const NUM_CHUNKS_IN_CHUNK_ENDORSEMENTS_CACHE: usize = 100;
@@ -64,10 +63,6 @@ impl ChunkEndorsementTracker {
     }
 
     /// This function is called by block producer potentially multiple times if there's not enough stake.
-    /// We return ChunkEndorsementsState::Endorsed if node has enough signed stake for the chunk represented
-    /// by chunk_header. Signatures have the same order as ordered_chunk_validators, thus ready to be included
-    /// in a block as is.
-    /// We returns ChunkEndorsementsState::NotEnoughStake if chunk doesn't have enough stake.
     pub(crate) fn collect_chunk_endorsements(
         &mut self,
         chunk_header: &ShardChunkHeader,
@@ -77,8 +72,11 @@ impl ChunkEndorsementTracker {
             self.epoch_manager.get_epoch_id_from_prev_block(chunk_header.prev_block_hash())?;
         let protocol_version = self.epoch_manager.get_epoch_protocol_version(&epoch_id)?;
         if !ProtocolFeature::StatelessValidation.enabled(protocol_version) {
-            // Return an empty array of chunk endorsements for older protocol versions.
-            return Ok(ChunkEndorsementsState::Endorsed(None, vec![]));
+            // Return an endorsed empty array of chunk endorsements for older protocol versions.
+            return Ok(ChunkEndorsementsState {
+                is_endorsed: true,
+                ..ChunkEndorsementsState::default()
+            });
         }
 
         let height_created = chunk_header.height_created();
@@ -96,30 +94,13 @@ impl ChunkEndorsementTracker {
         //    1. The chunk endorsements are from valid chunk_validator for this chunk.
         //    2. The chunk endorsements signatures are valid.
         //    3. We still need to validate if the chunk_hash matches the chunk_header.chunk_hash()
-        let Some(entry) = self.chunk_endorsements.get_mut(&key) else {
-            // Early return if no chunk_endorsements found in our cache.
-            return Ok(ChunkEndorsementsState::NotEnoughStake(None));
-        };
-        entry.retain(|_, endorsement| endorsement.chunk_hash() == &chunk_header.chunk_hash());
-
-        let endorsement_stats =
-            chunk_validator_assignments.compute_endorsement_stats(&entry.keys().collect());
-
-        // Check whether the current set of chunk_validators have enough stake to include chunk in block.
-        if !endorsement_stats.has_enough_stake() {
-            return Ok(ChunkEndorsementsState::NotEnoughStake(Some(endorsement_stats)));
-        }
-
-        // We've already verified the chunk_endorsements are valid, collect signatures.
-        let signatures = chunk_validator_assignments
-            .ordered_chunk_validators()
-            .iter()
-            .map(|account_id| {
-                // map Option<ChunkEndorsement> to Option<Box<Signature>>
-                entry.get(account_id).map(|endorsement| Box::new(endorsement.signature()))
-            })
+        let entry = self.chunk_endorsements.get_or_insert(key, || HashMap::new());
+        let validator_signatures = entry
+            .into_iter()
+            .filter(|(_, endorsement)| endorsement.chunk_hash() == &chunk_header.chunk_hash())
+            .map(|(account_id, endorsement)| (account_id, endorsement.signature()))
             .collect();
 
-        Ok(ChunkEndorsementsState::Endorsed(Some(endorsement_stats), signatures))
+        Ok(chunk_validator_assignments.compute_endorsement_state(validator_signatures))
     }
 }

chain/client/src/test_utils/test_env.rs

@@ -808,9 +808,12 @@ impl TestEnv {
     pub fn print_block_summary(&self, height: u64) {
         let client = &self.clients[0];
         let block = client.chain.get_block_by_height(height);
-        let Ok(block) = block else {
-            tracing::info!(target: "test", "Block {}: missing", height);
-            return;
+        let block = match block {
+            Ok(block) => block,
+            Err(err) => {
+                tracing::info!(target: "test", ?err, "Block {}: missing", height);
+                return;
+            }
         };
         let prev_hash = block.header().prev_hash();
         let epoch_id = client.epoch_manager.get_epoch_id_from_prev_block(prev_hash).unwrap();