diff --git a/.github/workflows/bitcoin-tests.yml b/.github/workflows/bitcoin-tests.yml index 919ebe5056..26fde85f0c 100644 --- a/.github/workflows/bitcoin-tests.yml +++ b/.github/workflows/bitcoin-tests.yml @@ -135,6 +135,7 @@ jobs: - tests::signer::v0::tenure_extend_after_bad_commit - tests::signer::v0::block_proposal_max_age_rejections - tests::signer::v0::global_acceptance_depends_on_block_announcement + - tests::signer::v0::no_reorg_due_to_successive_block_validation_ok - tests::nakamoto_integrations::burn_ops_integration_test - tests::nakamoto_integrations::check_block_heights - tests::nakamoto_integrations::clarity_burn_state diff --git a/libsigner/src/v0/messages.rs b/libsigner/src/v0/messages.rs index 4d27f0e9d1..5f716cea2f 100644 --- a/libsigner/src/v0/messages.rs +++ b/libsigner/src/v0/messages.rs @@ -686,6 +686,22 @@ impl BlockResponse { BlockResponse::Rejected(rejection) => rejection.signer_signature_hash, } } + + /// Get the block accept data from the block response + pub fn as_block_accepted(&self) -> Option<&BlockAccepted> { + match self { + BlockResponse::Accepted(accepted) => Some(accepted), + _ => None, + } + } + + /// Get the block accept data from the block response + pub fn as_block_rejection(&self) -> Option<&BlockRejection> { + match self { + BlockResponse::Rejected(rejection) => Some(rejection), + _ => None, + } + } } impl StacksMessageCodec for BlockResponse { diff --git a/stacks-signer/src/chainstate.rs b/stacks-signer/src/chainstate.rs index 462f3dc2d2..dbf03c1f91 100644 --- a/stacks-signer/src/chainstate.rs +++ b/stacks-signer/src/chainstate.rs @@ -505,7 +505,7 @@ impl SortitionsView { /// Get the last block from the given tenure /// Returns the last locally accepted block if it is not timed out, otherwise it will return the last globally accepted block. - fn get_tenure_last_block_info( + pub fn get_tenure_last_block_info( consensus_hash: &ConsensusHash, signer_db: &SignerDb, tenure_last_block_proposal_timeout: Duration, @@ -517,7 +517,7 @@ impl SortitionsView { if let Some(local_info) = last_locally_accepted_block { if let Some(signed_over_time) = local_info.signed_self { - if signed_over_time + tenure_last_block_proposal_timeout.as_secs() + if signed_over_time.saturating_add(tenure_last_block_proposal_timeout.as_secs()) > get_epoch_time_secs() { // The last locally accepted block is not timed out, return it diff --git a/stacks-signer/src/signerdb.rs b/stacks-signer/src/signerdb.rs index 4cdc61471a..0da2c1adcc 100644 --- a/stacks-signer/src/signerdb.rs +++ b/stacks-signer/src/signerdb.rs @@ -733,6 +733,18 @@ impl SignerDb { try_deserialize(result) } + /// Return the last accepted block the signer (highest stacks height). It will tie break a match based on which was more recently signed. + pub fn get_signer_last_accepted_block(&self) -> Result, DBError> { + let query = "SELECT block_info FROM blocks WHERE state IN (?1, ?2) ORDER BY stacks_height DESC, signed_group DESC, signed_self DESC LIMIT 1"; + let args = params![ + &BlockState::GloballyAccepted.to_string(), + &BlockState::LocallyAccepted.to_string() + ]; + let result: Option = query_row(&self.db, query, args)?; + + try_deserialize(result) + } + /// Return the last accepted block in a tenure (identified by its consensus hash). pub fn get_last_accepted_block( &self, @@ -1757,4 +1769,69 @@ mod tests { < block_infos[0].proposed_time ); } + + #[test] + fn signer_last_accepted_block() { + let db_path = tmp_db_path(); + let mut db = SignerDb::new(db_path).expect("Failed to create signer db"); + + let (mut block_info_1, _block_proposal_1) = create_block_override(|b| { + b.block.header.miner_signature = MessageSignature([0x01; 65]); + b.block.header.chain_length = 1; + b.burn_height = 1; + }); + + let (mut block_info_2, _block_proposal_2) = create_block_override(|b| { + b.block.header.miner_signature = MessageSignature([0x02; 65]); + b.block.header.chain_length = 2; + b.burn_height = 1; + }); + + let (mut block_info_3, _block_proposal_3) = create_block_override(|b| { + b.block.header.miner_signature = MessageSignature([0x02; 65]); + b.block.header.chain_length = 2; + b.burn_height = 4; + }); + block_info_3 + .mark_locally_accepted(false) + .expect("Failed to mark block as locally accepted"); + + db.insert_block(&block_info_1) + .expect("Unable to insert block into db"); + db.insert_block(&block_info_2) + .expect("Unable to insert block into db"); + + assert!(db.get_signer_last_accepted_block().unwrap().is_none()); + + block_info_1 + .mark_globally_accepted() + .expect("Failed to mark block as globally accepted"); + db.insert_block(&block_info_1) + .expect("Unable to insert block into db"); + + assert_eq!( + db.get_signer_last_accepted_block().unwrap().unwrap(), + block_info_1 + ); + + block_info_2 + .mark_globally_accepted() + .expect("Failed to mark block as globally accepted"); + block_info_2.signed_self = Some(get_epoch_time_secs()); + db.insert_block(&block_info_2) + .expect("Unable to insert block into db"); + + assert_eq!( + db.get_signer_last_accepted_block().unwrap().unwrap(), + block_info_2 + ); + + db.insert_block(&block_info_3) + .expect("Unable to insert block into db"); + + assert_eq!( + db.get_signer_last_accepted_block().unwrap().unwrap(), + block_info_3 + ); + } } diff --git a/stacks-signer/src/tests/chainstate.rs b/stacks-signer/src/tests/chainstate.rs index 2037a25def..92b7a6ed53 100644 --- a/stacks-signer/src/tests/chainstate.rs +++ b/stacks-signer/src/tests/chainstate.rs @@ -452,8 +452,12 @@ fn check_sortition_timeout() { fs::create_dir_all(signer_db_dir).unwrap(); let mut signer_db = SignerDb::new(signer_db_path).unwrap(); + let block_sk = StacksPrivateKey::from_seed(&[0, 1]); + let block_pk = StacksPublicKey::from_private(&block_sk); + let block_pkh = Hash160::from_node_public_key(&block_pk); + let mut sortition = SortitionState { - miner_pkh: Hash160([0; 20]), + miner_pkh: block_pkh, miner_pubkey: None, prior_sortition: ConsensusHash([0; 20]), parent_tenure_id: ConsensusHash([0; 20]), diff --git a/stacks-signer/src/v0/signer.rs b/stacks-signer/src/v0/signer.rs index 5a5128cce4..988cc8f4a5 100644 --- a/stacks-signer/src/v0/signer.rs +++ b/stacks-signer/src/v0/signer.rs @@ -298,36 +298,113 @@ impl Signer { let valid = block_info.valid?; let response = if valid { debug!("{self}: Accepting block {}", block_info.block.block_id()); - let signature = self - .private_key - .sign(block_info.signer_signature_hash().bits()) - .expect("Failed to sign block"); - BlockResponse::accepted( - block_info.signer_signature_hash(), - signature, - self.signer_db.calculate_tenure_extend_timestamp( - self.proposal_config.tenure_idle_timeout, - &block_info.block, - true, - ), - ) + self.create_block_acceptance(&block_info.block) } else { debug!("{self}: Rejecting block {}", block_info.block.block_id()); - BlockResponse::rejected( - block_info.signer_signature_hash(), - RejectCode::RejectedInPriorRound, - &self.private_key, - self.mainnet, - self.signer_db.calculate_tenure_extend_timestamp( - self.proposal_config.tenure_idle_timeout, - &block_info.block, - false, - ), - ) + self.create_block_rejection(RejectCode::RejectedInPriorRound, &block_info.block) }; Some(response) } + /// Create a block acceptance response for a block + pub fn create_block_acceptance(&self, block: &NakamotoBlock) -> BlockResponse { + let signature = self + .private_key + .sign(block.header.signer_signature_hash().bits()) + .expect("Failed to sign block"); + BlockResponse::accepted( + block.header.signer_signature_hash(), + signature, + self.signer_db.calculate_tenure_extend_timestamp( + self.proposal_config.tenure_idle_timeout, + block, + true, + ), + ) + } + /// Create a block rejection response for a block with the given reject code + pub fn create_block_rejection( + &self, + reject_code: RejectCode, + block: &NakamotoBlock, + ) -> BlockResponse { + BlockResponse::rejected( + block.header.signer_signature_hash(), + reject_code, + &self.private_key, + self.mainnet, + self.signer_db.calculate_tenure_extend_timestamp( + self.proposal_config.tenure_idle_timeout, + block, + false, + ), + ) + } + /// Check if block should be rejected based on sortition state + /// Will return a BlockResponse::Rejection if the block is invalid, none otherwise. + fn check_block_against_sortition_state( + &mut self, + stacks_client: &StacksClient, + sortition_state: &mut Option, + block: &NakamotoBlock, + miner_pubkey: &Secp256k1PublicKey, + ) -> Option { + let signer_signature_hash = block.header.signer_signature_hash(); + let block_id = block.block_id(); + // Get sortition view if we don't have it + if sortition_state.is_none() { + *sortition_state = + SortitionsView::fetch_view(self.proposal_config.clone(), stacks_client) + .inspect_err(|e| { + warn!( + "{self}: Failed to update sortition view: {e:?}"; + "signer_sighash" => %signer_signature_hash, + "block_id" => %block_id, + ) + }) + .ok(); + } + + // Check if proposal can be rejected now if not valid against sortition view + if let Some(sortition_state) = sortition_state { + match sortition_state.check_proposal( + stacks_client, + &mut self.signer_db, + block, + miner_pubkey, + true, + ) { + // Error validating block + Err(e) => { + warn!( + "{self}: Error checking block proposal: {e:?}"; + "signer_sighash" => %signer_signature_hash, + "block_id" => %block_id, + ); + Some(self.create_block_rejection(RejectCode::ConnectivityIssues, block)) + } + // Block proposal is bad + Ok(false) => { + warn!( + "{self}: Block proposal invalid"; + "signer_sighash" => %signer_signature_hash, + "block_id" => %block_id, + ); + Some(self.create_block_rejection(RejectCode::SortitionViewMismatch, block)) + } + // Block proposal passed check, still don't know if valid + Ok(true) => None, + } + } else { + warn!( + "{self}: Cannot validate block, no sortition view"; + "signer_sighash" => %signer_signature_hash, + "block_id" => %block_id, + ); + Some(self.create_block_rejection(RejectCode::NoSortitionView, block)) + } + } + /// Handle block proposal messages submitted to signers stackerdb fn handle_block_proposal( &mut self, @@ -425,73 +502,12 @@ impl Signer { } // Check if proposal can be rejected now if not valid against sortition view - let block_response = if let Some(sortition_state) = sortition_state { - match sortition_state.check_proposal( - stacks_client, - &mut self.signer_db, - &block_proposal.block, - miner_pubkey, - true, - ) { - // Error validating block - Err(e) => { - warn!( - "{self}: Error checking block proposal: {e:?}"; - "signer_sighash" => %signer_signature_hash, - "block_id" => %block_proposal.block.block_id(), - ); - Some(BlockResponse::rejected( - block_proposal.block.header.signer_signature_hash(), - RejectCode::ConnectivityIssues, - &self.private_key, - self.mainnet, - self.signer_db.calculate_tenure_extend_timestamp( - self.proposal_config.tenure_idle_timeout, - &block_proposal.block, - false, - ), - )) - } - // Block proposal is bad - Ok(false) => { - warn!( - "{self}: Block proposal invalid"; - "signer_sighash" => %signer_signature_hash, - "block_id" => %block_proposal.block.block_id(), - ); - Some(BlockResponse::rejected( - block_proposal.block.header.signer_signature_hash(), - RejectCode::SortitionViewMismatch, - &self.private_key, - self.mainnet, - self.signer_db.calculate_tenure_extend_timestamp( - self.proposal_config.tenure_idle_timeout, - &block_proposal.block, - false, - ), - )) - } - // Block proposal passed check, still don't know if valid - Ok(true) => None, - } - } else { - warn!( - "{self}: Cannot validate block, no sortition view"; - "signer_sighash" => %signer_signature_hash, - "block_id" => %block_proposal.block.block_id(), - ); - Some(BlockResponse::rejected( - block_proposal.block.header.signer_signature_hash(), - RejectCode::NoSortitionView, - &self.private_key, - self.mainnet, - self.signer_db.calculate_tenure_extend_timestamp( - self.proposal_config.tenure_idle_timeout, - &block_proposal.block, - false, - ), - )) - }; + let block_response = self.check_block_against_sortition_state( + stacks_client, + sortition_state, + &block_proposal.block, + miner_pubkey, + ); #[cfg(any(test, feature = "testing"))] let block_response = @@ -524,6 +540,8 @@ impl Signer { "block_height" => block_proposal.block.header.chain_length, "burn_height" => block_proposal.burn_height, ); + #[cfg(any(test, feature = "testing"))] + self.test_stall_block_validation_submission(); match stacks_client.submit_block_for_validation(block_info.block.clone()) { Ok(_) => { self.submitted_block_proposal = @@ -563,6 +581,63 @@ impl Signer { } } } + + /// WARNING: Do NOT call this function PRIOR to check_proposal or block_proposal validation succeeds. + /// + /// Re-verify a block's chain length against the last signed block within signerdb. + /// This is required in case a block has been approved since the initial checks of the block validation endpoint. + fn check_block_against_signer_db_state( + &self, + proposed_block: &NakamotoBlock, + ) -> Option { + let signer_signature_hash = proposed_block.header.signer_signature_hash(); + let proposed_block_consensus_hash = proposed_block.header.consensus_hash; + + match self.signer_db.get_signer_last_accepted_block() { + Ok(Some(last_block_info)) => { + if proposed_block.header.chain_length <= last_block_info.block.header.chain_length { + // We do not allow reorgs at any time within the same consensus hash OR of globally accepted blocks + let non_reorgable_block = last_block_info.block.header.consensus_hash + == proposed_block_consensus_hash + || last_block_info.state == BlockState::GloballyAccepted; + // Is the reorg timeout requirement exceeded? + let reorg_timeout_exceeded = last_block_info + .signed_self + .map(|signed_over_time| { + signed_over_time.saturating_add( + self.proposal_config + .tenure_last_block_proposal_timeout + .as_secs(), + ) <= get_epoch_time_secs() + }) + .unwrap_or(false); + if non_reorgable_block || !reorg_timeout_exceeded { + warn!( + "Miner's block proposal does not confirm as many blocks as we expect"; + "proposed_block_consensus_hash" => %proposed_block_consensus_hash, + "proposed_block_signer_sighash" => %signer_signature_hash, + "proposed_chain_length" => proposed_block.header.chain_length, + "expected_at_least" => last_block_info.block.header.chain_length + 1, + ); + return Some(self.create_block_rejection( + RejectCode::SortitionViewMismatch, + proposed_block, + )); + } + } + None + } + Ok(_) => None, + Err(e) => { + warn!("{self}: Failed to check block against signer db: {e}"; + "signer_sighash" => %signer_signature_hash, + "block_id" => %proposed_block.block_id() + ); + Some(self.create_block_rejection(RejectCode::ConnectivityIssues, proposed_block)) + } + } + } + /// Handle the block validate ok response. Returns our block response if we have one fn handle_block_validate_ok( &mut self, @@ -600,40 +675,54 @@ impl Signer { return None; } }; - if let Err(e) = block_info.mark_locally_accepted(false) { - if !block_info.has_reached_consensus() { - warn!("{self}: Failed to mark block as locally accepted: {e:?}",); - return None; + + if let Some(block_response) = self.check_block_against_signer_db_state(&block_info.block) { + // The signer db state has changed. We no longer view this block as valid. Override the validation response. + if let Err(e) = block_info.mark_locally_rejected() { + if !block_info.has_reached_consensus() { + warn!("{self}: Failed to mark block as locally rejected: {e:?}"); + } + }; + debug!("{self}: Broadcasting a block response to stacks node: {block_response:?}"); + let res = self + .stackerdb + .send_message_with_retry::(block_response.into()); + + match res { + Err(e) => warn!("{self}: Failed to send block rejection to stacker-db: {e:?}"), + Ok(ack) if !ack.accepted => warn!( + "{self}: Block rejection not accepted by stacker-db: {:?}", + ack.reason + ), + Ok(_) => debug!("{self}: Block rejection accepted by stacker-db"), } - block_info.signed_self.get_or_insert(get_epoch_time_secs()); - } - // Record the block validation time but do not consider stx transfers or boot contract calls - block_info.validation_time_ms = if block_validate_ok.cost.is_zero() { - Some(0) + self.signer_db + .insert_block(&block_info) + .unwrap_or_else(|e| self.handle_insert_block_error(e)); + None } else { - Some(block_validate_ok.validation_time_ms) - }; - - let signature = self - .private_key - .sign(&signer_signature_hash.0) - .expect("Failed to sign block"); + if let Err(e) = block_info.mark_locally_accepted(false) { + if !block_info.has_reached_consensus() { + warn!("{self}: Failed to mark block as locally accepted: {e:?}",); + return None; + } + block_info.signed_self.get_or_insert(get_epoch_time_secs()); + } + // Record the block validation time but do not consider stx transfers or boot contract calls + block_info.validation_time_ms = if block_validate_ok.cost.is_zero() { + Some(0) + } else { + Some(block_validate_ok.validation_time_ms) + }; - self.signer_db - .insert_block(&block_info) - .unwrap_or_else(|e| self.handle_insert_block_error(e)); - let accepted = BlockAccepted::new( - block_info.signer_signature_hash(), - signature, - self.signer_db.calculate_tenure_extend_timestamp( - self.proposal_config.tenure_idle_timeout, - &block_info.block, - true, - ), - ); - // have to save the signature _after_ the block info - self.handle_block_signature(stacks_client, &accepted); - Some(BlockResponse::Accepted(accepted)) + self.signer_db + .insert_block(&block_info) + .unwrap_or_else(|e| self.handle_insert_block_error(e)); + let block_response = self.create_block_acceptance(&block_info.block); + // have to save the signature _after_ the block info + self.handle_block_signature(stacks_client, block_response.as_block_accepted()?); + Some(block_response) + } } /// Handle the block validate reject response. Returns our block response if we have one @@ -775,19 +864,12 @@ impl Signer { "signer_sighash" => %signature_sighash, "block_id" => %block_proposal.block.block_id(), ); - let rejection = BlockResponse::rejected( - block_proposal.block.header.signer_signature_hash(), - RejectCode::ConnectivityIssues, - &self.private_key, - self.mainnet, - self.signer_db.calculate_tenure_extend_timestamp( - self.proposal_config.tenure_idle_timeout, - &block_proposal.block, - false, - ), - ); + let rejection = + self.create_block_rejection(RejectCode::ConnectivityIssues, &block_proposal.block); if let Err(e) = block_info.mark_locally_rejected() { - warn!("{self}: Failed to mark block as locally rejected: {e:?}",); + if !block_info.has_reached_consensus() { + warn!("{self}: Failed to mark block as locally rejected: {e:?}"); + } }; debug!("{self}: Broadcasting a block response to stacks node: {rejection:?}"); let res = self diff --git a/stacks-signer/src/v0/tests.rs b/stacks-signer/src/v0/tests.rs index 0b9cdcc569..b9ea43fae5 100644 --- a/stacks-signer/src/v0/tests.rs +++ b/stacks-signer/src/v0/tests.rs @@ -41,6 +41,10 @@ pub static TEST_PAUSE_BLOCK_BROADCAST: LazyLock> = LazyLock::new( /// A global variable that can be used to skip broadcasting the block to the network pub static TEST_SKIP_BLOCK_BROADCAST: LazyLock> = LazyLock::new(TestFlag::default); +/// A global variable that can be used to pause the block validation submission +pub static TEST_STALL_BLOCK_VALIDATION_SUBMISSION: LazyLock> = + LazyLock::new(TestFlag::default); + impl Signer { /// Skip the block broadcast if the TEST_SKIP_BLOCK_BROADCAST flag is set pub fn test_skip_block_broadcast(&self, block: &NakamotoBlock) -> bool { @@ -81,7 +85,9 @@ impl Signer { "consensus_hash" => %block_proposal.block.header.consensus_hash ); if let Err(e) = block_info.mark_locally_rejected() { - warn!("{self}: Failed to mark block as locally rejected: {e:?}",); + if !block_info.has_reached_consensus() { + warn!("{self}: Failed to mark block as locally rejected: {e:?}"); + } }; // We must insert the block into the DB to prevent subsequent repeat proposals being accepted (should reject // as invalid since we rejected in a prior round if this crops up again) @@ -89,17 +95,7 @@ impl Signer { self.signer_db .insert_block(block_info) .unwrap_or_else(|e| self.handle_insert_block_error(e)); - Some(BlockResponse::rejected( - block_proposal.block.header.signer_signature_hash(), - RejectCode::TestingDirective, - &self.private_key, - self.mainnet, - self.signer_db.calculate_tenure_extend_timestamp( - self.proposal_config.tenure_idle_timeout, - &block_proposal.block, - false, - ), - )) + Some(self.create_block_rejection(RejectCode::TestingDirective, &block_proposal.block)) } else { block_response } @@ -138,4 +134,16 @@ impl Signer { } false } + + /// Stall the block validation submission if the TEST_STALL_BLOCK_VALIDATION_SUBMISSION flag is set + pub fn test_stall_block_validation_submission(&self) { + if TEST_STALL_BLOCK_VALIDATION_SUBMISSION.get() { + // Do an extra check just so we don't log EVERY time. + warn!("{self}: Block validation submission is stalled due to testing directive"); + while TEST_STALL_BLOCK_VALIDATION_SUBMISSION.get() { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + warn!("{self}: Block validation submission is no longer stalled due to testing directive. Continuing..."); + } + } } diff --git a/stackslib/src/net/api/postblock_proposal.rs b/stackslib/src/net/api/postblock_proposal.rs index c832695103..ca0d71815f 100644 --- a/stackslib/src/net/api/postblock_proposal.rs +++ b/stackslib/src/net/api/postblock_proposal.rs @@ -359,7 +359,9 @@ impl NakamotoBlockProposal { while *TEST_VALIDATE_STALL.lock().unwrap() == Some(true) { std::thread::sleep(std::time::Duration::from_millis(10)); } - info!("Block validation is no longer stalled due to testing directive."); + info!( + "Block validation is no longer stalled due to testing directive. Continuing..." + ); } } let start = Instant::now(); diff --git a/testnet/stacks-node/src/event_dispatcher.rs b/testnet/stacks-node/src/event_dispatcher.rs index 2f71838adb..540c84f95b 100644 --- a/testnet/stacks-node/src/event_dispatcher.rs +++ b/testnet/stacks-node/src/event_dispatcher.rs @@ -162,6 +162,7 @@ pub struct MinedNakamotoBlockEvent { pub block_size: u64, pub cost: ExecutionCost, pub miner_signature: MessageSignature, + pub miner_signature_hash: Sha512Trunc256Sum, pub signer_signature_hash: Sha512Trunc256Sum, pub tx_events: Vec, pub signer_bitvec: String, @@ -1528,6 +1529,7 @@ impl EventDispatcher { cost: consumed.clone(), tx_events, miner_signature: block.header.miner_signature, + miner_signature_hash: block.header.miner_signature_hash(), signer_signature_hash: block.header.signer_signature_hash(), signer_signature: block.header.signer_signature.clone(), signer_bitvec, diff --git a/testnet/stacks-node/src/tests/signer/mod.rs b/testnet/stacks-node/src/tests/signer/mod.rs index 432b990667..6365579dfd 100644 --- a/testnet/stacks-node/src/tests/signer/mod.rs +++ b/testnet/stacks-node/src/tests/signer/mod.rs @@ -591,24 +591,21 @@ impl + Send + 'static, T: SignerEventTrait + 'static> SignerTest { - if accepted.signer_signature_hash == *signer_signature_hash - && expected_signers.iter().any(|pk| { - pk.verify( - accepted.signer_signature_hash.bits(), - &accepted.signature, - ) - .expect("Failed to verify signature") - }) - { - Some(accepted.signature) - } else { - None - } + if let SignerMessage::BlockResponse(BlockResponse::Accepted(accepted)) = message + { + if accepted.signer_signature_hash == *signer_signature_hash + && expected_signers.iter().any(|pk| { + pk.verify( + accepted.signer_signature_hash.bits(), + &accepted.signature, + ) + .expect("Failed to verify signature") + }) + { + return Some(accepted.signature); } - _ => None, } + None }) .collect::>(); Ok(signatures.len() > expected_signers.len() * 7 / 10) @@ -672,11 +669,10 @@ impl + Send + 'static, T: SignerEventTrait + 'static> SignerTest BlockAccepted { - let block_response = self.get_latest_block_response(slot_id); - match block_response { - BlockResponse::Accepted(accepted) => accepted, - _ => panic!("Latest block response from slot #{slot_id} isn't a block acceptance"), - } + self.get_latest_block_response(slot_id) + .as_block_accepted() + .expect("Latest block response from slot #{slot_id} isn't a block acceptance") + .clone() } /// Get /v2/info from the node diff --git a/testnet/stacks-node/src/tests/signer/v0.rs b/testnet/stacks-node/src/tests/signer/v0.rs index 5200883667..044e5f0cbc 100644 --- a/testnet/stacks-node/src/tests/signer/v0.rs +++ b/testnet/stacks-node/src/tests/signer/v0.rs @@ -42,7 +42,8 @@ use stacks::core::{StacksEpochId, CHAIN_ID_TESTNET}; use stacks::libstackerdb::StackerDBChunkData; use stacks::net::api::getsigner::GetSignerResponse; use stacks::net::api::postblock_proposal::{ - ValidateRejectCode, TEST_VALIDATE_DELAY_DURATION_SECS, TEST_VALIDATE_STALL, + BlockValidateResponse, ValidateRejectCode, TEST_VALIDATE_DELAY_DURATION_SECS, + TEST_VALIDATE_STALL, }; use stacks::net::relay::fault_injection::set_ignore_block; use stacks::types::chainstate::{StacksAddress, StacksBlockId, StacksPrivateKey, StacksPublicKey}; @@ -62,7 +63,7 @@ use stacks_signer::client::{SignerSlotID, StackerDB}; use stacks_signer::config::{build_signer_config_tomls, GlobalConfig as SignerConfig, Network}; use stacks_signer::v0::tests::{ TEST_IGNORE_ALL_BLOCK_PROPOSALS, TEST_PAUSE_BLOCK_BROADCAST, TEST_REJECT_ALL_BLOCK_PROPOSAL, - TEST_SKIP_BLOCK_BROADCAST, + TEST_SKIP_BLOCK_BROADCAST, TEST_STALL_BLOCK_VALIDATION_SUBMISSION, }; use stacks_signer::v0::SpawnedSigner; use tracing_subscriber::prelude::*; @@ -5795,15 +5796,13 @@ fn reorg_locally_accepted_blocks_across_tenures_succeeds() { .filter_map(|chunk| { let message = SignerMessage::consensus_deserialize(&mut chunk.data.as_slice()) .expect("Failed to deserialize SignerMessage"); - match message { - SignerMessage::BlockResponse(BlockResponse::Accepted(accepted)) => { - non_ignoring_signers.iter().find(|key| { - key.verify(accepted.signer_signature_hash.bits(), &accepted.signature) - .is_ok() - }) - } - _ => None, + if let SignerMessage::BlockResponse(BlockResponse::Accepted(accepted)) = message { + return non_ignoring_signers.iter().find(|key| { + key.verify(accepted.signer_signature_hash.bits(), &accepted.signature) + .is_ok() + }); } + None }) .collect::>(); Ok(accepted_signers.len() + ignoring_signers.len() == num_signers) @@ -6015,15 +6014,13 @@ fn reorg_locally_accepted_blocks_across_tenures_fails() { .filter_map(|chunk| { let message = SignerMessage::consensus_deserialize(&mut chunk.data.as_slice()) .expect("Failed to deserialize SignerMessage"); - match message { - SignerMessage::BlockResponse(BlockResponse::Accepted(accepted)) => { - non_ignoring_signers.iter().find(|key| { - key.verify(accepted.signer_signature_hash.bits(), &accepted.signature) - .is_ok() - }) - } - _ => None, + if let SignerMessage::BlockResponse(BlockResponse::Accepted(accepted)) = message { + return non_ignoring_signers.iter().find(|key| { + key.verify(accepted.signer_signature_hash.bits(), &accepted.signature) + .is_ok() + }); } + None }) .collect::>(); Ok(accepted_signers.len() + ignoring_signers.len() == num_signers) @@ -6257,16 +6254,12 @@ fn miner_recovers_when_broadcast_block_delay_across_tenures_occurs() { .filter_map(|chunk| { let message = SignerMessage::consensus_deserialize(&mut chunk.data.as_slice()) .expect("Failed to deserialize SignerMessage"); - match message { - SignerMessage::BlockResponse(BlockResponse::Accepted(accepted)) => { - if block.header.signer_signature_hash() == accepted.signer_signature_hash { - Some(accepted.signature) - } else { - None - } + if let SignerMessage::BlockResponse(BlockResponse::Accepted(accepted)) = message { + if block.header.signer_signature_hash() == accepted.signer_signature_hash { + return Some(accepted.signature); } - _ => None, } + None }) .collect::>(); Ok(signatures.len() >= num_signers * 7 / 10) @@ -9528,3 +9521,534 @@ fn global_acceptance_depends_on_block_announcement() { ); assert_ne!(sister_block, proposed_block); } + +/// Test a scenario where: +/// Two miners boot to Nakamoto. +/// Sortition occurs. Miner 1 wins. +/// Miner 1 proposes a block N +/// Signers accept and the stacks tip advances to N +/// Sortition occurs. Miner 2 wins. +/// Miner 2 proposes block N+1 +/// Sortition occurs. Miner 1 wins. +/// Miner 1 proposes block N+1' +/// N+1 passes signers initial checks and is submitted to the node for validation. +/// N+1' arrives at the signers and passes inital checks, but BEFORE N+1' can be submitted for validation: +/// N+1 finishes being processed at the node and sits in the signers queue. +/// Signers THEN submit N+1' for node validation. +/// Signers process N+1 validation response ok, followed immediately by the N+1' validation response ok. +/// Signers broadcast N+1 acceptance +/// Signers broadcast N+1' rejection +/// Miner 2 proposes a new N+2 block built upon N+1 +/// Asserts: +/// - N+1 is signed and broadcasted +/// - N+1' is rejected as a sortition view mismatch +/// - The tip advances to N+1 (Signed by Miner 1) +/// - The tip advances to N+2 (Signed by Miner 2) +#[test] +#[ignore] +fn no_reorg_due_to_successive_block_validation_ok() { + if env::var("BITCOIND_TEST") != Ok("1".into()) { + return; + } + + let num_signers = 5; + let recipient = PrincipalData::from(StacksAddress::burn_address(false)); + let sender_sk = Secp256k1PrivateKey::new(); + let sender_addr = tests::to_addr(&sender_sk); + let send_amt = 100; + let send_fee = 180; + let num_txs = 1; + let sender_nonce = 0; + + let btc_miner_1_seed = vec![1, 1, 1, 1]; + let btc_miner_2_seed = vec![2, 2, 2, 2]; + let btc_miner_1_pk = Keychain::default(btc_miner_1_seed.clone()).get_pub_key(); + let btc_miner_2_pk = Keychain::default(btc_miner_2_seed.clone()).get_pub_key(); + + let node_1_rpc = gen_random_port(); + let node_1_p2p = gen_random_port(); + let node_2_rpc = gen_random_port(); + let node_2_p2p = gen_random_port(); + + let localhost = "127.0.0.1"; + let node_1_rpc_bind = format!("{localhost}:{node_1_rpc}"); + let node_2_rpc_bind = format!("{localhost}:{node_2_rpc}"); + let mut node_2_listeners = Vec::new(); + + let max_nakamoto_tenures = 30; + + info!("------------------------- Test Setup -------------------------"); + // partition the signer set so that ~half are listening and using node 1 for RPC and events, + // and the rest are using node 2 + + let mut signer_test: SignerTest = SignerTest::new_with_config_modifications( + num_signers, + vec![(sender_addr, (send_amt + send_fee) * num_txs)], + |signer_config| { + // Lets make sure we never time out since we need to stall some things to force our scenario + signer_config.block_proposal_validation_timeout = Duration::from_secs(u64::MAX); + signer_config.tenure_last_block_proposal_timeout = Duration::from_secs(u64::MAX); + signer_config.first_proposal_burn_block_timing = Duration::from_secs(u64::MAX); + let node_host = if signer_config.endpoint.port() % 2 == 0 { + &node_1_rpc_bind + } else { + &node_2_rpc_bind + }; + signer_config.node_host = node_host.to_string(); + }, + |config| { + config.node.rpc_bind = format!("{localhost}:{node_1_rpc}"); + config.node.p2p_bind = format!("{localhost}:{node_1_p2p}"); + config.node.data_url = format!("http://{localhost}:{node_1_rpc}"); + config.node.p2p_address = format!("{localhost}:{node_1_p2p}"); + config.miner.wait_on_interim_blocks = Duration::from_secs(5); + config.node.pox_sync_sample_secs = 30; + config.burnchain.pox_reward_length = Some(max_nakamoto_tenures); + + config.node.seed = btc_miner_1_seed.clone(); + config.node.local_peer_seed = btc_miner_1_seed.clone(); + config.burnchain.local_mining_public_key = Some(btc_miner_1_pk.to_hex()); + config.miner.mining_key = Some(Secp256k1PrivateKey::from_seed(&[1])); + + config.events_observers.retain(|listener| { + let Ok(addr) = std::net::SocketAddr::from_str(&listener.endpoint) else { + warn!( + "Cannot parse {} to a socket, assuming it isn't a signer-listener binding", + listener.endpoint + ); + return true; + }; + if addr.port() % 2 == 0 || addr.port() == test_observer::EVENT_OBSERVER_PORT { + return true; + } + node_2_listeners.push(listener.clone()); + false + }) + }, + Some(vec![btc_miner_1_pk, btc_miner_2_pk]), + None, + ); + let conf = signer_test.running_nodes.conf.clone(); + let mut conf_node_2 = conf.clone(); + conf_node_2.node.rpc_bind = format!("{localhost}:{node_2_rpc}"); + conf_node_2.node.p2p_bind = format!("{localhost}:{node_2_p2p}"); + conf_node_2.node.data_url = format!("http://{localhost}:{node_2_rpc}"); + conf_node_2.node.p2p_address = format!("{localhost}:{node_2_p2p}"); + conf_node_2.node.seed = btc_miner_2_seed.clone(); + conf_node_2.burnchain.local_mining_public_key = Some(btc_miner_2_pk.to_hex()); + conf_node_2.node.local_peer_seed = btc_miner_2_seed.clone(); + conf_node_2.miner.mining_key = Some(Secp256k1PrivateKey::from_seed(&[2])); + conf_node_2.node.miner = true; + conf_node_2.events_observers.clear(); + conf_node_2.events_observers.extend(node_2_listeners); + assert!(!conf_node_2.events_observers.is_empty()); + + let node_1_sk = Secp256k1PrivateKey::from_seed(&conf.node.local_peer_seed); + let node_1_pk = StacksPublicKey::from_private(&node_1_sk); + + conf_node_2.node.working_dir = format!("{}-1", conf_node_2.node.working_dir); + + conf_node_2.node.set_bootstrap_nodes( + format!("{}@{}", &node_1_pk.to_hex(), conf.node.p2p_bind), + conf.burnchain.chain_id, + conf.burnchain.peer_version, + ); + let http_origin = format!("http://{}", &signer_test.running_nodes.conf.node.rpc_bind); + + let mut run_loop_2 = boot_nakamoto::BootRunLoop::new(conf_node_2.clone()).unwrap(); + let run_loop_stopper_2 = run_loop_2.get_termination_switch(); + let rl2_coord_channels = run_loop_2.coordinator_channels(); + let Counters { + naka_submitted_commits: rl2_commits, + naka_skip_commit_op: rl2_skip_commit_op, + naka_mined_blocks: blocks_mined2, + naka_rejected_blocks: rl2_rejections, + naka_proposed_blocks: rl2_proposals, + .. + } = run_loop_2.counters(); + + let blocks_mined1 = signer_test.running_nodes.nakamoto_blocks_mined.clone(); + + info!("------------------------- Pause Miner 2's Block Commits -------------------------"); + + // Make sure Miner 2 cannot win a sortition at first. + rl2_skip_commit_op.set(true); + + info!("------------------------- Boot to Epoch 3.0 -------------------------"); + + let run_loop_2_thread = thread::Builder::new() + .name("run_loop_2".into()) + .spawn(move || run_loop_2.start(None, 0)) + .unwrap(); + + signer_test.boot_to_epoch_3(); + + wait_for(120, || { + let Some(node_1_info) = get_chain_info_opt(&conf) else { + return Ok(false); + }; + let Some(node_2_info) = get_chain_info_opt(&conf_node_2) else { + return Ok(false); + }; + Ok(node_1_info.stacks_tip_height == node_2_info.stacks_tip_height) + }) + .expect("Timed out waiting for boostrapped node to catch up to the miner"); + + let mining_pk_1 = StacksPublicKey::from_private(&conf.miner.mining_key.unwrap()); + let mining_pk_2 = StacksPublicKey::from_private(&conf_node_2.miner.mining_key.unwrap()); + let mining_pkh_1 = Hash160::from_node_public_key(&mining_pk_1); + let mining_pkh_2 = Hash160::from_node_public_key(&mining_pk_2); + debug!("The mining key for miner 1 is {mining_pkh_1}"); + debug!("The mining key for miner 2 is {mining_pkh_2}"); + + info!("------------------------- Reached Epoch 3.0 -------------------------"); + + let burnchain = signer_test.running_nodes.conf.get_burnchain(); + let sortdb = burnchain.open_sortition_db(true).unwrap(); + + let get_burn_height = || { + SortitionDB::get_canonical_burn_chain_tip(sortdb.conn()) + .unwrap() + .block_height + }; + let starting_peer_height = get_chain_info(&conf).stacks_tip_height; + let starting_burn_height = get_burn_height(); + + info!("------------------------- Pause Miner 1's Block Commits -------------------------"); + signer_test + .running_nodes + .nakamoto_test_skip_commit_op + .set(true); + + info!("------------------------- Miner 1 Mines a Nakamoto Block N (Globally Accepted) -------------------------"); + let blocks_processed_before_1 = blocks_mined1.load(Ordering::SeqCst); + let stacks_height_before = signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip_height; + let info_before = get_chain_info(&conf); + let mined_before = test_observer::get_mined_nakamoto_blocks().len(); + + next_block_and( + &mut signer_test.running_nodes.btc_regtest_controller, + 30, + || { + Ok(get_burn_height() > starting_burn_height + && signer_test + .stacks_client + .get_peer_info() + .expect("Failed to get peer info") + .stacks_tip_height + > stacks_height_before + && blocks_mined1.load(Ordering::SeqCst) > blocks_processed_before_1 + && get_chain_info(&conf).stacks_tip_height > info_before.stacks_tip_height + && test_observer::get_mined_nakamoto_blocks().len() > mined_before) + }, + ) + .expect("Timed out waiting for Miner 1 to Mine Block N"); + + let blocks = test_observer::get_mined_nakamoto_blocks(); + let block_n = blocks.last().unwrap().clone(); + let block_n_signature_hash = block_n.signer_signature_hash; + + let info_after = get_chain_info(&conf); + assert_eq!(info_after.stacks_tip.to_string(), block_n.block_hash); + assert_eq!(block_n.signer_signature_hash, block_n_signature_hash); + assert_eq!( + info_after.stacks_tip_height, + info_before.stacks_tip_height + 1 + ); + + // assure we have a successful sortition that miner 1 won + let tip = SortitionDB::get_canonical_burn_chain_tip(sortdb.conn()).unwrap(); + assert!(tip.sortition); + assert_eq!(tip.miner_pk_hash.unwrap(), mining_pkh_1); + + debug!("Miner 1 mined block N: {block_n_signature_hash}"); + + info!("------------------------- Pause Block Validation Response of N+1 -------------------------"); + TEST_VALIDATE_STALL.lock().unwrap().replace(true); + let proposals_before_2 = rl2_proposals.load(Ordering::SeqCst); + let rejections_before_2 = rl2_rejections.load(Ordering::SeqCst); + let blocks_before = test_observer::get_blocks().len(); + let blocks_processed_before_1 = blocks_mined1.load(Ordering::SeqCst); + let blocks_processed_before_2 = blocks_mined2.load(Ordering::SeqCst); + + // Force miner 1 to submit a block + // submit a tx so that the miner will mine an extra block + let transfer_tx = make_stacks_transfer( + &sender_sk, + sender_nonce, + send_fee, + signer_test.running_nodes.conf.burnchain.chain_id, + &recipient, + send_amt, + ); + submit_tx(&http_origin, &transfer_tx); + + let mut block_n_1 = None; + wait_for(30, || { + let chunks = test_observer::get_stackerdb_chunks(); + for chunk in chunks.into_iter().flat_map(|chunk| chunk.modified_slots) { + let Ok(message) = SignerMessage::consensus_deserialize(&mut chunk.data.as_slice()) + else { + continue; + }; + if let SignerMessage::BlockProposal(proposal) = message { + if proposal.block.header.signer_signature_hash() != block_n_signature_hash + && proposal + .block + .header + .recover_miner_pk() + .map(|pk| pk == mining_pk_1) + .unwrap() + && proposal.block.header.chain_length == block_n.stacks_height + 1 + { + block_n_1 = Some(proposal.block.clone()); + return Ok(true); + } + } + } + Ok(false) + }) + .expect("Timed out waiting for Miner 1 to propose N+1"); + let block_n_1 = block_n_1.expect("Failed to find N+1 proposal"); + let block_n_1_signature_hash = block_n_1.header.signer_signature_hash(); + + assert_eq!( + block_n_1.header.parent_block_id.to_string(), + block_n.block_id + ); + debug!("Miner 1 proposed block N+1: {block_n_1_signature_hash}"); + + info!("------------------------- Unpause Miner 2's Block Commits -------------------------"); + let rl2_commits_before = rl2_commits.load(Ordering::SeqCst); + rl2_skip_commit_op.set(false); + + wait_for(30, || { + Ok(rl2_commits.load(Ordering::SeqCst) > rl2_commits_before) + }) + .expect("Timed out waiting for Miner 2 to submit its block commit"); + let rl2_commits_before = rl2_commits.load(Ordering::SeqCst); + + info!("------------------------- Pause Block Validation Submission of N+1'-------------------------"); + TEST_STALL_BLOCK_VALIDATION_SUBMISSION.set(true); + + info!("------------------------- Start Miner 2's Tenure-------------------------"); + let burn_height_before = get_burn_height(); + next_block_and( + &mut signer_test.running_nodes.btc_regtest_controller, + 30, + || { + Ok(get_burn_height() > burn_height_before + && rl2_proposals.load(Ordering::SeqCst) > proposals_before_2 + && rl2_commits.load(Ordering::SeqCst) > rl2_commits_before) + }, + ) + .expect("Timed out waiting for burn block height to advance and Miner 2 to propose a block"); + + let mut block_n_1_prime = None; + wait_for(30, || { + let chunks = test_observer::get_stackerdb_chunks(); + for chunk in chunks.into_iter().flat_map(|chunk| chunk.modified_slots) { + let Ok(message) = SignerMessage::consensus_deserialize(&mut chunk.data.as_slice()) + else { + continue; + }; + if let SignerMessage::BlockProposal(proposal) = message { + if proposal + .block + .header + .recover_miner_pk() + .map(|pk| pk == mining_pk_2) + .unwrap() + { + block_n_1_prime = Some(proposal.block.clone()); + return Ok(true); + } + } + } + Ok(false) + }) + .expect("Timed out waiting for Miner 2 to propose N+1'"); + + let block_n_1_prime = block_n_1_prime.expect("Failed to find N+1' proposal"); + let block_n_1_prime_signature_hash = block_n_1_prime.header.signer_signature_hash(); + + debug!("Miner 2 proposed N+1': {block_n_1_prime_signature_hash}"); + + // assure we have a successful sortition that miner 2 won + let tip = SortitionDB::get_canonical_burn_chain_tip(sortdb.conn()).unwrap(); + assert!(tip.sortition); + assert_eq!(tip.miner_pk_hash.unwrap(), mining_pkh_2); + // Make sure that the tip is still at block N + assert_eq!(tip.canonical_stacks_tip_height, block_n.stacks_height); + assert_eq!( + tip.canonical_stacks_tip_hash.to_string(), + block_n.block_hash + ); + + // Just a precaution to make sure no stacks blocks has been processed between now and our original pause + assert_eq!(rejections_before_2, rl2_rejections.load(Ordering::SeqCst)); + assert_eq!( + blocks_processed_before_1, + blocks_mined1.load(Ordering::SeqCst) + ); + assert_eq!( + blocks_processed_before_2, + blocks_mined2.load(Ordering::SeqCst) + ); + assert_eq!(blocks_before, test_observer::get_blocks().len()); + + info!("------------------------- Unpause Block Validation Response of N+1 -------------------------"); + + TEST_VALIDATE_STALL.lock().unwrap().replace(false); + + // Verify that the node accepted the proposed N+1, sending back a validate ok response + wait_for(30, || { + for proposal in test_observer::get_proposal_responses() { + if let BlockValidateResponse::Ok(response) = proposal { + if response.signer_signature_hash == block_n_1_signature_hash { + return Ok(true); + } + } + } + Ok(false) + }) + .expect("Timed out waiting for validation response for N+1"); + + debug!( + "Node finished processing proposal validation request for N+1: {block_n_1_signature_hash}" + ); + + // This is awful but I can't gurantee signers have reached the submission stall and we need to ensure the event order is as expected. + sleep_ms(5_000); + + info!("------------------------- Unpause Block Validation Submission and Response for N+1' -------------------------"); + TEST_STALL_BLOCK_VALIDATION_SUBMISSION.set(false); + + info!("------------------------- Confirm N+1 is Accepted ------------------------"); + wait_for(30, || { + let chunks = test_observer::get_stackerdb_chunks(); + for chunk in chunks.into_iter().flat_map(|chunk| chunk.modified_slots) { + let Ok(message) = SignerMessage::consensus_deserialize(&mut chunk.data.as_slice()) + else { + continue; + }; + if let SignerMessage::BlockResponse(BlockResponse::Accepted(BlockAccepted { + signer_signature_hash, + .. + })) = message + { + if signer_signature_hash == block_n_1_signature_hash { + return Ok(true); + } + } + } + Ok(false) + }) + .expect("Timed out waiting for N+1 acceptance."); + + debug!("Miner 1 mined block N+1: {block_n_1_signature_hash}"); + + info!("------------------------- Confirm N+1' is Rejected ------------------------"); + + wait_for(30, || { + let chunks = test_observer::get_stackerdb_chunks(); + for chunk in chunks.into_iter().flat_map(|chunk| chunk.modified_slots) { + let Ok(message) = SignerMessage::consensus_deserialize(&mut chunk.data.as_slice()) + else { + continue; + }; + if let SignerMessage::BlockResponse(BlockResponse::Rejected(BlockRejection { + signer_signature_hash, + .. + })) = message + { + if signer_signature_hash == block_n_1_prime_signature_hash { + return Ok(true); + } + } else if let SignerMessage::BlockResponse(BlockResponse::Accepted(BlockAccepted { + signer_signature_hash, + .. + })) = message + { + assert!( + signer_signature_hash != block_n_1_prime_signature_hash, + "N+1' was accepted after N+1 was accepted. This should not be possible." + ); + } + } + Ok(false) + }) + .expect("Timed out waiting for N+1' rejection."); + + info!("------------------------- Confirm N+2 Accepted ------------------------"); + + let mut block_n_2 = None; + wait_for(30, || { + let chunks = test_observer::get_stackerdb_chunks(); + for chunk in chunks.into_iter().flat_map(|chunk| chunk.modified_slots) { + let Ok(message) = SignerMessage::consensus_deserialize(&mut chunk.data.as_slice()) + else { + continue; + }; + if let SignerMessage::BlockProposal(proposal) = message { + if proposal.block.header.chain_length == block_n_1.header.chain_length + 1 + && proposal + .block + .header + .recover_miner_pk() + .map(|pk| pk == mining_pk_2) + .unwrap() + { + block_n_2 = Some(proposal.block.clone()); + return Ok(true); + } + } + } + Ok(false) + }) + .expect("Timed out waiting for Miner 1 to propose N+2"); + let block_n_2 = block_n_2.expect("Failed to find N+2 proposal"); + + wait_for(30, || { + Ok(get_chain_info(&conf).stacks_tip_height >= block_n_2.header.chain_length) + }) + .expect("Timed out waiting for the stacks tip height to advance"); + + info!("------------------------- Confirm Stacks Chain is As Expected ------------------------"); + let info_after = get_chain_info(&conf); + assert_eq!(info_after.stacks_tip_height, block_n_2.header.chain_length); + assert_eq!(info_after.stacks_tip_height, starting_peer_height + 3); + assert_eq!( + info_after.stacks_tip.to_string(), + block_n_2.header.block_hash().to_string() + ); + assert_ne!( + info_after.stacks_tip_consensus_hash, + block_n_1.header.consensus_hash + ); + assert_eq!( + info_after.stacks_tip_consensus_hash, + block_n_2.header.consensus_hash + ); + assert_eq!( + block_n_2.header.parent_block_id, + block_n_1.header.block_id() + ); + assert_eq!( + block_n_1.header.parent_block_id.to_string(), + block_n.block_id + ); + + info!("------------------------- Shutdown -------------------------"); + rl2_coord_channels + .lock() + .expect("Mutex poisoned") + .stop_chains_coordinator(); + run_loop_stopper_2.store(false, Ordering::SeqCst); + run_loop_2_thread.join().unwrap(); + signer_test.shutdown(); +}