From d8964df52ed36ed0cfbf9c82e615cbe01908c945 Mon Sep 17 00:00:00 2001
From: Adrian Catangiu <adrian@parity.io>
Date: Wed, 5 Jul 2023 12:02:07 +0300
Subject: [PATCH 001/188] sc-consensus-beefy: add BEEFY fisherman to gossip
 network

Signed-off-by: Adrian Catangiu <adrian@parity.io>
---
 .../beefy/src/communication/fisherman.rs      | 113 ++++++++++++++++++
 .../beefy/src/communication/gossip.rs         |  46 +++++--
 .../consensus/beefy/src/communication/mod.rs  |   1 +
 substrate/client/consensus/beefy/src/lib.rs   |  13 +-
 substrate/client/consensus/beefy/src/tests.rs |  27 ++++-
 .../client/consensus/beefy/src/worker.rs      |  26 +++-
 .../consensus/beefy/src/commitment.rs         |  15 +++
 .../primitives/consensus/beefy/src/mmr.rs     |   2 +-
 .../primitives/consensus/beefy/src/payload.rs |   2 +-
 9 files changed, 217 insertions(+), 28 deletions(-)
 create mode 100644 substrate/client/consensus/beefy/src/communication/fisherman.rs

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
new file mode 100644
index 000000000000..46e2458cd92c
--- /dev/null
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -0,0 +1,113 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::justification::BeefyVersionedFinalityProof;
+use sc_client_api::Backend;
+use sp_api::ProvideRuntimeApi;
+use sp_blockchain::HeaderBackend;
+use sp_consensus_beefy::{
+	crypto::{AuthorityId, Signature},
+	BeefyApi, Payload, PayloadProvider, VoteMessage,
+};
+use sp_runtime::{
+	generic::BlockId,
+	traits::{Block, NumberFor},
+};
+use std::{marker::PhantomData, sync::Arc};
+
+pub(crate) trait BeefyFisherman<B: Block>: Send + Sync {
+	/// Check `vote` for contained finalized block against expected payload.
+	///
+	/// Note: this fn expects `vote.commitment.block_number` to be finalized.
+	fn check_vote(
+		&self,
+		vote: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
+	) -> Result<(), sp_blockchain::Error>;
+
+	/// Check `proof` for contained finalized block against expected payload.
+	///
+	/// Note: this fn expects block referenced in `proof` to be finalized.
+	fn check_proof(
+		&self,
+		proof: BeefyVersionedFinalityProof<B>,
+	) -> Result<(), sp_blockchain::Error>;
+}
+
+/// Helper wrapper used to check gossiped votes for (historical) equivocations,
+/// and report any such protocol infringements.
+pub(crate) struct Fisherman<B: Block, BE, R, P> {
+	pub backend: Arc<BE>,
+	pub runtime: Arc<R>,
+	pub payload_provider: P,
+	pub _phantom: PhantomData<B>,
+}
+
+impl<B, BE, R, P> Fisherman<B, BE, R, P>
+where
+	B: Block,
+	BE: Backend<B>,
+	P: PayloadProvider<B>,
+{
+	fn expected_payload(&self, number: NumberFor<B>) -> Result<Payload, sp_blockchain::Error> {
+		// This should be un-ambiguous since `number` is finalized.
+		let hash = self.backend.blockchain().expect_block_hash_from_id(&BlockId::Number(number))?;
+		let header = self.backend.blockchain().expect_header(hash)?;
+		self.payload_provider
+			.payload(&header)
+			.ok_or_else(|| sp_blockchain::Error::Backend("BEEFY Payload not found".into()))
+	}
+}
+
+impl<B, BE, R, P> BeefyFisherman<B> for Fisherman<B, BE, R, P>
+where
+	B: Block,
+	BE: Backend<B>,
+	P: PayloadProvider<B>,
+	R: ProvideRuntimeApi<B> + Send + Sync,
+	R::Api: BeefyApi<B>,
+{
+	/// Check `vote` for contained finalized block against expected payload.
+	///
+	/// Note: this fn expects `vote.commitment.block_number` to be finalized.
+	fn check_vote(
+		&self,
+		vote: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
+	) -> Result<(), sp_blockchain::Error> {
+		let number = vote.commitment.block_number;
+		let expected = self.expected_payload(number)?;
+		if vote.commitment.payload != expected {
+			// TODO: report equivocation
+		}
+		Ok(())
+	}
+
+	/// Check `proof` for contained finalized block against expected payload.
+	///
+	/// Note: this fn expects block referenced in `proof` to be finalized.
+	fn check_proof(
+		&self,
+		proof: BeefyVersionedFinalityProof<B>,
+	) -> Result<(), sp_blockchain::Error> {
+		let payload = proof.payload();
+		let expected = self.expected_payload(*proof.number())?;
+		if payload != &expected {
+			// TODO: report equivocation
+		}
+		Ok(())
+	}
+}
diff --git a/substrate/client/consensus/beefy/src/communication/gossip.rs b/substrate/client/consensus/beefy/src/communication/gossip.rs
index 8c025ca06761..5b11b5884999 100644
--- a/substrate/client/consensus/beefy/src/communication/gossip.rs
+++ b/substrate/client/consensus/beefy/src/communication/gossip.rs
@@ -32,6 +32,7 @@ use wasm_timer::Instant;
 use crate::{
 	communication::{
 		benefit, cost,
+		fisherman::BeefyFisherman,
 		peers::{KnownPeers, PeerReport},
 	},
 	justification::{
@@ -225,26 +226,29 @@ impl<B: Block> Filter<B> {
 /// Allows messages for 'rounds >= last concluded' to flow, everything else gets
 /// rejected/expired.
 ///
+/// Messages for active and expired rounds are validated for expected payloads and attempts
+/// to create forks before head of GRANDPA are reported.
+///
 ///All messaging is handled in a single BEEFY global topic.
-pub(crate) struct GossipValidator<B>
-where
-	B: Block,
-{
+pub(crate) struct GossipValidator<B: Block, F> {
 	votes_topic: B::Hash,
 	justifs_topic: B::Hash,
 	gossip_filter: RwLock<Filter<B>>,
 	next_rebroadcast: Mutex<Instant>,
 	known_peers: Arc<Mutex<KnownPeers<B>>>,
 	report_sender: TracingUnboundedSender<PeerReport>,
+	fisherman: F,
 }
 
-impl<B> GossipValidator<B>
+impl<B, F> GossipValidator<B, F>
 where
 	B: Block,
+	F: BeefyFisherman<B>,
 {
 	pub(crate) fn new(
 		known_peers: Arc<Mutex<KnownPeers<B>>>,
-	) -> (GossipValidator<B>, TracingUnboundedReceiver<PeerReport>) {
+		fisherman: F,
+	) -> (GossipValidator<B, F>, TracingUnboundedReceiver<PeerReport>) {
 		let (tx, rx) = tracing_unbounded("mpsc_beefy_gossip_validator", 10_000);
 		let val = GossipValidator {
 			votes_topic: votes_topic::<B>(),
@@ -253,6 +257,7 @@ where
 			next_rebroadcast: Mutex::new(Instant::now() + REBROADCAST_AFTER),
 			known_peers,
 			report_sender: tx,
+			fisherman,
 		};
 		(val, rx)
 	}
@@ -287,9 +292,14 @@ where
 			let filter = self.gossip_filter.read();
 
 			match filter.consider_vote(round, set_id) {
-				Consider::RejectPast => return Action::Discard(cost::OUTDATED_MESSAGE),
 				Consider::RejectFuture => return Action::Discard(cost::FUTURE_MESSAGE),
 				Consider::RejectOutOfScope => return Action::Discard(cost::OUT_OF_SCOPE_MESSAGE),
+				Consider::RejectPast => {
+					// We know `vote` is for some past (finalized) block. Have fisherman check
+					// for equivocations. Best-effort, ignore errors such as state pruned.
+					let _ = self.fisherman.check_vote(vote);
+					return Action::Discard(cost::OUTDATED_MESSAGE)
+				},
 				Consider::Accept => {},
 			}
 
@@ -331,9 +341,14 @@ where
 		let guard = self.gossip_filter.read();
 		// Verify general usefulness of the justification.
 		match guard.consider_finality_proof(round, set_id) {
-			Consider::RejectPast => return Action::Discard(cost::OUTDATED_MESSAGE),
 			Consider::RejectFuture => return Action::Discard(cost::FUTURE_MESSAGE),
 			Consider::RejectOutOfScope => return Action::Discard(cost::OUT_OF_SCOPE_MESSAGE),
+			Consider::RejectPast => {
+				// We know `proof` is for some past (finalized) block. Have fisherman check
+				// for equivocations. Best-effort, ignore errors such as state pruned.
+				let _ = self.fisherman.check_proof(proof);
+				return Action::Discard(cost::OUTDATED_MESSAGE)
+			},
 			Consider::Accept => {},
 		}
 		// Verify justification signatures.
@@ -359,9 +374,10 @@ where
 	}
 }
 
-impl<B> Validator<B> for GossipValidator<B>
+impl<B, F> Validator<B> for GossipValidator<B, F>
 where
 	B: Block,
+	F: BeefyFisherman<B>,
 {
 	fn peer_disconnected(&self, _context: &mut dyn ValidatorContext<B>, who: &PeerId) {
 		self.known_peers.lock().remove(who);
@@ -474,7 +490,7 @@ where
 #[cfg(test)]
 pub(crate) mod tests {
 	use super::*;
-	use crate::keystore::BeefyKeystore;
+	use crate::{keystore::BeefyKeystore, tests::DummyFisherman};
 	use sc_network_test::Block;
 	use sp_application_crypto::key_types::BEEFY as BEEFY_KEY_TYPE;
 	use sp_consensus_beefy::{
@@ -482,6 +498,7 @@ pub(crate) mod tests {
 		SignedCommitment, VoteMessage,
 	};
 	use sp_keystore::{testing::MemoryKeystore, Keystore};
+	use std::marker::PhantomData;
 
 	#[test]
 	fn known_votes_insert_remove() {
@@ -577,8 +594,9 @@ pub(crate) mod tests {
 	fn should_validate_messages() {
 		let keys = vec![Keyring::Alice.public()];
 		let validator_set = ValidatorSet::<AuthorityId>::new(keys.clone(), 0).unwrap();
+		let fisherman = DummyFisherman { _phantom: PhantomData::<Block> };
 		let (gv, mut report_stream) =
-			GossipValidator::<Block>::new(Arc::new(Mutex::new(KnownPeers::new())));
+			GossipValidator::new(Arc::new(Mutex::new(KnownPeers::new())), fisherman);
 		let sender = PeerId::random();
 		let mut context = TestContext;
 
@@ -705,7 +723,8 @@ pub(crate) mod tests {
 	fn messages_allowed_and_expired() {
 		let keys = vec![Keyring::Alice.public()];
 		let validator_set = ValidatorSet::<AuthorityId>::new(keys.clone(), 0).unwrap();
-		let (gv, _) = GossipValidator::<Block>::new(Arc::new(Mutex::new(KnownPeers::new())));
+		let fisherman = DummyFisherman { _phantom: PhantomData::<Block> };
+		let (gv, _) = GossipValidator::new(Arc::new(Mutex::new(KnownPeers::new())), fisherman);
 		gv.update_filter(GossipFilterCfg { start: 0, end: 10, validator_set: &validator_set });
 		let sender = sc_network::PeerId::random();
 		let topic = Default::default();
@@ -782,7 +801,8 @@ pub(crate) mod tests {
 	fn messages_rebroadcast() {
 		let keys = vec![Keyring::Alice.public()];
 		let validator_set = ValidatorSet::<AuthorityId>::new(keys.clone(), 0).unwrap();
-		let (gv, _) = GossipValidator::<Block>::new(Arc::new(Mutex::new(KnownPeers::new())));
+		let fisherman = DummyFisherman { _phantom: PhantomData::<Block> };
+		let (gv, _) = GossipValidator::new(Arc::new(Mutex::new(KnownPeers::new())), fisherman);
 		gv.update_filter(GossipFilterCfg { start: 0, end: 10, validator_set: &validator_set });
 		let sender = sc_network::PeerId::random();
 		let topic = Default::default();
diff --git a/substrate/client/consensus/beefy/src/communication/mod.rs b/substrate/client/consensus/beefy/src/communication/mod.rs
index 7f9535bfc23f..c31447d08e56 100644
--- a/substrate/client/consensus/beefy/src/communication/mod.rs
+++ b/substrate/client/consensus/beefy/src/communication/mod.rs
@@ -21,6 +21,7 @@
 pub mod notification;
 pub mod request_response;
 
+pub(crate) mod fisherman;
 pub(crate) mod gossip;
 pub(crate) mod peers;
 
diff --git a/substrate/client/consensus/beefy/src/lib.rs b/substrate/client/consensus/beefy/src/lib.rs
index 0b3baa007c1c..a28cb4ea973f 100644
--- a/substrate/client/consensus/beefy/src/lib.rs
+++ b/substrate/client/consensus/beefy/src/lib.rs
@@ -18,6 +18,7 @@
 
 use crate::{
 	communication::{
+		fisherman::Fisherman,
 		notification::{
 			BeefyBestBlockSender, BeefyBestBlockStream, BeefyVersionedFinalityProofSender,
 			BeefyVersionedFinalityProofStream,
@@ -219,10 +220,10 @@ pub async fn start_beefy_gadget<B, BE, C, N, P, R, S>(
 	beefy_params: BeefyParams<B, BE, C, N, P, R, S>,
 ) where
 	B: Block,
-	BE: Backend<B>,
+	BE: Backend<B> + 'static,
 	C: Client<B, BE> + BlockBackend<B>,
 	P: PayloadProvider<B> + Clone,
-	R: ProvideRuntimeApi<B>,
+	R: ProvideRuntimeApi<B> + Send + Sync + 'static,
 	R::Api: BeefyApi<B, AuthorityId> + MmrApi<B, MmrRootHash, NumberFor<B>>,
 	N: GossipNetwork<B> + NetworkRequest + Send + Sync + 'static,
 	S: GossipSyncing<B> + SyncOracle + 'static,
@@ -259,10 +260,16 @@ pub async fn start_beefy_gadget<B, BE, C, N, P, R, S>(
 	// select recoverable errors.
 	loop {
 		let known_peers = Arc::new(Mutex::new(KnownPeers::new()));
+        let fisherman = Fisherman {
+            backend: backend.clone(),
+            runtime: runtime.clone(),
+            payload_provider: payload_provider.clone(),
+            _phantom: PhantomData,
+        };
 		// Default votes filter is to discard everything.
 		// Validator is updated later with correct starting round and set id.
 		let (gossip_validator, gossip_report_stream) =
-			communication::gossip::GossipValidator::new(known_peers.clone());
+			communication::gossip::GossipValidator::new(known_peers.clone(), fisherman);
 		let gossip_validator = Arc::new(gossip_validator);
 		let mut gossip_engine = GossipEngine::new(
 			network.clone(),
diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index 3bb65e9d57f4..80ac9224d1d4 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -22,6 +22,7 @@ use crate::{
 	aux_schema::{load_persistent, tests::verify_persisted_version},
 	beefy_block_import_and_links,
 	communication::{
+		fisherman::BeefyFisherman,
 		gossip::{
 			proofs_topic, tests::sign_commitment, votes_topic, GossipFilterCfg, GossipMessage,
 			GossipValidator,
@@ -47,7 +48,7 @@ use sc_network_test::{
 };
 use sc_utils::notification::NotificationReceiver;
 use serde::{Deserialize, Serialize};
-use sp_api::{ApiRef, ProvideRuntimeApi};
+use sp_api::{ApiRef, BlockT, ProvideRuntimeApi};
 use sp_application_crypto::key_types::BEEFY as BEEFY_KEY_TYPE;
 use sp_consensus::BlockOrigin;
 use sp_consensus_beefy::{
@@ -244,6 +245,22 @@ impl TestNetFactory for BeefyTestNet {
 	}
 }
 
+pub(crate) struct DummyFisherman<B> {
+	pub _phantom: PhantomData<B>,
+}
+
+impl<B: BlockT> BeefyFisherman<B> for DummyFisherman<B> {
+	fn check_proof(&self, _: BeefyVersionedFinalityProof<B>) -> Result<(), sp_blockchain::Error> {
+		Ok(())
+	}
+	fn check_vote(
+		&self,
+		_: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
+	) -> Result<(), sp_blockchain::Error> {
+		Ok(())
+	}
+}
+
 #[derive(Clone)]
 pub(crate) struct TestApi {
 	pub beefy_genesis: u64,
@@ -364,8 +381,9 @@ async fn voter_init_setup(
 	api: &TestApi,
 ) -> sp_blockchain::Result<PersistedState<Block>> {
 	let backend = net.peer(0).client().as_backend();
+	let fisherman = DummyFisherman { _phantom: PhantomData };
 	let known_peers = Arc::new(Mutex::new(KnownPeers::new()));
-	let (gossip_validator, _) = GossipValidator::new(known_peers);
+	let (gossip_validator, _) = GossipValidator::new(known_peers, fisherman);
 	let gossip_validator = Arc::new(gossip_validator);
 	let mut gossip_engine = sc_network_gossip::GossipEngine::new(
 		net.peer(0).network_service().clone(),
@@ -386,7 +404,7 @@ fn initialize_beefy<API>(
 	min_block_delta: u32,
 ) -> impl Future<Output = ()>
 where
-	API: ProvideRuntimeApi<Block> + Sync + Send,
+	API: ProvideRuntimeApi<Block> + Sync + Send +'static,
 	API::Api: BeefyApi<Block, AuthorityId> + MmrApi<Block, MmrRootHash, NumberFor<Block>>,
 {
 	let tasks = FuturesUnordered::new();
@@ -1310,9 +1328,10 @@ async fn gossipped_finality_proofs() {
 	let beefy_peers = peers.iter().enumerate().map(|(id, key)| (id, key, api.clone())).collect();
 
 	let charlie = &net.peers[2];
+	let fisherman = DummyFisherman { _phantom: PhantomData::<Block> };
 	let known_peers = Arc::new(Mutex::new(KnownPeers::<Block>::new()));
 	// Charlie will run just the gossip engine and not the full voter.
-	let (gossip_validator, _) = GossipValidator::new(known_peers);
+	let (gossip_validator, _) = GossipValidator::new(known_peers, fisherman);
 	let charlie_gossip_validator = Arc::new(gossip_validator);
 	charlie_gossip_validator.update_filter(GossipFilterCfg::<Block> {
 		start: 1,
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index 0d3845a27036..f0a486df901a 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -18,6 +18,7 @@
 
 use crate::{
 	communication::{
+		fisherman::BeefyFisherman,
 		gossip::{proofs_topic, votes_topic, GossipFilterCfg, GossipMessage, GossipValidator},
 		peers::PeerReport,
 		request_response::outgoing_requests_engine::{OnDemandJustificationsEngine, ResponseInfo},
@@ -314,7 +315,7 @@ impl<B: Block> PersistedState<B> {
 }
 
 /// A BEEFY worker plays the BEEFY protocol
-pub(crate) struct BeefyWorker<B: Block, BE, P, RuntimeApi, S> {
+pub(crate) struct BeefyWorker<B: Block, BE, P, RuntimeApi, S, F> {
 	// utilities
 	pub backend: Arc<BE>,
 	pub payload_provider: P,
@@ -324,7 +325,7 @@ pub(crate) struct BeefyWorker<B: Block, BE, P, RuntimeApi, S> {
 
 	// communication
 	pub gossip_engine: GossipEngine<B>,
-	pub gossip_validator: Arc<GossipValidator<B>>,
+	pub gossip_validator: Arc<GossipValidator<B, F>>,
 	pub gossip_report_stream: TracingUnboundedReceiver<PeerReport>,
 	pub on_demand_justifications: OnDemandJustificationsEngine<B>,
 
@@ -341,7 +342,7 @@ pub(crate) struct BeefyWorker<B: Block, BE, P, RuntimeApi, S> {
 	pub persisted_state: PersistedState<B>,
 }
 
-impl<B, BE, P, R, S> BeefyWorker<B, BE, P, R, S>
+impl<B, BE, P, R, S, F> BeefyWorker<B, BE, P, R, S, F>
 where
 	B: Block + Codec,
 	BE: Backend<B>,
@@ -349,6 +350,7 @@ where
 	S: SyncOracle,
 	R: ProvideRuntimeApi<B>,
 	R::Api: BeefyApi<B, AuthorityId>,
+	F: BeefyFisherman<B>,
 {
 	fn best_grandpa_block(&self) -> NumberFor<B> {
 		*self.persisted_state.voting_oracle.best_grandpa_block_header.number()
@@ -1029,7 +1031,10 @@ where
 pub(crate) mod tests {
 	use super::*;
 	use crate::{
-		communication::notification::{BeefyBestBlockStream, BeefyVersionedFinalityProofStream},
+		communication::{
+			fisherman::Fisherman,
+			notification::{BeefyBestBlockStream, BeefyVersionedFinalityProofStream},
+		},
 		tests::{
 			create_beefy_keystore, get_beefy_streams, make_beefy_ids, BeefyPeer, BeefyTestNet,
 			TestApi,
@@ -1048,6 +1053,7 @@ pub(crate) mod tests {
 		mmr::MmrRootProvider, Keyring, Payload, SignedCommitment,
 	};
 	use sp_runtime::traits::One;
+	use std::marker::PhantomData;
 	use substrate_test_runtime_client::{
 		runtime::{Block, Digest, DigestItem, Header},
 		Backend,
@@ -1088,6 +1094,7 @@ pub(crate) mod tests {
 		MmrRootProvider<Block, TestApi>,
 		TestApi,
 		Arc<SyncingService<Block>>,
+		Fisherman<Block, Backend, TestApi, MmrRootProvider<Block, TestApi>>,
 	> {
 		let keystore = create_beefy_keystore(*key);
 
@@ -1113,8 +1120,16 @@ pub(crate) mod tests {
 		let api = Arc::new(TestApi::with_validator_set(&genesis_validator_set));
 		let network = peer.network_service().clone();
 		let sync = peer.sync_service().clone();
+		let payload_provider = MmrRootProvider::new(api.clone());
+		let fisherman = Fisherman {
+			backend: backend.clone(),
+			runtime: api.clone(),
+			payload_provider: payload_provider.clone(),
+			_phantom: PhantomData,
+		};
 		let known_peers = Arc::new(Mutex::new(KnownPeers::new()));
-		let (gossip_validator, gossip_report_stream) = GossipValidator::new(known_peers.clone());
+		let (gossip_validator, gossip_report_stream) =
+			GossipValidator::new(known_peers.clone(), fisherman);
 		let gossip_validator = Arc::new(gossip_validator);
 		let gossip_engine = GossipEngine::new(
 			network.clone(),
@@ -1145,7 +1160,6 @@ pub(crate) mod tests {
 			beefy_genesis,
 		)
 		.unwrap();
-		let payload_provider = MmrRootProvider::new(api.clone());
 		BeefyWorker {
 			backend,
 			payload_provider,
diff --git a/substrate/primitives/consensus/beefy/src/commitment.rs b/substrate/primitives/consensus/beefy/src/commitment.rs
index 5b6ef9ae5ab3..aa2efdb10ca2 100644
--- a/substrate/primitives/consensus/beefy/src/commitment.rs
+++ b/substrate/primitives/consensus/beefy/src/commitment.rs
@@ -247,6 +247,21 @@ impl<N, S> From<SignedCommitment<N, S>> for VersionedFinalityProof<N, S> {
 	}
 }
 
+impl<N, S> VersionedFinalityProof<N, S> {
+	/// Provide reference to inner `Payload`.
+	pub fn payload(&self) -> &Payload {
+		match self {
+			VersionedFinalityProof::V1(inner) => &inner.commitment.payload,
+		}
+	}
+	/// Block number this proof is for.
+	pub fn number(&self) -> &N {
+		match self {
+			VersionedFinalityProof::V1(inner) => &inner.commitment.block_number,
+		}
+	}
+}
+
 #[cfg(test)]
 mod tests {
 
diff --git a/substrate/primitives/consensus/beefy/src/mmr.rs b/substrate/primitives/consensus/beefy/src/mmr.rs
index 660506b8763f..b318d84475f7 100644
--- a/substrate/primitives/consensus/beefy/src/mmr.rs
+++ b/substrate/primitives/consensus/beefy/src/mmr.rs
@@ -190,7 +190,7 @@ mod mmr_root_provider {
 	impl<B: Block, R> PayloadProvider<B> for MmrRootProvider<B, R>
 	where
 		B: Block,
-		R: ProvideRuntimeApi<B>,
+		R: ProvideRuntimeApi<B> + Send + Sync + 'static,
 		R::Api: MmrApi<B, MmrRootHash, NumberFor<B>>,
 	{
 		fn payload(&self, header: &B::Header) -> Option<Payload> {
diff --git a/substrate/primitives/consensus/beefy/src/payload.rs b/substrate/primitives/consensus/beefy/src/payload.rs
index d520de445c95..c80f00255281 100644
--- a/substrate/primitives/consensus/beefy/src/payload.rs
+++ b/substrate/primitives/consensus/beefy/src/payload.rs
@@ -75,7 +75,7 @@ impl Payload {
 }
 
 /// Trait for custom BEEFY payload providers.
-pub trait PayloadProvider<B: Block> {
+pub trait PayloadProvider<B: Block>: Clone + Send + Sync + 'static {
 	/// Provide BEEFY payload if available for `header`.
 	fn payload(&self, header: &B::Header) -> Option<Payload>;
 }

From a2678a00241f050f1292845ac8503b2acbf31c68 Mon Sep 17 00:00:00 2001
From: Adrian Catangiu <adrian@parity.io>
Date: Wed, 5 Jul 2023 14:10:40 +0300
Subject: [PATCH 002/188] sp-consensus-beefy: add invalid fork vote proof and
 equivalent BeefyApi

Signed-off-by: Adrian Catangiu <adrian@parity.io>
---
 .../primitives/consensus/beefy/src/lib.rs     | 70 +++++++++++++++++++
 1 file changed, 70 insertions(+)

diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index c69e26bf574d..fdfacae037d0 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -251,6 +251,31 @@ impl<Number, Id, Signature> EquivocationProof<Number, Id, Signature> {
 	}
 }
 
+/// Proof of voter misbehavior on a given set id.
+/// This proof shows voter voted on a different fork than finalized by GRANDPA.
+#[derive(Clone, Debug, Decode, Encode, PartialEq, TypeInfo)]
+pub struct InvalidForkVoteProof<Number, Id, Signature> {
+	/// Voting for a block on different fork than one finalized by GRANDPA.
+	pub vote: VoteMessage<Number, Id, Signature>,
+	/// TODO: GRANDPA proof that this block is final
+	pub grandpa_proof: (),
+}
+
+impl<Number, Id, Signature> InvalidForkVoteProof<Number, Id, Signature> {
+	/// Returns the authority id of the misbehaving voter.
+	pub fn offender_id(&self) -> &Id {
+		&self.vote.id
+	}
+	/// Returns the round number at which the infringement occurred.
+	pub fn round_number(&self) -> &Number {
+		&self.vote.commitment.block_number
+	}
+	/// Returns the set id at which the infringement occurred.
+	pub fn set_id(&self) -> ValidatorSetId {
+		self.vote.commitment.validator_set_id
+	}
+}
+
 /// Check a commitment signature by encoding the commitment and
 /// verifying the provided signature using the expected authority id.
 pub fn check_commitment_signature<Number, Id, MsgHash>(
@@ -302,6 +327,37 @@ where
 	return valid_first && valid_second
 }
 
+/// Validates [InvalidForkVoteProof] by checking:
+/// 1. `vote` is signed,
+/// 2. GRANDPA proof is provided for block number >= vote.commitment.block_number.
+/// 3. `vote.commitment.payload` != `expected_payload`.
+pub fn check_invalid_fork_proof<Number, Id, MsgHash>(
+	proof: &InvalidForkVoteProof<Number, Id, <Id as RuntimeAppPublic>::Signature>,
+	expected_payload: &Payload,
+) -> bool
+where
+	Id: BeefyAuthorityId<MsgHash> + PartialEq,
+	Number: Clone + Encode + PartialEq,
+	MsgHash: Hash,
+{
+	let InvalidForkVoteProof { vote, grandpa_proof: _ } = proof;
+
+	// check signature `vote`, if invalid, equivocation report is invalid
+	if !check_commitment_signature(&vote.commitment, &vote.id, &vote.signature) {
+		return false
+	}
+
+	// TODO: add GRANDPA proof
+	// if GRANDPA proof doesn't show `vote.commitment.block_number` has been finalized,
+	// we cannot safely prove that `vote` payload is invalid.
+	// if grandpa_proof.block_number < vote.commitment.block_number {
+	// 	return false
+	// }
+
+	// check that `payload` on the `vote` is different that the `expected_payload`.
+	&vote.commitment.payload != expected_payload
+}
+
 /// New BEEFY validator set notification hook.
 pub trait OnNewValidatorSet<AuthorityId> {
 	/// Function called by the pallet when BEEFY validator set changes.
@@ -364,6 +420,20 @@ sp_api::decl_runtime_apis! {
 			key_owner_proof: OpaqueKeyOwnershipProof,
 		) -> Option<()>;
 
+		/// Submits an unsigned extrinsic to report an vote for an invalid fork.
+		/// The caller must provide the invalid vote proof and a key ownership proof
+		/// (should be obtained using `generate_key_ownership_proof`). The
+		/// extrinsic will be unsigned and should only be accepted for local
+		/// authorship (not to be broadcast to the network). This method returns
+		/// `None` when creation of the extrinsic fails, e.g. if equivocation
+		/// reporting is disabled for the given runtime (i.e. this method is
+		/// hardcoded to return `None`). Only useful in an offchain context.
+		fn submit_report_invalid_fork_unsigned_extrinsic(
+			invalid_fork_proof:
+				InvalidForkVoteProof<NumberFor<Block>, crypto::AuthorityId, crypto::Signature>,
+			key_owner_proof: OpaqueKeyOwnershipProof,
+		) -> Option<()>;
+
 		/// Generates a proof of key ownership for the given authority in the
 		/// given set. An example usage of this module is coupled with the
 		/// session historical module to prove that a given authority key is

From 17398d856c2ae1199b83fb3da3bbc53816758988 Mon Sep 17 00:00:00 2001
From: Adrian Catangiu <adrian@parity.io>
Date: Wed, 5 Jul 2023 14:22:00 +0300
Subject: [PATCH 003/188] pallet-beefy: add stubs for reporting invalid fork
 votes

Signed-off-by: Adrian Catangiu <adrian@parity.io>
---
 substrate/frame/beefy/src/equivocation.rs |  1 +
 substrate/frame/beefy/src/lib.rs          | 63 +++++++++++++++++++++++
 2 files changed, 64 insertions(+)

diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 0a7ede327c9e..884802194bc4 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -122,6 +122,7 @@ where
 pub struct EquivocationReportSystem<T, R, P, L>(sp_std::marker::PhantomData<(T, R, P, L)>);
 
 /// Equivocation evidence convenience alias.
+// TODO: use an enum that takes either `EquivocationProof` or `InvalidForkVoteProof`
 pub type EquivocationEvidenceFor<T> = (
 	EquivocationProof<
 		BlockNumberFor<T>,
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 77e74436dd67..30641ea1ee6b 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -63,6 +63,7 @@ const LOG_TARGET: &str = "runtime::beefy";
 pub mod pallet {
 	use super::*;
 	use frame_system::pallet_prelude::BlockNumberFor;
+	use sp_consensus_beefy::InvalidForkVoteProof;
 
 	#[pallet::config]
 	pub trait Config: frame_system::Config {
@@ -111,6 +112,8 @@ pub mod pallet {
 		/// Defines methods to publish, check and process an equivocation offence.
 		type EquivocationReportSystem: OffenceReportSystem<
 			Option<Self::AccountId>,
+			// TODO: make below an enum that takes either `EquivocationProof` or
+			// `InvalidForkVoteProof`
 			EquivocationEvidenceFor<Self>,
 		>;
 	}
@@ -265,6 +268,66 @@ pub mod pallet {
 			)?;
 			Ok(Pays::No.into())
 		}
+
+		/// Report voter voting on invalid fork. This method will verify the
+		/// invalid fork proof and validate the given key ownership proof
+		/// against the extracted offender. If both are valid, the offence
+		/// will be reported.
+		#[pallet::call_index(2)]
+		#[pallet::weight(T::WeightInfo::report_equivocation(key_owner_proof.validator_count()))]
+		pub fn report_invalid_fork_vote(
+			origin: OriginFor<T>,
+			invalid_fork_proof: Box<
+				InvalidForkVoteProof<
+					BlockNumberFor<T>,
+					T::BeefyId,
+					<T::BeefyId as RuntimeAppPublic>::Signature,
+				>,
+			>,
+			key_owner_proof: T::KeyOwnerProof,
+		) -> DispatchResultWithPostInfo {
+			let reporter = ensure_signed(origin)?;
+
+			// TODO:
+			// T::EquivocationReportSystem::process_evidence(
+			// 	Some(reporter),
+			// 	(*invalid_fork_proof, key_owner_proof),
+			// )?;
+			// Waive the fee since the report is valid and beneficial
+			Ok(Pays::No.into())
+		}
+
+		/// Report voter voting on invalid fork. This method will verify the
+		/// invalid fork proof and validate the given key ownership proof
+		/// against the extracted offender. If both are valid, the offence
+		/// will be reported.
+		///
+		/// This extrinsic must be called unsigned and it is expected that only
+		/// block authors will call it (validated in `ValidateUnsigned`), as such
+		/// if the block author is defined it will be defined as the equivocation
+		/// reporter.
+		#[pallet::call_index(3)]
+		#[pallet::weight(T::WeightInfo::report_equivocation(key_owner_proof.validator_count()))]
+		pub fn report_invalid_fork_vote_unsigned(
+			origin: OriginFor<T>,
+			invalid_fork_proof: Box<
+				InvalidForkVoteProof<
+					BlockNumberFor<T>,
+					T::BeefyId,
+					<T::BeefyId as RuntimeAppPublic>::Signature,
+				>,
+			>,
+			key_owner_proof: T::KeyOwnerProof,
+		) -> DispatchResultWithPostInfo {
+			ensure_none(origin)?;
+
+			// TODO:
+			// T::EquivocationReportSystem::process_evidence(
+			// 	None,
+			// 	(*invalid_fork_proof, key_owner_proof),
+			// )?;
+			Ok(Pays::No.into())
+		}
 	}
 
 	#[pallet::validate_unsigned]

From df31624306261b576af8e04542e45d64af9573ff Mon Sep 17 00:00:00 2001
From: Adrian Catangiu <adrian@parity.io>
Date: Wed, 5 Jul 2023 15:52:27 +0300
Subject: [PATCH 004/188] sc-consensus-beefy: fisherman reports invalid votes
 and justifications

Signed-off-by: Adrian Catangiu <adrian@parity.io>
---
 .../beefy/src/communication/fisherman.rs      | 140 +++++++++++++++---
 .../beefy/src/communication/gossip.rs         |   8 +
 substrate/client/consensus/beefy/src/tests.rs |   5 +-
 .../consensus/beefy/src/commitment.rs         |   4 +-
 4 files changed, 129 insertions(+), 28 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 46e2458cd92c..dd778fa71137 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -16,17 +16,22 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-use crate::justification::BeefyVersionedFinalityProof;
+use crate::{
+	error::Error, justification::BeefyVersionedFinalityProof, keystore::BeefySignatureHasher,
+	LOG_TARGET,
+};
+use log::debug;
 use sc_client_api::Backend;
 use sp_api::ProvideRuntimeApi;
 use sp_blockchain::HeaderBackend;
 use sp_consensus_beefy::{
+	check_invalid_fork_proof,
 	crypto::{AuthorityId, Signature},
-	BeefyApi, Payload, PayloadProvider, VoteMessage,
+	BeefyApi, InvalidForkVoteProof, Payload, PayloadProvider, ValidatorSet, VoteMessage,
 };
 use sp_runtime::{
 	generic::BlockId,
-	traits::{Block, NumberFor},
+	traits::{Block, Header, NumberFor},
 };
 use std::{marker::PhantomData, sync::Arc};
 
@@ -37,15 +42,12 @@ pub(crate) trait BeefyFisherman<B: Block>: Send + Sync {
 	fn check_vote(
 		&self,
 		vote: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
-	) -> Result<(), sp_blockchain::Error>;
+	) -> Result<(), Error>;
 
 	/// Check `proof` for contained finalized block against expected payload.
 	///
 	/// Note: this fn expects block referenced in `proof` to be finalized.
-	fn check_proof(
-		&self,
-		proof: BeefyVersionedFinalityProof<B>,
-	) -> Result<(), sp_blockchain::Error>;
+	fn check_proof(&self, proof: BeefyVersionedFinalityProof<B>) -> Result<(), Error>;
 }
 
 /// Helper wrapper used to check gossiped votes for (historical) equivocations,
@@ -62,14 +64,79 @@ where
 	B: Block,
 	BE: Backend<B>,
 	P: PayloadProvider<B>,
+	R: ProvideRuntimeApi<B> + Send + Sync,
+	R::Api: BeefyApi<B>,
 {
-	fn expected_payload(&self, number: NumberFor<B>) -> Result<Payload, sp_blockchain::Error> {
+	fn expected_header_and_payload(&self, number: NumberFor<B>) -> Result<(B::Header, Payload), Error> {
 		// This should be un-ambiguous since `number` is finalized.
-		let hash = self.backend.blockchain().expect_block_hash_from_id(&BlockId::Number(number))?;
-		let header = self.backend.blockchain().expect_header(hash)?;
+		let hash = self
+			.backend
+			.blockchain()
+			.expect_block_hash_from_id(&BlockId::Number(number))
+			.map_err(|e| Error::Backend(e.to_string()))?;
+		let header = self
+			.backend
+			.blockchain()
+			.expect_header(hash)
+			.map_err(|e| Error::Backend(e.to_string()))?;
 		self.payload_provider
 			.payload(&header)
-			.ok_or_else(|| sp_blockchain::Error::Backend("BEEFY Payload not found".into()))
+			.map(|payload| (header, payload))
+			.ok_or_else(|| Error::Backend("BEEFY Payload not found".into()))
+	}
+
+	fn active_validator_set_at(
+		&self,
+		header: &B::Header,
+	) -> Result<ValidatorSet<AuthorityId>, Error> {
+		self.runtime
+			.runtime_api()
+			.validator_set(header.hash())
+			.map_err(Error::RuntimeApi)?
+			.ok_or_else(|| Error::Backend("could not get BEEFY validator set".into()))
+	}
+
+	fn report_invalid_fork_vote(
+		&self,
+		proof: InvalidForkVoteProof<NumberFor<B>, AuthorityId, Signature>,
+		correct_header: &B::Header,
+		correct_payload: &Payload,
+		validator_set: &ValidatorSet<AuthorityId>, // validator set active at the time
+	) -> Result<(), Error> {
+		let set_id = validator_set.id();
+
+		if proof.vote.commitment.validator_set_id != set_id ||
+			!check_invalid_fork_proof::<_, _, BeefySignatureHasher>(&proof, correct_payload)
+		{
+			debug!(target: LOG_TARGET, "🥩 Skip report for bad invalid fork proof {:?}", proof);
+			return Ok(())
+		}
+
+		let hash = correct_header.hash();
+		let offender_id = proof.vote.id.clone();
+		let runtime_api = self.runtime.runtime_api();
+		// generate key ownership proof at that block
+		let key_owner_proof = match runtime_api
+			.generate_key_ownership_proof(hash, set_id, offender_id)
+			.map_err(Error::RuntimeApi)?
+		{
+			Some(proof) => proof,
+			None => {
+				debug!(
+					target: LOG_TARGET,
+					"🥩 Invalid fork vote offender not part of the authority set."
+				);
+				return Ok(())
+			},
+		};
+
+		// submit invalid fork vote report at **best** block
+		let best_block_hash = self.backend.blockchain().info().best_hash;
+		runtime_api
+			.submit_report_invalid_fork_unsigned_extrinsic(best_block_hash, proof, key_owner_proof)
+			.map_err(Error::RuntimeApi)?;
+
+		Ok(())
 	}
 }
 
@@ -87,11 +154,15 @@ where
 	fn check_vote(
 		&self,
 		vote: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
-	) -> Result<(), sp_blockchain::Error> {
+	) -> Result<(), Error> {
 		let number = vote.commitment.block_number;
-		let expected = self.expected_payload(number)?;
-		if vote.commitment.payload != expected {
-			// TODO: report equivocation
+		let (header, expected_payload) = self.expected_header_and_payload(number)?;
+		if vote.commitment.payload != expected_payload {
+			let validator_set = self.active_validator_set_at(&header)?;
+			// TODO: create/get grandpa proof for block number
+			let grandpa_proof = ();
+			let proof = InvalidForkVoteProof { vote, grandpa_proof };
+			self.report_invalid_fork_vote(proof, &header, &expected_payload, &validator_set)?;
 		}
 		Ok(())
 	}
@@ -99,14 +170,35 @@ where
 	/// Check `proof` for contained finalized block against expected payload.
 	///
 	/// Note: this fn expects block referenced in `proof` to be finalized.
-	fn check_proof(
-		&self,
-		proof: BeefyVersionedFinalityProof<B>,
-	) -> Result<(), sp_blockchain::Error> {
-		let payload = proof.payload();
-		let expected = self.expected_payload(*proof.number())?;
-		if payload != &expected {
-			// TODO: report equivocation
+	fn check_proof(&self, proof: BeefyVersionedFinalityProof<B>) -> Result<(), Error> {
+		let (commitment, signatures) = match proof {
+			BeefyVersionedFinalityProof::<B>::V1(inner) => (inner.commitment, inner.signatures),
+		};
+		let number = commitment.block_number;
+		let (header, expected_payload) = self.expected_header_and_payload(number)?;
+		if commitment.payload != expected_payload {
+			// TODO: create/get grandpa proof for block number
+			let grandpa_proof = ();
+			let validator_set = self.active_validator_set_at(&header)?;
+			if signatures.len() != validator_set.validators().len() {
+				// invalid proof
+				return Ok(())
+			}
+			// report every signer of the bad justification
+			for signed_pair in validator_set.validators().iter().zip(signatures.into_iter()) {
+				let (id, signature) = signed_pair;
+				if let Some(signature) = signature {
+					let vote =
+						VoteMessage { commitment: commitment.clone(), id: id.clone(), signature };
+					let proof = InvalidForkVoteProof { vote, grandpa_proof };
+					self.report_invalid_fork_vote(
+						proof,
+						&header,
+						&expected_payload,
+						&validator_set,
+					)?;
+				}
+			}
 		}
 		Ok(())
 	}
diff --git a/substrate/client/consensus/beefy/src/communication/gossip.rs b/substrate/client/consensus/beefy/src/communication/gossip.rs
index 5b11b5884999..883f4e48f48e 100644
--- a/substrate/client/consensus/beefy/src/communication/gossip.rs
+++ b/substrate/client/consensus/beefy/src/communication/gossip.rs
@@ -298,6 +298,10 @@ where
 					// We know `vote` is for some past (finalized) block. Have fisherman check
 					// for equivocations. Best-effort, ignore errors such as state pruned.
 					let _ = self.fisherman.check_vote(vote);
+					// TODO: maybe raise cost reputation when seeing votes that are intentional
+					// spam: votes that trigger fisherman reports, but don't go through either
+					// because signer is/was not authority or similar reasons.
+					// The idea is to more quickly disconnect neighbors which are attempting DoS.
 					return Action::Discard(cost::OUTDATED_MESSAGE)
 				},
 				Consider::Accept => {},
@@ -347,6 +351,10 @@ where
 				// We know `proof` is for some past (finalized) block. Have fisherman check
 				// for equivocations. Best-effort, ignore errors such as state pruned.
 				let _ = self.fisherman.check_proof(proof);
+				// TODO: maybe raise cost reputation when seeing votes that are intentional
+				// spam: votes that trigger fisherman reports, but don't go through either because
+				// signer is/was not authority or similar reasons.
+				// The idea is to more quickly disconnect neighbors which are attempting DoS.
 				return Action::Discard(cost::OUTDATED_MESSAGE)
 			},
 			Consider::Accept => {},
diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index 80ac9224d1d4..3c1ad619335a 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -29,6 +29,7 @@ use crate::{
 		},
 		request_response::{on_demand_justifications_protocol_config, BeefyJustifsRequestHandler},
 	},
+	error::Error,
 	gossip_protocol_name,
 	justification::*,
 	load_or_init_voter_state, wait_for_runtime_pallet, BeefyRPCLinks, BeefyVoterLinks, KnownPeers,
@@ -250,13 +251,13 @@ pub(crate) struct DummyFisherman<B> {
 }
 
 impl<B: BlockT> BeefyFisherman<B> for DummyFisherman<B> {
-	fn check_proof(&self, _: BeefyVersionedFinalityProof<B>) -> Result<(), sp_blockchain::Error> {
+	fn check_proof(&self, _: BeefyVersionedFinalityProof<B>) -> Result<(), Error> {
 		Ok(())
 	}
 	fn check_vote(
 		&self,
 		_: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
-	) -> Result<(), sp_blockchain::Error> {
+	) -> Result<(), Error> {
 		Ok(())
 	}
 }
diff --git a/substrate/primitives/consensus/beefy/src/commitment.rs b/substrate/primitives/consensus/beefy/src/commitment.rs
index aa2efdb10ca2..3c451b70afb0 100644
--- a/substrate/primitives/consensus/beefy/src/commitment.rs
+++ b/substrate/primitives/consensus/beefy/src/commitment.rs
@@ -81,7 +81,7 @@ where
 	}
 }
 
-/// A commitment with matching GRANDPA validators' signatures.
+/// A commitment with matching BEEFY validators' signatures.
 ///
 /// Note that SCALE-encoding of the structure is optimized for size efficiency over the wire,
 /// please take a look at custom [`Encode`] and [`Decode`] implementations and
@@ -90,7 +90,7 @@ where
 pub struct SignedCommitment<TBlockNumber, TSignature> {
 	/// The commitment signatures are collected for.
 	pub commitment: Commitment<TBlockNumber>,
-	/// GRANDPA validators' signatures for the commitment.
+	/// BEEFY validators' signatures for the commitment.
 	///
 	/// The length of this `Vec` must match number of validators in the current set (see
 	/// [Commitment::validator_set_id]).

From 938cde1d84949b09250d4f3621f3e5b0f93cffee Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 20 Jul 2023 09:49:31 +0200
Subject: [PATCH 005/188] don't check GRANDPA finality

GRANDPA finalization proof is not checked, which leads to slashing on
forks. This is fine since honest validators will not be slashed on the
chain finalized by GRANDPA, which is the only chain that ultimately
matters. The only material difference not checking GRANDPA proofs makes
is that validators are not slashed for signing BEEFY commitments prior
to the blocks committed to being finalized by GRANDPA. This is fine too,
since the slashing risk of committing to an incorrect block implies
validators will only sign blocks they *know* will be finalized by
GRANDPA.
---
 .../beefy/src/communication/fisherman.rs      |  9 +++----
 .../primitives/consensus/beefy/src/lib.rs     | 24 +++++++++----------
 2 files changed, 14 insertions(+), 19 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index dd778fa71137..1f63fe58a294 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -148,7 +148,7 @@ where
 	R: ProvideRuntimeApi<B> + Send + Sync,
 	R::Api: BeefyApi<B>,
 {
-	/// Check `vote` for contained finalized block against expected payload.
+	/// Check `vote` for contained block against expected payload.
 	///
 	/// Note: this fn expects `vote.commitment.block_number` to be finalized.
 	fn check_vote(
@@ -159,9 +159,7 @@ where
 		let (header, expected_payload) = self.expected_header_and_payload(number)?;
 		if vote.commitment.payload != expected_payload {
 			let validator_set = self.active_validator_set_at(&header)?;
-			// TODO: create/get grandpa proof for block number
-			let grandpa_proof = ();
-			let proof = InvalidForkVoteProof { vote, grandpa_proof };
+			let proof = InvalidForkVoteProof { vote };
 			self.report_invalid_fork_vote(proof, &header, &expected_payload, &validator_set)?;
 		}
 		Ok(())
@@ -178,7 +176,6 @@ where
 		let (header, expected_payload) = self.expected_header_and_payload(number)?;
 		if commitment.payload != expected_payload {
 			// TODO: create/get grandpa proof for block number
-			let grandpa_proof = ();
 			let validator_set = self.active_validator_set_at(&header)?;
 			if signatures.len() != validator_set.validators().len() {
 				// invalid proof
@@ -190,7 +187,7 @@ where
 				if let Some(signature) = signature {
 					let vote =
 						VoteMessage { commitment: commitment.clone(), id: id.clone(), signature };
-					let proof = InvalidForkVoteProof { vote, grandpa_proof };
+					let proof = InvalidForkVoteProof { vote };
 					self.report_invalid_fork_vote(
 						proof,
 						&header,
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index fdfacae037d0..7913adc03de8 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -252,13 +252,11 @@ impl<Number, Id, Signature> EquivocationProof<Number, Id, Signature> {
 }
 
 /// Proof of voter misbehavior on a given set id.
-/// This proof shows voter voted on a different fork than finalized by GRANDPA.
+/// This proof shows voter voted on a different fork than what is included in our chain.
 #[derive(Clone, Debug, Decode, Encode, PartialEq, TypeInfo)]
 pub struct InvalidForkVoteProof<Number, Id, Signature> {
 	/// Voting for a block on different fork than one finalized by GRANDPA.
 	pub vote: VoteMessage<Number, Id, Signature>,
-	/// TODO: GRANDPA proof that this block is final
-	pub grandpa_proof: (),
 }
 
 impl<Number, Id, Signature> InvalidForkVoteProof<Number, Id, Signature> {
@@ -329,8 +327,15 @@ where
 
 /// Validates [InvalidForkVoteProof] by checking:
 /// 1. `vote` is signed,
-/// 2. GRANDPA proof is provided for block number >= vote.commitment.block_number.
-/// 3. `vote.commitment.payload` != `expected_payload`.
+/// 2. `vote.commitment.payload` != `expected_payload`.
+/// NOTE: GRANDPA finalization proof is not checked, which leads to slashing on forks.
+/// This is fine since honest validators will not be slashed on the chain finalized
+/// by GRANDPA, which is the only chain that ultimately matters.
+/// The only material difference not checking GRANDPA proofs makes is that validators
+/// are not slashed for signing BEEFY commitments prior to the blocks committed to being
+/// finalized by GRANDPA. This is fine too, since the slashing risk of committing to
+/// an incorrect block implies validators will only sign blocks they *know* will be
+/// finalized by GRANDPA.
 pub fn check_invalid_fork_proof<Number, Id, MsgHash>(
 	proof: &InvalidForkVoteProof<Number, Id, <Id as RuntimeAppPublic>::Signature>,
 	expected_payload: &Payload,
@@ -340,20 +345,13 @@ where
 	Number: Clone + Encode + PartialEq,
 	MsgHash: Hash,
 {
-	let InvalidForkVoteProof { vote, grandpa_proof: _ } = proof;
+	let InvalidForkVoteProof { vote } = proof;
 
 	// check signature `vote`, if invalid, equivocation report is invalid
 	if !check_commitment_signature(&vote.commitment, &vote.id, &vote.signature) {
 		return false
 	}
 
-	// TODO: add GRANDPA proof
-	// if GRANDPA proof doesn't show `vote.commitment.block_number` has been finalized,
-	// we cannot safely prove that `vote` payload is invalid.
-	// if grandpa_proof.block_number < vote.commitment.block_number {
-	// 	return false
-	// }
-
 	// check that `payload` on the `vote` is different that the `expected_payload`.
 	&vote.commitment.payload != expected_payload
 }

From fb0855300a94cb0595b180a944374d834385feb3 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 20 Jul 2023 23:00:09 +0200
Subject: [PATCH 006/188] change primitive: vote -> commitment

instead of using votes as the underlying primitive, rather use
commitments since they're a more universal container for signed payloads
(for instance `SignedCommitment` is also the primitive used by ethereum relayers).
SignedCommitments are already aggregates of multiple signatures. Will
use SignedCommitment directly next.
---
 .../beefy/src/communication/fisherman.rs      | 71 +++++++++----------
 substrate/frame/beefy/src/lib.rs              | 14 ++--
 .../primitives/consensus/beefy/src/lib.rs     | 62 +++++++++-------
 3 files changed, 78 insertions(+), 69 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 1f63fe58a294..9820f1fcfbab 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -27,7 +27,7 @@ use sp_blockchain::HeaderBackend;
 use sp_consensus_beefy::{
 	check_invalid_fork_proof,
 	crypto::{AuthorityId, Signature},
-	BeefyApi, InvalidForkVoteProof, Payload, PayloadProvider, ValidatorSet, VoteMessage,
+	BeefyApi, InvalidForkCommitmentProof, Payload, PayloadProvider, ValidatorSet, VoteMessage, Commitment, OpaqueKeyOwnershipProof,
 };
 use sp_runtime::{
 	generic::BlockId,
@@ -96,44 +96,46 @@ where
 			.ok_or_else(|| Error::Backend("could not get BEEFY validator set".into()))
 	}
 
-	fn report_invalid_fork_vote(
+	fn report_invalid_fork_commitments(
 		&self,
-		proof: InvalidForkVoteProof<NumberFor<B>, AuthorityId, Signature>,
+		proof: InvalidForkCommitmentProof<NumberFor<B>, AuthorityId, Signature>,
 		correct_header: &B::Header,
-		correct_payload: &Payload,
 		validator_set: &ValidatorSet<AuthorityId>, // validator set active at the time
 	) -> Result<(), Error> {
 		let set_id = validator_set.id();
 
-		if proof.vote.commitment.validator_set_id != set_id ||
-			!check_invalid_fork_proof::<_, _, BeefySignatureHasher>(&proof, correct_payload)
+		if proof.commitment.validator_set_id != set_id ||
+			!check_invalid_fork_proof::<NumberFor<B>, AuthorityId, BeefySignatureHasher>(&proof)
 		{
 			debug!(target: LOG_TARGET, "🥩 Skip report for bad invalid fork proof {:?}", proof);
 			return Ok(())
 		}
 
 		let hash = correct_header.hash();
-		let offender_id = proof.vote.id.clone();
+		let offender_ids = proof.signatories.iter().cloned().map(|(id, _sig)| id).collect::<Vec<_>>();
 		let runtime_api = self.runtime.runtime_api();
+
 		// generate key ownership proof at that block
-		let key_owner_proof = match runtime_api
-			.generate_key_ownership_proof(hash, set_id, offender_id)
-			.map_err(Error::RuntimeApi)?
-		{
-			Some(proof) => proof,
-			None => {
-				debug!(
-					target: LOG_TARGET,
-					"🥩 Invalid fork vote offender not part of the authority set."
-				);
-				return Ok(())
-			},
-		};
+		let key_owner_proofs = offender_ids.iter()
+										   .filter_map(|id| {
+											   match runtime_api.generate_key_ownership_proof(hash, set_id, id.clone()) {
+												   Ok(Some(proof)) => Some(Ok(proof)),
+												   Ok(None) => {
+													   debug!(
+														   target: LOG_TARGET,
+														   "🥩 Invalid fork vote offender not part of the authority set."
+													   );
+													   None
+												   },
+												   Err(e) => Some(Err(Error::RuntimeApi(e))),
+											   }
+										   })
+										   .collect::<Result<_, _>>()?;
 
 		// submit invalid fork vote report at **best** block
 		let best_block_hash = self.backend.blockchain().info().best_hash;
 		runtime_api
-			.submit_report_invalid_fork_unsigned_extrinsic(best_block_hash, proof, key_owner_proof)
+			.submit_report_invalid_fork_unsigned_extrinsic(best_block_hash, proof, key_owner_proofs)
 			.map_err(Error::RuntimeApi)?;
 
 		Ok(())
@@ -159,8 +161,8 @@ where
 		let (header, expected_payload) = self.expected_header_and_payload(number)?;
 		if vote.commitment.payload != expected_payload {
 			let validator_set = self.active_validator_set_at(&header)?;
-			let proof = InvalidForkVoteProof { vote };
-			self.report_invalid_fork_vote(proof, &header, &expected_payload, &validator_set)?;
+			let proof = InvalidForkCommitmentProof { commitment: vote.commitment, signatories: vec![(vote.id, vote.signature)], expected_payload };
+			self.report_invalid_fork_commitments(proof, &header, &validator_set)?;
 		}
 		Ok(())
 	}
@@ -182,20 +184,15 @@ where
 				return Ok(())
 			}
 			// report every signer of the bad justification
-			for signed_pair in validator_set.validators().iter().zip(signatures.into_iter()) {
-				let (id, signature) = signed_pair;
-				if let Some(signature) = signature {
-					let vote =
-						VoteMessage { commitment: commitment.clone(), id: id.clone(), signature };
-					let proof = InvalidForkVoteProof { vote };
-					self.report_invalid_fork_vote(
-						proof,
-						&header,
-						&expected_payload,
-						&validator_set,
-					)?;
-				}
-			}
+			let signatories = validator_set.validators().iter().cloned().zip(signatures.into_iter())
+																		.filter_map(|(id, signature)| signature.map(|sig| (id, sig))).collect();
+
+			let proof = InvalidForkCommitmentProof { commitment, signatories, expected_payload };
+			self.report_invalid_fork_commitments(
+				proof,
+				&header,
+				&validator_set,
+			)?;
 		}
 		Ok(())
 	}
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 30641ea1ee6b..0bd586017605 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -63,7 +63,7 @@ const LOG_TARGET: &str = "runtime::beefy";
 pub mod pallet {
 	use super::*;
 	use frame_system::pallet_prelude::BlockNumberFor;
-	use sp_consensus_beefy::InvalidForkVoteProof;
+	use sp_consensus_beefy::InvalidForkCommitmentProof;
 
 	#[pallet::config]
 	pub trait Config: frame_system::Config {
@@ -275,10 +275,10 @@ pub mod pallet {
 		/// will be reported.
 		#[pallet::call_index(2)]
 		#[pallet::weight(T::WeightInfo::report_equivocation(key_owner_proof.validator_count()))]
-		pub fn report_invalid_fork_vote(
+		pub fn report_invalid_fork_commitment(
 			origin: OriginFor<T>,
 			invalid_fork_proof: Box<
-				InvalidForkVoteProof<
+				InvalidForkCommitmentProof<
 					BlockNumberFor<T>,
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
@@ -297,9 +297,9 @@ pub mod pallet {
 			Ok(Pays::No.into())
 		}
 
-		/// Report voter voting on invalid fork. This method will verify the
+		/// Report commitment on invalid fork. This method will verify the
 		/// invalid fork proof and validate the given key ownership proof
-		/// against the extracted offender. If both are valid, the offence
+		/// against the extracted offenders. If both are valid, the offence
 		/// will be reported.
 		///
 		/// This extrinsic must be called unsigned and it is expected that only
@@ -308,10 +308,10 @@ pub mod pallet {
 		/// reporter.
 		#[pallet::call_index(3)]
 		#[pallet::weight(T::WeightInfo::report_equivocation(key_owner_proof.validator_count()))]
-		pub fn report_invalid_fork_vote_unsigned(
+		pub fn report_invalid_fork_commitment_unsigned(
 			origin: OriginFor<T>,
 			invalid_fork_proof: Box<
-				InvalidForkVoteProof<
+				InvalidForkCommitmentProof<
 					BlockNumberFor<T>,
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 7913adc03de8..32d43c6fb95c 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -251,26 +251,35 @@ impl<Number, Id, Signature> EquivocationProof<Number, Id, Signature> {
 	}
 }
 
-/// Proof of voter misbehavior on a given set id.
-/// This proof shows voter voted on a different fork than what is included in our chain.
+/// Proof of invalid commitment on a given set id.
+/// This proof shows commitment signed on a different fork than finalized by GRANDPA.
 #[derive(Clone, Debug, Decode, Encode, PartialEq, TypeInfo)]
-pub struct InvalidForkVoteProof<Number, Id, Signature> {
-	/// Voting for a block on different fork than one finalized by GRANDPA.
-	pub vote: VoteMessage<Number, Id, Signature>,
+pub struct InvalidForkCommitmentProof<Number, Id, Signature> {
+	/// Commitment for a block on different fork than one at the same height in
+	/// this client's chain.
+	/// TODO: maybe replace {commitment, signatories} with SignedCommitment
+	/// (tradeoff: SignedCommitment not ideal since sigs optional, but fewer
+	/// types to juggle around) - check once usage pattern is clear
+	pub commitment: Commitment<Number>,
+	/// Signatures on this block
+	/// TODO: maybe change to HashMap - check once usage pattern is clear
+	pub signatories: Vec<(Id, Signature)>,
+	/// Expected payload
+	pub expected_payload: Payload,
 }
 
-impl<Number, Id, Signature> InvalidForkVoteProof<Number, Id, Signature> {
+impl<Number, Id, Signature> InvalidForkCommitmentProof<Number, Id, Signature> {
 	/// Returns the authority id of the misbehaving voter.
-	pub fn offender_id(&self) -> &Id {
-		&self.vote.id
+	pub fn offender_ids(&self) -> Vec<&Id> {
+		self.signatories.iter().map(|(id, _)| id).collect()
 	}
 	/// Returns the round number at which the infringement occurred.
 	pub fn round_number(&self) -> &Number {
-		&self.vote.commitment.block_number
+		&self.commitment.block_number
 	}
 	/// Returns the set id at which the infringement occurred.
 	pub fn set_id(&self) -> ValidatorSetId {
-		self.vote.commitment.validator_set_id
+		self.commitment.validator_set_id
 	}
 }
 
@@ -337,23 +346,26 @@ where
 /// an incorrect block implies validators will only sign blocks they *know* will be
 /// finalized by GRANDPA.
 pub fn check_invalid_fork_proof<Number, Id, MsgHash>(
-	proof: &InvalidForkVoteProof<Number, Id, <Id as RuntimeAppPublic>::Signature>,
-	expected_payload: &Payload,
+	proof: &InvalidForkCommitmentProof<Number, Id, <Id as RuntimeAppPublic>::Signature>,
 ) -> bool
 where
 	Id: BeefyAuthorityId<MsgHash> + PartialEq,
 	Number: Clone + Encode + PartialEq,
 	MsgHash: Hash,
 {
-	let InvalidForkVoteProof { vote } = proof;
-
-	// check signature `vote`, if invalid, equivocation report is invalid
-	if !check_commitment_signature(&vote.commitment, &vote.id, &vote.signature) {
-		return false
+	let InvalidForkCommitmentProof { commitment, signatories, expected_payload } = proof;
+
+	// check that `payload` on the `vote` is different that the `expected_payload` (checked first since cheap failfast).
+	if &commitment.payload != expected_payload {
+		// check check each signatory's signature on the commitment.
+		// if any are invalid, equivocation report is invalid
+		// TODO: refactor check_commitment_signature to take a slice of signatories
+		return signatories.iter().all(|(authority_id, signature)| {
+			check_commitment_signature(&commitment, authority_id, signature)
+		})
+	} else {
+		false
 	}
-
-	// check that `payload` on the `vote` is different that the `expected_payload`.
-	&vote.commitment.payload != expected_payload
 }
 
 /// New BEEFY validator set notification hook.
@@ -418,9 +430,9 @@ sp_api::decl_runtime_apis! {
 			key_owner_proof: OpaqueKeyOwnershipProof,
 		) -> Option<()>;
 
-		/// Submits an unsigned extrinsic to report an vote for an invalid fork.
-		/// The caller must provide the invalid vote proof and a key ownership proof
-		/// (should be obtained using `generate_key_ownership_proof`). The
+		/// Submits an unsigned extrinsic to report commitments to an invalid fork.
+		/// The caller must provide the invalid commitments proof and key ownership proofs
+		/// (should be obtained using `generate_key_ownership_proof`) for the offenders. The
 		/// extrinsic will be unsigned and should only be accepted for local
 		/// authorship (not to be broadcast to the network). This method returns
 		/// `None` when creation of the extrinsic fails, e.g. if equivocation
@@ -428,8 +440,8 @@ sp_api::decl_runtime_apis! {
 		/// hardcoded to return `None`). Only useful in an offchain context.
 		fn submit_report_invalid_fork_unsigned_extrinsic(
 			invalid_fork_proof:
-				InvalidForkVoteProof<NumberFor<Block>, crypto::AuthorityId, crypto::Signature>,
-			key_owner_proof: OpaqueKeyOwnershipProof,
+				InvalidForkCommitmentProof<NumberFor<Block>, crypto::AuthorityId, crypto::Signature>,
+			key_owner_proofs: Vec<OpaqueKeyOwnershipProof>,
 		) -> Option<()>;
 
 		/// Generates a proof of key ownership for the given authority in the

From 39799b2180eef14517a25164f38c9e4fb54211fe Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 27 Jul 2023 11:05:18 +0200
Subject: [PATCH 007/188] add check_signed_commitment/report_invalid_payload

---
 .../beefy/src/communication/fisherman.rs      | 78 ++++++++++++++++++-
 substrate/client/consensus/beefy/src/tests.rs |  3 +
 2 files changed, 77 insertions(+), 4 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 9820f1fcfbab..bb9554bc8ca0 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -27,7 +27,7 @@ use sp_blockchain::HeaderBackend;
 use sp_consensus_beefy::{
 	check_invalid_fork_proof,
 	crypto::{AuthorityId, Signature},
-	BeefyApi, InvalidForkCommitmentProof, Payload, PayloadProvider, ValidatorSet, VoteMessage, Commitment, OpaqueKeyOwnershipProof,
+	BeefyApi, InvalidForkCommitmentProof, Payload, PayloadProvider, ValidatorSet, VoteMessage, Commitment, OpaqueKeyOwnershipProof, SignedCommitment,
 };
 use sp_runtime::{
 	generic::BlockId,
@@ -44,6 +44,11 @@ pub(crate) trait BeefyFisherman<B: Block>: Send + Sync {
 		vote: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
 	) -> Result<(), Error>;
 
+	fn check_signed_commitment(
+		&self,
+		signed_commitment: SignedCommitment<NumberFor<B>, Signature>,
+	) -> Result<(), Error>;
+
 	/// Check `proof` for contained finalized block against expected payload.
 	///
 	/// Note: this fn expects block referenced in `proof` to be finalized.
@@ -96,12 +101,64 @@ where
 			.ok_or_else(|| Error::Backend("could not get BEEFY validator set".into()))
 	}
 
+	fn report_invalid_payload(
+		&self,
+		signed_commitment: SignedCommitment<NumberFor<B>, Signature>,
+		correct_payload: &Payload,
+		correct_header: &B::Header,
+	) -> Result<(), Error> {
+		let validator_set = self.active_validator_set_at(correct_header)?;
+		let set_id = validator_set.id();
+
+		let proof = InvalidForkCommitmentProof {
+			commitment: signed_commitment.commitment.clone(),
+			signatories: vec![],
+			expected_payload: correct_payload.clone(),
+		};
+
+		if signed_commitment.commitment.validator_set_id != set_id ||
+			signed_commitment.commitment.payload != *correct_payload ||
+			!check_invalid_fork_proof::<NumberFor<B>, AuthorityId, BeefySignatureHasher>(&proof)
+		{
+			debug!(target: LOG_TARGET, "🥩 Skip report for bad invalid fork proof {:?}", proof);
+			return Ok(())
+		}
+
+		let offender_ids = proof.signatories.iter().cloned().map(|(id, _sig)| id).collect::<Vec<_>>();
+		let runtime_api = self.runtime.runtime_api();
+
+		// generate key ownership proof at that block
+		let key_owner_proofs = offender_ids.iter()
+										   .filter_map(|id| {
+											   match runtime_api.generate_key_ownership_proof(correct_header.hash(), set_id, id.clone()) {
+												   Ok(Some(proof)) => Some(Ok(proof)),
+												   Ok(None) => {
+													   debug!(
+														   target: LOG_TARGET,
+														   "🥩 Invalid fork vote offender not part of the authority set."
+													   );
+													   None
+												   },
+												   Err(e) => Some(Err(Error::RuntimeApi(e))),
+											   }
+										   })
+										   .collect::<Result<_, _>>()?;
+
+		// submit invalid fork vote report at **best** block
+		let best_block_hash = self.backend.blockchain().info().best_hash;
+		runtime_api
+			.submit_report_invalid_fork_unsigned_extrinsic(best_block_hash, proof, key_owner_proofs)
+			.map_err(Error::RuntimeApi)?;
+
+		Ok(())
+	}
+
 	fn report_invalid_fork_commitments(
 		&self,
 		proof: InvalidForkCommitmentProof<NumberFor<B>, AuthorityId, Signature>,
 		correct_header: &B::Header,
-		validator_set: &ValidatorSet<AuthorityId>, // validator set active at the time
 	) -> Result<(), Error> {
+		let validator_set = self.active_validator_set_at(correct_header)?;
 		let set_id = validator_set.id();
 
 		if proof.commitment.validator_set_id != set_id ||
@@ -162,7 +219,21 @@ where
 		if vote.commitment.payload != expected_payload {
 			let validator_set = self.active_validator_set_at(&header)?;
 			let proof = InvalidForkCommitmentProof { commitment: vote.commitment, signatories: vec![(vote.id, vote.signature)], expected_payload };
-			self.report_invalid_fork_commitments(proof, &header, &validator_set)?;
+			self.report_invalid_fork_commitments(proof, &header)?;
+		}
+		Ok(())
+	}
+
+	/// Check `commitment` for contained block against expected payload.
+	fn check_signed_commitment(
+		&self,
+		signed_commitment: SignedCommitment<NumberFor<B>, Signature>,
+	) -> Result<(), Error> {
+		let number = signed_commitment.commitment.block_number;
+		let (header, expected_payload) = self.expected_header_and_payload(number)?;
+		if signed_commitment.commitment.payload != expected_payload {
+			let validator_set = self.active_validator_set_at(&header)?;
+			self.report_invalid_payload(signed_commitment, &expected_payload, &header)?;
 		}
 		Ok(())
 	}
@@ -191,7 +262,6 @@ where
 			self.report_invalid_fork_commitments(
 				proof,
 				&header,
-				&validator_set,
 			)?;
 		}
 		Ok(())
diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index 3c1ad619335a..f291b5674a02 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -254,6 +254,9 @@ impl<B: BlockT> BeefyFisherman<B> for DummyFisherman<B> {
 	fn check_proof(&self, _: BeefyVersionedFinalityProof<B>) -> Result<(), Error> {
 		Ok(())
 	}
+	fn check_signed_commitment(&self, _: SignedCommitment<NumberFor<B>, Signature>) -> Result<(), Error> {
+		Ok(())
+	}
 	fn check_vote(
 		&self,
 		_: VoteMessage<NumberFor<B>, AuthorityId, Signature>,

From 4aa602a176d11487e1f0fd44124e800ebb84ea17 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Wed, 2 Aug 2023 10:36:54 +0200
Subject: [PATCH 008/188] update comments

---
 .../client/consensus/beefy/src/communication/fisherman.rs   | 2 +-
 substrate/frame/beefy/src/lib.rs                            | 2 +-
 substrate/primitives/consensus/beefy/src/lib.rs             | 6 ++++--
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index bb9554bc8ca0..fd62d4851453 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -238,7 +238,7 @@ where
 		Ok(())
 	}
 
-	/// Check `proof` for contained finalized block against expected payload.
+	/// Check `proof` for contained block against expected payload.
 	///
 	/// Note: this fn expects block referenced in `proof` to be finalized.
 	fn check_proof(&self, proof: BeefyVersionedFinalityProof<B>) -> Result<(), Error> {
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 0bd586017605..07b399211416 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -113,7 +113,7 @@ pub mod pallet {
 		type EquivocationReportSystem: OffenceReportSystem<
 			Option<Self::AccountId>,
 			// TODO: make below an enum that takes either `EquivocationProof` or
-			// `InvalidForkVoteProof`
+			// `InvalidForkCommitmentProof`
 			EquivocationEvidenceFor<Self>,
 		>;
 	}
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 32d43c6fb95c..1ae5517d86a8 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -355,7 +355,8 @@ where
 {
 	let InvalidForkCommitmentProof { commitment, signatories, expected_payload } = proof;
 
-	// check that `payload` on the `vote` is different that the `expected_payload` (checked first since cheap failfast).
+	// check that `payload` on the `vote` is different that the `expected_payload` (checked first
+	// since cheap failfast).
 	if &commitment.payload != expected_payload {
 		// check check each signatory's signature on the commitment.
 		// if any are invalid, equivocation report is invalid
@@ -405,7 +406,8 @@ impl OpaqueKeyOwnershipProof {
 }
 
 sp_api::decl_runtime_apis! {
-	/// API necessary for BEEFY voters.
+	/// API necessary for BEEFY voters. Due to the significant conceptual
+	/// overlap, in large part, this is lifted from the GRANDPA API.
 	#[api_version(3)]
 	pub trait BeefyApi<AuthorityId> where
 		AuthorityId : Codec + RuntimeAppPublic,

From e0e13d9309d5eb8a5c71b11f4e8cd7c1c85bc7bc Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Wed, 2 Aug 2023 10:52:56 +0200
Subject: [PATCH 009/188] add dummy for report of invalid fork commitments

---
 substrate/frame/beefy/src/lib.rs                | 14 ++++++++++++++
 substrate/primitives/consensus/beefy/src/lib.rs |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 07b399211416..017c5a692607 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -366,6 +366,20 @@ impl<T: Config> Pallet<T> {
 		T::EquivocationReportSystem::publish_evidence((equivocation_proof, key_owner_proof)).ok()
 	}
 
+	/// Submits an extrinsic to report an invalid fork signed by potentially
+	/// multiple signatories. This method will create an unsigned extrinsic with
+	/// a call to `report_invalid_fork_unsigned` and will push the transaction
+	/// to the pool. Only useful in an offchain context.
+	pub fn submit_unsigned_invalid_fork_report(
+		_invalid_fork_proof: sp_consensus_beefy::InvalidForkCommitmentProof<BlockNumberFor<T>, T::BeefyId,
+			<T::BeefyId as RuntimeAppPublic>::Signature,
+>,
+		_key_owner_proofs: Vec<T::KeyOwnerProof>,
+	) -> Option<()> {
+		// T::EquivocationReportSystem::publish_evidence((invalid_fork_proof, key_owner_proofs)).ok()
+		None
+	}
+
 	fn change_authorities(
 		new: BoundedVec<T::BeefyId, T::MaxAuthorities>,
 		queued: BoundedVec<T::BeefyId, T::MaxAuthorities>,
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 1ae5517d86a8..ac68a7c31f9a 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -389,7 +389,7 @@ impl<AuthorityId> OnNewValidatorSet<AuthorityId> for () {
 /// the runtime API boundary this type is unknown and as such we keep this
 /// opaque representation, implementors of the runtime API will have to make
 /// sure that all usages of `OpaqueKeyOwnershipProof` refer to the same type.
-#[derive(Decode, Encode, PartialEq, TypeInfo)]
+#[derive(Decode, Encode, PartialEq, TypeInfo, Clone)]
 pub struct OpaqueKeyOwnershipProof(Vec<u8>);
 impl OpaqueKeyOwnershipProof {
 	/// Create a new `OpaqueKeyOwnershipProof` using the given encoded

From 165d7fcc3c6963d1534e08d13eebee7cd5439fab Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 1 Aug 2023 20:16:11 +0200
Subject: [PATCH 010/188] account for #14471

---
 substrate/frame/beefy/src/lib.rs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 017c5a692607..8c47db824f4f 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -274,7 +274,10 @@ pub mod pallet {
 		/// against the extracted offender. If both are valid, the offence
 		/// will be reported.
 		#[pallet::call_index(2)]
-		#[pallet::weight(T::WeightInfo::report_equivocation(key_owner_proof.validator_count()))]
+		#[pallet::weight(T::WeightInfo::report_equivocation(
+			key_owner_proof.validator_count(),
+			T::MaxNominators::get(),
+		))]
 		pub fn report_invalid_fork_commitment(
 			origin: OriginFor<T>,
 			invalid_fork_proof: Box<
@@ -307,7 +310,7 @@ pub mod pallet {
 		/// if the block author is defined it will be defined as the equivocation
 		/// reporter.
 		#[pallet::call_index(3)]
-		#[pallet::weight(T::WeightInfo::report_equivocation(key_owner_proof.validator_count()))]
+		#[pallet::weight(T::WeightInfo::report_equivocation(key_owner_proof.validator_count(), T::MaxNominators::get(),))]
 		pub fn report_invalid_fork_commitment_unsigned(
 			origin: OriginFor<T>,
 			invalid_fork_proof: Box<

From c7df83b0d555e6b7b13fae9c41995a43b252a05a Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 7 Aug 2023 17:44:39 +0200
Subject: [PATCH 011/188] account for #14373

---
 .../client/consensus/beefy/src/communication/fisherman.rs   | 6 +++---
 substrate/primitives/consensus/beefy/src/lib.rs             | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index fd62d4851453..2fda6ef17185 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -26,7 +26,7 @@ use sp_api::ProvideRuntimeApi;
 use sp_blockchain::HeaderBackend;
 use sp_consensus_beefy::{
 	check_invalid_fork_proof,
-	crypto::{AuthorityId, Signature},
+	ecdsa_crypto::{AuthorityId, Signature},
 	BeefyApi, InvalidForkCommitmentProof, Payload, PayloadProvider, ValidatorSet, VoteMessage, Commitment, OpaqueKeyOwnershipProof, SignedCommitment,
 };
 use sp_runtime::{
@@ -70,7 +70,7 @@ where
 	BE: Backend<B>,
 	P: PayloadProvider<B>,
 	R: ProvideRuntimeApi<B> + Send + Sync,
-	R::Api: BeefyApi<B>,
+	R::Api: BeefyApi<B, AuthorityId>,
 {
 	fn expected_header_and_payload(&self, number: NumberFor<B>) -> Result<(B::Header, Payload), Error> {
 		// This should be un-ambiguous since `number` is finalized.
@@ -205,7 +205,7 @@ where
 	BE: Backend<B>,
 	P: PayloadProvider<B>,
 	R: ProvideRuntimeApi<B> + Send + Sync,
-	R::Api: BeefyApi<B>,
+	R::Api: BeefyApi<B, AuthorityId>,
 {
 	/// Check `vote` for contained block against expected payload.
 	///
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index ac68a7c31f9a..06a921e021fa 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -442,7 +442,7 @@ sp_api::decl_runtime_apis! {
 		/// hardcoded to return `None`). Only useful in an offchain context.
 		fn submit_report_invalid_fork_unsigned_extrinsic(
 			invalid_fork_proof:
-				InvalidForkCommitmentProof<NumberFor<Block>, crypto::AuthorityId, crypto::Signature>,
+				InvalidForkCommitmentProof<NumberFor<Block>, AuthorityId, <AuthorityId as RuntimeAppPublic>::Signature>,
 			key_owner_proofs: Vec<OpaqueKeyOwnershipProof>,
 		) -> Option<()>;
 

From 754ae802adc25af07a7ab7e8be9d4a1f69fc7433 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 7 Aug 2023 17:45:52 +0200
Subject: [PATCH 012/188] EquivocationOffence.{offender->offenders}

previously assumed equivocation report for singular malicious party.
With fork equivocations, the expectation should be that most
equivocation reports will be for multiple simultaneous equivocators.
---
 substrate/frame/beefy/src/equivocation.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 884802194bc4..3ef8ff9029c3 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -76,8 +76,8 @@ where
 	pub session_index: SessionIndex,
 	/// The size of the validator set at the time of the offence.
 	pub validator_set_count: u32,
-	/// The authority which produced this equivocation.
-	pub offender: Offender,
+	/// The authorities which produced this equivocation.
+	pub offenders: Vec<Offender>,
 }
 
 impl<Offender: Clone, N> Offence<Offender> for EquivocationOffence<Offender, N>
@@ -88,7 +88,7 @@ where
 	type TimeSlot = TimeSlot<N>;
 
 	fn offenders(&self) -> Vec<Offender> {
-		vec![self.offender.clone()]
+		self.offenders.clone()
 	}
 
 	fn session_index(&self) -> SessionIndex {
@@ -224,7 +224,7 @@ where
 			time_slot: TimeSlot { set_id, round },
 			session_index,
 			validator_set_count,
-			offender,
+			offenders: vec![offender],
 		};
 
 		R::report_offence(reporter.into_iter().collect(), offence)

From 6ed4e998b99405b4bfff6ca59024de8e2bf0769d Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 7 Aug 2023 17:57:00 +0200
Subject: [PATCH 013/188] EquivocationProof->VoteEquivocationProof

---
 substrate/client/consensus/beefy/src/round.rs          | 10 +++++-----
 substrate/client/consensus/beefy/src/tests.rs          |  6 +++---
 substrate/client/consensus/beefy/src/worker.rs         |  4 ++--
 substrate/frame/beefy/src/equivocation.rs              |  6 +++---
 substrate/frame/beefy/src/lib.rs                       | 10 +++++-----
 substrate/frame/beefy/src/mock.rs                      |  2 +-
 substrate/primitives/consensus/beefy/src/lib.rs        |  8 ++++----
 substrate/primitives/consensus/beefy/src/test_utils.rs |  6 +++---
 8 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/round.rs b/substrate/client/consensus/beefy/src/round.rs
index 6f400ce47843..81e0d0634dcd 100644
--- a/substrate/client/consensus/beefy/src/round.rs
+++ b/substrate/client/consensus/beefy/src/round.rs
@@ -22,7 +22,7 @@ use codec::{Decode, Encode};
 use log::debug;
 use sp_consensus_beefy::{
 	ecdsa_crypto::{AuthorityId, Signature},
-	Commitment, EquivocationProof, SignedCommitment, ValidatorSet, ValidatorSetId, VoteMessage,
+	Commitment, VoteEquivocationProof, SignedCommitment, ValidatorSet, ValidatorSetId, VoteMessage,
 };
 use sp_runtime::traits::{Block, NumberFor};
 use std::collections::BTreeMap;
@@ -61,7 +61,7 @@ pub fn threshold(authorities: usize) -> usize {
 pub enum VoteImportResult<B: Block> {
 	Ok,
 	RoundConcluded(SignedCommitment<NumberFor<B>, Signature>),
-	Equivocation(EquivocationProof<NumberFor<B>, AuthorityId, Signature>),
+	Equivocation(VoteEquivocationProof<NumberFor<B>, AuthorityId, Signature>),
 	Invalid,
 	Stale,
 }
@@ -153,7 +153,7 @@ where
 					target: LOG_TARGET,
 					"🥩 detected equivocated vote: 1st: {:?}, 2nd: {:?}", previous_vote, vote
 				);
-				return VoteImportResult::Equivocation(EquivocationProof {
+				return VoteImportResult::Equivocation(VoteEquivocationProof {
 					first: previous_vote.clone(),
 					second: vote,
 				})
@@ -203,7 +203,7 @@ mod tests {
 	use sc_network_test::Block;
 
 	use sp_consensus_beefy::{
-		known_payloads::MMR_ROOT_ID, Commitment, EquivocationProof, Keyring, Payload,
+		known_payloads::MMR_ROOT_ID, Commitment, VoteEquivocationProof, Keyring, Payload,
 		SignedCommitment, ValidatorSet, VoteMessage,
 	};
 
@@ -485,7 +485,7 @@ mod tests {
 		let mut alice_vote2 = alice_vote1.clone();
 		alice_vote2.commitment = commitment2;
 
-		let expected_result = VoteImportResult::Equivocation(EquivocationProof {
+		let expected_result = VoteImportResult::Equivocation(VoteEquivocationProof {
 			first: alice_vote1.clone(),
 			second: alice_vote2.clone(),
 		});
diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index f291b5674a02..1fa6c036a527 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -56,7 +56,7 @@ use sp_consensus_beefy::{
 	ecdsa_crypto::{AuthorityId, Signature},
 	known_payloads,
 	mmr::{find_mmr_root_digest, MmrRootProvider},
-	BeefyApi, Commitment, ConsensusLog, EquivocationProof, Keyring as BeefyKeyring, MmrRootHash,
+	BeefyApi, Commitment, ConsensusLog, VoteEquivocationProof, Keyring as BeefyKeyring, MmrRootHash,
 	OpaqueKeyOwnershipProof, Payload, SignedCommitment, ValidatorSet, ValidatorSetId,
 	VersionedFinalityProof, VoteMessage, BEEFY_ENGINE_ID,
 };
@@ -271,7 +271,7 @@ pub(crate) struct TestApi {
 	pub validator_set: BeefyValidatorSet,
 	pub mmr_root_hash: MmrRootHash,
 	pub reported_equivocations:
-		Option<Arc<Mutex<Vec<EquivocationProof<NumberFor<Block>, AuthorityId, Signature>>>>>,
+		Option<Arc<Mutex<Vec<VoteEquivocationProof<NumberFor<Block>, AuthorityId, Signature>>>>>,
 }
 
 impl TestApi {
@@ -325,7 +325,7 @@ sp_api::mock_impl_runtime_apis! {
 		}
 
 		fn submit_report_equivocation_unsigned_extrinsic(
-			proof: EquivocationProof<NumberFor<Block>, AuthorityId, Signature>,
+			proof: VoteEquivocationProof<NumberFor<Block>, AuthorityId, Signature>,
 			_dummy: OpaqueKeyOwnershipProof,
 		) -> Option<()> {
 			if let Some(equivocations_buf) = self.inner.reported_equivocations.as_ref() {
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index f0a486df901a..a8523f079c11 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -43,7 +43,7 @@ use sp_consensus::SyncOracle;
 use sp_consensus_beefy::{
 	check_equivocation_proof,
 	ecdsa_crypto::{AuthorityId, Signature},
-	BeefyApi, Commitment, ConsensusLog, EquivocationProof, PayloadProvider, ValidatorSet,
+	BeefyApi, Commitment, ConsensusLog, VoteEquivocationProof, PayloadProvider, ValidatorSet,
 	VersionedFinalityProof, VoteMessage, BEEFY_ENGINE_ID,
 };
 use sp_runtime::{
@@ -922,7 +922,7 @@ where
 	/// isn't necessarily the best block if there are pending authority set changes.
 	pub(crate) fn report_equivocation(
 		&self,
-		proof: EquivocationProof<NumberFor<B>, AuthorityId, Signature>,
+		proof: VoteEquivocationProof<NumberFor<B>, AuthorityId, Signature>,
 	) -> Result<(), Error> {
 		let rounds = self.persisted_state.voting_oracle.active_rounds()?;
 		let (validators, validator_set_id) = (rounds.validators(), rounds.validator_set_id());
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 3ef8ff9029c3..b85453f9f63d 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -38,7 +38,7 @@ use codec::{self as codec, Decode, Encode};
 use frame_support::traits::{Get, KeyOwnerProofSystem};
 use frame_system::pallet_prelude::BlockNumberFor;
 use log::{error, info};
-use sp_consensus_beefy::{EquivocationProof, ValidatorSetId, KEY_TYPE as BEEFY_KEY_TYPE};
+use sp_consensus_beefy::{VoteEquivocationProof, ValidatorSetId, KEY_TYPE as BEEFY_KEY_TYPE};
 use sp_runtime::{
 	transaction_validity::{
 		InvalidTransaction, TransactionPriority, TransactionSource, TransactionValidity,
@@ -122,9 +122,9 @@ where
 pub struct EquivocationReportSystem<T, R, P, L>(sp_std::marker::PhantomData<(T, R, P, L)>);
 
 /// Equivocation evidence convenience alias.
-// TODO: use an enum that takes either `EquivocationProof` or `InvalidForkVoteProof`
+// TODO: use an enum that takes either `VoteEquivocationProof` or `ForkEquivocationProof`
 pub type EquivocationEvidenceFor<T> = (
-	EquivocationProof<
+	VoteEquivocationProof<
 		BlockNumberFor<T>,
 		<T as Config>::BeefyId,
 		<<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 8c47db824f4f..069e8b0d06cc 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -41,7 +41,7 @@ use sp_staking::{offence::OffenceReportSystem, SessionIndex};
 use sp_std::prelude::*;
 
 use sp_consensus_beefy::{
-	AuthorityIndex, BeefyAuthorityId, ConsensusLog, EquivocationProof, OnNewValidatorSet,
+	AuthorityIndex, BeefyAuthorityId, ConsensusLog, VoteEquivocationProof, OnNewValidatorSet,
 	ValidatorSet, BEEFY_ENGINE_ID, GENESIS_AUTHORITY_SET_ID,
 };
 
@@ -112,7 +112,7 @@ pub mod pallet {
 		/// Defines methods to publish, check and process an equivocation offence.
 		type EquivocationReportSystem: OffenceReportSystem<
 			Option<Self::AccountId>,
-			// TODO: make below an enum that takes either `EquivocationProof` or
+			// TODO: make below an enum that takes either `VoteEquivocationProof` or
 			// `InvalidForkCommitmentProof`
 			EquivocationEvidenceFor<Self>,
 		>;
@@ -217,7 +217,7 @@ pub mod pallet {
 		pub fn report_equivocation(
 			origin: OriginFor<T>,
 			equivocation_proof: Box<
-				EquivocationProof<
+				VoteEquivocationProof<
 					BlockNumberFor<T>,
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
@@ -252,7 +252,7 @@ pub mod pallet {
 		pub fn report_equivocation_unsigned(
 			origin: OriginFor<T>,
 			equivocation_proof: Box<
-				EquivocationProof<
+				VoteEquivocationProof<
 					BlockNumberFor<T>,
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
@@ -359,7 +359,7 @@ impl<T: Config> Pallet<T> {
 	/// an unsigned extrinsic with a call to `report_equivocation_unsigned` and
 	/// will push the transaction to the pool. Only useful in an offchain context.
 	pub fn submit_unsigned_equivocation_report(
-		equivocation_proof: EquivocationProof<
+		equivocation_proof: VoteEquivocationProof<
 			BlockNumberFor<T>,
 			T::BeefyId,
 			<T::BeefyId as RuntimeAppPublic>::Signature,
diff --git a/substrate/frame/beefy/src/mock.rs b/substrate/frame/beefy/src/mock.rs
index b55a65dbd73a..d0abdf52c388 100644
--- a/substrate/frame/beefy/src/mock.rs
+++ b/substrate/frame/beefy/src/mock.rs
@@ -43,7 +43,7 @@ use crate as pallet_beefy;
 
 pub use sp_consensus_beefy::{
 	ecdsa_crypto::{AuthorityId as BeefyId, AuthoritySignature as BeefySignature},
-	ConsensusLog, EquivocationProof, BEEFY_ENGINE_ID,
+	ConsensusLog, VoteEquivocationProof, BEEFY_ENGINE_ID,
 };
 
 impl_opaque_keys! {
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 06a921e021fa..3e588f83a199 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -229,14 +229,14 @@ pub struct VoteMessage<Number, Id, Signature> {
 /// BEEFY happens when a voter votes on the same round/block for different payloads.
 /// Proving is achieved by collecting the signed commitments of conflicting votes.
 #[derive(Clone, Debug, Decode, Encode, PartialEq, TypeInfo)]
-pub struct EquivocationProof<Number, Id, Signature> {
+pub struct VoteEquivocationProof<Number, Id, Signature> {
 	/// The first vote in the equivocation.
 	pub first: VoteMessage<Number, Id, Signature>,
 	/// The second vote in the equivocation.
 	pub second: VoteMessage<Number, Id, Signature>,
 }
 
-impl<Number, Id, Signature> EquivocationProof<Number, Id, Signature> {
+impl<Number, Id, Signature> VoteEquivocationProof<Number, Id, Signature> {
 	/// Returns the authority id of the equivocator.
 	pub fn offender_id(&self) -> &Id {
 		&self.first.id
@@ -302,7 +302,7 @@ where
 /// Verifies the equivocation proof by making sure that both votes target
 /// different blocks and that its signatures are valid.
 pub fn check_equivocation_proof<Number, Id, MsgHash>(
-	report: &EquivocationProof<Number, Id, <Id as RuntimeAppPublic>::Signature>,
+	report: &VoteEquivocationProof<Number, Id, <Id as RuntimeAppPublic>::Signature>,
 ) -> bool
 where
 	Id: BeefyAuthorityId<MsgHash> + PartialEq,
@@ -428,7 +428,7 @@ sp_api::decl_runtime_apis! {
 		/// hardcoded to return `None`). Only useful in an offchain context.
 		fn submit_report_equivocation_unsigned_extrinsic(
 			equivocation_proof:
-				EquivocationProof<NumberFor<Block>, AuthorityId, <AuthorityId as RuntimeAppPublic>::Signature>,
+				VoteEquivocationProof<NumberFor<Block>, AuthorityId, <AuthorityId as RuntimeAppPublic>::Signature>,
 			key_owner_proof: OpaqueKeyOwnershipProof,
 		) -> Option<()>;
 
diff --git a/substrate/primitives/consensus/beefy/src/test_utils.rs b/substrate/primitives/consensus/beefy/src/test_utils.rs
index b83f657af38e..12190a701ab2 100644
--- a/substrate/primitives/consensus/beefy/src/test_utils.rs
+++ b/substrate/primitives/consensus/beefy/src/test_utils.rs
@@ -17,7 +17,7 @@
 
 #![cfg(feature = "std")]
 
-use crate::{ecdsa_crypto, Commitment, EquivocationProof, Payload, ValidatorSetId, VoteMessage};
+use crate::{ecdsa_crypto, Commitment, VoteEquivocationProof, Payload, ValidatorSetId, VoteMessage};
 use codec::Encode;
 use sp_core::{ecdsa, keccak_256, Pair};
 use std::collections::HashMap;
@@ -95,7 +95,7 @@ impl From<Keyring> for ecdsa_crypto::Public {
 pub fn generate_equivocation_proof(
 	vote1: (u64, Payload, ValidatorSetId, &Keyring),
 	vote2: (u64, Payload, ValidatorSetId, &Keyring),
-) -> EquivocationProof<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature> {
+) -> VoteEquivocationProof<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature> {
 	let signed_vote = |block_number: u64,
 	                   payload: Payload,
 	                   validator_set_id: ValidatorSetId,
@@ -106,5 +106,5 @@ pub fn generate_equivocation_proof(
 	};
 	let first = signed_vote(vote1.0, vote1.1, vote1.2, vote1.3);
 	let second = signed_vote(vote2.0, vote2.1, vote2.2, vote2.3);
-	EquivocationProof { first, second }
+	VoteEquivocationProof { first, second }
 }

From 1e41551e05fcb1486b9f605382d89045fb069986 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 7 Aug 2023 18:07:10 +0200
Subject: [PATCH 014/188] Invalid{""->Vote}EquivocationProof

---
 substrate/frame/beefy/src/equivocation.rs | 6 +++---
 substrate/frame/beefy/src/lib.rs          | 4 ++--
 substrate/frame/beefy/src/tests.rs        | 6 +++---
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index b85453f9f63d..07d14c536df2 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -209,15 +209,15 @@ where
 
 		// Validate equivocation proof (check votes are different and signatures are valid).
 		if !sp_consensus_beefy::check_equivocation_proof(&equivocation_proof) {
-			return Err(Error::<T>::InvalidEquivocationProof.into())
+			return Err(Error::<T>::InvalidVoteEquivocationProof.into())
 		}
 
 		// Check that the session id for the membership proof is within the
 		// bounds of the set id reported in the equivocation.
 		let set_id_session_index =
-			crate::SetIdSession::<T>::get(set_id).ok_or(Error::<T>::InvalidEquivocationProof)?;
+			crate::SetIdSession::<T>::get(set_id).ok_or(Error::<T>::InvalidVoteEquivocationProof)?;
 		if session_index != set_id_session_index {
-			return Err(Error::<T>::InvalidEquivocationProof.into())
+			return Err(Error::<T>::InvalidVoteEquivocationProof.into())
 		}
 
 		let offence = EquivocationOffence {
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 069e8b0d06cc..f2b210675e08 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -197,8 +197,8 @@ pub mod pallet {
 	pub enum Error<T> {
 		/// A key ownership proof provided as part of an equivocation report is invalid.
 		InvalidKeyOwnershipProof,
-		/// An equivocation proof provided as part of an equivocation report is invalid.
-		InvalidEquivocationProof,
+		/// An equivocation proof provided as part of a voter equivocation report is invalid.
+		InvalidVoteEquivocationProof,
 		/// A given equivocation report is valid but already previously reported.
 		DuplicateOffenceReport,
 	}
diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index e04dc330d0c0..d59bc053a849 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -451,7 +451,7 @@ fn report_equivocation_invalid_set_id() {
 				Box::new(equivocation_proof),
 				key_owner_proof,
 			),
-			Error::<Test>::InvalidEquivocationProof,
+			Error::<Test>::InvalidVoteEquivocationProof,
 		);
 	});
 }
@@ -494,7 +494,7 @@ fn report_equivocation_invalid_session() {
 				Box::new(equivocation_proof),
 				key_owner_proof,
 			),
-			Error::<Test>::InvalidEquivocationProof,
+			Error::<Test>::InvalidVoteEquivocationProof,
 		);
 	});
 }
@@ -573,7 +573,7 @@ fn report_equivocation_invalid_equivocation_proof() {
 					Box::new(equivocation_proof),
 					key_owner_proof.clone(),
 				),
-				Error::<Test>::InvalidEquivocationProof,
+				Error::<Test>::InvalidVoteEquivocationProof,
 			);
 		};
 

From de2ae90f551d9a280244fb9ac773d514a326da1e Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 7 Aug 2023 18:10:36 +0200
Subject: [PATCH 015/188] check_{""->vote}_equivocation_proof

---
 substrate/client/consensus/beefy/src/worker.rs  |  4 ++--
 substrate/frame/beefy/src/equivocation.rs       |  2 +-
 substrate/frame/beefy/src/tests.rs              | 12 ++++++------
 substrate/primitives/consensus/beefy/src/lib.rs |  4 ++--
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index a8523f079c11..83fbf5b6f352 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -41,7 +41,7 @@ use sp_api::{BlockId, ProvideRuntimeApi};
 use sp_arithmetic::traits::{AtLeast32Bit, Saturating};
 use sp_consensus::SyncOracle;
 use sp_consensus_beefy::{
-	check_equivocation_proof,
+	check_vote_equivocation_proof,
 	ecdsa_crypto::{AuthorityId, Signature},
 	BeefyApi, Commitment, ConsensusLog, VoteEquivocationProof, PayloadProvider, ValidatorSet,
 	VersionedFinalityProof, VoteMessage, BEEFY_ENGINE_ID,
@@ -928,7 +928,7 @@ where
 		let (validators, validator_set_id) = (rounds.validators(), rounds.validator_set_id());
 		let offender_id = proof.offender_id().clone();
 
-		if !check_equivocation_proof::<_, _, BeefySignatureHasher>(&proof) {
+		if !check_vote_equivocation_proof::<_, _, BeefySignatureHasher>(&proof) {
 			debug!(target: LOG_TARGET, "🥩 Skip report for bad equivocation {:?}", proof);
 			return Ok(())
 		} else if let Some(local_id) = self.key_store.authority_id(validators) {
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 07d14c536df2..c0b99ee0e4aa 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -208,7 +208,7 @@ where
 			.ok_or(Error::<T>::InvalidKeyOwnershipProof)?;
 
 		// Validate equivocation proof (check votes are different and signatures are valid).
-		if !sp_consensus_beefy::check_equivocation_proof(&equivocation_proof) {
+		if !sp_consensus_beefy::check_vote_equivocation_proof(&equivocation_proof) {
 			return Err(Error::<T>::InvalidVoteEquivocationProof.into())
 		}
 
diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index d59bc053a849..a493286decf5 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -19,7 +19,7 @@ use std::vec;
 
 use codec::Encode;
 use sp_consensus_beefy::{
-	check_equivocation_proof, generate_equivocation_proof, known_payloads::MMR_ROOT_ID,
+	check_vote_equivocation_proof, generate_equivocation_proof, known_payloads::MMR_ROOT_ID,
 	Keyring as BeefyKeyring, Payload, ValidatorSet, KEY_TYPE as BEEFY_KEY_TYPE,
 };
 
@@ -217,7 +217,7 @@ fn should_sign_and_verify() {
 		(1, payload1.clone(), set_id, &BeefyKeyring::Bob),
 	);
 	// expect invalid equivocation proof
-	assert!(!check_equivocation_proof::<_, _, Keccak256>(&equivocation_proof));
+	assert!(!check_vote_equivocation_proof::<_, _, Keccak256>(&equivocation_proof));
 
 	// generate an equivocation proof, with two votes in different rounds for
 	// different payloads signed by the same key
@@ -226,7 +226,7 @@ fn should_sign_and_verify() {
 		(2, payload2.clone(), set_id, &BeefyKeyring::Bob),
 	);
 	// expect invalid equivocation proof
-	assert!(!check_equivocation_proof::<_, _, Keccak256>(&equivocation_proof));
+	assert!(!check_vote_equivocation_proof::<_, _, Keccak256>(&equivocation_proof));
 
 	// generate an equivocation proof, with two votes by different authorities
 	let equivocation_proof = generate_equivocation_proof(
@@ -234,7 +234,7 @@ fn should_sign_and_verify() {
 		(1, payload2.clone(), set_id, &BeefyKeyring::Bob),
 	);
 	// expect invalid equivocation proof
-	assert!(!check_equivocation_proof::<_, _, Keccak256>(&equivocation_proof));
+	assert!(!check_vote_equivocation_proof::<_, _, Keccak256>(&equivocation_proof));
 
 	// generate an equivocation proof, with two votes in different set ids
 	let equivocation_proof = generate_equivocation_proof(
@@ -242,7 +242,7 @@ fn should_sign_and_verify() {
 		(1, payload2.clone(), set_id + 1, &BeefyKeyring::Bob),
 	);
 	// expect invalid equivocation proof
-	assert!(!check_equivocation_proof::<_, _, Keccak256>(&equivocation_proof));
+	assert!(!check_vote_equivocation_proof::<_, _, Keccak256>(&equivocation_proof));
 
 	// generate an equivocation proof, with two votes in the same round for
 	// different payloads signed by the same key
@@ -252,7 +252,7 @@ fn should_sign_and_verify() {
 		(1, payload2, set_id, &BeefyKeyring::Bob),
 	);
 	// expect valid equivocation proof
-	assert!(check_equivocation_proof::<_, _, Keccak256>(&equivocation_proof));
+	assert!(check_vote_equivocation_proof::<_, _, Keccak256>(&equivocation_proof));
 }
 
 #[test]
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 3e588f83a199..2aaef24dae5f 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -299,9 +299,9 @@ where
 	BeefyAuthorityId::<MsgHash>::verify(authority_id, signature, &encoded_commitment)
 }
 
-/// Verifies the equivocation proof by making sure that both votes target
+/// Verifies the vote equivocation proof by making sure that both votes target
 /// different blocks and that its signatures are valid.
-pub fn check_equivocation_proof<Number, Id, MsgHash>(
+pub fn check_vote_equivocation_proof<Number, Id, MsgHash>(
 	report: &VoteEquivocationProof<Number, Id, <Id as RuntimeAppPublic>::Signature>,
 ) -> bool
 where

From 273f34bf78497079863335c158289f30b0c0ba9d Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 7 Aug 2023 18:17:53 +0200
Subject: [PATCH 016/188] InvalidForkCommitmentProof->ForkEquivocationProof

---
 .../beefy/src/communication/fisherman.rs         | 16 ++++++++--------
 substrate/frame/beefy/src/lib.rs                 |  8 ++++----
 substrate/primitives/consensus/beefy/src/lib.rs  | 16 ++++++++--------
 3 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 2fda6ef17185..c228e4bd2a7d 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -25,9 +25,9 @@ use sc_client_api::Backend;
 use sp_api::ProvideRuntimeApi;
 use sp_blockchain::HeaderBackend;
 use sp_consensus_beefy::{
-	check_invalid_fork_proof,
+	check_fork_equivocation_proof,
 	ecdsa_crypto::{AuthorityId, Signature},
-	BeefyApi, InvalidForkCommitmentProof, Payload, PayloadProvider, ValidatorSet, VoteMessage, Commitment, OpaqueKeyOwnershipProof, SignedCommitment,
+	BeefyApi, ForkEquivocationProof, Payload, PayloadProvider, ValidatorSet, VoteMessage, Commitment, OpaqueKeyOwnershipProof, SignedCommitment,
 };
 use sp_runtime::{
 	generic::BlockId,
@@ -110,7 +110,7 @@ where
 		let validator_set = self.active_validator_set_at(correct_header)?;
 		let set_id = validator_set.id();
 
-		let proof = InvalidForkCommitmentProof {
+		let proof = ForkEquivocationProof {
 			commitment: signed_commitment.commitment.clone(),
 			signatories: vec![],
 			expected_payload: correct_payload.clone(),
@@ -118,7 +118,7 @@ where
 
 		if signed_commitment.commitment.validator_set_id != set_id ||
 			signed_commitment.commitment.payload != *correct_payload ||
-			!check_invalid_fork_proof::<NumberFor<B>, AuthorityId, BeefySignatureHasher>(&proof)
+			!check_fork_equivocation_proof::<NumberFor<B>, AuthorityId, BeefySignatureHasher>(&proof)
 		{
 			debug!(target: LOG_TARGET, "🥩 Skip report for bad invalid fork proof {:?}", proof);
 			return Ok(())
@@ -155,14 +155,14 @@ where
 
 	fn report_invalid_fork_commitments(
 		&self,
-		proof: InvalidForkCommitmentProof<NumberFor<B>, AuthorityId, Signature>,
+		proof: ForkEquivocationProof<NumberFor<B>, AuthorityId, Signature>,
 		correct_header: &B::Header,
 	) -> Result<(), Error> {
 		let validator_set = self.active_validator_set_at(correct_header)?;
 		let set_id = validator_set.id();
 
 		if proof.commitment.validator_set_id != set_id ||
-			!check_invalid_fork_proof::<NumberFor<B>, AuthorityId, BeefySignatureHasher>(&proof)
+			!check_fork_equivocation_proof::<NumberFor<B>, AuthorityId, BeefySignatureHasher>(&proof)
 		{
 			debug!(target: LOG_TARGET, "🥩 Skip report for bad invalid fork proof {:?}", proof);
 			return Ok(())
@@ -218,7 +218,7 @@ where
 		let (header, expected_payload) = self.expected_header_and_payload(number)?;
 		if vote.commitment.payload != expected_payload {
 			let validator_set = self.active_validator_set_at(&header)?;
-			let proof = InvalidForkCommitmentProof { commitment: vote.commitment, signatories: vec![(vote.id, vote.signature)], expected_payload };
+			let proof = ForkEquivocationProof { commitment: vote.commitment, signatories: vec![(vote.id, vote.signature)], expected_payload };
 			self.report_invalid_fork_commitments(proof, &header)?;
 		}
 		Ok(())
@@ -258,7 +258,7 @@ where
 			let signatories = validator_set.validators().iter().cloned().zip(signatures.into_iter())
 																		.filter_map(|(id, signature)| signature.map(|sig| (id, sig))).collect();
 
-			let proof = InvalidForkCommitmentProof { commitment, signatories, expected_payload };
+			let proof = ForkEquivocationProof { commitment, signatories, expected_payload };
 			self.report_invalid_fork_commitments(
 				proof,
 				&header,
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index f2b210675e08..ba1130963a38 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -63,7 +63,7 @@ const LOG_TARGET: &str = "runtime::beefy";
 pub mod pallet {
 	use super::*;
 	use frame_system::pallet_prelude::BlockNumberFor;
-	use sp_consensus_beefy::InvalidForkCommitmentProof;
+	use sp_consensus_beefy::ForkEquivocationProof;
 
 	#[pallet::config]
 	pub trait Config: frame_system::Config {
@@ -281,7 +281,7 @@ pub mod pallet {
 		pub fn report_invalid_fork_commitment(
 			origin: OriginFor<T>,
 			invalid_fork_proof: Box<
-				InvalidForkCommitmentProof<
+				ForkEquivocationProof<
 					BlockNumberFor<T>,
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
@@ -314,7 +314,7 @@ pub mod pallet {
 		pub fn report_invalid_fork_commitment_unsigned(
 			origin: OriginFor<T>,
 			invalid_fork_proof: Box<
-				InvalidForkCommitmentProof<
+				ForkEquivocationProof<
 					BlockNumberFor<T>,
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
@@ -374,7 +374,7 @@ impl<T: Config> Pallet<T> {
 	/// a call to `report_invalid_fork_unsigned` and will push the transaction
 	/// to the pool. Only useful in an offchain context.
 	pub fn submit_unsigned_invalid_fork_report(
-		_invalid_fork_proof: sp_consensus_beefy::InvalidForkCommitmentProof<BlockNumberFor<T>, T::BeefyId,
+		_invalid_fork_proof: sp_consensus_beefy::ForkEquivocationProof<BlockNumberFor<T>, T::BeefyId,
 			<T::BeefyId as RuntimeAppPublic>::Signature,
 >,
 		_key_owner_proofs: Vec<T::KeyOwnerProof>,
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 2aaef24dae5f..de59ec20dc2e 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -251,10 +251,10 @@ impl<Number, Id, Signature> VoteEquivocationProof<Number, Id, Signature> {
 	}
 }
 
-/// Proof of invalid commitment on a given set id.
-/// This proof shows commitment signed on a different fork than finalized by GRANDPA.
+/// Proof of authority misbehavior on a given set id.
+/// This proof shows commitment signed on a different fork.
 #[derive(Clone, Debug, Decode, Encode, PartialEq, TypeInfo)]
-pub struct InvalidForkCommitmentProof<Number, Id, Signature> {
+pub struct ForkEquivocationProof<Number, Id, Signature> {
 	/// Commitment for a block on different fork than one at the same height in
 	/// this client's chain.
 	/// TODO: maybe replace {commitment, signatories} with SignedCommitment
@@ -268,7 +268,7 @@ pub struct InvalidForkCommitmentProof<Number, Id, Signature> {
 	pub expected_payload: Payload,
 }
 
-impl<Number, Id, Signature> InvalidForkCommitmentProof<Number, Id, Signature> {
+impl<Number, Id, Signature> ForkEquivocationProof<Number, Id, Signature> {
 	/// Returns the authority id of the misbehaving voter.
 	pub fn offender_ids(&self) -> Vec<&Id> {
 		self.signatories.iter().map(|(id, _)| id).collect()
@@ -345,15 +345,15 @@ where
 /// finalized by GRANDPA. This is fine too, since the slashing risk of committing to
 /// an incorrect block implies validators will only sign blocks they *know* will be
 /// finalized by GRANDPA.
-pub fn check_invalid_fork_proof<Number, Id, MsgHash>(
-	proof: &InvalidForkCommitmentProof<Number, Id, <Id as RuntimeAppPublic>::Signature>,
+pub fn check_fork_equivocation_proof<Number, Id, MsgHash>(
+	proof: &ForkEquivocationProof<Number, Id, <Id as RuntimeAppPublic>::Signature>,
 ) -> bool
 where
 	Id: BeefyAuthorityId<MsgHash> + PartialEq,
 	Number: Clone + Encode + PartialEq,
 	MsgHash: Hash,
 {
-	let InvalidForkCommitmentProof { commitment, signatories, expected_payload } = proof;
+	let ForkEquivocationProof { commitment, signatories, expected_payload } = proof;
 
 	// check that `payload` on the `vote` is different that the `expected_payload` (checked first
 	// since cheap failfast).
@@ -442,7 +442,7 @@ sp_api::decl_runtime_apis! {
 		/// hardcoded to return `None`). Only useful in an offchain context.
 		fn submit_report_invalid_fork_unsigned_extrinsic(
 			invalid_fork_proof:
-				InvalidForkCommitmentProof<NumberFor<Block>, AuthorityId, <AuthorityId as RuntimeAppPublic>::Signature>,
+				ForkEquivocationProof<NumberFor<Block>, AuthorityId, <AuthorityId as RuntimeAppPublic>::Signature>,
 			key_owner_proofs: Vec<OpaqueKeyOwnershipProof>,
 		) -> Option<()>;
 

From eb1e549f4c07d4a59183bb9e7c2df08f25777561 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 7 Aug 2023 18:42:38 +0200
Subject: [PATCH 017/188] renames across submit_report_...

---
 .../beefy/src/communication/fisherman.rs      | 10 ++++----
 substrate/client/consensus/beefy/src/tests.rs |  2 +-
 .../client/consensus/beefy/src/worker.rs      |  2 +-
 substrate/frame/beefy/src/equivocation.rs     |  8 +++----
 substrate/frame/beefy/src/lib.rs              | 24 +++++++++----------
 substrate/frame/beefy/src/tests.rs            | 22 ++++++++---------
 .../primitives/consensus/beefy/src/lib.rs     |  8 +++----
 7 files changed, 38 insertions(+), 38 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index c228e4bd2a7d..4b79a20eea8e 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -147,13 +147,13 @@ where
 		// submit invalid fork vote report at **best** block
 		let best_block_hash = self.backend.blockchain().info().best_hash;
 		runtime_api
-			.submit_report_invalid_fork_unsigned_extrinsic(best_block_hash, proof, key_owner_proofs)
+			.submit_report_fork_equivocation_unsigned_extrinsic(best_block_hash, proof, key_owner_proofs)
 			.map_err(Error::RuntimeApi)?;
 
 		Ok(())
 	}
 
-	fn report_invalid_fork_commitments(
+	fn report_fork_equivocation(
 		&self,
 		proof: ForkEquivocationProof<NumberFor<B>, AuthorityId, Signature>,
 		correct_header: &B::Header,
@@ -192,7 +192,7 @@ where
 		// submit invalid fork vote report at **best** block
 		let best_block_hash = self.backend.blockchain().info().best_hash;
 		runtime_api
-			.submit_report_invalid_fork_unsigned_extrinsic(best_block_hash, proof, key_owner_proofs)
+			.submit_report_fork_equivocation_unsigned_extrinsic(best_block_hash, proof, key_owner_proofs)
 			.map_err(Error::RuntimeApi)?;
 
 		Ok(())
@@ -219,7 +219,7 @@ where
 		if vote.commitment.payload != expected_payload {
 			let validator_set = self.active_validator_set_at(&header)?;
 			let proof = ForkEquivocationProof { commitment: vote.commitment, signatories: vec![(vote.id, vote.signature)], expected_payload };
-			self.report_invalid_fork_commitments(proof, &header)?;
+			self.report_fork_equivocation(proof, &header)?;
 		}
 		Ok(())
 	}
@@ -259,7 +259,7 @@ where
 																		.filter_map(|(id, signature)| signature.map(|sig| (id, sig))).collect();
 
 			let proof = ForkEquivocationProof { commitment, signatories, expected_payload };
-			self.report_invalid_fork_commitments(
+			self.report_fork_equivocation(
 				proof,
 				&header,
 			)?;
diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index 1fa6c036a527..8858aa6cfdcf 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -324,7 +324,7 @@ sp_api::mock_impl_runtime_apis! {
 			Some(self.inner.validator_set.clone())
 		}
 
-		fn submit_report_equivocation_unsigned_extrinsic(
+		fn submit_report_vote_equivocation_unsigned_extrinsic(
 			proof: VoteEquivocationProof<NumberFor<Block>, AuthorityId, Signature>,
 			_dummy: OpaqueKeyOwnershipProof,
 		) -> Option<()> {
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index 83fbf5b6f352..c7430767b159 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -969,7 +969,7 @@ where
 		// submit equivocation report at **best** block
 		let best_block_hash = self.backend.blockchain().info().best_hash;
 		runtime_api
-			.submit_report_equivocation_unsigned_extrinsic(best_block_hash, proof, key_owner_proof)
+			.submit_report_vote_equivocation_unsigned_extrinsic(best_block_hash, proof, key_owner_proof)
 			.map_err(Error::RuntimeApi)?;
 
 		Ok(())
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index c0b99ee0e4aa..04605d64336b 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -151,7 +151,7 @@ where
 		use frame_system::offchain::SubmitTransaction;
 		let (equivocation_proof, key_owner_proof) = evidence;
 
-		let call = Call::report_equivocation_unsigned {
+		let call = Call::report_vote_equivocation_unsigned {
 			equivocation_proof: Box::new(equivocation_proof),
 			key_owner_proof,
 		};
@@ -235,12 +235,12 @@ where
 }
 
 /// Methods for the `ValidateUnsigned` implementation:
-/// It restricts calls to `report_equivocation_unsigned` to local calls (i.e. extrinsics generated
+/// It restricts calls to `report_vote_equivocation_unsigned` to local calls (i.e. extrinsics generated
 /// on this node) or that already in a block. This guarantees that only block authors can include
 /// unsigned equivocation reports.
 impl<T: Config> Pallet<T> {
 	pub fn validate_unsigned(source: TransactionSource, call: &Call<T>) -> TransactionValidity {
-		if let Call::report_equivocation_unsigned { equivocation_proof, key_owner_proof } = call {
+		if let Call::report_vote_equivocation_unsigned { equivocation_proof, key_owner_proof } = call {
 			// discard equivocation report not coming from the local node
 			match source {
 				TransactionSource::Local | TransactionSource::InBlock => { /* allowed */ },
@@ -278,7 +278,7 @@ impl<T: Config> Pallet<T> {
 	}
 
 	pub fn pre_dispatch(call: &Call<T>) -> Result<(), TransactionValidityError> {
-		if let Call::report_equivocation_unsigned { equivocation_proof, key_owner_proof } = call {
+		if let Call::report_vote_equivocation_unsigned { equivocation_proof, key_owner_proof } = call {
 			let evidence = (*equivocation_proof.clone(), key_owner_proof.clone());
 			T::EquivocationReportSystem::check_evidence(evidence)
 		} else {
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index ba1130963a38..c4dcfd4576eb 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -249,7 +249,7 @@ pub mod pallet {
 			key_owner_proof.validator_count(),
 			T::MaxNominators::get(),
 		))]
-		pub fn report_equivocation_unsigned(
+		pub fn report_vote_equivocation_unsigned(
 			origin: OriginFor<T>,
 			equivocation_proof: Box<
 				VoteEquivocationProof<
@@ -278,9 +278,9 @@ pub mod pallet {
 			key_owner_proof.validator_count(),
 			T::MaxNominators::get(),
 		))]
-		pub fn report_invalid_fork_commitment(
+		pub fn report_fork_equivocation(
 			origin: OriginFor<T>,
-			invalid_fork_proof: Box<
+			fork_equivocation_proof: Box<
 				ForkEquivocationProof<
 					BlockNumberFor<T>,
 					T::BeefyId,
@@ -294,7 +294,7 @@ pub mod pallet {
 			// TODO:
 			// T::EquivocationReportSystem::process_evidence(
 			// 	Some(reporter),
-			// 	(*invalid_fork_proof, key_owner_proof),
+			// 	(*fork_equivocation_proof, key_owner_proof),
 			// )?;
 			// Waive the fee since the report is valid and beneficial
 			Ok(Pays::No.into())
@@ -311,9 +311,9 @@ pub mod pallet {
 		/// reporter.
 		#[pallet::call_index(3)]
 		#[pallet::weight(T::WeightInfo::report_equivocation(key_owner_proof.validator_count(), T::MaxNominators::get(),))]
-		pub fn report_invalid_fork_commitment_unsigned(
+		pub fn report_fork_equivocation_unsigned(
 			origin: OriginFor<T>,
-			invalid_fork_proof: Box<
+			fork_equivocation_proof: Box<
 				ForkEquivocationProof<
 					BlockNumberFor<T>,
 					T::BeefyId,
@@ -327,7 +327,7 @@ pub mod pallet {
 			// TODO:
 			// T::EquivocationReportSystem::process_evidence(
 			// 	None,
-			// 	(*invalid_fork_proof, key_owner_proof),
+			// 	(*fork_equivocation_proof, key_owner_proof),
 			// )?;
 			Ok(Pays::No.into())
 		}
@@ -358,7 +358,7 @@ impl<T: Config> Pallet<T> {
 	/// Submits an extrinsic to report an equivocation. This method will create
 	/// an unsigned extrinsic with a call to `report_equivocation_unsigned` and
 	/// will push the transaction to the pool. Only useful in an offchain context.
-	pub fn submit_unsigned_equivocation_report(
+	pub fn submit_unsigned_vote_equivocation_report(
 		equivocation_proof: VoteEquivocationProof<
 			BlockNumberFor<T>,
 			T::BeefyId,
@@ -371,15 +371,15 @@ impl<T: Config> Pallet<T> {
 
 	/// Submits an extrinsic to report an invalid fork signed by potentially
 	/// multiple signatories. This method will create an unsigned extrinsic with
-	/// a call to `report_invalid_fork_unsigned` and will push the transaction
+	/// a call to `report_fork_equivocation_unsigned` and will push the transaction
 	/// to the pool. Only useful in an offchain context.
-	pub fn submit_unsigned_invalid_fork_report(
-		_invalid_fork_proof: sp_consensus_beefy::ForkEquivocationProof<BlockNumberFor<T>, T::BeefyId,
+	pub fn submit_unsigned_fork_equivocation_report(
+		_fork_equivocation_proof: sp_consensus_beefy::ForkEquivocationProof<BlockNumberFor<T>, T::BeefyId,
 			<T::BeefyId as RuntimeAppPublic>::Signature,
 >,
 		_key_owner_proofs: Vec<T::KeyOwnerProof>,
 	) -> Option<()> {
-		// T::EquivocationReportSystem::publish_evidence((invalid_fork_proof, key_owner_proofs)).ok()
+		// T::EquivocationReportSystem::publish_evidence((fork_equivocation_proof, key_owner_proofs)).ok()
 		None
 	}
 
diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index a493286decf5..45a708098076 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -300,7 +300,7 @@ fn report_equivocation_current_set_works() {
 		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
 
 		// report the equivocation and the tx should be dispatched successfully
-		assert_ok!(Beefy::report_equivocation_unsigned(
+		assert_ok!(Beefy::report_vote_equivocation_unsigned(
 			RuntimeOrigin::none(),
 			Box::new(equivocation_proof),
 			key_owner_proof,
@@ -383,7 +383,7 @@ fn report_equivocation_old_set_works() {
 		);
 
 		// report the equivocation and the tx should be dispatched successfully
-		assert_ok!(Beefy::report_equivocation_unsigned(
+		assert_ok!(Beefy::report_vote_equivocation_unsigned(
 			RuntimeOrigin::none(),
 			Box::new(equivocation_proof),
 			key_owner_proof,
@@ -446,7 +446,7 @@ fn report_equivocation_invalid_set_id() {
 
 		// the call for reporting the equivocation should error
 		assert_err!(
-			Beefy::report_equivocation_unsigned(
+			Beefy::report_vote_equivocation_unsigned(
 				RuntimeOrigin::none(),
 				Box::new(equivocation_proof),
 				key_owner_proof,
@@ -489,7 +489,7 @@ fn report_equivocation_invalid_session() {
 		// report an equivocation for the current set using an key ownership
 		// proof from the previous set, the session should be invalid.
 		assert_err!(
-			Beefy::report_equivocation_unsigned(
+			Beefy::report_vote_equivocation_unsigned(
 				RuntimeOrigin::none(),
 				Box::new(equivocation_proof),
 				key_owner_proof,
@@ -537,7 +537,7 @@ fn report_equivocation_invalid_key_owner_proof() {
 		// report an equivocation for the current set using a key ownership
 		// proof for a different key than the one in the equivocation proof.
 		assert_err!(
-			Beefy::report_equivocation_unsigned(
+			Beefy::report_vote_equivocation_unsigned(
 				RuntimeOrigin::none(),
 				Box::new(equivocation_proof),
 				invalid_key_owner_proof,
@@ -568,7 +568,7 @@ fn report_equivocation_invalid_equivocation_proof() {
 
 		let assert_invalid_equivocation_proof = |equivocation_proof| {
 			assert_err!(
-				Beefy::report_equivocation_unsigned(
+				Beefy::report_vote_equivocation_unsigned(
 					RuntimeOrigin::none(),
 					Box::new(equivocation_proof),
 					key_owner_proof.clone(),
@@ -646,7 +646,7 @@ fn report_equivocation_validate_unsigned_prevents_duplicates() {
 
 		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
 
-		let call = Call::report_equivocation_unsigned {
+		let call = Call::report_vote_equivocation_unsigned {
 			equivocation_proof: Box::new(equivocation_proof.clone()),
 			key_owner_proof: key_owner_proof.clone(),
 		};
@@ -681,7 +681,7 @@ fn report_equivocation_validate_unsigned_prevents_duplicates() {
 		assert_ok!(<Beefy as sp_runtime::traits::ValidateUnsigned>::pre_dispatch(&call));
 
 		// we submit the report
-		Beefy::report_equivocation_unsigned(
+		Beefy::report_vote_equivocation_unsigned(
 			RuntimeOrigin::none(),
 			Box::new(equivocation_proof),
 			key_owner_proof,
@@ -752,7 +752,7 @@ fn valid_equivocation_reports_dont_pay_fees() {
 		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
 
 		// check the dispatch info for the call.
-		let info = Call::<Test>::report_equivocation_unsigned {
+		let info = Call::<Test>::report_vote_equivocation_unsigned {
 			equivocation_proof: Box::new(equivocation_proof.clone()),
 			key_owner_proof: key_owner_proof.clone(),
 		}
@@ -763,7 +763,7 @@ fn valid_equivocation_reports_dont_pay_fees() {
 		assert_eq!(info.pays_fee, Pays::Yes);
 
 		// report the equivocation.
-		let post_info = Beefy::report_equivocation_unsigned(
+		let post_info = Beefy::report_vote_equivocation_unsigned(
 			RuntimeOrigin::none(),
 			Box::new(equivocation_proof.clone()),
 			key_owner_proof.clone(),
@@ -777,7 +777,7 @@ fn valid_equivocation_reports_dont_pay_fees() {
 
 		// report the equivocation again which is invalid now since it is
 		// duplicate.
-		let post_info = Beefy::report_equivocation_unsigned(
+		let post_info = Beefy::report_vote_equivocation_unsigned(
 			RuntimeOrigin::none(),
 			Box::new(equivocation_proof),
 			key_owner_proof,
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index de59ec20dc2e..bf1196adce2f 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -426,8 +426,8 @@ sp_api::decl_runtime_apis! {
 		/// `None` when creation of the extrinsic fails, e.g. if equivocation
 		/// reporting is disabled for the given runtime (i.e. this method is
 		/// hardcoded to return `None`). Only useful in an offchain context.
-		fn submit_report_equivocation_unsigned_extrinsic(
-			equivocation_proof:
+		fn submit_report_vote_equivocation_unsigned_extrinsic(
+			vote_equivocation_proof:
 				VoteEquivocationProof<NumberFor<Block>, AuthorityId, <AuthorityId as RuntimeAppPublic>::Signature>,
 			key_owner_proof: OpaqueKeyOwnershipProof,
 		) -> Option<()>;
@@ -440,8 +440,8 @@ sp_api::decl_runtime_apis! {
 		/// `None` when creation of the extrinsic fails, e.g. if equivocation
 		/// reporting is disabled for the given runtime (i.e. this method is
 		/// hardcoded to return `None`). Only useful in an offchain context.
-		fn submit_report_invalid_fork_unsigned_extrinsic(
-			invalid_fork_proof:
+		fn submit_report_fork_equivocation_unsigned_extrinsic(
+			fork_equivocation_proof:
 				ForkEquivocationProof<NumberFor<Block>, AuthorityId, <AuthorityId as RuntimeAppPublic>::Signature>,
 			key_owner_proofs: Vec<OpaqueKeyOwnershipProof>,
 		) -> Option<()>;

From 336978355789a35029dbf03b1bf11c51c5822621 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 7 Aug 2023 19:07:19 +0200
Subject: [PATCH 018/188] convert EquivocationEvidenceFor to enum (minimal)

---
 substrate/frame/beefy/src/equivocation.rs | 81 +++++++++++++++--------
 substrate/frame/beefy/src/lib.rs          | 18 ++++-
 2 files changed, 68 insertions(+), 31 deletions(-)

diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 04605d64336b..7fb6277f3a31 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -122,15 +122,21 @@ where
 pub struct EquivocationReportSystem<T, R, P, L>(sp_std::marker::PhantomData<(T, R, P, L)>);
 
 /// Equivocation evidence convenience alias.
-// TODO: use an enum that takes either `VoteEquivocationProof` or `ForkEquivocationProof`
-pub type EquivocationEvidenceFor<T> = (
-	VoteEquivocationProof<
-		BlockNumberFor<T>,
-		<T as Config>::BeefyId,
-		<<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
-	>,
-	<T as Config>::KeyOwnerProof,
-);
+pub enum EquivocationEvidenceFor<T: Config> {
+	VoteEquivocationProof(
+		VoteEquivocationProof<
+				BlockNumberFor<T>,
+			<T as Config>::BeefyId,
+			<<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
+			>,
+		<T as Config>::KeyOwnerProof,
+	),
+	// ForkEquivocationProof(sp_consensus_beefy::ForkEquivocationProof<
+	// 		BlockNumberFor<T>,
+	// 	<T as Config>::BeefyId,
+	// 	<<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
+	// 	>)
+}
 
 impl<T, R, P, L> OffenceReportSystem<Option<T::AccountId>, EquivocationEvidenceFor<T>>
 	for EquivocationReportSystem<T, R, P, L>
@@ -149,11 +155,14 @@ where
 
 	fn publish_evidence(evidence: EquivocationEvidenceFor<T>) -> Result<(), ()> {
 		use frame_system::offchain::SubmitTransaction;
-		let (equivocation_proof, key_owner_proof) = evidence;
 
-		let call = Call::report_vote_equivocation_unsigned {
-			equivocation_proof: Box::new(equivocation_proof),
-			key_owner_proof,
+		let call = match evidence {
+			EquivocationEvidenceFor::VoteEquivocationProof(equivocation_proof, key_owner_proof) => {
+				Call::report_vote_equivocation_unsigned {
+					equivocation_proof: Box::new(equivocation_proof),
+					key_owner_proof,
+				}
+			}
 		};
 
 		let res = SubmitTransaction::<T, Call<T>>::submit_unsigned_transaction(call.into());
@@ -167,16 +176,17 @@ where
 	fn check_evidence(
 		evidence: EquivocationEvidenceFor<T>,
 	) -> Result<(), TransactionValidityError> {
-		let (equivocation_proof, key_owner_proof) = evidence;
-
-		// Check the membership proof to extract the offender's id
-		let key = (BEEFY_KEY_TYPE, equivocation_proof.offender_id().clone());
-		let offender = P::check_proof(key, key_owner_proof).ok_or(InvalidTransaction::BadProof)?;
-
-		// Check if the offence has already been reported, and if so then we can discard the report.
-		let time_slot = TimeSlot {
-			set_id: equivocation_proof.set_id(),
-			round: *equivocation_proof.round_number(),
+		let (offender, time_slot) = match evidence {
+			EquivocationEvidenceFor::VoteEquivocationProof(equivocation_proof, key_owner_proof) => {
+				let key = (BEEFY_KEY_TYPE, equivocation_proof.offender_id().clone());
+
+				// Check if the offence has already been reported, and if so then we can discard the report.
+				let time_slot = TimeSlot {
+					set_id: equivocation_proof.set_id(),
+					round: *equivocation_proof.round_number(),
+				};
+				(P::check_proof(key, key_owner_proof).ok_or(InvalidTransaction::BadProof)?, time_slot)
+			}
 		};
 
 		if R::is_known_offence(&[offender], &time_slot) {
@@ -190,9 +200,16 @@ where
 		reporter: Option<T::AccountId>,
 		evidence: EquivocationEvidenceFor<T>,
 	) -> Result<(), DispatchError> {
-		let (equivocation_proof, key_owner_proof) = evidence;
 		let reporter = reporter.or_else(|| <pallet_authorship::Pallet<T>>::author());
-		let offender = equivocation_proof.offender_id().clone();
+
+		let (equivocation_proof, key_owner_proof, offender) = match evidence {
+			EquivocationEvidenceFor::VoteEquivocationProof(equivocation_proof, key_owner_proof) =>
+				(
+					equivocation_proof.clone(),
+					key_owner_proof,
+					equivocation_proof.offender_id().clone(),
+				),
+		};
 
 		// We check the equivocation within the context of its set id (and
 		// associated session) and round. We also need to know the validator
@@ -253,7 +270,10 @@ impl<T: Config> Pallet<T> {
 				},
 			}
 
-			let evidence = (*equivocation_proof.clone(), key_owner_proof.clone());
+			let evidence = EquivocationEvidenceFor::<T>::VoteEquivocationProof(
+				*equivocation_proof.clone(),
+				key_owner_proof.clone(),
+			);
 			T::EquivocationReportSystem::check_evidence(evidence)?;
 
 			let longevity =
@@ -278,8 +298,13 @@ impl<T: Config> Pallet<T> {
 	}
 
 	pub fn pre_dispatch(call: &Call<T>) -> Result<(), TransactionValidityError> {
-		if let Call::report_vote_equivocation_unsigned { equivocation_proof, key_owner_proof } = call {
-			let evidence = (*equivocation_proof.clone(), key_owner_proof.clone());
+		if let Call::report_vote_equivocation_unsigned { equivocation_proof, key_owner_proof } =
+			call
+		{
+			let evidence = EquivocationEvidenceFor::<T>::VoteEquivocationProof(
+				*equivocation_proof.clone(),
+				key_owner_proof.clone(),
+			);
 			T::EquivocationReportSystem::check_evidence(evidence)
 		} else {
 			Err(InvalidTransaction::Call.into())
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index c4dcfd4576eb..4efd0fcb791a 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -229,7 +229,10 @@ pub mod pallet {
 
 			T::EquivocationReportSystem::process_evidence(
 				Some(reporter),
-				(*equivocation_proof, key_owner_proof),
+				EquivocationEvidenceFor::VoteEquivocationProof(
+					*equivocation_proof,
+					key_owner_proof,
+				),
 			)?;
 			// Waive the fee since the report is valid and beneficial
 			Ok(Pays::No.into())
@@ -264,7 +267,10 @@ pub mod pallet {
 
 			T::EquivocationReportSystem::process_evidence(
 				None,
-				(*equivocation_proof, key_owner_proof),
+				EquivocationEvidenceFor::<T>::VoteEquivocationProof(
+					*equivocation_proof,
+					key_owner_proof,
+				),
 			)?;
 			Ok(Pays::No.into())
 		}
@@ -366,7 +372,13 @@ impl<T: Config> Pallet<T> {
 		>,
 		key_owner_proof: T::KeyOwnerProof,
 	) -> Option<()> {
-		T::EquivocationReportSystem::publish_evidence((equivocation_proof, key_owner_proof)).ok()
+		T::EquivocationReportSystem::publish_evidence(
+			EquivocationEvidenceFor::<T>::VoteEquivocationProof(
+				equivocation_proof,
+				key_owner_proof,
+			),
+		)
+		.ok()
 	}
 
 	/// Submits an extrinsic to report an invalid fork signed by potentially

From 7abbddc5b056fc50f3b831f2037d033da0336989 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 7 Aug 2023 21:22:49 +0200
Subject: [PATCH 019/188] handle ForkEquivocationProof enum variant

---
 substrate/frame/beefy/src/equivocation.rs | 96 ++++++++++++++++-------
 substrate/frame/beefy/src/lib.rs          | 13 +--
 2 files changed, 73 insertions(+), 36 deletions(-)

diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 7fb6277f3a31..debb10807995 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -38,7 +38,7 @@ use codec::{self as codec, Decode, Encode};
 use frame_support::traits::{Get, KeyOwnerProofSystem};
 use frame_system::pallet_prelude::BlockNumberFor;
 use log::{error, info};
-use sp_consensus_beefy::{VoteEquivocationProof, ValidatorSetId, KEY_TYPE as BEEFY_KEY_TYPE};
+use sp_consensus_beefy::{VoteEquivocationProof, ForkEquivocationProof, ValidatorSetId, KEY_TYPE as BEEFY_KEY_TYPE};
 use sp_runtime::{
 	transaction_validity::{
 		InvalidTransaction, TransactionPriority, TransactionSource, TransactionValidity,
@@ -131,11 +131,13 @@ pub enum EquivocationEvidenceFor<T: Config> {
 			>,
 		<T as Config>::KeyOwnerProof,
 	),
-	// ForkEquivocationProof(sp_consensus_beefy::ForkEquivocationProof<
-	// 		BlockNumberFor<T>,
-	// 	<T as Config>::BeefyId,
-	// 	<<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
-	// 	>)
+	ForkEquivocationProof(
+		ForkEquivocationProof<BlockNumberFor<T>,
+							  <T as Config>::BeefyId,
+							  <<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
+							  >,
+		Vec<<T as Config>::KeyOwnerProof>,
+	)
 }
 
 impl<T, R, P, L> OffenceReportSystem<Option<T::AccountId>, EquivocationEvidenceFor<T>>
@@ -163,6 +165,12 @@ where
 					key_owner_proof,
 				}
 			}
+			EquivocationEvidenceFor::ForkEquivocationProof(equivocation_proof, key_owner_proofs) => {
+				Call::report_fork_equivocation_unsigned {
+					equivocation_proof: Box::new(equivocation_proof),
+					key_owner_proofs,
+				}
+			}
 		};
 
 		let res = SubmitTransaction::<T, Call<T>>::submit_unsigned_transaction(call.into());
@@ -176,20 +184,35 @@ where
 	fn check_evidence(
 		evidence: EquivocationEvidenceFor<T>,
 	) -> Result<(), TransactionValidityError> {
-		let (offender, time_slot) = match evidence {
+		let (offenders, key_owner_proofs, time_slot) = match &evidence {
 			EquivocationEvidenceFor::VoteEquivocationProof(equivocation_proof, key_owner_proof) => {
-				let key = (BEEFY_KEY_TYPE, equivocation_proof.offender_id().clone());
-
 				// Check if the offence has already been reported, and if so then we can discard the report.
 				let time_slot = TimeSlot {
 					set_id: equivocation_proof.set_id(),
 					round: *equivocation_proof.round_number(),
 				};
-				(P::check_proof(key, key_owner_proof).ok_or(InvalidTransaction::BadProof)?, time_slot)
-			}
+				(vec![equivocation_proof.offender_id()], vec![key_owner_proof.clone()], time_slot)
+			},
+			EquivocationEvidenceFor::ForkEquivocationProof(equivocation_proof, key_owner_proofs) => {
+				// Check if the offence has already been reported, and if so then we can discard the report.
+				let time_slot = TimeSlot {
+					set_id: equivocation_proof.set_id(),
+					round: *equivocation_proof.round_number(),
+				};
+				let offenders = equivocation_proof.offender_ids(); // clone data here
+				(offenders, key_owner_proofs.to_owned(), time_slot)
+			},
 		};
 
-		if R::is_known_offence(&[offender], &time_slot) {
+		// Validate the key ownership proof extracting the id of the offender.
+		let offenders = offenders
+			.into_iter()
+			.zip(key_owner_proofs.iter())
+			.map(|(key, key_owner_proof)| P::check_proof((BEEFY_KEY_TYPE, key.clone()), key_owner_proof.clone()))
+			.collect::<Option<Vec<_>>>()
+			.ok_or(InvalidTransaction::BadProof)?;
+
+		if R::is_known_offence(&offenders, &time_slot) {
 			Err(InvalidTransaction::Stale.into())
 		} else {
 			Ok(())
@@ -202,31 +225,44 @@ where
 	) -> Result<(), DispatchError> {
 		let reporter = reporter.or_else(|| <pallet_authorship::Pallet<T>>::author());
 
-		let (equivocation_proof, key_owner_proof, offender) = match evidence {
-			EquivocationEvidenceFor::VoteEquivocationProof(equivocation_proof, key_owner_proof) =>
-				(
-					equivocation_proof.clone(),
-					key_owner_proof,
-					equivocation_proof.offender_id().clone(),
-				),
+		let (offenders, key_owner_proofs, set_id, round) = match &evidence {
+			EquivocationEvidenceFor::VoteEquivocationProof(equivocation_proof, key_owner_proof) => {
+				(vec![equivocation_proof.offender_id()], vec![key_owner_proof.clone()], equivocation_proof.set_id(), *equivocation_proof.round_number())
+			},
+			EquivocationEvidenceFor::ForkEquivocationProof(equivocation_proof, key_owner_proofs) => {
+				let offenders = equivocation_proof.offender_ids(); // clone data here
+				(offenders, key_owner_proofs.to_owned(), equivocation_proof.set_id(), *equivocation_proof.round_number())
+			},
 		};
 
 		// We check the equivocation within the context of its set id (and
 		// associated session) and round. We also need to know the validator
 		// set count at the time of the offence since it is required to calculate
 		// the slash amount.
-		let set_id = equivocation_proof.set_id();
-		let round = *equivocation_proof.round_number();
-		let session_index = key_owner_proof.session();
-		let validator_set_count = key_owner_proof.validator_count();
-
-		// Validate the key ownership proof extracting the id of the offender.
-		let offender = P::check_proof((BEEFY_KEY_TYPE, offender), key_owner_proof)
+		let session_index = key_owner_proofs[0].session();
+		let validator_set_count = key_owner_proofs[0].validator_count();
+
+		// Validate the key ownership proof extracting the ids of the offenders.
+		let offenders = offenders
+			.into_iter()
+			.zip(key_owner_proofs.iter())
+			.map(|(key, key_owner_proof)| P::check_proof((BEEFY_KEY_TYPE, key.clone()), key_owner_proof.clone()))
+			.collect::<Option<Vec<_>>>()
 			.ok_or(Error::<T>::InvalidKeyOwnershipProof)?;
 
-		// Validate equivocation proof (check votes are different and signatures are valid).
-		if !sp_consensus_beefy::check_vote_equivocation_proof(&equivocation_proof) {
-			return Err(Error::<T>::InvalidVoteEquivocationProof.into())
+		match &evidence {
+			EquivocationEvidenceFor::VoteEquivocationProof(equivocation_proof, _) => {
+				// Validate equivocation proof (check votes are different and signatures are valid).
+				if !sp_consensus_beefy::check_vote_equivocation_proof(&equivocation_proof) {
+					return Err(Error::<T>::InvalidVoteEquivocationProof.into())
+				}
+			},
+			EquivocationEvidenceFor::ForkEquivocationProof(equivocation_proof, _) => {
+				// Validate equivocation proof (check commitment is to unexpected payload and signatures are valid).
+				if !sp_consensus_beefy::check_fork_equivocation_proof(&equivocation_proof) {
+					return Err(Error::<T>::InvalidForkEquivocationProof.into())
+				}
+			},
 		}
 
 		// Check that the session id for the membership proof is within the
@@ -241,7 +277,7 @@ where
 			time_slot: TimeSlot { set_id, round },
 			session_index,
 			validator_set_count,
-			offenders: vec![offender],
+			offenders,
 		};
 
 		R::report_offence(reporter.into_iter().collect(), offence)
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 4efd0fcb791a..37152bf207d4 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -112,8 +112,6 @@ pub mod pallet {
 		/// Defines methods to publish, check and process an equivocation offence.
 		type EquivocationReportSystem: OffenceReportSystem<
 			Option<Self::AccountId>,
-			// TODO: make below an enum that takes either `VoteEquivocationProof` or
-			// `InvalidForkCommitmentProof`
 			EquivocationEvidenceFor<Self>,
 		>;
 	}
@@ -199,6 +197,8 @@ pub mod pallet {
 		InvalidKeyOwnershipProof,
 		/// An equivocation proof provided as part of a voter equivocation report is invalid.
 		InvalidVoteEquivocationProof,
+		/// An equivocation proof provided as part of a fork equivocation report is invalid.
+		InvalidForkEquivocationProof,
 		/// A given equivocation report is valid but already previously reported.
 		DuplicateOffenceReport,
 	}
@@ -286,7 +286,7 @@ pub mod pallet {
 		))]
 		pub fn report_fork_equivocation(
 			origin: OriginFor<T>,
-			fork_equivocation_proof: Box<
+			equivocation_proof: Box<
 				ForkEquivocationProof<
 					BlockNumberFor<T>,
 					T::BeefyId,
@@ -315,18 +315,19 @@ pub mod pallet {
 		/// block authors will call it (validated in `ValidateUnsigned`), as such
 		/// if the block author is defined it will be defined as the equivocation
 		/// reporter.
+		/// TODO: fix key_owner_proofs[0].validator_count()
 		#[pallet::call_index(3)]
-		#[pallet::weight(T::WeightInfo::report_equivocation(key_owner_proof.validator_count(), T::MaxNominators::get(),))]
+		#[pallet::weight(T::WeightInfo::report_equivocation(key_owner_proofs[0].validator_count(), T::MaxNominators::get(),))]
 		pub fn report_fork_equivocation_unsigned(
 			origin: OriginFor<T>,
-			fork_equivocation_proof: Box<
+			equivocation_proof: Box<
 				ForkEquivocationProof<
 					BlockNumberFor<T>,
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
 				>,
 			>,
-			key_owner_proof: T::KeyOwnerProof,
+			key_owner_proofs: Vec<T::KeyOwnerProof>,
 		) -> DispatchResultWithPostInfo {
 			ensure_none(origin)?;
 

From 8acea22f122fec334f434fbc0effca6dc55479b6 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 8 Aug 2023 11:40:51 +0200
Subject: [PATCH 020/188] reduce find_mmr_root_digest trait constraint

reduce from Block to Header: less restrictive.
---
 substrate/client/consensus/beefy/src/tests.rs   | 4 ++--
 substrate/primitives/consensus/beefy/src/mmr.rs | 9 ++++-----
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index 8858aa6cfdcf..dc9ce46e8c31 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -44,7 +44,7 @@ use sc_consensus::{
 };
 use sc_network::{config::RequestResponseConfig, ProtocolName};
 use sc_network_test::{
-	Block, BlockImportAdapter, FullPeerConfig, PassThroughVerifier, Peer, PeersClient,
+	Block, BlockImportAdapter, Header, FullPeerConfig, PassThroughVerifier, Peer, PeersClient,
 	PeersFullClient, TestNetFactory,
 };
 use sc_utils::notification::NotificationReceiver;
@@ -1406,7 +1406,7 @@ async fn gossipped_finality_proofs() {
 
 	// Simulate Charlie vote on #2
 	let header = net.lock().peer(2).client().as_client().expect_header(finalize).unwrap();
-	let mmr_root = find_mmr_root_digest::<Block>(&header).unwrap();
+	let mmr_root = find_mmr_root_digest::<Header>(&header).unwrap();
 	let payload = Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode());
 	let commitment = Commitment { payload, block_number, validator_set_id: validator_set.id() };
 	let signature = sign_commitment(&BeefyKeyring::Charlie, &commitment);
diff --git a/substrate/primitives/consensus/beefy/src/mmr.rs b/substrate/primitives/consensus/beefy/src/mmr.rs
index b318d84475f7..6bee2c041923 100644
--- a/substrate/primitives/consensus/beefy/src/mmr.rs
+++ b/substrate/primitives/consensus/beefy/src/mmr.rs
@@ -134,7 +134,7 @@ pub struct BeefyAuthoritySet<AuthoritySetCommitment> {
 pub type BeefyNextAuthoritySet<MerkleRoot> = BeefyAuthoritySet<MerkleRoot>;
 
 /// Extract the MMR root hash from a digest in the given header, if it exists.
-pub fn find_mmr_root_digest<B: Block>(header: &B::Header) -> Option<MmrRootHash> {
+pub fn find_mmr_root_digest<H: Header>(header: &H) -> Option<MmrRootHash> {
 	let id = OpaqueDigestItemId::Consensus(&BEEFY_ENGINE_ID);
 
 	let filter = |log: ConsensusLog<AuthorityId>| match log {
@@ -181,7 +181,7 @@ mod mmr_root_provider {
 
 		/// Simple wrapper that gets MMR root from header digests or from client state.
 		fn mmr_root_from_digest_or_runtime(&self, header: &B::Header) -> Option<MmrRootHash> {
-			find_mmr_root_digest::<B>(header).or_else(|| {
+			find_mmr_root_digest::<B::Header>(header).or_else(|| {
 				self.runtime.runtime_api().mmr_root(header.hash()).ok().and_then(|r| r.ok())
 			})
 		}
@@ -233,7 +233,6 @@ mod tests {
 	#[test]
 	fn extract_mmr_root_digest() {
 		type Header = sp_runtime::generic::Header<u64, BlakeTwo256>;
-		type Block = sp_runtime::generic::Block<Header, OpaqueExtrinsic>;
 		let mut header = Header::new(
 			1u64,
 			Default::default(),
@@ -243,7 +242,7 @@ mod tests {
 		);
 
 		// verify empty digest shows nothing
-		assert!(find_mmr_root_digest::<Block>(&header).is_none());
+		assert!(find_mmr_root_digest::<Header>(&header).is_none());
 
 		let mmr_root_hash = H256::random();
 		header.digest_mut().push(DigestItem::Consensus(
@@ -252,7 +251,7 @@ mod tests {
 		));
 
 		// verify validator set is correctly extracted from digest
-		let extracted = find_mmr_root_digest::<Block>(&header);
+		let extracted = find_mmr_root_digest::<Header>(&header);
 		assert_eq!(extracted, Some(mmr_root_hash));
 	}
 }

From 103fc533d527f6f0f9d616cb7672057eabe9f474 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 8 Aug 2023 12:08:20 +0200
Subject: [PATCH 021/188] fix fork equiv. call interfaces (vecs of sigs)

---
 substrate/frame/beefy/src/lib.rs                 | 16 ++++++++--------
 substrate/primitives/consensus/beefy/src/mmr.rs  |  2 +-
 .../primitives/consensus/beefy/src/payload.rs    |  2 +-
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 37152bf207d4..9f1738ec8406 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -279,9 +279,10 @@ pub mod pallet {
 		/// invalid fork proof and validate the given key ownership proof
 		/// against the extracted offender. If both are valid, the offence
 		/// will be reported.
+		// TODO: fix key_owner_proofs[0].validator_count()
 		#[pallet::call_index(2)]
 		#[pallet::weight(T::WeightInfo::report_equivocation(
-			key_owner_proof.validator_count(),
+			key_owner_proofs[0].validator_count(),
 			T::MaxNominators::get(),
 		))]
 		pub fn report_fork_equivocation(
@@ -293,15 +294,14 @@ pub mod pallet {
 					<T::BeefyId as RuntimeAppPublic>::Signature,
 				>,
 			>,
-			key_owner_proof: T::KeyOwnerProof,
+			key_owner_proofs: Vec<T::KeyOwnerProof>,
 		) -> DispatchResultWithPostInfo {
 			let reporter = ensure_signed(origin)?;
 
-			// TODO:
-			// T::EquivocationReportSystem::process_evidence(
-			// 	Some(reporter),
-			// 	(*fork_equivocation_proof, key_owner_proof),
-			// )?;
+			T::EquivocationReportSystem::process_evidence(
+				Some(reporter),
+				EquivocationEvidenceFor::ForkEquivocationProof(*equivocation_proof, key_owner_proofs),
+			)?;
 			// Waive the fee since the report is valid and beneficial
 			Ok(Pays::No.into())
 		}
@@ -315,7 +315,7 @@ pub mod pallet {
 		/// block authors will call it (validated in `ValidateUnsigned`), as such
 		/// if the block author is defined it will be defined as the equivocation
 		/// reporter.
-		/// TODO: fix key_owner_proofs[0].validator_count()
+		// TODO: fix key_owner_proofs[0].validator_count()
 		#[pallet::call_index(3)]
 		#[pallet::weight(T::WeightInfo::report_equivocation(key_owner_proofs[0].validator_count(), T::MaxNominators::get(),))]
 		pub fn report_fork_equivocation_unsigned(
diff --git a/substrate/primitives/consensus/beefy/src/mmr.rs b/substrate/primitives/consensus/beefy/src/mmr.rs
index 6bee2c041923..6df5925acc19 100644
--- a/substrate/primitives/consensus/beefy/src/mmr.rs
+++ b/substrate/primitives/consensus/beefy/src/mmr.rs
@@ -205,7 +205,7 @@ mod mmr_root_provider {
 mod tests {
 	use super::*;
 	use crate::H256;
-	use sp_runtime::{traits::BlakeTwo256, Digest, DigestItem, OpaqueExtrinsic};
+	use sp_runtime::{traits::BlakeTwo256, Digest, DigestItem};
 
 	#[test]
 	fn should_construct_version_correctly() {
diff --git a/substrate/primitives/consensus/beefy/src/payload.rs b/substrate/primitives/consensus/beefy/src/payload.rs
index c80f00255281..43b83911dd62 100644
--- a/substrate/primitives/consensus/beefy/src/payload.rs
+++ b/substrate/primitives/consensus/beefy/src/payload.rs
@@ -43,7 +43,7 @@ pub mod known_payloads {
 pub struct Payload(Vec<(BeefyPayloadId, Vec<u8>)>);
 
 impl Payload {
-	/// Construct a new payload given an initial vallue
+	/// Construct a new payload given an initial value
 	pub fn from_single_entry(id: BeefyPayloadId, value: Vec<u8>) -> Self {
 		Self(vec![(id, value)])
 	}

From 1509a26b22b2c5fea2ed7da17937d7a42a7543e9 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 8 Aug 2023 12:14:15 +0200
Subject: [PATCH 022/188] check proof's payload against correct header's

---
 .../beefy/src/communication/fisherman.rs      | 24 +++++-----
 substrate/frame/beefy/src/equivocation.rs     | 20 +++++++--
 substrate/frame/beefy/src/lib.rs              |  6 ++-
 .../primitives/consensus/beefy/src/lib.rs     | 45 ++++++++++++-------
 4 files changed, 62 insertions(+), 33 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 4b79a20eea8e..714195835605 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -113,12 +113,12 @@ where
 		let proof = ForkEquivocationProof {
 			commitment: signed_commitment.commitment.clone(),
 			signatories: vec![],
-			expected_payload: correct_payload.clone(),
+			correct_header: correct_header.clone(),
 		};
 
 		if signed_commitment.commitment.validator_set_id != set_id ||
 			signed_commitment.commitment.payload != *correct_payload ||
-			!check_fork_equivocation_proof::<NumberFor<B>, AuthorityId, BeefySignatureHasher>(&proof)
+			!check_fork_equivocation_proof::<NumberFor<B>, AuthorityId, BeefySignatureHasher, B::Header>(&proof)
 		{
 			debug!(target: LOG_TARGET, "🥩 Skip report for bad invalid fork proof {:?}", proof);
 			return Ok(())
@@ -155,14 +155,14 @@ where
 
 	fn report_fork_equivocation(
 		&self,
-		proof: ForkEquivocationProof<NumberFor<B>, AuthorityId, Signature>,
+		proof: ForkEquivocationProof<NumberFor<B>, AuthorityId, Signature, B::Header>,
 		correct_header: &B::Header,
 	) -> Result<(), Error> {
 		let validator_set = self.active_validator_set_at(correct_header)?;
 		let set_id = validator_set.id();
 
 		if proof.commitment.validator_set_id != set_id ||
-			!check_fork_equivocation_proof::<NumberFor<B>, AuthorityId, BeefySignatureHasher>(&proof)
+			!check_fork_equivocation_proof::<NumberFor<B>, AuthorityId, BeefySignatureHasher, B::Header>(&proof)
 		{
 			debug!(target: LOG_TARGET, "🥩 Skip report for bad invalid fork proof {:?}", proof);
 			return Ok(())
@@ -215,11 +215,11 @@ where
 		vote: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
 	) -> Result<(), Error> {
 		let number = vote.commitment.block_number;
-		let (header, expected_payload) = self.expected_header_and_payload(number)?;
+		let (correct_header, expected_payload) = self.expected_header_and_payload(number)?;
 		if vote.commitment.payload != expected_payload {
-			let validator_set = self.active_validator_set_at(&header)?;
-			let proof = ForkEquivocationProof { commitment: vote.commitment, signatories: vec![(vote.id, vote.signature)], expected_payload };
-			self.report_fork_equivocation(proof, &header)?;
+			let validator_set = self.active_validator_set_at(&correct_header)?;
+			let proof = ForkEquivocationProof { commitment: vote.commitment, signatories: vec![(vote.id, vote.signature)], correct_header: correct_header.clone() };
+			self.report_fork_equivocation(proof, &correct_header)?;
 		}
 		Ok(())
 	}
@@ -246,10 +246,10 @@ where
 			BeefyVersionedFinalityProof::<B>::V1(inner) => (inner.commitment, inner.signatures),
 		};
 		let number = commitment.block_number;
-		let (header, expected_payload) = self.expected_header_and_payload(number)?;
+		let (correct_header, expected_payload) = self.expected_header_and_payload(number)?;
 		if commitment.payload != expected_payload {
 			// TODO: create/get grandpa proof for block number
-			let validator_set = self.active_validator_set_at(&header)?;
+			let validator_set = self.active_validator_set_at(&correct_header)?;
 			if signatures.len() != validator_set.validators().len() {
 				// invalid proof
 				return Ok(())
@@ -258,10 +258,10 @@ where
 			let signatories = validator_set.validators().iter().cloned().zip(signatures.into_iter())
 																		.filter_map(|(id, signature)| signature.map(|sig| (id, sig))).collect();
 
-			let proof = ForkEquivocationProof { commitment, signatories, expected_payload };
+			let proof = ForkEquivocationProof { commitment, signatories, correct_header: correct_header.clone() };
 			self.report_fork_equivocation(
 				proof,
-				&header,
+				&correct_header,
 			)?;
 		}
 		Ok(())
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index debb10807995..5a2b43128a06 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -36,10 +36,11 @@
 
 use codec::{self as codec, Decode, Encode};
 use frame_support::traits::{Get, KeyOwnerProofSystem};
-use frame_system::pallet_prelude::BlockNumberFor;
+use frame_system::pallet_prelude::{BlockNumberFor, HeaderFor};
 use log::{error, info};
 use sp_consensus_beefy::{VoteEquivocationProof, ForkEquivocationProof, ValidatorSetId, KEY_TYPE as BEEFY_KEY_TYPE};
 use sp_runtime::{
+	traits::Zero,
 	transaction_validity::{
 		InvalidTransaction, TransactionPriority, TransactionSource, TransactionValidity,
 		TransactionValidityError, ValidTransaction,
@@ -135,6 +136,7 @@ pub enum EquivocationEvidenceFor<T: Config> {
 		ForkEquivocationProof<BlockNumberFor<T>,
 							  <T as Config>::BeefyId,
 							  <<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
+							  HeaderFor<T>,
 							  >,
 		Vec<<T as Config>::KeyOwnerProof>,
 	)
@@ -258,6 +260,18 @@ where
 				}
 			},
 			EquivocationEvidenceFor::ForkEquivocationProof(equivocation_proof, _) => {
+				use sp_runtime::traits::Header;
+				let block_number = equivocation_proof.commitment.block_number;
+				let correct_header = &equivocation_proof.correct_header;
+
+				// Check that the provided header is correct.
+				if block_number <= BlockNumberFor::<T>::zero() ||
+					<frame_system::Pallet<T>>::block_hash(block_number) != correct_header.hash()
+				{
+					// TODO: maybe have a specific error for this
+					return Err(Error::<T>::InvalidForkEquivocationProof.into())
+				}
+
 				// Validate equivocation proof (check commitment is to unexpected payload and signatures are valid).
 				if !sp_consensus_beefy::check_fork_equivocation_proof(&equivocation_proof) {
 					return Err(Error::<T>::InvalidForkEquivocationProof.into())
@@ -274,7 +288,7 @@ where
 		}
 
 		let offence = EquivocationOffence {
-			time_slot: TimeSlot { set_id, round },
+			time_slot: TimeSlot set_id, round,
 			session_index,
 			validator_set_count,
 			offenders,
@@ -293,7 +307,7 @@ where
 /// unsigned equivocation reports.
 impl<T: Config> Pallet<T> {
 	pub fn validate_unsigned(source: TransactionSource, call: &Call<T>) -> TransactionValidity {
-		if let Call::report_vote_equivocation_unsigned { equivocation_proof, key_owner_proof } = call {
+		if let Call::report_vote_equivocation_unsigned equivocation_proof, key_owner_proof = call {
 			// discard equivocation report not coming from the local node
 			match source {
 				TransactionSource::Local | TransactionSource::InBlock => { /* allowed */ },
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 9f1738ec8406..f6cc6bf78ff8 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -28,7 +28,7 @@ use frame_support::{
 };
 use frame_system::{
 	ensure_none, ensure_signed,
-	pallet_prelude::{BlockNumberFor, OriginFor},
+	pallet_prelude::{BlockNumberFor, OriginFor, HeaderFor},
 };
 use log;
 use sp_runtime::{
@@ -292,6 +292,7 @@ pub mod pallet {
 					BlockNumberFor<T>,
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
+					HeaderFor<T>
 				>,
 			>,
 			key_owner_proofs: Vec<T::KeyOwnerProof>,
@@ -325,6 +326,7 @@ pub mod pallet {
 					BlockNumberFor<T>,
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
+					HeaderFor<T>,
 				>,
 			>,
 			key_owner_proofs: Vec<T::KeyOwnerProof>,
@@ -388,7 +390,7 @@ impl<T: Config> Pallet<T> {
 	/// to the pool. Only useful in an offchain context.
 	pub fn submit_unsigned_fork_equivocation_report(
 		_fork_equivocation_proof: sp_consensus_beefy::ForkEquivocationProof<BlockNumberFor<T>, T::BeefyId,
-			<T::BeefyId as RuntimeAppPublic>::Signature,
+			<T::BeefyId as RuntimeAppPublic>::Signature, HeaderFor<T>
 >,
 		_key_owner_proofs: Vec<T::KeyOwnerProof>,
 	) -> Option<()> {
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index bf1196adce2f..2ace78a29cc9 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -47,7 +47,7 @@ use codec::{Codec, Decode, Encode};
 use scale_info::TypeInfo;
 use sp_application_crypto::RuntimeAppPublic;
 use sp_core::H256;
-use sp_runtime::traits::{Hash, Keccak256, NumberFor};
+use sp_runtime::traits::{Hash, Keccak256, NumberFor, Header};
 use sp_std::prelude::*;
 
 /// Key type for BEEFY module.
@@ -254,7 +254,7 @@ impl<Number, Id, Signature> VoteEquivocationProof<Number, Id, Signature> {
 /// Proof of authority misbehavior on a given set id.
 /// This proof shows commitment signed on a different fork.
 #[derive(Clone, Debug, Decode, Encode, PartialEq, TypeInfo)]
-pub struct ForkEquivocationProof<Number, Id, Signature> {
+pub struct ForkEquivocationProof<Number, Id, Signature, Header> {
 	/// Commitment for a block on different fork than one at the same height in
 	/// this client's chain.
 	/// TODO: maybe replace {commitment, signatories} with SignedCommitment
@@ -264,11 +264,14 @@ pub struct ForkEquivocationProof<Number, Id, Signature> {
 	/// Signatures on this block
 	/// TODO: maybe change to HashMap - check once usage pattern is clear
 	pub signatories: Vec<(Id, Signature)>,
-	/// Expected payload
-	pub expected_payload: Payload,
+	/// The proof is valid if
+	/// 1. the header is in our chain
+	/// 2. its digest's payload != commitment.payload
+	/// 3. commitment is signed by signatories
+	pub correct_header: Header,
 }
 
-impl<Number, Id, Signature> ForkEquivocationProof<Number, Id, Signature> {
+impl<Number, Id, Signature, H: Header> ForkEquivocationProof<Number, Id, Signature, H> {
 	/// Returns the authority id of the misbehaving voter.
 	pub fn offender_ids(&self) -> Vec<&Id> {
 		self.signatories.iter().map(|(id, _)| id).collect()
@@ -334,9 +337,10 @@ where
 	return valid_first && valid_second
 }
 
-/// Validates [InvalidForkVoteProof] by checking:
-/// 1. `vote` is signed,
-/// 2. `vote.commitment.payload` != `expected_payload`.
+/// Validates [ForkEquivocationProof] by checking:
+/// 1. `commitment` is signed,
+/// 2. `correct_header` is valid and matches `commitment.block_number`.
+/// 2. `commitment.payload` != `expected_payload(correct_header)`.
 /// NOTE: GRANDPA finalization proof is not checked, which leads to slashing on forks.
 /// This is fine since honest validators will not be slashed on the chain finalized
 /// by GRANDPA, which is the only chain that ultimately matters.
@@ -345,19 +349,28 @@ where
 /// finalized by GRANDPA. This is fine too, since the slashing risk of committing to
 /// an incorrect block implies validators will only sign blocks they *know* will be
 /// finalized by GRANDPA.
-pub fn check_fork_equivocation_proof<Number, Id, MsgHash>(
-	proof: &ForkEquivocationProof<Number, Id, <Id as RuntimeAppPublic>::Signature>,
+pub fn check_fork_equivocation_proof<Number, Id, MsgHash, Header>(
+	proof: &ForkEquivocationProof<Number, Id, <Id as RuntimeAppPublic>::Signature, Header>,
 ) -> bool
 where
 	Id: BeefyAuthorityId<MsgHash> + PartialEq,
 	Number: Clone + Encode + PartialEq,
 	MsgHash: Hash,
+	Header: sp_api::HeaderT,
 {
-	let ForkEquivocationProof { commitment, signatories, expected_payload } = proof;
-
-	// check that `payload` on the `vote` is different that the `expected_payload` (checked first
-	// since cheap failfast).
-	if &commitment.payload != expected_payload {
+	let ForkEquivocationProof { commitment, signatories, correct_header } = proof;
+
+	let expected_mmr_root_digest = mmr::find_mmr_root_digest::<Header>(correct_header);
+	let expected_payload = expected_mmr_root_digest.map(|mmr_root| {
+		Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode())
+	});
+
+	// cheap failfasts:
+	// 1. check that `payload` on the `vote` is different that the `expected_payload`
+	// 2. if the signatories signed a payload when there should be none (for
+	// instance for a block prior to BEEFY activation), they should likewise be
+	// slashed
+	if Some(&commitment.payload) != expected_payload.as_ref() || expected_mmr_root_digest.is_none() {
 		// check check each signatory's signature on the commitment.
 		// if any are invalid, equivocation report is invalid
 		// TODO: refactor check_commitment_signature to take a slice of signatories
@@ -442,7 +455,7 @@ sp_api::decl_runtime_apis! {
 		/// hardcoded to return `None`). Only useful in an offchain context.
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
 			fork_equivocation_proof:
-				ForkEquivocationProof<NumberFor<Block>, AuthorityId, <AuthorityId as RuntimeAppPublic>::Signature>,
+				ForkEquivocationProof<NumberFor<Block>, AuthorityId, <AuthorityId as RuntimeAppPublic>::Signature, Block::Header>,
 			key_owner_proofs: Vec<OpaqueKeyOwnershipProof>,
 		) -> Option<()>;
 

From a62cc7fd8c62c74a6d393b49197630ffad1aafd9 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 8 Aug 2023 12:22:42 +0200
Subject: [PATCH 023/188] rm superfluous report_fork_equiv.correct_header

---
 .../consensus/beefy/src/communication/fisherman.rs   | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 714195835605..023d8edcecc0 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -156,9 +156,8 @@ where
 	fn report_fork_equivocation(
 		&self,
 		proof: ForkEquivocationProof<NumberFor<B>, AuthorityId, Signature, B::Header>,
-		correct_header: &B::Header,
 	) -> Result<(), Error> {
-		let validator_set = self.active_validator_set_at(correct_header)?;
+		let validator_set = self.active_validator_set_at(&proof.correct_header)?;
 		let set_id = validator_set.id();
 
 		if proof.commitment.validator_set_id != set_id ||
@@ -168,7 +167,7 @@ where
 			return Ok(())
 		}
 
-		let hash = correct_header.hash();
+		let hash = proof.correct_header.hash();
 		let offender_ids = proof.signatories.iter().cloned().map(|(id, _sig)| id).collect::<Vec<_>>();
 		let runtime_api = self.runtime.runtime_api();
 
@@ -219,7 +218,7 @@ where
 		if vote.commitment.payload != expected_payload {
 			let validator_set = self.active_validator_set_at(&correct_header)?;
 			let proof = ForkEquivocationProof { commitment: vote.commitment, signatories: vec![(vote.id, vote.signature)], correct_header: correct_header.clone() };
-			self.report_fork_equivocation(proof, &correct_header)?;
+			self.report_fork_equivocation(proof)?;
 		}
 		Ok(())
 	}
@@ -259,10 +258,7 @@ where
 																		.filter_map(|(id, signature)| signature.map(|sig| (id, sig))).collect();
 
 			let proof = ForkEquivocationProof { commitment, signatories, correct_header: correct_header.clone() };
-			self.report_fork_equivocation(
-				proof,
-				&correct_header,
-			)?;
+			self.report_fork_equivocation(proof)?;
 		}
 		Ok(())
 	}

From 544597764f9e0f8c417389d3908cdff16d32d4a8 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 8 Aug 2023 12:53:36 +0200
Subject: [PATCH 024/188] remove duplic. in check_{signed_commitment, proof}

check_signed commitment wasn't complete anyway.
good to have both interfaces since BeefyVersionedFinalityProof is what's
passed on the gossip layer, and SignedCommitments are naturally
reconstructed from multiple sources, for instance submit_initial calls
on Ethereum.
---
 .../beefy/src/communication/fisherman.rs      | 29 +++++++------------
 1 file changed, 11 insertions(+), 18 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 023d8edcecc0..deba4876e05e 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -223,31 +223,15 @@ where
 		Ok(())
 	}
 
-	/// Check `commitment` for contained block against expected payload.
+	/// Check `signed_commitment` for contained block against expected payload.
 	fn check_signed_commitment(
 		&self,
 		signed_commitment: SignedCommitment<NumberFor<B>, Signature>,
 	) -> Result<(), Error> {
-		let number = signed_commitment.commitment.block_number;
-		let (header, expected_payload) = self.expected_header_and_payload(number)?;
-		if signed_commitment.commitment.payload != expected_payload {
-			let validator_set = self.active_validator_set_at(&header)?;
-			self.report_invalid_payload(signed_commitment, &expected_payload, &header)?;
-		}
-		Ok(())
-	}
-
-	/// Check `proof` for contained block against expected payload.
-	///
-	/// Note: this fn expects block referenced in `proof` to be finalized.
-	fn check_proof(&self, proof: BeefyVersionedFinalityProof<B>) -> Result<(), Error> {
-		let (commitment, signatures) = match proof {
-			BeefyVersionedFinalityProof::<B>::V1(inner) => (inner.commitment, inner.signatures),
-		};
+		let SignedCommitment {commitment, signatures} = signed_commitment;
 		let number = commitment.block_number;
 		let (correct_header, expected_payload) = self.expected_header_and_payload(number)?;
 		if commitment.payload != expected_payload {
-			// TODO: create/get grandpa proof for block number
 			let validator_set = self.active_validator_set_at(&correct_header)?;
 			if signatures.len() != validator_set.validators().len() {
 				// invalid proof
@@ -262,4 +246,13 @@ where
 		}
 		Ok(())
 	}
+
+	/// Check `proof` for contained block against expected payload.
+	fn check_proof(&self, proof: BeefyVersionedFinalityProof<B>) -> Result<(), Error> {
+		match proof {
+			BeefyVersionedFinalityProof::<B>::V1(signed_commitment) => {
+				self.check_signed_commitment(signed_commitment)
+			}
+		}
+	}
 }

From d35a97c32b80d1e05adfb7514f8c0dc535d1808b Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 8 Aug 2023 15:50:37 +0200
Subject: [PATCH 025/188] update outdated comments

---
 .../consensus/beefy/src/communication/fisherman.rs    | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index deba4876e05e..d92b64f62751 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -36,22 +36,19 @@ use sp_runtime::{
 use std::{marker::PhantomData, sync::Arc};
 
 pub(crate) trait BeefyFisherman<B: Block>: Send + Sync {
-	/// Check `vote` for contained finalized block against expected payload.
-	///
-	/// Note: this fn expects `vote.commitment.block_number` to be finalized.
+	/// Check `vote` for contained block against expected payload.
 	fn check_vote(
 		&self,
 		vote: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
 	) -> Result<(), Error>;
 
+	/// Check `signed_commitment` for contained block against expected payload.
 	fn check_signed_commitment(
 		&self,
 		signed_commitment: SignedCommitment<NumberFor<B>, Signature>,
 	) -> Result<(), Error>;
 
-	/// Check `proof` for contained finalized block against expected payload.
-	///
-	/// Note: this fn expects block referenced in `proof` to be finalized.
+	/// Check `proof` for contained block against expected payload.
 	fn check_proof(&self, proof: BeefyVersionedFinalityProof<B>) -> Result<(), Error>;
 }
 
@@ -207,8 +204,6 @@ where
 	R::Api: BeefyApi<B, AuthorityId>,
 {
 	/// Check `vote` for contained block against expected payload.
-	///
-	/// Note: this fn expects `vote.commitment.block_number` to be finalized.
 	fn check_vote(
 		&self,
 		vote: VoteMessage<NumberFor<B>, AuthorityId, Signature>,

From 4e3e1cccbd4e30d2632d664cb4360ea1c155ca1e Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 8 Aug 2023 23:21:21 +0200
Subject: [PATCH 026/188] remove report_invalid_payload

redundant vs report_fork_equivocation
---
 .../beefy/src/communication/fisherman.rs      | 52 -------------------
 1 file changed, 52 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index d92b64f62751..757de88563c0 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -98,58 +98,6 @@ where
 			.ok_or_else(|| Error::Backend("could not get BEEFY validator set".into()))
 	}
 
-	fn report_invalid_payload(
-		&self,
-		signed_commitment: SignedCommitment<NumberFor<B>, Signature>,
-		correct_payload: &Payload,
-		correct_header: &B::Header,
-	) -> Result<(), Error> {
-		let validator_set = self.active_validator_set_at(correct_header)?;
-		let set_id = validator_set.id();
-
-		let proof = ForkEquivocationProof {
-			commitment: signed_commitment.commitment.clone(),
-			signatories: vec![],
-			correct_header: correct_header.clone(),
-		};
-
-		if signed_commitment.commitment.validator_set_id != set_id ||
-			signed_commitment.commitment.payload != *correct_payload ||
-			!check_fork_equivocation_proof::<NumberFor<B>, AuthorityId, BeefySignatureHasher, B::Header>(&proof)
-		{
-			debug!(target: LOG_TARGET, "🥩 Skip report for bad invalid fork proof {:?}", proof);
-			return Ok(())
-		}
-
-		let offender_ids = proof.signatories.iter().cloned().map(|(id, _sig)| id).collect::<Vec<_>>();
-		let runtime_api = self.runtime.runtime_api();
-
-		// generate key ownership proof at that block
-		let key_owner_proofs = offender_ids.iter()
-										   .filter_map(|id| {
-											   match runtime_api.generate_key_ownership_proof(correct_header.hash(), set_id, id.clone()) {
-												   Ok(Some(proof)) => Some(Ok(proof)),
-												   Ok(None) => {
-													   debug!(
-														   target: LOG_TARGET,
-														   "🥩 Invalid fork vote offender not part of the authority set."
-													   );
-													   None
-												   },
-												   Err(e) => Some(Err(Error::RuntimeApi(e))),
-											   }
-										   })
-										   .collect::<Result<_, _>>()?;
-
-		// submit invalid fork vote report at **best** block
-		let best_block_hash = self.backend.blockchain().info().best_hash;
-		runtime_api
-			.submit_report_fork_equivocation_unsigned_extrinsic(best_block_hash, proof, key_owner_proofs)
-			.map_err(Error::RuntimeApi)?;
-
-		Ok(())
-	}
-
 	fn report_fork_equivocation(
 		&self,
 		proof: ForkEquivocationProof<NumberFor<B>, AuthorityId, Signature, B::Header>,

From f27a8765c607091c26ebbcf3e267456cb130481b Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Wed, 9 Aug 2023 00:37:09 +0200
Subject: [PATCH 027/188] .+report.+{""->_vote}_equivocations

---
 substrate/client/consensus/beefy/src/tests.rs | 24 +++++++++++--------
 .../client/consensus/beefy/src/worker.rs      | 18 +++++++-------
 .../consensus/beefy/src/test_utils.rs         |  4 ++--
 3 files changed, 25 insertions(+), 21 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index dc9ce46e8c31..f6ae401ab838 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -270,7 +270,9 @@ pub(crate) struct TestApi {
 	pub beefy_genesis: u64,
 	pub validator_set: BeefyValidatorSet,
 	pub mmr_root_hash: MmrRootHash,
-	pub reported_equivocations:
+	pub reported_vote_equivocations:
+		Option<Arc<Mutex<Vec<VoteEquivocationProof<NumberFor<Block>, AuthorityId, Signature>>>>>,
+	pub reported_fork_equivocations:
 		Option<Arc<Mutex<Vec<VoteEquivocationProof<NumberFor<Block>, AuthorityId, Signature>>>>>,
 }
 
@@ -284,7 +286,8 @@ impl TestApi {
 			beefy_genesis,
 			validator_set: validator_set.clone(),
 			mmr_root_hash,
-			reported_equivocations: None,
+			reported_vote_equivocations: None,
+			reported_fork_equivocations: None,
 		}
 	}
 
@@ -293,12 +296,13 @@ impl TestApi {
 			beefy_genesis: 1,
 			validator_set: validator_set.clone(),
 			mmr_root_hash: GOOD_MMR_ROOT,
-			reported_equivocations: None,
+			reported_vote_equivocations: None,
+			reported_fork_equivocations: None,
 		}
 	}
 
 	pub fn allow_equivocations(&mut self) {
-		self.reported_equivocations = Some(Arc::new(Mutex::new(vec![])));
+		self.reported_vote_equivocations = Some(Arc::new(Mutex::new(vec![])));
 	}
 }
 
@@ -328,7 +332,7 @@ sp_api::mock_impl_runtime_apis! {
 			proof: VoteEquivocationProof<NumberFor<Block>, AuthorityId, Signature>,
 			_dummy: OpaqueKeyOwnershipProof,
 		) -> Option<()> {
-			if let Some(equivocations_buf) = self.inner.reported_equivocations.as_ref() {
+			if let Some(equivocations_buf) = self.inner.reported_vote_equivocations.as_ref() {
 				equivocations_buf.lock().push(proof);
 				None
 			} else {
@@ -1245,7 +1249,7 @@ async fn beefy_finalizing_after_pallet_genesis() {
 }
 
 #[tokio::test]
-async fn beefy_reports_equivocations() {
+async fn beefy_reports_vote_equivocations() {
 	sp_tracing::try_init_simple();
 
 	let peers = [BeefyKeyring::Alice, BeefyKeyring::Bob, BeefyKeyring::Charlie];
@@ -1295,21 +1299,21 @@ async fn beefy_reports_equivocations() {
 	// run for up to 5 seconds waiting for Alice's report of Bob/Bob_Prime equivocation.
 	for wait_ms in [250, 500, 1250, 3000] {
 		run_for(Duration::from_millis(wait_ms), &net).await;
-		if !api_alice.reported_equivocations.as_ref().unwrap().lock().is_empty() {
+		if !api_alice.reported_vote_equivocations.as_ref().unwrap().lock().is_empty() {
 			break
 		}
 	}
 
 	// Verify expected equivocation
-	let alice_reported_equivocations = api_alice.reported_equivocations.as_ref().unwrap().lock();
+	let alice_reported_equivocations = api_alice.reported_vote_equivocations.as_ref().unwrap().lock();
 	assert_eq!(alice_reported_equivocations.len(), 1);
 	let equivocation_proof = alice_reported_equivocations.get(0).unwrap();
 	assert_eq!(equivocation_proof.first.id, BeefyKeyring::Bob.public());
 	assert_eq!(equivocation_proof.first.commitment.block_number, 1);
 
 	// Verify neither Bob or Bob_Prime report themselves as equivocating.
-	assert!(api_bob.reported_equivocations.as_ref().unwrap().lock().is_empty());
-	assert!(api_bob_prime.reported_equivocations.as_ref().unwrap().lock().is_empty());
+	assert!(api_bob.reported_vote_equivocations.as_ref().unwrap().lock().is_empty());
+	assert!(api_bob_prime.reported_vote_equivocations.as_ref().unwrap().lock().is_empty());
 
 	// sanity verify no new blocks have been finalized by BEEFY
 	streams_empty_after_timeout(best_blocks, &net, None).await;
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index c7430767b159..f904ae4a753b 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -1049,7 +1049,7 @@ pub(crate) mod tests {
 	use sp_api::HeaderT;
 	use sp_blockchain::Backend as BlockchainBackendT;
 	use sp_consensus_beefy::{
-		generate_equivocation_proof, known_payloads, known_payloads::MMR_ROOT_ID,
+		generate_vote_equivocation_proof, known_payloads, known_payloads::MMR_ROOT_ID,
 		mmr::MmrRootProvider, Keyring, Payload, SignedCommitment,
 	};
 	use sp_runtime::traits::One;
@@ -1597,7 +1597,7 @@ pub(crate) mod tests {
 	}
 
 	#[tokio::test]
-	async fn should_not_report_bad_old_or_self_equivocations() {
+	async fn should_not_report_bad_old_or_self_vote_equivocations() {
 		let block_num = 1;
 		let set_id = 1;
 		let keys = [Keyring::Alice];
@@ -1618,7 +1618,7 @@ pub(crate) mod tests {
 		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
 
 		// generate an equivocation proof, with Bob as perpetrator
-		let good_proof = generate_equivocation_proof(
+		let good_proof = generate_vote_equivocation_proof(
 			(block_num, payload1.clone(), set_id, &Keyring::Bob),
 			(block_num, payload2.clone(), set_id, &Keyring::Bob),
 		);
@@ -1626,11 +1626,11 @@ pub(crate) mod tests {
 			// expect voter (Alice) to successfully report it
 			assert_eq!(worker.report_equivocation(good_proof.clone()), Ok(()));
 			// verify Alice reports Bob equivocation to runtime
-			let reported = api_alice.reported_equivocations.as_ref().unwrap().lock();
+			let reported = api_alice.reported_vote_equivocations.as_ref().unwrap().lock();
 			assert_eq!(reported.len(), 1);
 			assert_eq!(*reported.get(0).unwrap(), good_proof);
 		}
-		api_alice.reported_equivocations.as_ref().unwrap().lock().clear();
+		api_alice.reported_vote_equivocations.as_ref().unwrap().lock().clear();
 
 		// now let's try with a bad proof
 		let mut bad_proof = good_proof.clone();
@@ -1638,7 +1638,7 @@ pub(crate) mod tests {
 		// bad proofs are simply ignored
 		assert_eq!(worker.report_equivocation(bad_proof), Ok(()));
 		// verify nothing reported to runtime
-		assert!(api_alice.reported_equivocations.as_ref().unwrap().lock().is_empty());
+		assert!(api_alice.reported_vote_equivocations.as_ref().unwrap().lock().is_empty());
 
 		// now let's try with old set it
 		let mut old_proof = good_proof.clone();
@@ -1647,16 +1647,16 @@ pub(crate) mod tests {
 		// old proofs are simply ignored
 		assert_eq!(worker.report_equivocation(old_proof), Ok(()));
 		// verify nothing reported to runtime
-		assert!(api_alice.reported_equivocations.as_ref().unwrap().lock().is_empty());
+		assert!(api_alice.reported_vote_equivocations.as_ref().unwrap().lock().is_empty());
 
 		// now let's try reporting a self-equivocation
-		let self_proof = generate_equivocation_proof(
+		let self_proof = generate_vote_equivocation_proof(
 			(block_num, payload1.clone(), set_id, &Keyring::Alice),
 			(block_num, payload2.clone(), set_id, &Keyring::Alice),
 		);
 		// equivocations done by 'self' are simply ignored (not reported)
 		assert_eq!(worker.report_equivocation(self_proof), Ok(()));
 		// verify nothing reported to runtime
-		assert!(api_alice.reported_equivocations.as_ref().unwrap().lock().is_empty());
+		assert!(api_alice.reported_vote_equivocations.as_ref().unwrap().lock().is_empty());
 	}
 }
diff --git a/substrate/primitives/consensus/beefy/src/test_utils.rs b/substrate/primitives/consensus/beefy/src/test_utils.rs
index 12190a701ab2..36fe05f3aa31 100644
--- a/substrate/primitives/consensus/beefy/src/test_utils.rs
+++ b/substrate/primitives/consensus/beefy/src/test_utils.rs
@@ -91,8 +91,8 @@ impl From<Keyring> for ecdsa_crypto::Public {
 	}
 }
 
-/// Create a new `EquivocationProof` based on given arguments.
-pub fn generate_equivocation_proof(
+/// Create a new `VoteEquivocationProof` based on given arguments.
+pub fn generate_vote_equivocation_proof(
 	vote1: (u64, Payload, ValidatorSetId, &Keyring),
 	vote2: (u64, Payload, ValidatorSetId, &Keyring),
 ) -> VoteEquivocationProof<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature> {

From f64175936cb9e6bdae7f764b4f359a749b4b02e3 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Wed, 9 Aug 2023 11:05:09 +0200
Subject: [PATCH 028/188] move correct_header hash check into primitives

---
 .../consensus/beefy/src/communication/fisherman.rs |  9 +++++++--
 substrate/frame/beefy/src/equivocation.rs          | 14 ++------------
 substrate/primitives/consensus/beefy/src/lib.rs    |  5 +++++
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 757de88563c0..36c4f4cd6482 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -105,8 +105,14 @@ where
 		let validator_set = self.active_validator_set_at(&proof.correct_header)?;
 		let set_id = validator_set.id();
 
+		let expected_header_hash = self
+			.backend
+			.blockchain()
+			.expect_block_hash_from_id(&BlockId::Number(proof.commitment.block_number))
+			.map_err(|e| Error::Backend(e.to_string()))?;
+
 		if proof.commitment.validator_set_id != set_id ||
-			!check_fork_equivocation_proof::<NumberFor<B>, AuthorityId, BeefySignatureHasher, B::Header>(&proof)
+			!check_fork_equivocation_proof::<NumberFor<B>, AuthorityId, BeefySignatureHasher, B::Header>(&proof, &expected_header_hash)
 		{
 			debug!(target: LOG_TARGET, "🥩 Skip report for bad invalid fork proof {:?}", proof);
 			return Ok(())
@@ -159,7 +165,6 @@ where
 		let number = vote.commitment.block_number;
 		let (correct_header, expected_payload) = self.expected_header_and_payload(number)?;
 		if vote.commitment.payload != expected_payload {
-			let validator_set = self.active_validator_set_at(&correct_header)?;
 			let proof = ForkEquivocationProof { commitment: vote.commitment, signatories: vec![(vote.id, vote.signature)], correct_header: correct_header.clone() };
 			self.report_fork_equivocation(proof)?;
 		}
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 5a2b43128a06..2f55c807391d 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -40,7 +40,6 @@ use frame_system::pallet_prelude::{BlockNumberFor, HeaderFor};
 use log::{error, info};
 use sp_consensus_beefy::{VoteEquivocationProof, ForkEquivocationProof, ValidatorSetId, KEY_TYPE as BEEFY_KEY_TYPE};
 use sp_runtime::{
-	traits::Zero,
 	transaction_validity::{
 		InvalidTransaction, TransactionPriority, TransactionSource, TransactionValidity,
 		TransactionValidityError, ValidTransaction,
@@ -260,20 +259,11 @@ where
 				}
 			},
 			EquivocationEvidenceFor::ForkEquivocationProof(equivocation_proof, _) => {
-				use sp_runtime::traits::Header;
 				let block_number = equivocation_proof.commitment.block_number;
-				let correct_header = &equivocation_proof.correct_header;
-
-				// Check that the provided header is correct.
-				if block_number <= BlockNumberFor::<T>::zero() ||
-					<frame_system::Pallet<T>>::block_hash(block_number) != correct_header.hash()
-				{
-					// TODO: maybe have a specific error for this
-					return Err(Error::<T>::InvalidForkEquivocationProof.into())
-				}
+				let expected_block_hash = <frame_system::Pallet<T>>::block_hash(block_number);
 
 				// Validate equivocation proof (check commitment is to unexpected payload and signatures are valid).
-				if !sp_consensus_beefy::check_fork_equivocation_proof(&equivocation_proof) {
+				if !sp_consensus_beefy::check_fork_equivocation_proof(&equivocation_proof, &expected_block_hash) {
 					return Err(Error::<T>::InvalidForkEquivocationProof.into())
 				}
 			},
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 2ace78a29cc9..ab14837d3191 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -351,6 +351,7 @@ where
 /// finalized by GRANDPA.
 pub fn check_fork_equivocation_proof<Number, Id, MsgHash, Header>(
 	proof: &ForkEquivocationProof<Number, Id, <Id as RuntimeAppPublic>::Signature, Header>,
+	expected_header_hash: &Header::Hash,
 ) -> bool
 where
 	Id: BeefyAuthorityId<MsgHash> + PartialEq,
@@ -360,6 +361,10 @@ where
 {
 	let ForkEquivocationProof { commitment, signatories, correct_header } = proof;
 
+	if correct_header.hash() != *expected_header_hash {
+		return false
+	}
+
 	let expected_mmr_root_digest = mmr::find_mmr_root_digest::<Header>(correct_header);
 	let expected_payload = expected_mmr_root_digest.map(|mmr_root| {
 		Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode())

From 5c4104c1657f26bdab0b0794b285018d3f83b212 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Wed, 9 Aug 2023 20:18:12 +0200
Subject: [PATCH 029/188] create_beefy_worker: only push block if at genesis

No need to trigger first session if chain's already had blocks built,
such as with generate_blocks_and_sync, which needs to build on genesis.
If chain is not at genesis and create_beefy_worker, it will panic trying
to canonicalize an invalid (first) block.
---
 substrate/client/consensus/beefy/src/worker.rs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index f904ae4a753b..b552eef06ab4 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -1145,9 +1145,11 @@ pub(crate) mod tests {
 			known_peers,
 			None,
 		);
-		// Push 1 block - will start first session.
-		let hashes = peer.push_blocks(1, false);
-		backend.finalize_block(hashes[0], None).unwrap();
+		// If chain's still at genesis, push 1 block to start first session.
+		if backend.blockchain().info().best_hash == backend.blockchain().info().genesis_hash {
+			let hashes = peer.push_blocks(1, false);
+			backend.finalize_block(hashes[0], None).unwrap();
+		}
 		let first_header = backend
 			.blockchain()
 			.expect_header(backend.blockchain().info().best_hash)

From a0d8b87ae027b42de87bfb53a8b04f1daed3daef Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Wed, 9 Aug 2023 20:33:41 +0200
Subject: [PATCH 030/188] create_beefy_worker: opt. instantiate with TestApi

can pass in Arc of TestApi now. Required since fisherman reports should be
pushed to `reported_fork_equivocations`, and should be logged with the
same instance (see upcoming commit).
---
 substrate/client/consensus/beefy/src/worker.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index b552eef06ab4..1c82dd0c39fb 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -1088,6 +1088,7 @@ pub(crate) mod tests {
 		key: &Keyring,
 		min_block_delta: u32,
 		genesis_validator_set: ValidatorSet<AuthorityId>,
+		runtime_api: Option<Arc<TestApi>>,
 	) -> BeefyWorker<
 		Block,
 		Backend,
@@ -1117,7 +1118,7 @@ pub(crate) mod tests {
 
 		let backend = peer.client().as_backend();
 		let beefy_genesis = 1;
-		let api = Arc::new(TestApi::with_validator_set(&genesis_validator_set));
+		let api = runtime_api.unwrap_or_else(|| Arc::new(TestApi::with_validator_set(&genesis_validator_set)));
 		let network = peer.network_service().clone();
 		let sync = peer.sync_service().clone();
 		let payload_provider = MmrRootProvider::new(api.clone());
@@ -1444,7 +1445,7 @@ pub(crate) mod tests {
 		let keys = &[Keyring::Alice];
 		let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap();
 		let mut net = BeefyTestNet::new(1);
-		let mut worker = create_beefy_worker(net.peer(0), &keys[0], 1, validator_set.clone());
+		let mut worker = create_beefy_worker(net.peer(0), &keys[0], 1, validator_set.clone(), None);
 
 		// keystore doesn't contain other keys than validators'
 		assert_eq!(worker.verify_validator_set(&1, &validator_set), Ok(()));
@@ -1468,7 +1469,7 @@ pub(crate) mod tests {
 		let validator_set = ValidatorSet::new(make_beefy_ids(&keys), 0).unwrap();
 		let mut net = BeefyTestNet::new(1);
 		let backend = net.peer(0).client().as_backend();
-		let mut worker = create_beefy_worker(net.peer(0), &keys[0], 1, validator_set.clone());
+		let mut worker = create_beefy_worker(net.peer(0), &keys[0], 1, validator_set.clone(), None);
 		// remove default session, will manually add custom one.
 		worker.persisted_state.voting_oracle.sessions.clear();
 
@@ -1572,7 +1573,7 @@ pub(crate) mod tests {
 		let keys = &[Keyring::Alice, Keyring::Bob];
 		let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap();
 		let mut net = BeefyTestNet::new(1);
-		let mut worker = create_beefy_worker(net.peer(0), &keys[0], 1, validator_set.clone());
+		let mut worker = create_beefy_worker(net.peer(0), &keys[0], 1, validator_set.clone(), None);
 
 		let worker_rounds = worker.active_rounds().unwrap();
 		assert_eq!(worker_rounds.session_start(), 1);
@@ -1610,8 +1611,7 @@ pub(crate) mod tests {
 		let api_alice = Arc::new(api_alice);
 
 		let mut net = BeefyTestNet::new(1);
-		let mut worker = create_beefy_worker(net.peer(0), &keys[0], 1, validator_set.clone());
-		worker.runtime = api_alice.clone();
+		let worker = create_beefy_worker(net.peer(0), &keys[0], 1, validator_set.clone(), Some(api_alice.clone()));
 
 		// let there be a block with num = 1:
 		let _ = net.peer(0).push_blocks(1, false);

From 912690450bf312e5c13843deaa1756bc4494c21f Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Wed, 9 Aug 2023 20:43:27 +0200
Subject: [PATCH 031/188] push to reported_fork_equivocations

mock api's `submit_report_fork_equivocation_unsigned_extrinsic` pushes
reported equivocations to runtime_api.reported_fork_equivocations
---
 substrate/client/consensus/beefy/src/tests.rs | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index f6ae401ab838..129ee5f2e72f 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -273,7 +273,7 @@ pub(crate) struct TestApi {
 	pub reported_vote_equivocations:
 		Option<Arc<Mutex<Vec<VoteEquivocationProof<NumberFor<Block>, AuthorityId, Signature>>>>>,
 	pub reported_fork_equivocations:
-		Option<Arc<Mutex<Vec<VoteEquivocationProof<NumberFor<Block>, AuthorityId, Signature>>>>>,
+		Option<Arc<Mutex<Vec<ForkEquivocationProof<NumberFor<Block>, AuthorityId, Signature, Header>>>>>,
 }
 
 impl TestApi {
@@ -303,6 +303,7 @@ impl TestApi {
 
 	pub fn allow_equivocations(&mut self) {
 		self.reported_vote_equivocations = Some(Arc::new(Mutex::new(vec![])));
+		self.reported_fork_equivocations = Some(Arc::new(Mutex::new(vec![])));
 	}
 }
 
@@ -340,6 +341,18 @@ sp_api::mock_impl_runtime_apis! {
 			}
 		}
 
+		fn submit_report_fork_equivocation_unsigned_extrinsic(
+			proof: ForkEquivocationProof<NumberFor<Block>, AuthorityId, Signature, Header>,
+			_dummy: Vec<OpaqueKeyOwnershipProof>,
+		) -> Option<()> {
+			if let Some(equivocations_buf) = self.inner.reported_fork_equivocations.as_ref() {
+				equivocations_buf.lock().push(proof);
+				None
+			} else {
+				panic!("Equivocations not expected, but following proof was reported: {:?}", proof);
+			}
+		}
+
 		fn generate_key_ownership_proof(
 			_dummy1: ValidatorSetId,
 			_dummy2: AuthorityId,

From 02da07eaf2753b6ef4abfe29af7647276c811081 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 10 Aug 2023 00:11:13 +0200
Subject: [PATCH 032/188] add generate_fork_equivocation_proof_vote to tests

Generates fork equivocation proofs from a vote and a header.
---
 substrate/client/consensus/beefy/src/tests.rs |  8 ++---
 substrate/frame/beefy/src/tests.rs            | 36 +++++++++----------
 .../consensus/beefy/src/test_utils.rs         | 26 +++++++++-----
 3 files changed, 39 insertions(+), 31 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index 129ee5f2e72f..12b7884c4a02 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -56,7 +56,7 @@ use sp_consensus_beefy::{
 	ecdsa_crypto::{AuthorityId, Signature},
 	known_payloads,
 	mmr::{find_mmr_root_digest, MmrRootProvider},
-	BeefyApi, Commitment, ConsensusLog, VoteEquivocationProof, Keyring as BeefyKeyring, MmrRootHash,
+	BeefyApi, Commitment, ConsensusLog, ForkEquivocationProof, VoteEquivocationProof, Keyring as BeefyKeyring, MmrRootHash,
 	OpaqueKeyOwnershipProof, Payload, SignedCommitment, ValidatorSet, ValidatorSetId,
 	VersionedFinalityProof, VoteMessage, BEEFY_ENGINE_ID,
 };
@@ -1318,9 +1318,9 @@ async fn beefy_reports_vote_equivocations() {
 	}
 
 	// Verify expected equivocation
-	let alice_reported_equivocations = api_alice.reported_vote_equivocations.as_ref().unwrap().lock();
-	assert_eq!(alice_reported_equivocations.len(), 1);
-	let equivocation_proof = alice_reported_equivocations.get(0).unwrap();
+	let alice_reported_vote_equivocations = api_alice.reported_vote_equivocations.as_ref().unwrap().lock();
+	assert_eq!(alice_reported_vote_equivocations.len(), 1);
+	let equivocation_proof = alice_reported_vote_equivocations.get(0).unwrap();
 	assert_eq!(equivocation_proof.first.id, BeefyKeyring::Bob.public());
 	assert_eq!(equivocation_proof.first.commitment.block_number, 1);
 
diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index 45a708098076..fd99343c7746 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -19,7 +19,7 @@ use std::vec;
 
 use codec::Encode;
 use sp_consensus_beefy::{
-	check_vote_equivocation_proof, generate_equivocation_proof, known_payloads::MMR_ROOT_ID,
+	check_vote_equivocation_proof, generate_vote_equivocation_proof, known_payloads::MMR_ROOT_ID,
 	Keyring as BeefyKeyring, Payload, ValidatorSet, KEY_TYPE as BEEFY_KEY_TYPE,
 };
 
@@ -212,7 +212,7 @@ fn should_sign_and_verify() {
 
 	// generate an equivocation proof, with two votes in the same round for
 	// same payload signed by the same key
-	let equivocation_proof = generate_equivocation_proof(
+	let equivocation_proof = generate_vote_equivocation_proof(
 		(1, payload1.clone(), set_id, &BeefyKeyring::Bob),
 		(1, payload1.clone(), set_id, &BeefyKeyring::Bob),
 	);
@@ -221,7 +221,7 @@ fn should_sign_and_verify() {
 
 	// generate an equivocation proof, with two votes in different rounds for
 	// different payloads signed by the same key
-	let equivocation_proof = generate_equivocation_proof(
+	let equivocation_proof = generate_vote_equivocation_proof(
 		(1, payload1.clone(), set_id, &BeefyKeyring::Bob),
 		(2, payload2.clone(), set_id, &BeefyKeyring::Bob),
 	);
@@ -229,7 +229,7 @@ fn should_sign_and_verify() {
 	assert!(!check_vote_equivocation_proof::<_, _, Keccak256>(&equivocation_proof));
 
 	// generate an equivocation proof, with two votes by different authorities
-	let equivocation_proof = generate_equivocation_proof(
+	let equivocation_proof = generate_vote_equivocation_proof(
 		(1, payload1.clone(), set_id, &BeefyKeyring::Alice),
 		(1, payload2.clone(), set_id, &BeefyKeyring::Bob),
 	);
@@ -237,7 +237,7 @@ fn should_sign_and_verify() {
 	assert!(!check_vote_equivocation_proof::<_, _, Keccak256>(&equivocation_proof));
 
 	// generate an equivocation proof, with two votes in different set ids
-	let equivocation_proof = generate_equivocation_proof(
+	let equivocation_proof = generate_vote_equivocation_proof(
 		(1, payload1.clone(), set_id, &BeefyKeyring::Bob),
 		(1, payload2.clone(), set_id + 1, &BeefyKeyring::Bob),
 	);
@@ -247,7 +247,7 @@ fn should_sign_and_verify() {
 	// generate an equivocation proof, with two votes in the same round for
 	// different payloads signed by the same key
 	let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
-	let equivocation_proof = generate_equivocation_proof(
+	let equivocation_proof = generate_vote_equivocation_proof(
 		(1, payload1, set_id, &BeefyKeyring::Bob),
 		(1, payload2, set_id, &BeefyKeyring::Bob),
 	);
@@ -291,7 +291,7 @@ fn report_equivocation_current_set_works() {
 		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
 		// generate an equivocation proof, with two votes in the same round for
 		// different payloads signed by the same key
-		let equivocation_proof = generate_equivocation_proof(
+		let equivocation_proof = generate_vote_equivocation_proof(
 			(block_num, payload1, set_id, &equivocation_keyring),
 			(block_num, payload2, set_id, &equivocation_keyring),
 		);
@@ -377,7 +377,7 @@ fn report_equivocation_old_set_works() {
 		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
 		// generate an equivocation proof for the old set,
-		let equivocation_proof = generate_equivocation_proof(
+		let equivocation_proof = generate_vote_equivocation_proof(
 			(block_num, payload1, old_set_id, &equivocation_keyring),
 			(block_num, payload2, old_set_id, &equivocation_keyring),
 		);
@@ -439,7 +439,7 @@ fn report_equivocation_invalid_set_id() {
 		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
 		// generate an equivocation for a future set
-		let equivocation_proof = generate_equivocation_proof(
+		let equivocation_proof = generate_vote_equivocation_proof(
 			(block_num, payload1, set_id + 1, &equivocation_keyring),
 			(block_num, payload2, set_id + 1, &equivocation_keyring),
 		);
@@ -481,7 +481,7 @@ fn report_equivocation_invalid_session() {
 		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
 		// generate an equivocation proof at following era set id = 2
-		let equivocation_proof = generate_equivocation_proof(
+		let equivocation_proof = generate_vote_equivocation_proof(
 			(block_num, payload1, set_id, &equivocation_keyring),
 			(block_num, payload2, set_id, &equivocation_keyring),
 		);
@@ -525,7 +525,7 @@ fn report_equivocation_invalid_key_owner_proof() {
 		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
 		// generate an equivocation proof for the authority at index 0
-		let equivocation_proof = generate_equivocation_proof(
+		let equivocation_proof = generate_vote_equivocation_proof(
 			(block_num, payload1, set_id + 1, &equivocation_keyring),
 			(block_num, payload2, set_id + 1, &equivocation_keyring),
 		);
@@ -584,31 +584,31 @@ fn report_equivocation_invalid_equivocation_proof() {
 
 		// both votes target the same block number and payload,
 		// there is no equivocation.
-		assert_invalid_equivocation_proof(generate_equivocation_proof(
+		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
 			(block_num, payload1.clone(), set_id, &equivocation_keyring),
 			(block_num, payload1.clone(), set_id, &equivocation_keyring),
 		));
 
 		// votes targeting different rounds, there is no equivocation.
-		assert_invalid_equivocation_proof(generate_equivocation_proof(
+		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
 			(block_num, payload1.clone(), set_id, &equivocation_keyring),
 			(block_num + 1, payload2.clone(), set_id, &equivocation_keyring),
 		));
 
 		// votes signed with different authority keys
-		assert_invalid_equivocation_proof(generate_equivocation_proof(
+		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
 			(block_num, payload1.clone(), set_id, &equivocation_keyring),
 			(block_num, payload1.clone(), set_id, &BeefyKeyring::Charlie),
 		));
 
 		// votes signed with a key that isn't part of the authority set
-		assert_invalid_equivocation_proof(generate_equivocation_proof(
+		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
 			(block_num, payload1.clone(), set_id, &equivocation_keyring),
 			(block_num, payload1.clone(), set_id, &BeefyKeyring::Dave),
 		));
 
 		// votes targeting different set ids
-		assert_invalid_equivocation_proof(generate_equivocation_proof(
+		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
 			(block_num, payload1, set_id, &equivocation_keyring),
 			(block_num, payload2, set_id + 1, &equivocation_keyring),
 		));
@@ -639,7 +639,7 @@ fn report_equivocation_validate_unsigned_prevents_duplicates() {
 
 		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
-		let equivocation_proof = generate_equivocation_proof(
+		let equivocation_proof = generate_vote_equivocation_proof(
 			(block_num, payload1, set_id, &equivocation_keyring),
 			(block_num, payload2, set_id, &equivocation_keyring),
 		);
@@ -743,7 +743,7 @@ fn valid_equivocation_reports_dont_pay_fees() {
 		// generate equivocation proof
 		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
-		let equivocation_proof = generate_equivocation_proof(
+		let equivocation_proof = generate_vote_equivocation_proof(
 			(block_num, payload1, set_id, &equivocation_keyring),
 			(block_num, payload2, set_id, &equivocation_keyring),
 		);
diff --git a/substrate/primitives/consensus/beefy/src/test_utils.rs b/substrate/primitives/consensus/beefy/src/test_utils.rs
index 36fe05f3aa31..a680d997dc9b 100644
--- a/substrate/primitives/consensus/beefy/src/test_utils.rs
+++ b/substrate/primitives/consensus/beefy/src/test_utils.rs
@@ -17,7 +17,7 @@
 
 #![cfg(feature = "std")]
 
-use crate::{ecdsa_crypto, Commitment, VoteEquivocationProof, Payload, ValidatorSetId, VoteMessage};
+use crate::{ecdsa_crypto, Commitment, ForkEquivocationProof, VoteEquivocationProof, Payload, ValidatorSetId, VoteMessage};
 use codec::Encode;
 use sp_core::{ecdsa, keccak_256, Pair};
 use std::collections::HashMap;
@@ -91,20 +91,28 @@ impl From<Keyring> for ecdsa_crypto::Public {
 	}
 }
 
+fn signed_vote(block_number: u64, payload: Payload, validator_set_id: ValidatorSetId, keyring: &Keyring) -> VoteMessage<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature> {
+	let commitment = Commitment { validator_set_id, block_number, payload };
+	let signature = keyring.sign(&commitment.encode());
+	VoteMessage { commitment, id: keyring.public(), signature }
+}
+
 /// Create a new `VoteEquivocationProof` based on given arguments.
 pub fn generate_vote_equivocation_proof(
 	vote1: (u64, Payload, ValidatorSetId, &Keyring),
 	vote2: (u64, Payload, ValidatorSetId, &Keyring),
 ) -> VoteEquivocationProof<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature> {
-	let signed_vote = |block_number: u64,
-	                   payload: Payload,
-	                   validator_set_id: ValidatorSetId,
-	                   keyring: &Keyring| {
-		let commitment = Commitment { validator_set_id, block_number, payload };
-		let signature = keyring.sign(&commitment.encode());
-		VoteMessage { commitment, id: keyring.public(), signature }
-	};
 	let first = signed_vote(vote1.0, vote1.1, vote1.2, vote1.3);
 	let second = signed_vote(vote2.0, vote2.1, vote2.2, vote2.3);
 	VoteEquivocationProof { first, second }
 }
+
+/// Create a new `ForkEquivocationProof` based on given arguments.
+pub fn generate_fork_equivocation_proof_vote<Header>(
+	vote: (u64, Payload, ValidatorSetId, &Keyring),
+	correct_header: Header,
+) -> ForkEquivocationProof<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature, Header> {
+	let signed_vote = signed_vote(vote.0, vote.1, vote.2, vote.3);
+	let signatories = vec![(signed_vote.id, signed_vote.signature)];
+	ForkEquivocationProof { commitment: signed_vote.commitment, signatories, correct_header }
+}

From a3b4d3ffa2bf0eb6b465970ca4000cea7629a140 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 10 Aug 2023 00:15:40 +0200
Subject: [PATCH 033/188] test: Alice snitches on Bob's vote equivocation

i.e. a fork equivocation triggered by a vote
---
 .../beefy/src/communication/fisherman.rs      |  2 +-
 .../beefy/src/communication/gossip.rs         |  2 +-
 .../client/consensus/beefy/src/worker.rs      | 47 ++++++++++++++++---
 3 files changed, 42 insertions(+), 9 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 36c4f4cd6482..ac73323582e8 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -98,7 +98,7 @@ where
 			.ok_or_else(|| Error::Backend("could not get BEEFY validator set".into()))
 	}
 
-	fn report_fork_equivocation(
+	pub(crate) fn report_fork_equivocation(
 		&self,
 		proof: ForkEquivocationProof<NumberFor<B>, AuthorityId, Signature, B::Header>,
 	) -> Result<(), Error> {
diff --git a/substrate/client/consensus/beefy/src/communication/gossip.rs b/substrate/client/consensus/beefy/src/communication/gossip.rs
index 883f4e48f48e..64da63136213 100644
--- a/substrate/client/consensus/beefy/src/communication/gossip.rs
+++ b/substrate/client/consensus/beefy/src/communication/gossip.rs
@@ -237,7 +237,7 @@ pub(crate) struct GossipValidator<B: Block, F> {
 	next_rebroadcast: Mutex<Instant>,
 	known_peers: Arc<Mutex<KnownPeers<B>>>,
 	report_sender: TracingUnboundedSender<PeerReport>,
-	fisherman: F,
+	pub(crate) fisherman: F,
 }
 
 impl<B, F> GossipValidator<B, F>
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index 1c82dd0c39fb..e28e94a76015 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -575,7 +575,7 @@ where
 			},
 			VoteImportResult::Equivocation(proof) => {
 				metric_inc!(self, beefy_equivocation_votes);
-				self.report_equivocation(proof)?;
+				self.report_vote_equivocation(proof)?;
 			},
 			VoteImportResult::Invalid => metric_inc!(self, beefy_invalid_votes),
 			VoteImportResult::Stale => metric_inc!(self, beefy_stale_votes),
@@ -920,7 +920,7 @@ where
 	/// extrinsic to report the equivocation. In particular, the session membership
 	/// proof must be generated at the block at which the given set was active which
 	/// isn't necessarily the best block if there are pending authority set changes.
-	pub(crate) fn report_equivocation(
+	pub(crate) fn report_vote_equivocation(
 		&self,
 		proof: VoteEquivocationProof<NumberFor<B>, AuthorityId, Signature>,
 	) -> Result<(), Error> {
@@ -1049,7 +1049,7 @@ pub(crate) mod tests {
 	use sp_api::HeaderT;
 	use sp_blockchain::Backend as BlockchainBackendT;
 	use sp_consensus_beefy::{
-		generate_vote_equivocation_proof, known_payloads, known_payloads::MMR_ROOT_ID,
+		generate_vote_equivocation_proof, generate_fork_equivocation_proof_vote, known_payloads, known_payloads::MMR_ROOT_ID,
 		mmr::MmrRootProvider, Keyring, Payload, SignedCommitment,
 	};
 	use sp_runtime::traits::One;
@@ -1626,7 +1626,7 @@ pub(crate) mod tests {
 		);
 		{
 			// expect voter (Alice) to successfully report it
-			assert_eq!(worker.report_equivocation(good_proof.clone()), Ok(()));
+			assert_eq!(worker.report_vote_equivocation(good_proof.clone()), Ok(()));
 			// verify Alice reports Bob equivocation to runtime
 			let reported = api_alice.reported_vote_equivocations.as_ref().unwrap().lock();
 			assert_eq!(reported.len(), 1);
@@ -1638,7 +1638,7 @@ pub(crate) mod tests {
 		let mut bad_proof = good_proof.clone();
 		bad_proof.first.id = Keyring::Charlie.public();
 		// bad proofs are simply ignored
-		assert_eq!(worker.report_equivocation(bad_proof), Ok(()));
+		assert_eq!(worker.report_vote_equivocation(bad_proof), Ok(()));
 		// verify nothing reported to runtime
 		assert!(api_alice.reported_vote_equivocations.as_ref().unwrap().lock().is_empty());
 
@@ -1647,7 +1647,7 @@ pub(crate) mod tests {
 		old_proof.first.commitment.validator_set_id = 0;
 		old_proof.second.commitment.validator_set_id = 0;
 		// old proofs are simply ignored
-		assert_eq!(worker.report_equivocation(old_proof), Ok(()));
+		assert_eq!(worker.report_vote_equivocation(old_proof), Ok(()));
 		// verify nothing reported to runtime
 		assert!(api_alice.reported_vote_equivocations.as_ref().unwrap().lock().is_empty());
 
@@ -1657,8 +1657,41 @@ pub(crate) mod tests {
 			(block_num, payload2.clone(), set_id, &Keyring::Alice),
 		);
 		// equivocations done by 'self' are simply ignored (not reported)
-		assert_eq!(worker.report_equivocation(self_proof), Ok(()));
+		assert_eq!(worker.report_vote_equivocation(self_proof), Ok(()));
 		// verify nothing reported to runtime
 		assert!(api_alice.reported_vote_equivocations.as_ref().unwrap().lock().is_empty());
 	}
+
+	#[tokio::test]
+	async fn should_report_valid_fork_equivocations() {
+		let peers = [Keyring::Alice, Keyring::Bob];
+		let validator_set = ValidatorSet::new(make_beefy_ids(&peers), 0).unwrap();
+		let mut api_alice = TestApi::with_validator_set(&validator_set);
+		api_alice.allow_equivocations();
+		let api_alice = Arc::new(api_alice);
+
+		// instantiate network with Alice and Bob running full voters.
+		let mut net = BeefyTestNet::new(2);
+
+		let session_len = 10;
+		let hashes = net.generate_blocks_and_sync(50, session_len, &validator_set, true).await;
+		let alice_worker = create_beefy_worker(net.peer(0), &peers[0], 1, validator_set.clone(), Some(api_alice));
+
+		let block_number = 1;
+		let header = net.peer(1).client().as_backend().blockchain().header(hashes[block_number as usize]).unwrap().unwrap();
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, "amievil".encode());
+
+		let validator_set_id = 0;
+
+		let proof = generate_fork_equivocation_proof_vote((block_number as u64, payload, validator_set_id, &Keyring::Bob), header);
+		{
+			// expect fisher (Alice) to successfully report it
+			assert_eq!(alice_worker.gossip_validator.fisherman.report_fork_equivocation(proof.clone()), Ok(()));
+			// verify Alice reports Bob's equivocation to runtime
+			let reported = alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
+			assert_eq!(reported.len(), 1);
+			assert_eq!(*reported.get(0).unwrap(), proof);
+		}
+
+	}
 }

From d44e3c8576dee0fee0be8128999a16d104f6d99d Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 10 Aug 2023 10:22:39 +0200
Subject: [PATCH 034/188] store reference to key_store in fisherman

required for fisherman to *not* report own equivocations - see next commit.
---
 .../consensus/beefy/src/communication/fisherman.rs       | 3 ++-
 substrate/client/consensus/beefy/src/lib.rs              | 3 +++
 substrate/client/consensus/beefy/src/worker.rs           | 9 +++++----
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index ac73323582e8..1bd099cf2931 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -17,7 +17,7 @@
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 use crate::{
-	error::Error, justification::BeefyVersionedFinalityProof, keystore::BeefySignatureHasher,
+	error::Error, justification::BeefyVersionedFinalityProof, keystore::{BeefyKeystore, BeefySignatureHasher},
 	LOG_TARGET,
 };
 use log::debug;
@@ -57,6 +57,7 @@ pub(crate) trait BeefyFisherman<B: Block>: Send + Sync {
 pub(crate) struct Fisherman<B: Block, BE, R, P> {
 	pub backend: Arc<BE>,
 	pub runtime: Arc<R>,
+	pub key_store: Arc<BeefyKeystore>,
 	pub payload_provider: P,
 	pub _phantom: PhantomData<B>,
 }
diff --git a/substrate/client/consensus/beefy/src/lib.rs b/substrate/client/consensus/beefy/src/lib.rs
index a28cb4ea973f..d6eeb08618b4 100644
--- a/substrate/client/consensus/beefy/src/lib.rs
+++ b/substrate/client/consensus/beefy/src/lib.rs
@@ -32,6 +32,7 @@ use crate::{
 	metrics::register_metrics,
 	round::Rounds,
 	worker::PersistedState,
+	keystore::BeefyKeystore,
 };
 use futures::{stream::Fuse, StreamExt};
 use log::{error, info};
@@ -241,6 +242,8 @@ pub async fn start_beefy_gadget<B, BE, C, N, P, R, S>(
 		mut on_demand_justifications_handler,
 	} = beefy_params;
 
+	let key_store: Arc<BeefyKeystore> = Arc::new(key_store.into());
+
 	let BeefyNetworkParams {
 		network,
 		sync,
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index e28e94a76015..b27171d220bc 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -321,7 +321,7 @@ pub(crate) struct BeefyWorker<B: Block, BE, P, RuntimeApi, S, F> {
 	pub payload_provider: P,
 	pub runtime: Arc<RuntimeApi>,
 	pub sync: Arc<S>,
-	pub key_store: BeefyKeystore,
+	pub key_store: Arc<BeefyKeystore>,
 
 	// communication
 	pub gossip_engine: GossipEngine<B>,
@@ -1097,7 +1097,7 @@ pub(crate) mod tests {
 		Arc<SyncingService<Block>>,
 		Fisherman<Block, Backend, TestApi, MmrRootProvider<Block, TestApi>>,
 	> {
-		let keystore = create_beefy_keystore(*key);
+		let key_store: Arc<BeefyKeystore> = Arc::new(Some(create_beefy_keystore(*key)).into());
 
 		let (to_rpc_justif_sender, from_voter_justif_stream) =
 			BeefyVersionedFinalityProofStream::<Block>::channel();
@@ -1125,6 +1125,7 @@ pub(crate) mod tests {
 		let fisherman = Fisherman {
 			backend: backend.clone(),
 			runtime: api.clone(),
+			key_store: key_store.clone(),
 			payload_provider: payload_provider.clone(),
 			_phantom: PhantomData,
 		};
@@ -1167,7 +1168,7 @@ pub(crate) mod tests {
 			backend,
 			payload_provider,
 			runtime: api,
-			key_store: Some(keystore).into(),
+			key_store,
 			links,
 			gossip_engine,
 			gossip_validator,
@@ -1458,7 +1459,7 @@ pub(crate) mod tests {
 		assert_eq!(worker.verify_validator_set(&1, &validator_set), expected);
 
 		// worker has no keystore
-		worker.key_store = None.into();
+		worker.key_store = Arc::new(None.into());
 		let expected_err = Err(Error::Keystore("no Keystore".into()));
 		assert_eq!(worker.verify_validator_set(&1, &validator_set), expected_err);
 	}

From c27579cbfcbd23a417df37816fb2a09c2a3b4a25 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 10 Aug 2023 10:24:09 +0200
Subject: [PATCH 035/188] test: Alice doesn't snitch *own* vote equivocation

i.e. a fork equivocation triggered by a vote
---
 .../consensus/beefy/src/communication/fisherman.rs | 10 +++++++++-
 substrate/client/consensus/beefy/src/worker.rs     | 14 ++++++++++++--
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 1bd099cf2931..2a45a749737a 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -119,8 +119,16 @@ where
 			return Ok(())
 		}
 
-		let hash = proof.correct_header.hash();
 		let offender_ids = proof.signatories.iter().cloned().map(|(id, _sig)| id).collect::<Vec<_>>();
+		if let Some(local_id) = self.key_store.authority_id(validator_set.validators()) {
+			if offender_ids.contains(&local_id) {
+				debug!(target: LOG_TARGET, "🥩 Skip equivocation report for own equivocation");
+				// TODO: maybe error here instead?
+				return Ok(())
+			}
+		}
+
+		let hash = proof.correct_header.hash();
 		let runtime_api = self.runtime.runtime_api();
 
 		// generate key ownership proof at that block
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index b27171d220bc..96de3718ef14 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -1684,9 +1684,9 @@ pub(crate) mod tests {
 
 		let validator_set_id = 0;
 
-		let proof = generate_fork_equivocation_proof_vote((block_number as u64, payload, validator_set_id, &Keyring::Bob), header);
+		let proof = generate_fork_equivocation_proof_vote((block_number as u64, payload.clone(), validator_set_id, &Keyring::Bob), header.clone());
 		{
-			// expect fisher (Alice) to successfully report it
+			// expect fisher (Alice) to successfully process it
 			assert_eq!(alice_worker.gossip_validator.fisherman.report_fork_equivocation(proof.clone()), Ok(()));
 			// verify Alice reports Bob's equivocation to runtime
 			let reported = alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
@@ -1694,5 +1694,15 @@ pub(crate) mod tests {
 			assert_eq!(*reported.get(0).unwrap(), proof);
 		}
 
+		let proof = generate_fork_equivocation_proof_vote((block_number as u64, payload, validator_set_id, &Keyring::Alice), header);
+		{
+			// expect fisher (Alice) to successfully process it
+			assert_eq!(alice_worker.gossip_validator.fisherman.report_fork_equivocation(proof.clone()), Ok(()));
+			// verify Alice does not report her own equivocation to runtime
+			let reported = alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
+			assert_eq!(reported.len(), 1);
+			assert!(*reported.get(0).unwrap() != proof);
+		}
+
 	}
 }

From 5516837f57b96e0e9349a8094ae4ceeb79f906ab Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 10 Aug 2023 11:09:55 +0200
Subject: [PATCH 036/188] un-stub submit_unsigned_fork_equivocation_report

---
 substrate/frame/beefy/src/lib.rs | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index f6cc6bf78ff8..af869ae26bbc 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -389,13 +389,18 @@ impl<T: Config> Pallet<T> {
 	/// a call to `report_fork_equivocation_unsigned` and will push the transaction
 	/// to the pool. Only useful in an offchain context.
 	pub fn submit_unsigned_fork_equivocation_report(
-		_fork_equivocation_proof: sp_consensus_beefy::ForkEquivocationProof<BlockNumberFor<T>, T::BeefyId,
+		fork_equivocation_proof: sp_consensus_beefy::ForkEquivocationProof<BlockNumberFor<T>, T::BeefyId,
 			<T::BeefyId as RuntimeAppPublic>::Signature, HeaderFor<T>
 >,
-		_key_owner_proofs: Vec<T::KeyOwnerProof>,
+		key_owner_proofs: Vec<T::KeyOwnerProof>,
 	) -> Option<()> {
-		// T::EquivocationReportSystem::publish_evidence((fork_equivocation_proof, key_owner_proofs)).ok()
-		None
+		T::EquivocationReportSystem::publish_evidence(
+			EquivocationEvidenceFor::<T>::ForkEquivocationProof(
+				fork_equivocation_proof,
+				key_owner_proofs,
+			),
+		)
+		.ok()
 	}
 
 	fn change_authorities(

From 60a71c7aac89e391fdb17183c50327897ece1188 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <lederstrumpf@MacBook-Pro-M1.local>
Date: Thu, 24 Aug 2023 10:23:56 +0200
Subject: [PATCH 037/188] Alice reports Bob & Charlie's signed commitment

---
 .../client/consensus/beefy/src/worker.rs      | 36 +++++++++++++++----
 .../consensus/beefy/src/test_utils.rs         |  5 +--
 2 files changed, 32 insertions(+), 9 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index 96de3718ef14..c50a0ada180b 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -1050,7 +1050,7 @@ pub(crate) mod tests {
 	use sp_blockchain::Backend as BlockchainBackendT;
 	use sp_consensus_beefy::{
 		generate_vote_equivocation_proof, generate_fork_equivocation_proof_vote, known_payloads, known_payloads::MMR_ROOT_ID,
-		mmr::MmrRootProvider, Keyring, Payload, SignedCommitment,
+		mmr::MmrRootProvider, Keyring, Payload, SignedCommitment, ForkEquivocationProof,
 	};
 	use sp_runtime::traits::One;
 	use std::marker::PhantomData;
@@ -1665,14 +1665,14 @@ pub(crate) mod tests {
 
 	#[tokio::test]
 	async fn should_report_valid_fork_equivocations() {
-		let peers = [Keyring::Alice, Keyring::Bob];
+		let peers = [Keyring::Alice, Keyring::Bob, Keyring::Charlie];
 		let validator_set = ValidatorSet::new(make_beefy_ids(&peers), 0).unwrap();
 		let mut api_alice = TestApi::with_validator_set(&validator_set);
 		api_alice.allow_equivocations();
 		let api_alice = Arc::new(api_alice);
 
 		// instantiate network with Alice and Bob running full voters.
-		let mut net = BeefyTestNet::new(2);
+		let mut net = BeefyTestNet::new(3);
 
 		let session_len = 10;
 		let hashes = net.generate_blocks_and_sync(50, session_len, &validator_set, true).await;
@@ -1681,10 +1681,14 @@ pub(crate) mod tests {
 		let block_number = 1;
 		let header = net.peer(1).client().as_backend().blockchain().header(hashes[block_number as usize]).unwrap().unwrap();
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, "amievil".encode());
+		let votes: Vec<_> = peers.iter().map(|k| {
+			// signed_vote(block_number as u64, payload.clone(), validator_set.id(), k)
+			(block_number as u64, payload.clone(), validator_set.id(), k)
+		}).collect();
 
-		let validator_set_id = 0;
 
-		let proof = generate_fork_equivocation_proof_vote((block_number as u64, payload.clone(), validator_set_id, &Keyring::Bob), header.clone());
+		// verify: Alice reports Bob
+		let proof = generate_fork_equivocation_proof_vote(votes[1].clone(), header.clone());
 		{
 			// expect fisher (Alice) to successfully process it
 			assert_eq!(alice_worker.gossip_validator.fisherman.report_fork_equivocation(proof.clone()), Ok(()));
@@ -1694,15 +1698,33 @@ pub(crate) mod tests {
 			assert_eq!(*reported.get(0).unwrap(), proof);
 		}
 
-		let proof = generate_fork_equivocation_proof_vote((block_number as u64, payload, validator_set_id, &Keyring::Alice), header);
+		// verify: Alice does not self-report
+		let proof = generate_fork_equivocation_proof_vote(votes[0].clone(), header.clone());
 		{
 			// expect fisher (Alice) to successfully process it
 			assert_eq!(alice_worker.gossip_validator.fisherman.report_fork_equivocation(proof.clone()), Ok(()));
-			// verify Alice does not report her own equivocation to runtime
+			// verify Alice does *not* report her own equivocation to runtime
 			let reported = alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
 			assert_eq!(reported.len(), 1);
 			assert!(*reported.get(0).unwrap() != proof);
 		}
 
+		// verify: Alice reports VersionedFinalityProof equivocation
+		let commitment = Commitment {
+			payload: payload.clone(),
+			block_number: block_number as u64,
+			validator_set_id: validator_set.id(),
+		};
+		// only Bob and Charlie sign
+		let signatories: Vec<_> = vec![Keyring::Bob, Keyring::Charlie].iter().map(|k| (k.public(), k.sign(&commitment.encode()))).collect();
+		let proof = ForkEquivocationProof {commitment, signatories, correct_header: header};
+		{
+			// expect fisher (Alice) to successfully process it
+			assert_eq!(alice_worker.gossip_validator.fisherman.report_fork_equivocation(proof.clone()), Ok(()));
+			// verify Alice report Bob's and Charlie's equivocation to runtime
+			let reported = alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
+			assert_eq!(reported.len(), 2);
+			assert_eq!(*reported.get(1).unwrap(), proof);
+		}
 	}
 }
diff --git a/substrate/primitives/consensus/beefy/src/test_utils.rs b/substrate/primitives/consensus/beefy/src/test_utils.rs
index a680d997dc9b..20a4277eb9dd 100644
--- a/substrate/primitives/consensus/beefy/src/test_utils.rs
+++ b/substrate/primitives/consensus/beefy/src/test_utils.rs
@@ -17,7 +17,7 @@
 
 #![cfg(feature = "std")]
 
-use crate::{ecdsa_crypto, Commitment, ForkEquivocationProof, VoteEquivocationProof, Payload, ValidatorSetId, VoteMessage};
+use crate::{ecdsa_crypto, ForkEquivocationProof, Commitment, SignedCommitment, VoteEquivocationProof, Payload, ValidatorSetId, VoteMessage};
 use codec::Encode;
 use sp_core::{ecdsa, keccak_256, Pair};
 use std::collections::HashMap;
@@ -91,6 +91,7 @@ impl From<Keyring> for ecdsa_crypto::Public {
 	}
 }
 
+/// Create a new `VoteMessage` from commitment primitives and keyring
 fn signed_vote(block_number: u64, payload: Payload, validator_set_id: ValidatorSetId, keyring: &Keyring) -> VoteMessage<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature> {
 	let commitment = Commitment { validator_set_id, block_number, payload };
 	let signature = keyring.sign(&commitment.encode());
@@ -107,7 +108,7 @@ pub fn generate_vote_equivocation_proof(
 	VoteEquivocationProof { first, second }
 }
 
-/// Create a new `ForkEquivocationProof` based on given arguments.
+/// Create a new `ForkEquivocationProof` based on vote & correct header.
 pub fn generate_fork_equivocation_proof_vote<Header>(
 	vote: (u64, Payload, ValidatorSetId, &Keyring),
 	correct_header: Header,

From ec99b1aca692ce47afb56720ea3e166373635312 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 24 Aug 2023 10:42:53 +0200
Subject: [PATCH 038/188] cleanup

---
 .../client/consensus/beefy/src/communication/fisherman.rs     | 2 +-
 substrate/primitives/consensus/beefy/src/lib.rs               | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 2a45a749737a..75f4ffe67057 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -27,7 +27,7 @@ use sp_blockchain::HeaderBackend;
 use sp_consensus_beefy::{
 	check_fork_equivocation_proof,
 	ecdsa_crypto::{AuthorityId, Signature},
-	BeefyApi, ForkEquivocationProof, Payload, PayloadProvider, ValidatorSet, VoteMessage, Commitment, OpaqueKeyOwnershipProof, SignedCommitment,
+	BeefyApi, ForkEquivocationProof, Payload, PayloadProvider, ValidatorSet, VoteMessage, SignedCommitment,
 };
 use sp_runtime::{
 	generic::BlockId,
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index ab14837d3191..a1c007241d81 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -34,13 +34,13 @@
 mod commitment;
 pub mod mmr;
 mod payload;
-#[cfg(feature = "std")]
+#[cfg(all(test, feature = "std"))]
 mod test_utils;
 pub mod witness;
 
 pub use commitment::{Commitment, SignedCommitment, VersionedFinalityProof};
 pub use payload::{known_payloads, BeefyPayloadId, Payload, PayloadProvider};
-#[cfg(feature = "std")]
+#[cfg(all(test, feature = "std"))]
 pub use test_utils::*;
 
 use codec::{Codec, Decode, Encode};

From ea7dd35630551e141fd8f301d33aae7dcde579dd Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 24 Aug 2023 12:30:42 +0200
Subject: [PATCH 039/188] remove superfluous None check

Co-authored-by: Adrian Catangiu <adrian@parity.io>
---
 substrate/primitives/consensus/beefy/src/lib.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index a1c007241d81..b7dc418a182d 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -373,9 +373,9 @@ where
 	// cheap failfasts:
 	// 1. check that `payload` on the `vote` is different that the `expected_payload`
 	// 2. if the signatories signed a payload when there should be none (for
-	// instance for a block prior to BEEFY activation), they should likewise be
-	// slashed
-	if Some(&commitment.payload) != expected_payload.as_ref() || expected_mmr_root_digest.is_none() {
+	// instance for a block prior to BEEFY activation), then expected_payload =
+	// None, and they will likewise be slashed
+	if Some(&commitment.payload) != expected_payload.as_ref() {
 		// check check each signatory's signature on the commitment.
 		// if any are invalid, equivocation report is invalid
 		// TODO: refactor check_commitment_signature to take a slice of signatories

From 318577640db9e97ec8d8459dc9bcb57d51029fba Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 29 Aug 2023 00:44:47 +0200
Subject: [PATCH 040/188] fixup! check proof's payload against correct header's

---
 substrate/frame/beefy/src/equivocation.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 2f55c807391d..b7a63981d4a5 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -278,7 +278,7 @@ where
 		}
 
 		let offence = EquivocationOffence {
-			time_slot: TimeSlot set_id, round,
+			time_slot:  TimeSlot { set_id, round },
 			session_index,
 			validator_set_count,
 			offenders,
@@ -297,7 +297,7 @@ where
 /// unsigned equivocation reports.
 impl<T: Config> Pallet<T> {
 	pub fn validate_unsigned(source: TransactionSource, call: &Call<T>) -> TransactionValidity {
-		if let Call::report_vote_equivocation_unsigned equivocation_proof, key_owner_proof = call {
+		if let Call::report_vote_equivocation_unsigned { equivocation_proof, key_owner_proof } = call {
 			// discard equivocation report not coming from the local node
 			match source {
 				TransactionSource::Local | TransactionSource::InBlock => { /* allowed */ },

From 59c1c2ac9ba53a06877170d281a97781c47e8522 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 29 Aug 2023 01:33:59 +0200
Subject: [PATCH 041/188] fmt

---
 .../beefy/src/communication/fisherman.rs      |  89 ++++++++++-----
 substrate/client/consensus/beefy/src/lib.rs   |  14 +--
 substrate/client/consensus/beefy/src/round.rs |   6 +-
 substrate/client/consensus/beefy/src/tests.rs |  23 ++--
 .../client/consensus/beefy/src/worker.rs      |  81 ++++++++++----
 substrate/frame/beefy/src/equivocation.rs     | 104 +++++++++++-------
 substrate/frame/beefy/src/lib.rs              |  22 ++--
 .../primitives/consensus/beefy/src/lib.rs     |   7 +-
 .../consensus/beefy/src/test_utils.rs         |  12 +-
 9 files changed, 236 insertions(+), 122 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 75f4ffe67057..2fa91ce78396 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -17,7 +17,9 @@
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 use crate::{
-	error::Error, justification::BeefyVersionedFinalityProof, keystore::{BeefyKeystore, BeefySignatureHasher},
+	error::Error,
+	justification::BeefyVersionedFinalityProof,
+	keystore::{BeefyKeystore, BeefySignatureHasher},
 	LOG_TARGET,
 };
 use log::debug;
@@ -27,7 +29,8 @@ use sp_blockchain::HeaderBackend;
 use sp_consensus_beefy::{
 	check_fork_equivocation_proof,
 	ecdsa_crypto::{AuthorityId, Signature},
-	BeefyApi, ForkEquivocationProof, Payload, PayloadProvider, ValidatorSet, VoteMessage, SignedCommitment,
+	BeefyApi, ForkEquivocationProof, Payload, PayloadProvider, SignedCommitment, ValidatorSet,
+	VoteMessage,
 };
 use sp_runtime::{
 	generic::BlockId,
@@ -70,7 +73,10 @@ where
 	R: ProvideRuntimeApi<B> + Send + Sync,
 	R::Api: BeefyApi<B, AuthorityId>,
 {
-	fn expected_header_and_payload(&self, number: NumberFor<B>) -> Result<(B::Header, Payload), Error> {
+	fn expected_header_and_payload(
+		&self,
+		number: NumberFor<B>,
+	) -> Result<(B::Header, Payload), Error> {
 		// This should be un-ambiguous since `number` is finalized.
 		let hash = self
 			.backend
@@ -113,13 +119,19 @@ where
 			.map_err(|e| Error::Backend(e.to_string()))?;
 
 		if proof.commitment.validator_set_id != set_id ||
-			!check_fork_equivocation_proof::<NumberFor<B>, AuthorityId, BeefySignatureHasher, B::Header>(&proof, &expected_header_hash)
+			!check_fork_equivocation_proof::<
+				NumberFor<B>,
+				AuthorityId,
+				BeefySignatureHasher,
+				B::Header,
+			>(&proof, &expected_header_hash)
 		{
 			debug!(target: LOG_TARGET, "🥩 Skip report for bad invalid fork proof {:?}", proof);
 			return Ok(())
 		}
 
-		let offender_ids = proof.signatories.iter().cloned().map(|(id, _sig)| id).collect::<Vec<_>>();
+		let offender_ids =
+			proof.signatories.iter().cloned().map(|(id, _sig)| id).collect::<Vec<_>>();
 		if let Some(local_id) = self.key_store.authority_id(validator_set.validators()) {
 			if offender_ids.contains(&local_id) {
 				debug!(target: LOG_TARGET, "🥩 Skip equivocation report for own equivocation");
@@ -132,26 +144,31 @@ where
 		let runtime_api = self.runtime.runtime_api();
 
 		// generate key ownership proof at that block
-		let key_owner_proofs = offender_ids.iter()
-										   .filter_map(|id| {
-											   match runtime_api.generate_key_ownership_proof(hash, set_id, id.clone()) {
-												   Ok(Some(proof)) => Some(Ok(proof)),
-												   Ok(None) => {
-													   debug!(
-														   target: LOG_TARGET,
-														   "🥩 Invalid fork vote offender not part of the authority set."
-													   );
-													   None
-												   },
-												   Err(e) => Some(Err(Error::RuntimeApi(e))),
-											   }
-										   })
-										   .collect::<Result<_, _>>()?;
+		let key_owner_proofs = offender_ids
+			.iter()
+			.filter_map(|id| {
+				match runtime_api.generate_key_ownership_proof(hash, set_id, id.clone()) {
+					Ok(Some(proof)) => Some(Ok(proof)),
+					Ok(None) => {
+						debug!(
+							target: LOG_TARGET,
+							"🥩 Invalid fork vote offender not part of the authority set."
+						);
+						None
+					},
+					Err(e) => Some(Err(Error::RuntimeApi(e))),
+				}
+			})
+			.collect::<Result<_, _>>()?;
 
 		// submit invalid fork vote report at **best** block
 		let best_block_hash = self.backend.blockchain().info().best_hash;
 		runtime_api
-			.submit_report_fork_equivocation_unsigned_extrinsic(best_block_hash, proof, key_owner_proofs)
+			.submit_report_fork_equivocation_unsigned_extrinsic(
+				best_block_hash,
+				proof,
+				key_owner_proofs,
+			)
 			.map_err(Error::RuntimeApi)?;
 
 		Ok(())
@@ -174,7 +191,11 @@ where
 		let number = vote.commitment.block_number;
 		let (correct_header, expected_payload) = self.expected_header_and_payload(number)?;
 		if vote.commitment.payload != expected_payload {
-			let proof = ForkEquivocationProof { commitment: vote.commitment, signatories: vec![(vote.id, vote.signature)], correct_header: correct_header.clone() };
+			let proof = ForkEquivocationProof {
+				commitment: vote.commitment,
+				signatories: vec![(vote.id, vote.signature)],
+				correct_header: correct_header.clone(),
+			};
 			self.report_fork_equivocation(proof)?;
 		}
 		Ok(())
@@ -185,7 +206,7 @@ where
 		&self,
 		signed_commitment: SignedCommitment<NumberFor<B>, Signature>,
 	) -> Result<(), Error> {
-		let SignedCommitment {commitment, signatures} = signed_commitment;
+		let SignedCommitment { commitment, signatures } = signed_commitment;
 		let number = commitment.block_number;
 		let (correct_header, expected_payload) = self.expected_header_and_payload(number)?;
 		if commitment.payload != expected_payload {
@@ -195,10 +216,19 @@ where
 				return Ok(())
 			}
 			// report every signer of the bad justification
-			let signatories = validator_set.validators().iter().cloned().zip(signatures.into_iter())
-																		.filter_map(|(id, signature)| signature.map(|sig| (id, sig))).collect();
-
-			let proof = ForkEquivocationProof { commitment, signatories, correct_header: correct_header.clone() };
+			let signatories = validator_set
+				.validators()
+				.iter()
+				.cloned()
+				.zip(signatures.into_iter())
+				.filter_map(|(id, signature)| signature.map(|sig| (id, sig)))
+				.collect();
+
+			let proof = ForkEquivocationProof {
+				commitment,
+				signatories,
+				correct_header: correct_header.clone(),
+			};
 			self.report_fork_equivocation(proof)?;
 		}
 		Ok(())
@@ -207,9 +237,8 @@ where
 	/// Check `proof` for contained block against expected payload.
 	fn check_proof(&self, proof: BeefyVersionedFinalityProof<B>) -> Result<(), Error> {
 		match proof {
-			BeefyVersionedFinalityProof::<B>::V1(signed_commitment) => {
-				self.check_signed_commitment(signed_commitment)
-			}
+			BeefyVersionedFinalityProof::<B>::V1(signed_commitment) =>
+				self.check_signed_commitment(signed_commitment),
 		}
 	}
 }
diff --git a/substrate/client/consensus/beefy/src/lib.rs b/substrate/client/consensus/beefy/src/lib.rs
index d6eeb08618b4..eef618d13ada 100644
--- a/substrate/client/consensus/beefy/src/lib.rs
+++ b/substrate/client/consensus/beefy/src/lib.rs
@@ -29,10 +29,10 @@ use crate::{
 		},
 	},
 	import::BeefyBlockImport,
+	keystore::BeefyKeystore,
 	metrics::register_metrics,
 	round::Rounds,
 	worker::PersistedState,
-	keystore::BeefyKeystore,
 };
 use futures::{stream::Fuse, StreamExt};
 use log::{error, info};
@@ -263,12 +263,12 @@ pub async fn start_beefy_gadget<B, BE, C, N, P, R, S>(
 	// select recoverable errors.
 	loop {
 		let known_peers = Arc::new(Mutex::new(KnownPeers::new()));
-        let fisherman = Fisherman {
-            backend: backend.clone(),
-            runtime: runtime.clone(),
-            payload_provider: payload_provider.clone(),
-            _phantom: PhantomData,
-        };
+		let fisherman = Fisherman {
+			backend: backend.clone(),
+			runtime: runtime.clone(),
+			payload_provider: payload_provider.clone(),
+			_phantom: PhantomData,
+		};
 		// Default votes filter is to discard everything.
 		// Validator is updated later with correct starting round and set id.
 		let (gossip_validator, gossip_report_stream) =
diff --git a/substrate/client/consensus/beefy/src/round.rs b/substrate/client/consensus/beefy/src/round.rs
index 81e0d0634dcd..3abe662fc27f 100644
--- a/substrate/client/consensus/beefy/src/round.rs
+++ b/substrate/client/consensus/beefy/src/round.rs
@@ -22,7 +22,7 @@ use codec::{Decode, Encode};
 use log::debug;
 use sp_consensus_beefy::{
 	ecdsa_crypto::{AuthorityId, Signature},
-	Commitment, VoteEquivocationProof, SignedCommitment, ValidatorSet, ValidatorSetId, VoteMessage,
+	Commitment, SignedCommitment, ValidatorSet, ValidatorSetId, VoteEquivocationProof, VoteMessage,
 };
 use sp_runtime::traits::{Block, NumberFor};
 use std::collections::BTreeMap;
@@ -203,8 +203,8 @@ mod tests {
 	use sc_network_test::Block;
 
 	use sp_consensus_beefy::{
-		known_payloads::MMR_ROOT_ID, Commitment, VoteEquivocationProof, Keyring, Payload,
-		SignedCommitment, ValidatorSet, VoteMessage,
+		known_payloads::MMR_ROOT_ID, Commitment, Keyring, Payload, SignedCommitment, ValidatorSet,
+		VoteEquivocationProof, VoteMessage,
 	};
 
 	use super::{threshold, AuthorityId, Block as BlockT, RoundTracker, Rounds};
diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index 12b7884c4a02..5522e782fde8 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -44,7 +44,7 @@ use sc_consensus::{
 };
 use sc_network::{config::RequestResponseConfig, ProtocolName};
 use sc_network_test::{
-	Block, BlockImportAdapter, Header, FullPeerConfig, PassThroughVerifier, Peer, PeersClient,
+	Block, BlockImportAdapter, FullPeerConfig, Header, PassThroughVerifier, Peer, PeersClient,
 	PeersFullClient, TestNetFactory,
 };
 use sc_utils::notification::NotificationReceiver;
@@ -56,9 +56,9 @@ use sp_consensus_beefy::{
 	ecdsa_crypto::{AuthorityId, Signature},
 	known_payloads,
 	mmr::{find_mmr_root_digest, MmrRootProvider},
-	BeefyApi, Commitment, ConsensusLog, ForkEquivocationProof, VoteEquivocationProof, Keyring as BeefyKeyring, MmrRootHash,
-	OpaqueKeyOwnershipProof, Payload, SignedCommitment, ValidatorSet, ValidatorSetId,
-	VersionedFinalityProof, VoteMessage, BEEFY_ENGINE_ID,
+	BeefyApi, Commitment, ConsensusLog, ForkEquivocationProof, Keyring as BeefyKeyring,
+	MmrRootHash, OpaqueKeyOwnershipProof, Payload, SignedCommitment, ValidatorSet, ValidatorSetId,
+	VersionedFinalityProof, VoteEquivocationProof, VoteMessage, BEEFY_ENGINE_ID,
 };
 use sp_core::H256;
 use sp_keystore::{testing::MemoryKeystore, Keystore, KeystorePtr};
@@ -254,7 +254,10 @@ impl<B: BlockT> BeefyFisherman<B> for DummyFisherman<B> {
 	fn check_proof(&self, _: BeefyVersionedFinalityProof<B>) -> Result<(), Error> {
 		Ok(())
 	}
-	fn check_signed_commitment(&self, _: SignedCommitment<NumberFor<B>, Signature>) -> Result<(), Error> {
+	fn check_signed_commitment(
+		&self,
+		_: SignedCommitment<NumberFor<B>, Signature>,
+	) -> Result<(), Error> {
 		Ok(())
 	}
 	fn check_vote(
@@ -272,8 +275,9 @@ pub(crate) struct TestApi {
 	pub mmr_root_hash: MmrRootHash,
 	pub reported_vote_equivocations:
 		Option<Arc<Mutex<Vec<VoteEquivocationProof<NumberFor<Block>, AuthorityId, Signature>>>>>,
-	pub reported_fork_equivocations:
-		Option<Arc<Mutex<Vec<ForkEquivocationProof<NumberFor<Block>, AuthorityId, Signature, Header>>>>>,
+	pub reported_fork_equivocations: Option<
+		Arc<Mutex<Vec<ForkEquivocationProof<NumberFor<Block>, AuthorityId, Signature, Header>>>>,
+	>,
 }
 
 impl TestApi {
@@ -425,7 +429,7 @@ fn initialize_beefy<API>(
 	min_block_delta: u32,
 ) -> impl Future<Output = ()>
 where
-	API: ProvideRuntimeApi<Block> + Sync + Send +'static,
+	API: ProvideRuntimeApi<Block> + Sync + Send + 'static,
 	API::Api: BeefyApi<Block, AuthorityId> + MmrApi<Block, MmrRootHash, NumberFor<Block>>,
 {
 	let tasks = FuturesUnordered::new();
@@ -1318,7 +1322,8 @@ async fn beefy_reports_vote_equivocations() {
 	}
 
 	// Verify expected equivocation
-	let alice_reported_vote_equivocations = api_alice.reported_vote_equivocations.as_ref().unwrap().lock();
+	let alice_reported_vote_equivocations =
+		api_alice.reported_vote_equivocations.as_ref().unwrap().lock();
 	assert_eq!(alice_reported_vote_equivocations.len(), 1);
 	let equivocation_proof = alice_reported_vote_equivocations.get(0).unwrap();
 	assert_eq!(equivocation_proof.first.id, BeefyKeyring::Bob.public());
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index c50a0ada180b..838a15b91786 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -43,8 +43,8 @@ use sp_consensus::SyncOracle;
 use sp_consensus_beefy::{
 	check_vote_equivocation_proof,
 	ecdsa_crypto::{AuthorityId, Signature},
-	BeefyApi, Commitment, ConsensusLog, VoteEquivocationProof, PayloadProvider, ValidatorSet,
-	VersionedFinalityProof, VoteMessage, BEEFY_ENGINE_ID,
+	BeefyApi, Commitment, ConsensusLog, PayloadProvider, ValidatorSet, VersionedFinalityProof,
+	VoteEquivocationProof, VoteMessage, BEEFY_ENGINE_ID,
 };
 use sp_runtime::{
 	generic::OpaqueDigestItemId,
@@ -969,7 +969,11 @@ where
 		// submit equivocation report at **best** block
 		let best_block_hash = self.backend.blockchain().info().best_hash;
 		runtime_api
-			.submit_report_vote_equivocation_unsigned_extrinsic(best_block_hash, proof, key_owner_proof)
+			.submit_report_vote_equivocation_unsigned_extrinsic(
+				best_block_hash,
+				proof,
+				key_owner_proof,
+			)
 			.map_err(Error::RuntimeApi)?;
 
 		Ok(())
@@ -1049,8 +1053,9 @@ pub(crate) mod tests {
 	use sp_api::HeaderT;
 	use sp_blockchain::Backend as BlockchainBackendT;
 	use sp_consensus_beefy::{
-		generate_vote_equivocation_proof, generate_fork_equivocation_proof_vote, known_payloads, known_payloads::MMR_ROOT_ID,
-		mmr::MmrRootProvider, Keyring, Payload, SignedCommitment, ForkEquivocationProof,
+		generate_fork_equivocation_proof_vote, generate_vote_equivocation_proof, known_payloads,
+		known_payloads::MMR_ROOT_ID, mmr::MmrRootProvider, ForkEquivocationProof, Keyring, Payload,
+		SignedCommitment,
 	};
 	use sp_runtime::traits::One;
 	use std::marker::PhantomData;
@@ -1118,7 +1123,8 @@ pub(crate) mod tests {
 
 		let backend = peer.client().as_backend();
 		let beefy_genesis = 1;
-		let api = runtime_api.unwrap_or_else(|| Arc::new(TestApi::with_validator_set(&genesis_validator_set)));
+		let api = runtime_api
+			.unwrap_or_else(|| Arc::new(TestApi::with_validator_set(&genesis_validator_set)));
 		let network = peer.network_service().clone();
 		let sync = peer.sync_service().clone();
 		let payload_provider = MmrRootProvider::new(api.clone());
@@ -1612,7 +1618,13 @@ pub(crate) mod tests {
 		let api_alice = Arc::new(api_alice);
 
 		let mut net = BeefyTestNet::new(1);
-		let worker = create_beefy_worker(net.peer(0), &keys[0], 1, validator_set.clone(), Some(api_alice.clone()));
+		let worker = create_beefy_worker(
+			net.peer(0),
+			&keys[0],
+			1,
+			validator_set.clone(),
+			Some(api_alice.clone()),
+		);
 
 		// let there be a block with num = 1:
 		let _ = net.peer(0).push_blocks(1, false);
@@ -1676,24 +1688,38 @@ pub(crate) mod tests {
 
 		let session_len = 10;
 		let hashes = net.generate_blocks_and_sync(50, session_len, &validator_set, true).await;
-		let alice_worker = create_beefy_worker(net.peer(0), &peers[0], 1, validator_set.clone(), Some(api_alice));
+		let alice_worker =
+			create_beefy_worker(net.peer(0), &peers[0], 1, validator_set.clone(), Some(api_alice));
 
 		let block_number = 1;
-		let header = net.peer(1).client().as_backend().blockchain().header(hashes[block_number as usize]).unwrap().unwrap();
+		let header = net
+			.peer(1)
+			.client()
+			.as_backend()
+			.blockchain()
+			.header(hashes[block_number as usize])
+			.unwrap()
+			.unwrap();
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, "amievil".encode());
-		let votes: Vec<_> = peers.iter().map(|k| {
-			// signed_vote(block_number as u64, payload.clone(), validator_set.id(), k)
-			(block_number as u64, payload.clone(), validator_set.id(), k)
-		}).collect();
-
+		let votes: Vec<_> = peers
+			.iter()
+			.map(|k| {
+				// signed_vote(block_number as u64, payload.clone(), validator_set.id(), k)
+				(block_number as u64, payload.clone(), validator_set.id(), k)
+			})
+			.collect();
 
 		// verify: Alice reports Bob
 		let proof = generate_fork_equivocation_proof_vote(votes[1].clone(), header.clone());
 		{
 			// expect fisher (Alice) to successfully process it
-			assert_eq!(alice_worker.gossip_validator.fisherman.report_fork_equivocation(proof.clone()), Ok(()));
+			assert_eq!(
+				alice_worker.gossip_validator.fisherman.report_fork_equivocation(proof.clone()),
+				Ok(())
+			);
 			// verify Alice reports Bob's equivocation to runtime
-			let reported = alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
+			let reported =
+				alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
 			assert_eq!(reported.len(), 1);
 			assert_eq!(*reported.get(0).unwrap(), proof);
 		}
@@ -1702,9 +1728,13 @@ pub(crate) mod tests {
 		let proof = generate_fork_equivocation_proof_vote(votes[0].clone(), header.clone());
 		{
 			// expect fisher (Alice) to successfully process it
-			assert_eq!(alice_worker.gossip_validator.fisherman.report_fork_equivocation(proof.clone()), Ok(()));
+			assert_eq!(
+				alice_worker.gossip_validator.fisherman.report_fork_equivocation(proof.clone()),
+				Ok(())
+			);
 			// verify Alice does *not* report her own equivocation to runtime
-			let reported = alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
+			let reported =
+				alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
 			assert_eq!(reported.len(), 1);
 			assert!(*reported.get(0).unwrap() != proof);
 		}
@@ -1716,13 +1746,20 @@ pub(crate) mod tests {
 			validator_set_id: validator_set.id(),
 		};
 		// only Bob and Charlie sign
-		let signatories: Vec<_> = vec![Keyring::Bob, Keyring::Charlie].iter().map(|k| (k.public(), k.sign(&commitment.encode()))).collect();
-		let proof = ForkEquivocationProof {commitment, signatories, correct_header: header};
+		let signatories: Vec<_> = vec![Keyring::Bob, Keyring::Charlie]
+			.iter()
+			.map(|k| (k.public(), k.sign(&commitment.encode())))
+			.collect();
+		let proof = ForkEquivocationProof { commitment, signatories, correct_header: header };
 		{
 			// expect fisher (Alice) to successfully process it
-			assert_eq!(alice_worker.gossip_validator.fisherman.report_fork_equivocation(proof.clone()), Ok(()));
+			assert_eq!(
+				alice_worker.gossip_validator.fisherman.report_fork_equivocation(proof.clone()),
+				Ok(())
+			);
 			// verify Alice report Bob's and Charlie's equivocation to runtime
-			let reported = alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
+			let reported =
+				alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
 			assert_eq!(reported.len(), 2);
 			assert_eq!(*reported.get(1).unwrap(), proof);
 		}
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index b7a63981d4a5..625d12ec2132 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -38,7 +38,9 @@ use codec::{self as codec, Decode, Encode};
 use frame_support::traits::{Get, KeyOwnerProofSystem};
 use frame_system::pallet_prelude::{BlockNumberFor, HeaderFor};
 use log::{error, info};
-use sp_consensus_beefy::{VoteEquivocationProof, ForkEquivocationProof, ValidatorSetId, KEY_TYPE as BEEFY_KEY_TYPE};
+use sp_consensus_beefy::{
+	ForkEquivocationProof, ValidatorSetId, VoteEquivocationProof, KEY_TYPE as BEEFY_KEY_TYPE,
+};
 use sp_runtime::{
 	transaction_validity::{
 		InvalidTransaction, TransactionPriority, TransactionSource, TransactionValidity,
@@ -125,20 +127,21 @@ pub struct EquivocationReportSystem<T, R, P, L>(sp_std::marker::PhantomData<(T,
 pub enum EquivocationEvidenceFor<T: Config> {
 	VoteEquivocationProof(
 		VoteEquivocationProof<
-				BlockNumberFor<T>,
+			BlockNumberFor<T>,
 			<T as Config>::BeefyId,
 			<<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
-			>,
+		>,
 		<T as Config>::KeyOwnerProof,
 	),
 	ForkEquivocationProof(
-		ForkEquivocationProof<BlockNumberFor<T>,
-							  <T as Config>::BeefyId,
-							  <<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
-							  HeaderFor<T>,
-							  >,
+		ForkEquivocationProof<
+			BlockNumberFor<T>,
+			<T as Config>::BeefyId,
+			<<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
+			HeaderFor<T>,
+		>,
 		Vec<<T as Config>::KeyOwnerProof>,
-	)
+	),
 }
 
 impl<T, R, P, L> OffenceReportSystem<Option<T::AccountId>, EquivocationEvidenceFor<T>>
@@ -160,18 +163,18 @@ where
 		use frame_system::offchain::SubmitTransaction;
 
 		let call = match evidence {
-			EquivocationEvidenceFor::VoteEquivocationProof(equivocation_proof, key_owner_proof) => {
+			EquivocationEvidenceFor::VoteEquivocationProof(equivocation_proof, key_owner_proof) =>
 				Call::report_vote_equivocation_unsigned {
 					equivocation_proof: Box::new(equivocation_proof),
 					key_owner_proof,
-				}
-			}
-			EquivocationEvidenceFor::ForkEquivocationProof(equivocation_proof, key_owner_proofs) => {
-				Call::report_fork_equivocation_unsigned {
-					equivocation_proof: Box::new(equivocation_proof),
-					key_owner_proofs,
-				}
-			}
+				},
+			EquivocationEvidenceFor::ForkEquivocationProof(
+				equivocation_proof,
+				key_owner_proofs,
+			) => Call::report_fork_equivocation_unsigned {
+				equivocation_proof: Box::new(equivocation_proof),
+				key_owner_proofs,
+			},
 		};
 
 		let res = SubmitTransaction::<T, Call<T>>::submit_unsigned_transaction(call.into());
@@ -187,15 +190,20 @@ where
 	) -> Result<(), TransactionValidityError> {
 		let (offenders, key_owner_proofs, time_slot) = match &evidence {
 			EquivocationEvidenceFor::VoteEquivocationProof(equivocation_proof, key_owner_proof) => {
-				// Check if the offence has already been reported, and if so then we can discard the report.
+				// Check if the offence has already been reported, and if so then we can discard the
+				// report.
 				let time_slot = TimeSlot {
 					set_id: equivocation_proof.set_id(),
 					round: *equivocation_proof.round_number(),
 				};
 				(vec![equivocation_proof.offender_id()], vec![key_owner_proof.clone()], time_slot)
 			},
-			EquivocationEvidenceFor::ForkEquivocationProof(equivocation_proof, key_owner_proofs) => {
-				// Check if the offence has already been reported, and if so then we can discard the report.
+			EquivocationEvidenceFor::ForkEquivocationProof(
+				equivocation_proof,
+				key_owner_proofs,
+			) => {
+				// Check if the offence has already been reported, and if so then we can discard the
+				// report.
 				let time_slot = TimeSlot {
 					set_id: equivocation_proof.set_id(),
 					round: *equivocation_proof.round_number(),
@@ -209,7 +217,9 @@ where
 		let offenders = offenders
 			.into_iter()
 			.zip(key_owner_proofs.iter())
-			.map(|(key, key_owner_proof)| P::check_proof((BEEFY_KEY_TYPE, key.clone()), key_owner_proof.clone()))
+			.map(|(key, key_owner_proof)| {
+				P::check_proof((BEEFY_KEY_TYPE, key.clone()), key_owner_proof.clone())
+			})
 			.collect::<Option<Vec<_>>>()
 			.ok_or(InvalidTransaction::BadProof)?;
 
@@ -227,12 +237,24 @@ where
 		let reporter = reporter.or_else(|| <pallet_authorship::Pallet<T>>::author());
 
 		let (offenders, key_owner_proofs, set_id, round) = match &evidence {
-			EquivocationEvidenceFor::VoteEquivocationProof(equivocation_proof, key_owner_proof) => {
-				(vec![equivocation_proof.offender_id()], vec![key_owner_proof.clone()], equivocation_proof.set_id(), *equivocation_proof.round_number())
-			},
-			EquivocationEvidenceFor::ForkEquivocationProof(equivocation_proof, key_owner_proofs) => {
+			EquivocationEvidenceFor::VoteEquivocationProof(equivocation_proof, key_owner_proof) =>
+				(
+					vec![equivocation_proof.offender_id()],
+					vec![key_owner_proof.clone()],
+					equivocation_proof.set_id(),
+					*equivocation_proof.round_number(),
+				),
+			EquivocationEvidenceFor::ForkEquivocationProof(
+				equivocation_proof,
+				key_owner_proofs,
+			) => {
 				let offenders = equivocation_proof.offender_ids(); // clone data here
-				(offenders, key_owner_proofs.to_owned(), equivocation_proof.set_id(), *equivocation_proof.round_number())
+				(
+					offenders,
+					key_owner_proofs.to_owned(),
+					equivocation_proof.set_id(),
+					*equivocation_proof.round_number(),
+				)
 			},
 		};
 
@@ -247,7 +269,9 @@ where
 		let offenders = offenders
 			.into_iter()
 			.zip(key_owner_proofs.iter())
-			.map(|(key, key_owner_proof)| P::check_proof((BEEFY_KEY_TYPE, key.clone()), key_owner_proof.clone()))
+			.map(|(key, key_owner_proof)| {
+				P::check_proof((BEEFY_KEY_TYPE, key.clone()), key_owner_proof.clone())
+			})
 			.collect::<Option<Vec<_>>>()
 			.ok_or(Error::<T>::InvalidKeyOwnershipProof)?;
 
@@ -262,8 +286,12 @@ where
 				let block_number = equivocation_proof.commitment.block_number;
 				let expected_block_hash = <frame_system::Pallet<T>>::block_hash(block_number);
 
-				// Validate equivocation proof (check commitment is to unexpected payload and signatures are valid).
-				if !sp_consensus_beefy::check_fork_equivocation_proof(&equivocation_proof, &expected_block_hash) {
+				// Validate equivocation proof (check commitment is to unexpected payload and
+				// signatures are valid).
+				if !sp_consensus_beefy::check_fork_equivocation_proof(
+					&equivocation_proof,
+					&expected_block_hash,
+				) {
 					return Err(Error::<T>::InvalidForkEquivocationProof.into())
 				}
 			},
@@ -271,14 +299,14 @@ where
 
 		// Check that the session id for the membership proof is within the
 		// bounds of the set id reported in the equivocation.
-		let set_id_session_index =
-			crate::SetIdSession::<T>::get(set_id).ok_or(Error::<T>::InvalidVoteEquivocationProof)?;
+		let set_id_session_index = crate::SetIdSession::<T>::get(set_id)
+			.ok_or(Error::<T>::InvalidVoteEquivocationProof)?;
 		if session_index != set_id_session_index {
 			return Err(Error::<T>::InvalidVoteEquivocationProof.into())
 		}
 
 		let offence = EquivocationOffence {
-			time_slot:  TimeSlot { set_id, round },
+			time_slot: TimeSlot { set_id, round },
 			session_index,
 			validator_set_count,
 			offenders,
@@ -292,12 +320,14 @@ where
 }
 
 /// Methods for the `ValidateUnsigned` implementation:
-/// It restricts calls to `report_vote_equivocation_unsigned` to local calls (i.e. extrinsics generated
-/// on this node) or that already in a block. This guarantees that only block authors can include
-/// unsigned equivocation reports.
+/// It restricts calls to `report_vote_equivocation_unsigned` to local calls (i.e. extrinsics
+/// generated on this node) or that already in a block. This guarantees that only block authors can
+/// include unsigned equivocation reports.
 impl<T: Config> Pallet<T> {
 	pub fn validate_unsigned(source: TransactionSource, call: &Call<T>) -> TransactionValidity {
-		if let Call::report_vote_equivocation_unsigned { equivocation_proof, key_owner_proof } = call {
+		if let Call::report_vote_equivocation_unsigned { equivocation_proof, key_owner_proof } =
+			call
+		{
 			// discard equivocation report not coming from the local node
 			match source {
 				TransactionSource::Local | TransactionSource::InBlock => { /* allowed */ },
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index af869ae26bbc..e54098f99e35 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -28,7 +28,7 @@ use frame_support::{
 };
 use frame_system::{
 	ensure_none, ensure_signed,
-	pallet_prelude::{BlockNumberFor, OriginFor, HeaderFor},
+	pallet_prelude::{BlockNumberFor, HeaderFor, OriginFor},
 };
 use log;
 use sp_runtime::{
@@ -41,8 +41,8 @@ use sp_staking::{offence::OffenceReportSystem, SessionIndex};
 use sp_std::prelude::*;
 
 use sp_consensus_beefy::{
-	AuthorityIndex, BeefyAuthorityId, ConsensusLog, VoteEquivocationProof, OnNewValidatorSet,
-	ValidatorSet, BEEFY_ENGINE_ID, GENESIS_AUTHORITY_SET_ID,
+	AuthorityIndex, BeefyAuthorityId, ConsensusLog, OnNewValidatorSet, ValidatorSet,
+	VoteEquivocationProof, BEEFY_ENGINE_ID, GENESIS_AUTHORITY_SET_ID,
 };
 
 mod default_weights;
@@ -292,7 +292,7 @@ pub mod pallet {
 					BlockNumberFor<T>,
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
-					HeaderFor<T>
+					HeaderFor<T>,
 				>,
 			>,
 			key_owner_proofs: Vec<T::KeyOwnerProof>,
@@ -301,7 +301,10 @@ pub mod pallet {
 
 			T::EquivocationReportSystem::process_evidence(
 				Some(reporter),
-				EquivocationEvidenceFor::ForkEquivocationProof(*equivocation_proof, key_owner_proofs),
+				EquivocationEvidenceFor::ForkEquivocationProof(
+					*equivocation_proof,
+					key_owner_proofs,
+				),
 			)?;
 			// Waive the fee since the report is valid and beneficial
 			Ok(Pays::No.into())
@@ -389,9 +392,12 @@ impl<T: Config> Pallet<T> {
 	/// a call to `report_fork_equivocation_unsigned` and will push the transaction
 	/// to the pool. Only useful in an offchain context.
 	pub fn submit_unsigned_fork_equivocation_report(
-		fork_equivocation_proof: sp_consensus_beefy::ForkEquivocationProof<BlockNumberFor<T>, T::BeefyId,
-			<T::BeefyId as RuntimeAppPublic>::Signature, HeaderFor<T>
->,
+		fork_equivocation_proof: sp_consensus_beefy::ForkEquivocationProof<
+			BlockNumberFor<T>,
+			T::BeefyId,
+			<T::BeefyId as RuntimeAppPublic>::Signature,
+			HeaderFor<T>,
+		>,
 		key_owner_proofs: Vec<T::KeyOwnerProof>,
 	) -> Option<()> {
 		T::EquivocationReportSystem::publish_evidence(
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index b7dc418a182d..09da46cdf113 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -47,7 +47,7 @@ use codec::{Codec, Decode, Encode};
 use scale_info::TypeInfo;
 use sp_application_crypto::RuntimeAppPublic;
 use sp_core::H256;
-use sp_runtime::traits::{Hash, Keccak256, NumberFor, Header};
+use sp_runtime::traits::{Hash, Header, Keccak256, NumberFor};
 use sp_std::prelude::*;
 
 /// Key type for BEEFY module.
@@ -366,9 +366,8 @@ where
 	}
 
 	let expected_mmr_root_digest = mmr::find_mmr_root_digest::<Header>(correct_header);
-	let expected_payload = expected_mmr_root_digest.map(|mmr_root| {
-		Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode())
-	});
+	let expected_payload = expected_mmr_root_digest
+		.map(|mmr_root| Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode()));
 
 	// cheap failfasts:
 	// 1. check that `payload` on the `vote` is different that the `expected_payload`
diff --git a/substrate/primitives/consensus/beefy/src/test_utils.rs b/substrate/primitives/consensus/beefy/src/test_utils.rs
index 20a4277eb9dd..d52148073efd 100644
--- a/substrate/primitives/consensus/beefy/src/test_utils.rs
+++ b/substrate/primitives/consensus/beefy/src/test_utils.rs
@@ -17,7 +17,10 @@
 
 #![cfg(feature = "std")]
 
-use crate::{ecdsa_crypto, ForkEquivocationProof, Commitment, SignedCommitment, VoteEquivocationProof, Payload, ValidatorSetId, VoteMessage};
+use crate::{
+	ecdsa_crypto, Commitment, ForkEquivocationProof, Payload, SignedCommitment, ValidatorSetId,
+	VoteEquivocationProof, VoteMessage,
+};
 use codec::Encode;
 use sp_core::{ecdsa, keccak_256, Pair};
 use std::collections::HashMap;
@@ -92,7 +95,12 @@ impl From<Keyring> for ecdsa_crypto::Public {
 }
 
 /// Create a new `VoteMessage` from commitment primitives and keyring
-fn signed_vote(block_number: u64, payload: Payload, validator_set_id: ValidatorSetId, keyring: &Keyring) -> VoteMessage<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature> {
+fn signed_vote(
+	block_number: u64,
+	payload: Payload,
+	validator_set_id: ValidatorSetId,
+	keyring: &Keyring,
+) -> VoteMessage<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature> {
 	let commitment = Commitment { validator_set_id, block_number, payload };
 	let signature = keyring.sign(&commitment.encode());
 	VoteMessage { commitment, id: keyring.public(), signature }

From ab3eb028f08f86fb8cc62460a2abcec36b077813 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 29 Aug 2023 09:59:19 +0200
Subject: [PATCH 042/188] missing keystore

---
 substrate/client/consensus/beefy/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/substrate/client/consensus/beefy/src/lib.rs b/substrate/client/consensus/beefy/src/lib.rs
index eef618d13ada..3846b6b08011 100644
--- a/substrate/client/consensus/beefy/src/lib.rs
+++ b/substrate/client/consensus/beefy/src/lib.rs
@@ -265,6 +265,7 @@ pub async fn start_beefy_gadget<B, BE, C, N, P, R, S>(
 		let known_peers = Arc::new(Mutex::new(KnownPeers::new()));
 		let fisherman = Fisherman {
 			backend: backend.clone(),
+			key_store: key_store.clone(),
 			runtime: runtime.clone(),
 			payload_provider: payload_provider.clone(),
 			_phantom: PhantomData,

From 844ed67e9f748c91d39d3f28344f9bb5f684bf8f Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 29 Aug 2023 10:24:11 +0200
Subject: [PATCH 043/188] graft
 https://github.com/paritytech/polkadot/pull/7634

---
 polkadot/node/service/src/fake_runtime_api.rs | 11 ++++++++--
 polkadot/runtime/polkadot/src/lib.rs          | 14 +++++++++++--
 polkadot/runtime/rococo/src/lib.rs            | 20 +++++++++++++++----
 polkadot/runtime/test-runtime/src/lib.rs      | 11 ++++++++--
 polkadot/runtime/westend/src/lib.rs           |  7 +++++++
 5 files changed, 53 insertions(+), 10 deletions(-)

diff --git a/polkadot/node/service/src/fake_runtime_api.rs b/polkadot/node/service/src/fake_runtime_api.rs
index d9553afa024b..418217997385 100644
--- a/polkadot/node/service/src/fake_runtime_api.rs
+++ b/polkadot/node/service/src/fake_runtime_api.rs
@@ -240,8 +240,8 @@ sp_api::impl_runtime_apis! {
 			unimplemented!()
 		}
 
-		fn submit_report_equivocation_unsigned_extrinsic(
-			_: beefy_primitives::EquivocationProof<
+		fn submit_report_vote_equivocation_unsigned_extrinsic(
+			_: beefy_primitives::VoteEquivocationProof<
 				BlockNumber,
 				BeefyId,
 				BeefySignature,
@@ -251,6 +251,13 @@ sp_api::impl_runtime_apis! {
 			unimplemented!()
 		}
 
+		fn submit_report_fork_equivocation_unsigned_extrinsic(
+			_: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, <Block as BlockT>::Header>,
+			_: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
+		) -> Option<()> {
+			unimplemented!()
+		}
+
 		fn generate_key_ownership_proof(
 			_: beefy_primitives::ValidatorSetId,
 			_: BeefyId,
diff --git a/polkadot/runtime/polkadot/src/lib.rs b/polkadot/runtime/polkadot/src/lib.rs
index c4458076cb3d..4b7b2cf6d942 100644
--- a/polkadot/runtime/polkadot/src/lib.rs
+++ b/polkadot/runtime/polkadot/src/lib.rs
@@ -1833,14 +1833,23 @@ sp_api::impl_runtime_apis! {
 			None
 		}
 
-		fn submit_report_equivocation_unsigned_extrinsic(
-			_equivocation_proof: beefy_primitives::EquivocationProof<
+		fn submit_report_vote_equivocation_unsigned_extrinsic(
+			_vote_equivocation_proof: beefy_primitives::VoteEquivocationProof<
 				BlockNumber,
 				BeefyId,
 				BeefySignature,
 			>,
 			_key_owner_proof: beefy_primitives::OpaqueKeyOwnershipProof,
 		) -> Option<()> {
+			// dummy implementation due to lack of BEEFY pallet.
+			None
+		}
+
+		fn submit_report_fork_equivocation_unsigned_extrinsic(
+			_fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, Header>,
+			_key_owner_proofs: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
+		) -> Option<()> {
+			// dummy implementation due to lack of BEEFY pallet.
 			None
 		}
 
@@ -1848,6 +1857,7 @@ sp_api::impl_runtime_apis! {
 			_set_id: beefy_primitives::ValidatorSetId,
 			_authority_id: BeefyId,
 		) -> Option<beefy_primitives::OpaqueKeyOwnershipProof> {
+			// dummy implementation due to lack of BEEFY pallet.
 			None
 		}
 	}
diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs
index 6894bd7bbf44..33a581ece87d 100644
--- a/polkadot/runtime/rococo/src/lib.rs
+++ b/polkadot/runtime/rococo/src/lib.rs
@@ -1856,8 +1856,8 @@ sp_api::impl_runtime_apis! {
 			Beefy::validator_set()
 		}
 
-		fn submit_report_equivocation_unsigned_extrinsic(
-			equivocation_proof: beefy_primitives::EquivocationProof<
+		fn submit_report_vote_equivocation_unsigned_extrinsic(
+			vote_equivocation_proof: beefy_primitives::VoteEquivocationProof<
 				BlockNumber,
 				BeefyId,
 				BeefySignature,
@@ -1866,12 +1866,24 @@ sp_api::impl_runtime_apis! {
 		) -> Option<()> {
 			let key_owner_proof = key_owner_proof.decode()?;
 
-			Beefy::submit_unsigned_equivocation_report(
-				equivocation_proof,
+			Beefy::submit_unsigned_vote_equivocation_report(
+				vote_equivocation_proof,
 				key_owner_proof,
 			)
 		}
 
+		fn submit_report_fork_equivocation_unsigned_extrinsic(
+			fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, Header>,
+			key_owner_proofs: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
+		) -> Option<()> {
+			let key_owner_proofs = key_owner_proofs.iter().cloned().map(|p| p.decode()).collect::<Option<Vec<_>>>()?;
+
+			Beefy::submit_unsigned_fork_equivocation_report(
+				fork_equivocation_proof,
+				key_owner_proofs,
+			)
+		}
+
 		fn generate_key_ownership_proof(
 			_set_id: beefy_primitives::ValidatorSetId,
 			authority_id: BeefyId,
diff --git a/polkadot/runtime/test-runtime/src/lib.rs b/polkadot/runtime/test-runtime/src/lib.rs
index b2397299430d..1dd069a6c974 100644
--- a/polkadot/runtime/test-runtime/src/lib.rs
+++ b/polkadot/runtime/test-runtime/src/lib.rs
@@ -961,8 +961,8 @@ sp_api::impl_runtime_apis! {
 			None
 		}
 
-		fn submit_report_equivocation_unsigned_extrinsic(
-			_equivocation_proof: beefy_primitives::EquivocationProof<
+		fn submit_report_vote_equivocation_unsigned_extrinsic(
+			_vote_equivocation_proof: beefy_primitives::VoteEquivocationProof<
 				BlockNumber,
 				BeefyId,
 				BeefySignature,
@@ -972,6 +972,13 @@ sp_api::impl_runtime_apis! {
 			None
 		}
 
+		fn submit_report_fork_equivocation_unsigned_extrinsic(
+			_fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, Header>,
+			_key_owner_proofs: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
+		) -> Option<()> {
+			None
+		}
+
 		fn generate_key_ownership_proof(
 			_set_id: beefy_primitives::ValidatorSetId,
 			_authority_id: BeefyId,
diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs
index 9ae30c376010..ad345118f129 100644
--- a/polkadot/runtime/westend/src/lib.rs
+++ b/polkadot/runtime/westend/src/lib.rs
@@ -1718,6 +1718,13 @@ sp_api::impl_runtime_apis! {
 			)
 		}
 
+		fn submit_report_fork_equivocation_unsigned_extrinsic(
+			_fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, Header>,
+			_key_owner_proofs: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
+		) -> Option<()> {
+			None
+		}
+
 		fn generate_key_ownership_proof(
 			_set_id: beefy_primitives::ValidatorSetId,
 			authority_id: BeefyId,

From 6ebebedd209c900e39249c753fb9dc623347d046 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 29 Aug 2023 10:33:52 +0200
Subject: [PATCH 044/188] impl equiv. runtime interfaces kusama/westend

---
 polkadot/runtime/kusama/src/lib.rs  | 20 ++++++++++++++++----
 polkadot/runtime/westend/src/lib.rs | 19 ++++++++++++-------
 2 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/polkadot/runtime/kusama/src/lib.rs b/polkadot/runtime/kusama/src/lib.rs
index e9e3fb2d2026..3ce1a61c2f21 100644
--- a/polkadot/runtime/kusama/src/lib.rs
+++ b/polkadot/runtime/kusama/src/lib.rs
@@ -2028,8 +2028,8 @@ sp_api::impl_runtime_apis! {
 			Beefy::validator_set()
 		}
 
-		fn submit_report_equivocation_unsigned_extrinsic(
-			equivocation_proof: beefy_primitives::EquivocationProof<
+		fn submit_report_vote_equivocation_unsigned_extrinsic(
+			vote_equivocation_proof: beefy_primitives::VoteEquivocationProof<
 				BlockNumber,
 				BeefyId,
 				BeefySignature,
@@ -2038,12 +2038,24 @@ sp_api::impl_runtime_apis! {
 		) -> Option<()> {
 			let key_owner_proof = key_owner_proof.decode()?;
 
-			Beefy::submit_unsigned_equivocation_report(
-				equivocation_proof,
+			Beefy::submit_unsigned_vote_equivocation_report(
+				vote_equivocation_proof,
 				key_owner_proof,
 			)
 		}
 
+		fn submit_report_fork_equivocation_unsigned_extrinsic(
+			fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, Header>,
+			key_owner_proofs: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
+		) -> Option<()> {
+			let key_owner_proofs = key_owner_proofs.iter().cloned().map(|p| p.decode()).collect::<Option<Vec<_>>>()?;
+
+			Beefy::submit_unsigned_fork_equivocation_report(
+				fork_equivocation_proof,
+				key_owner_proofs,
+			)
+		}
+
 		fn generate_key_ownership_proof(
 			_set_id: beefy_primitives::ValidatorSetId,
 			authority_id: BeefyId,
diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs
index ad345118f129..697704b8f083 100644
--- a/polkadot/runtime/westend/src/lib.rs
+++ b/polkadot/runtime/westend/src/lib.rs
@@ -1702,8 +1702,8 @@ sp_api::impl_runtime_apis! {
 			Beefy::validator_set()
 		}
 
-		fn submit_report_equivocation_unsigned_extrinsic(
-			equivocation_proof: beefy_primitives::EquivocationProof<
+		fn submit_report_vote_equivocation_unsigned_extrinsic(
+			vote_equivocation_proof: beefy_primitives::VoteEquivocationProof<
 				BlockNumber,
 				BeefyId,
 				BeefySignature,
@@ -1712,17 +1712,22 @@ sp_api::impl_runtime_apis! {
 		) -> Option<()> {
 			let key_owner_proof = key_owner_proof.decode()?;
 
-			Beefy::submit_unsigned_equivocation_report(
-				equivocation_proof,
+			Beefy::submit_unsigned_vote_equivocation_report(
+				vote_equivocation_proof,
 				key_owner_proof,
 			)
 		}
 
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
-			_fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, Header>,
-			_key_owner_proofs: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
+			fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, Header>,
+			key_owner_proofs: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
-			None
+			let key_owner_proofs = key_owner_proofs.iter().cloned().map(|p| p.decode()).collect::<Option<Vec<_>>>()?;
+
+			Beefy::submit_unsigned_fork_equivocation_report(
+				fork_equivocation_proof,
+				key_owner_proofs,
+			)
 		}
 
 		fn generate_key_ownership_proof(

From efe4b2a896d12475ced62a320f971302dc76b87a Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 31 Aug 2023 09:50:17 +0200
Subject: [PATCH 045/188] clone beefy pallet equivocation tests

---
 substrate/frame/beefy/src/lib.rs   |  10 +-
 substrate/frame/beefy/src/tests.rs | 541 ++++++++++++++++++++++++++++-
 2 files changed, 545 insertions(+), 6 deletions(-)

diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index e54098f99e35..5e529ad64b27 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -336,11 +336,11 @@ pub mod pallet {
 		) -> DispatchResultWithPostInfo {
 			ensure_none(origin)?;
 
-			// TODO:
-			// T::EquivocationReportSystem::process_evidence(
-			// 	None,
-			// 	(*fork_equivocation_proof, key_owner_proof),
-			// )?;
+			T::EquivocationReportSystem::process_evidence(
+				None,
+				EquivocationEvidenceFor::ForkEquivocationProof(*equivocation_proof, key_owner_proofs),
+			)?;
+			// Waive the fee since the report is valid and beneficial
 			Ok(Pays::No.into())
 		}
 	}
diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index fd99343c7746..28930f69a7fe 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -19,7 +19,7 @@ use std::vec;
 
 use codec::Encode;
 use sp_consensus_beefy::{
-	check_vote_equivocation_proof, generate_vote_equivocation_proof, known_payloads::MMR_ROOT_ID,
+	check_vote_equivocation_proof, generate_vote_equivocation_proof, generate_fork_equivocation_proof_vote, known_payloads::MMR_ROOT_ID,
 	Keyring as BeefyKeyring, Payload, ValidatorSet, KEY_TYPE as BEEFY_KEY_TYPE,
 };
 
@@ -255,6 +255,545 @@ fn should_sign_and_verify() {
 	assert!(check_vote_equivocation_proof::<_, _, Keccak256>(&equivocation_proof));
 }
 
+// vote equivocation report tests
+#[test]
+fn report_equivocation_current_set_works() {
+	let authorities = test_authorities();
+
+	new_test_ext_raw_authorities(authorities).execute_with(|| {
+		assert_eq!(Staking::current_era(), Some(0));
+		assert_eq!(Session::current_index(), 0);
+
+		start_era(1);
+
+		let block_num = System::block_number();
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+		let set_id = validator_set.id();
+		let validators = Session::validators();
+
+		// make sure that all validators have the same balance
+		for validator in &validators {
+			assert_eq!(Balances::total_balance(validator), 10_000_000);
+			assert_eq!(Staking::slashable_balance_of(validator), 10_000);
+
+			assert_eq!(
+				Staking::eras_stakers(1, validator),
+				pallet_staking::Exposure { total: 10_000, own: 10_000, others: vec![] },
+			);
+		}
+
+		assert_eq!(authorities.len(), 2);
+		let equivocation_authority_index = 1;
+		let equivocation_key = &authorities[equivocation_authority_index];
+		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+
+		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
+		// generate an equivocation proof, with two votes in the same round for
+		// different payloads signed by the same key
+		let equivocation_proof = generate_vote_equivocation_proof(
+			(block_num, payload1, set_id, &equivocation_keyring),
+			(block_num, payload2, set_id, &equivocation_keyring),
+		);
+
+		// create the key ownership proof
+		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+
+		// report the equivocation and the tx should be dispatched successfully
+		assert_ok!(Beefy::report_vote_equivocation_unsigned(
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof),
+			key_owner_proof,
+		),);
+
+		start_era(2);
+
+		// check that the balance of 0-th validator is slashed 100%.
+		let equivocation_validator_id = validators[equivocation_authority_index];
+
+		assert_eq!(Balances::total_balance(&equivocation_validator_id), 10_000_000 - 10_000);
+		assert_eq!(Staking::slashable_balance_of(&equivocation_validator_id), 0);
+		assert_eq!(
+			Staking::eras_stakers(2, equivocation_validator_id),
+			pallet_staking::Exposure { total: 0, own: 0, others: vec![] },
+		);
+
+		// check that the balances of all other validators are left intact.
+		for validator in &validators {
+			if *validator == equivocation_validator_id {
+				continue
+			}
+
+			assert_eq!(Balances::total_balance(validator), 10_000_000);
+			assert_eq!(Staking::slashable_balance_of(validator), 10_000);
+
+			assert_eq!(
+				Staking::eras_stakers(2, validator),
+				pallet_staking::Exposure { total: 10_000, own: 10_000, others: vec![] },
+			);
+		}
+	});
+}
+
+#[test]
+fn report_equivocation_old_set_works() {
+	let authorities = test_authorities();
+
+	new_test_ext_raw_authorities(authorities).execute_with(|| {
+		start_era(1);
+
+		let block_num = System::block_number();
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+		let validators = Session::validators();
+		let old_set_id = validator_set.id();
+
+		assert_eq!(authorities.len(), 2);
+		let equivocation_authority_index = 0;
+		let equivocation_key = &authorities[equivocation_authority_index];
+
+		// create the key ownership proof in the "old" set
+		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+
+		start_era(2);
+
+		// make sure that all authorities have the same balance
+		for validator in &validators {
+			assert_eq!(Balances::total_balance(validator), 10_000_000);
+			assert_eq!(Staking::slashable_balance_of(validator), 10_000);
+
+			assert_eq!(
+				Staking::eras_stakers(2, validator),
+				pallet_staking::Exposure { total: 10_000, own: 10_000, others: vec![] },
+			);
+		}
+
+		let validator_set = Beefy::validator_set().unwrap();
+		let new_set_id = validator_set.id();
+		assert_eq!(old_set_id + 3, new_set_id);
+
+		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+
+		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
+		// generate an equivocation proof for the old set,
+		let equivocation_proof = generate_vote_equivocation_proof(
+			(block_num, payload1, old_set_id, &equivocation_keyring),
+			(block_num, payload2, old_set_id, &equivocation_keyring),
+		);
+
+		// report the equivocation and the tx should be dispatched successfully
+		assert_ok!(Beefy::report_vote_equivocation_unsigned(
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof),
+			key_owner_proof,
+		),);
+
+		start_era(3);
+
+		// check that the balance of 0-th validator is slashed 100%.
+		let equivocation_validator_id = validators[equivocation_authority_index];
+
+		assert_eq!(Balances::total_balance(&equivocation_validator_id), 10_000_000 - 10_000);
+		assert_eq!(Staking::slashable_balance_of(&equivocation_validator_id), 0);
+		assert_eq!(
+			Staking::eras_stakers(3, equivocation_validator_id),
+			pallet_staking::Exposure { total: 0, own: 0, others: vec![] },
+		);
+
+		// check that the balances of all other validators are left intact.
+		for validator in &validators {
+			if *validator == equivocation_validator_id {
+				continue
+			}
+
+			assert_eq!(Balances::total_balance(validator), 10_000_000);
+			assert_eq!(Staking::slashable_balance_of(validator), 10_000);
+
+			assert_eq!(
+				Staking::eras_stakers(3, validator),
+				pallet_staking::Exposure { total: 10_000, own: 10_000, others: vec![] },
+			);
+		}
+	});
+}
+
+#[test]
+fn report_equivocation_invalid_set_id() {
+	let authorities = test_authorities();
+
+	new_test_ext_raw_authorities(authorities).execute_with(|| {
+		start_era(1);
+
+		let block_num = System::block_number();
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+		let set_id = validator_set.id();
+
+		let equivocation_authority_index = 0;
+		let equivocation_key = &authorities[equivocation_authority_index];
+		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+
+		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+
+		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
+		// generate an equivocation for a future set
+		let equivocation_proof = generate_vote_equivocation_proof(
+			(block_num, payload1, set_id + 1, &equivocation_keyring),
+			(block_num, payload2, set_id + 1, &equivocation_keyring),
+		);
+
+		// the call for reporting the equivocation should error
+		assert_err!(
+			Beefy::report_vote_equivocation_unsigned(
+				RuntimeOrigin::none(),
+				Box::new(equivocation_proof),
+				key_owner_proof,
+			),
+			Error::<Test>::InvalidVoteEquivocationProof,
+		);
+	});
+}
+
+#[test]
+fn report_equivocation_invalid_session() {
+	let authorities = test_authorities();
+
+	new_test_ext_raw_authorities(authorities).execute_with(|| {
+		start_era(1);
+
+		let block_num = System::block_number();
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+
+		let equivocation_authority_index = 0;
+		let equivocation_key = &authorities[equivocation_authority_index];
+		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+
+		// generate a key ownership proof at current era set id
+		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+
+		start_era(2);
+
+		let set_id = Beefy::validator_set().unwrap().id();
+
+		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
+		// generate an equivocation proof at following era set id = 2
+		let equivocation_proof = generate_vote_equivocation_proof(
+			(block_num, payload1, set_id, &equivocation_keyring),
+			(block_num, payload2, set_id, &equivocation_keyring),
+		);
+
+		// report an equivocation for the current set using an key ownership
+		// proof from the previous set, the session should be invalid.
+		assert_err!(
+			Beefy::report_vote_equivocation_unsigned(
+				RuntimeOrigin::none(),
+				Box::new(equivocation_proof),
+				key_owner_proof,
+			),
+			Error::<Test>::InvalidVoteEquivocationProof,
+		);
+	});
+}
+
+#[test]
+fn report_equivocation_invalid_key_owner_proof() {
+	let authorities = test_authorities();
+
+	new_test_ext_raw_authorities(authorities).execute_with(|| {
+		start_era(1);
+
+		let block_num = System::block_number();
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+		let set_id = validator_set.id();
+
+		let invalid_owner_authority_index = 1;
+		let invalid_owner_key = &authorities[invalid_owner_authority_index];
+
+		// generate a key ownership proof for the authority at index 1
+		let invalid_key_owner_proof =
+			Historical::prove((BEEFY_KEY_TYPE, &invalid_owner_key)).unwrap();
+
+		let equivocation_authority_index = 0;
+		let equivocation_key = &authorities[equivocation_authority_index];
+		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+
+		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
+		// generate an equivocation proof for the authority at index 0
+		let equivocation_proof = generate_vote_equivocation_proof(
+			(block_num, payload1, set_id + 1, &equivocation_keyring),
+			(block_num, payload2, set_id + 1, &equivocation_keyring),
+		);
+
+		// we need to start a new era otherwise the key ownership proof won't be
+		// checked since the authorities are part of the current session
+		start_era(2);
+
+		// report an equivocation for the current set using a key ownership
+		// proof for a different key than the one in the equivocation proof.
+		assert_err!(
+			Beefy::report_vote_equivocation_unsigned(
+				RuntimeOrigin::none(),
+				Box::new(equivocation_proof),
+				invalid_key_owner_proof,
+			),
+			Error::<Test>::InvalidKeyOwnershipProof,
+		);
+	});
+}
+
+#[test]
+fn report_equivocation_invalid_equivocation_proof() {
+	let authorities = test_authorities();
+
+	new_test_ext_raw_authorities(authorities).execute_with(|| {
+		start_era(1);
+
+		let block_num = System::block_number();
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+		let set_id = validator_set.id();
+
+		let equivocation_authority_index = 0;
+		let equivocation_key = &authorities[equivocation_authority_index];
+		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+
+		// generate a key ownership proof at set id in era 1
+		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+
+		let assert_invalid_equivocation_proof = |equivocation_proof| {
+			assert_err!(
+				Beefy::report_vote_equivocation_unsigned(
+					RuntimeOrigin::none(),
+					Box::new(equivocation_proof),
+					key_owner_proof.clone(),
+				),
+				Error::<Test>::InvalidVoteEquivocationProof,
+			);
+		};
+
+		start_era(2);
+
+		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
+
+		// both votes target the same block number and payload,
+		// there is no equivocation.
+		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
+			(block_num, payload1.clone(), set_id, &equivocation_keyring),
+			(block_num, payload1.clone(), set_id, &equivocation_keyring),
+		));
+
+		// votes targeting different rounds, there is no equivocation.
+		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
+			(block_num, payload1.clone(), set_id, &equivocation_keyring),
+			(block_num + 1, payload2.clone(), set_id, &equivocation_keyring),
+		));
+
+		// votes signed with different authority keys
+		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
+			(block_num, payload1.clone(), set_id, &equivocation_keyring),
+			(block_num, payload1.clone(), set_id, &BeefyKeyring::Charlie),
+		));
+
+		// votes signed with a key that isn't part of the authority set
+		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
+			(block_num, payload1.clone(), set_id, &equivocation_keyring),
+			(block_num, payload1.clone(), set_id, &BeefyKeyring::Dave),
+		));
+
+		// votes targeting different set ids
+		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
+			(block_num, payload1, set_id, &equivocation_keyring),
+			(block_num, payload2, set_id + 1, &equivocation_keyring),
+		));
+	});
+}
+
+#[test]
+fn report_equivocation_validate_unsigned_prevents_duplicates() {
+	use sp_runtime::transaction_validity::{
+		InvalidTransaction, TransactionPriority, TransactionSource, TransactionValidity,
+		ValidTransaction,
+	};
+
+	let authorities = test_authorities();
+
+	new_test_ext_raw_authorities(authorities).execute_with(|| {
+		start_era(1);
+
+		let block_num = System::block_number();
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+		let set_id = validator_set.id();
+
+		// generate and report an equivocation for the validator at index 0
+		let equivocation_authority_index = 0;
+		let equivocation_key = &authorities[equivocation_authority_index];
+		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+
+		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
+		let equivocation_proof = generate_vote_equivocation_proof(
+			(block_num, payload1, set_id, &equivocation_keyring),
+			(block_num, payload2, set_id, &equivocation_keyring),
+		);
+
+		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+
+		let call = Call::report_vote_equivocation_unsigned {
+			equivocation_proof: Box::new(equivocation_proof.clone()),
+			key_owner_proof: key_owner_proof.clone(),
+		};
+
+		// only local/inblock reports are allowed
+		assert_eq!(
+			<Beefy as sp_runtime::traits::ValidateUnsigned>::validate_unsigned(
+				TransactionSource::External,
+				&call,
+			),
+			InvalidTransaction::Call.into(),
+		);
+
+		// the transaction is valid when passed as local
+		let tx_tag = (equivocation_key, set_id, 3u64);
+
+		assert_eq!(
+			<Beefy as sp_runtime::traits::ValidateUnsigned>::validate_unsigned(
+				TransactionSource::Local,
+				&call,
+			),
+			TransactionValidity::Ok(ValidTransaction {
+				priority: TransactionPriority::max_value(),
+				requires: vec![],
+				provides: vec![("BeefyEquivocation", tx_tag).encode()],
+				longevity: ReportLongevity::get(),
+				propagate: false,
+			})
+		);
+
+		// the pre dispatch checks should also pass
+		assert_ok!(<Beefy as sp_runtime::traits::ValidateUnsigned>::pre_dispatch(&call));
+
+		// we submit the report
+		Beefy::report_vote_equivocation_unsigned(
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof),
+			key_owner_proof,
+		)
+		.unwrap();
+
+		// the report should now be considered stale and the transaction is invalid
+		// the check for staleness should be done on both `validate_unsigned` and on `pre_dispatch`
+		assert_err!(
+			<Beefy as sp_runtime::traits::ValidateUnsigned>::validate_unsigned(
+				TransactionSource::Local,
+				&call,
+			),
+			InvalidTransaction::Stale,
+		);
+
+		assert_err!(
+			<Beefy as sp_runtime::traits::ValidateUnsigned>::pre_dispatch(&call),
+			InvalidTransaction::Stale,
+		);
+	});
+}
+
+#[test]
+fn report_equivocation_has_valid_weight() {
+	// the weight depends on the size of the validator set,
+	// but there's a lower bound of 100 validators.
+	assert!((1..=100)
+		.map(|validators| <Test as Config>::WeightInfo::report_equivocation(validators, 1000))
+		.collect::<Vec<_>>()
+		.windows(2)
+		.all(|w| w[0] == w[1]));
+
+	// after 100 validators the weight should keep increasing
+	// with every extra validator.
+	assert!((100..=1000)
+		.map(|validators| <Test as Config>::WeightInfo::report_equivocation(validators, 1000))
+		.collect::<Vec<_>>()
+		.windows(2)
+		.all(|w| w[0].ref_time() < w[1].ref_time()));
+}
+
+#[test]
+fn valid_equivocation_reports_dont_pay_fees() {
+	let authorities = test_authorities();
+
+	new_test_ext_raw_authorities(authorities).execute_with(|| {
+		start_era(1);
+
+		let block_num = System::block_number();
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+		let set_id = validator_set.id();
+
+		let equivocation_authority_index = 0;
+		let equivocation_key = &authorities[equivocation_authority_index];
+		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+
+		// generate equivocation proof
+		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
+		let equivocation_proof = generate_vote_equivocation_proof(
+			(block_num, payload1, set_id, &equivocation_keyring),
+			(block_num, payload2, set_id, &equivocation_keyring),
+		);
+
+		// create the key ownership proof.
+		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+
+		// check the dispatch info for the call.
+		let info = Call::<Test>::report_vote_equivocation_unsigned {
+			equivocation_proof: Box::new(equivocation_proof.clone()),
+			key_owner_proof: key_owner_proof.clone(),
+		}
+		.get_dispatch_info();
+
+		// it should have non-zero weight and the fee has to be paid.
+		assert!(info.weight.any_gt(Weight::zero()));
+		assert_eq!(info.pays_fee, Pays::Yes);
+
+		// report the equivocation.
+		let post_info = Beefy::report_vote_equivocation_unsigned(
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof.clone()),
+			key_owner_proof.clone(),
+		)
+		.unwrap();
+
+		// the original weight should be kept, but given that the report
+		// is valid the fee is waived.
+		assert!(post_info.actual_weight.is_none());
+		assert_eq!(post_info.pays_fee, Pays::No);
+
+		// report the equivocation again which is invalid now since it is
+		// duplicate.
+		let post_info = Beefy::report_vote_equivocation_unsigned(
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof),
+			key_owner_proof,
+		)
+		.err()
+		.unwrap()
+		.post_info;
+
+		// the fee is not waived and the original weight is kept.
+		assert!(post_info.actual_weight.is_none());
+		assert_eq!(post_info.pays_fee, Pays::Yes);
+	})
+}
+
+// fork equivocation report tests
 #[test]
 fn report_equivocation_current_set_works() {
 	let authorities = test_authorities();

From d0dba3b8999bfc6edf782828e62fd98aa3cf9a40 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 31 Aug 2023 09:52:47 +0200
Subject: [PATCH 046/188] test renames

---
 substrate/frame/beefy/src/tests.rs | 36 +++++++++++++++---------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index 28930f69a7fe..39e2ed0a57e8 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -257,7 +257,7 @@ fn should_sign_and_verify() {
 
 // vote equivocation report tests
 #[test]
-fn report_equivocation_current_set_works() {
+fn report_vote_equivocation_current_set_works() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
@@ -337,7 +337,7 @@ fn report_equivocation_current_set_works() {
 }
 
 #[test]
-fn report_equivocation_old_set_works() {
+fn report_vote_equivocation_old_set_works() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
@@ -420,7 +420,7 @@ fn report_equivocation_old_set_works() {
 }
 
 #[test]
-fn report_equivocation_invalid_set_id() {
+fn report_vote_equivocation_invalid_set_id() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
@@ -458,7 +458,7 @@ fn report_equivocation_invalid_set_id() {
 }
 
 #[test]
-fn report_equivocation_invalid_session() {
+fn report_vote_equivocation_invalid_session() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
@@ -501,7 +501,7 @@ fn report_equivocation_invalid_session() {
 }
 
 #[test]
-fn report_equivocation_invalid_key_owner_proof() {
+fn report_vote_equivocation_invalid_key_owner_proof() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
@@ -549,7 +549,7 @@ fn report_equivocation_invalid_key_owner_proof() {
 }
 
 #[test]
-fn report_equivocation_invalid_equivocation_proof() {
+fn report_vote_equivocation_invalid_equivocation_proof() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
@@ -617,7 +617,7 @@ fn report_equivocation_invalid_equivocation_proof() {
 }
 
 #[test]
-fn report_equivocation_validate_unsigned_prevents_duplicates() {
+fn report_vote_equivocation_validate_unsigned_prevents_duplicates() {
 	use sp_runtime::transaction_validity::{
 		InvalidTransaction, TransactionPriority, TransactionSource, TransactionValidity,
 		ValidTransaction,
@@ -707,7 +707,7 @@ fn report_equivocation_validate_unsigned_prevents_duplicates() {
 }
 
 #[test]
-fn report_equivocation_has_valid_weight() {
+fn report_vote_equivocation_has_valid_weight() {
 	// the weight depends on the size of the validator set,
 	// but there's a lower bound of 100 validators.
 	assert!((1..=100)
@@ -726,7 +726,7 @@ fn report_equivocation_has_valid_weight() {
 }
 
 #[test]
-fn valid_equivocation_reports_dont_pay_fees() {
+fn valid_vote_equivocation_reports_dont_pay_fees() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
@@ -795,7 +795,7 @@ fn valid_equivocation_reports_dont_pay_fees() {
 
 // fork equivocation report tests
 #[test]
-fn report_equivocation_current_set_works() {
+fn report_fork_equivocation_current_set_works() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
@@ -875,7 +875,7 @@ fn report_equivocation_current_set_works() {
 }
 
 #[test]
-fn report_equivocation_old_set_works() {
+fn report_fork_equivocation_old_set_works() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
@@ -958,7 +958,7 @@ fn report_equivocation_old_set_works() {
 }
 
 #[test]
-fn report_equivocation_invalid_set_id() {
+fn report_fork_equivocation_invalid_set_id() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
@@ -996,7 +996,7 @@ fn report_equivocation_invalid_set_id() {
 }
 
 #[test]
-fn report_equivocation_invalid_session() {
+fn report_fork_equivocation_invalid_session() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
@@ -1039,7 +1039,7 @@ fn report_equivocation_invalid_session() {
 }
 
 #[test]
-fn report_equivocation_invalid_key_owner_proof() {
+fn report_fork_equivocation_invalid_key_owner_proof() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
@@ -1087,7 +1087,7 @@ fn report_equivocation_invalid_key_owner_proof() {
 }
 
 #[test]
-fn report_equivocation_invalid_equivocation_proof() {
+fn report_fork_equivocation_invalid_equivocation_proof() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
@@ -1155,7 +1155,7 @@ fn report_equivocation_invalid_equivocation_proof() {
 }
 
 #[test]
-fn report_equivocation_validate_unsigned_prevents_duplicates() {
+fn report_fork_equivocation_validate_unsigned_prevents_duplicates() {
 	use sp_runtime::transaction_validity::{
 		InvalidTransaction, TransactionPriority, TransactionSource, TransactionValidity,
 		ValidTransaction,
@@ -1245,7 +1245,7 @@ fn report_equivocation_validate_unsigned_prevents_duplicates() {
 }
 
 #[test]
-fn report_equivocation_has_valid_weight() {
+fn report_fork_equivocation_has_valid_weight() {
 	// the weight depends on the size of the validator set,
 	// but there's a lower bound of 100 validators.
 	assert!((1..=100)
@@ -1264,7 +1264,7 @@ fn report_equivocation_has_valid_weight() {
 }
 
 #[test]
-fn valid_equivocation_reports_dont_pay_fees() {
+fn valid_fork_equivocation_reports_dont_pay_fees() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {

From b43886628af3f1c5b76b49c4f05af75aa63d6ecd Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 31 Aug 2023 10:54:51 +0200
Subject: [PATCH 047/188] test fork equivocation (via vote) reports

skip weight test for now
---
 substrate/frame/beefy/src/equivocation.rs |   4 +-
 substrate/frame/beefy/src/lib.rs          |   2 +
 substrate/frame/beefy/src/tests.rs        | 314 ++++++++++++----------
 3 files changed, 178 insertions(+), 142 deletions(-)

diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 625d12ec2132..bae1f282a8b8 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -300,9 +300,9 @@ where
 		// Check that the session id for the membership proof is within the
 		// bounds of the set id reported in the equivocation.
 		let set_id_session_index = crate::SetIdSession::<T>::get(set_id)
-			.ok_or(Error::<T>::InvalidVoteEquivocationProof)?;
+			.ok_or(Error::<T>::InvalidEquivocationProofSession)?;
 		if session_index != set_id_session_index {
-			return Err(Error::<T>::InvalidVoteEquivocationProof.into())
+			return Err(Error::<T>::InvalidEquivocationProofSession.into())
 		}
 
 		let offence = EquivocationOffence {
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 5e529ad64b27..2575c8d55487 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -199,6 +199,8 @@ pub mod pallet {
 		InvalidVoteEquivocationProof,
 		/// An equivocation proof provided as part of a fork equivocation report is invalid.
 		InvalidForkEquivocationProof,
+		/// The session of the equivocation proof is invalid
+		InvalidEquivocationProofSession,
 		/// A given equivocation report is valid but already previously reported.
 		DuplicateOffenceReport,
 	}
diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index 39e2ed0a57e8..995ff70abf6f 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -452,7 +452,7 @@ fn report_vote_equivocation_invalid_set_id() {
 				Box::new(equivocation_proof),
 				key_owner_proof,
 			),
-			Error::<Test>::InvalidVoteEquivocationProof,
+			Error::<Test>::InvalidEquivocationProofSession,
 		);
 	});
 }
@@ -495,7 +495,7 @@ fn report_vote_equivocation_invalid_session() {
 				Box::new(equivocation_proof),
 				key_owner_proof,
 			),
-			Error::<Test>::InvalidVoteEquivocationProof,
+			Error::<Test>::InvalidEquivocationProofSession,
 		);
 	});
 }
@@ -802,9 +802,14 @@ fn report_fork_equivocation_current_set_works() {
 		assert_eq!(Staking::current_era(), Some(0));
 		assert_eq!(Session::current_index(), 0);
 
-		start_era(1);
-
+		let mut era = 1;
+		start_era(era);
 		let block_num = System::block_number();
+		let header = System::finalize();
+
+		era += 1;
+		start_era(era);
+
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
@@ -816,7 +821,7 @@ fn report_fork_equivocation_current_set_works() {
 			assert_eq!(Staking::slashable_balance_of(validator), 10_000);
 
 			assert_eq!(
-				Staking::eras_stakers(1, validator),
+				Staking::eras_stakers(era, validator),
 				pallet_staking::Exposure { total: 10_000, own: 10_000, others: vec![] },
 			);
 		}
@@ -826,26 +831,26 @@ fn report_fork_equivocation_current_set_works() {
 		let equivocation_key = &authorities[equivocation_authority_index];
 		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
 
-		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
-		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
-		// generate an equivocation proof, with two votes in the same round for
-		// different payloads signed by the same key
-		let equivocation_proof = generate_vote_equivocation_proof(
-			(block_num, payload1, set_id, &equivocation_keyring),
-			(block_num, payload2, set_id, &equivocation_keyring),
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		// generate an fork equivocation proof, with a vote in the same round for a
+		// different payload than finalized
+		let equivocation_proof = generate_fork_equivocation_proof_vote(
+			(block_num, payload, set_id, &equivocation_keyring),
+			header,
 		);
 
 		// create the key ownership proof
 		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
 
 		// report the equivocation and the tx should be dispatched successfully
-		assert_ok!(Beefy::report_vote_equivocation_unsigned(
+		assert_ok!(Beefy::report_fork_equivocation_unsigned(
 			RuntimeOrigin::none(),
 			Box::new(equivocation_proof),
-			key_owner_proof,
+			vec![key_owner_proof],
 		),);
 
-		start_era(2);
+		era += 1;
+		start_era(era);
 
 		// check that the balance of 0-th validator is slashed 100%.
 		let equivocation_validator_id = validators[equivocation_authority_index];
@@ -853,7 +858,7 @@ fn report_fork_equivocation_current_set_works() {
 		assert_eq!(Balances::total_balance(&equivocation_validator_id), 10_000_000 - 10_000);
 		assert_eq!(Staking::slashable_balance_of(&equivocation_validator_id), 0);
 		assert_eq!(
-			Staking::eras_stakers(2, equivocation_validator_id),
+			Staking::eras_stakers(era, equivocation_validator_id),
 			pallet_staking::Exposure { total: 0, own: 0, others: vec![] },
 		);
 
@@ -867,7 +872,7 @@ fn report_fork_equivocation_current_set_works() {
 			assert_eq!(Staking::slashable_balance_of(validator), 10_000);
 
 			assert_eq!(
-				Staking::eras_stakers(2, validator),
+				Staking::eras_stakers(era, validator),
 				pallet_staking::Exposure { total: 10_000, own: 10_000, others: vec![] },
 			);
 		}
@@ -879,9 +884,14 @@ fn report_fork_equivocation_old_set_works() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		start_era(1);
-
+		let mut era = 1;
+		start_era(era);
 		let block_num = System::block_number();
+		let header = System::finalize();
+
+		era += 1;
+		start_era(era);
+
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let validators = Session::validators();
@@ -894,7 +904,8 @@ fn report_fork_equivocation_old_set_works() {
 		// create the key ownership proof in the "old" set
 		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
 
-		start_era(2);
+		era += 1;
+		start_era(era);
 
 		// make sure that all authorities have the same balance
 		for validator in &validators {
@@ -913,22 +924,23 @@ fn report_fork_equivocation_old_set_works() {
 
 		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
 
-		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
-		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
-		// generate an equivocation proof for the old set,
-		let equivocation_proof = generate_vote_equivocation_proof(
-			(block_num, payload1, old_set_id, &equivocation_keyring),
-			(block_num, payload2, old_set_id, &equivocation_keyring),
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		// generate an fork equivocation proof, with a vote in the same round for a
+		// different payload than finalized
+		let equivocation_proof = generate_fork_equivocation_proof_vote(
+			(block_num, payload, old_set_id, &equivocation_keyring),
+			header,
 		);
 
 		// report the equivocation and the tx should be dispatched successfully
-		assert_ok!(Beefy::report_vote_equivocation_unsigned(
+		assert_ok!(Beefy::report_fork_equivocation_unsigned(
 			RuntimeOrigin::none(),
 			Box::new(equivocation_proof),
-			key_owner_proof,
+			vec![key_owner_proof],
 		),);
 
-		start_era(3);
+		era += 1;
+		start_era(era);
 
 		// check that the balance of 0-th validator is slashed 100%.
 		let equivocation_validator_id = validators[equivocation_authority_index];
@@ -936,7 +948,7 @@ fn report_fork_equivocation_old_set_works() {
 		assert_eq!(Balances::total_balance(&equivocation_validator_id), 10_000_000 - 10_000);
 		assert_eq!(Staking::slashable_balance_of(&equivocation_validator_id), 0);
 		assert_eq!(
-			Staking::eras_stakers(3, equivocation_validator_id),
+			Staking::eras_stakers(era, equivocation_validator_id),
 			pallet_staking::Exposure { total: 0, own: 0, others: vec![] },
 		);
 
@@ -962,9 +974,14 @@ fn report_fork_equivocation_invalid_set_id() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		start_era(1);
-
+		let mut era = 1;
+		start_era(era);
 		let block_num = System::block_number();
+		let header = System::finalize();
+
+		era += 1;
+		start_era(era);
+
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
@@ -975,22 +992,20 @@ fn report_fork_equivocation_invalid_set_id() {
 
 		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
 
-		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
-		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		// generate an equivocation for a future set
-		let equivocation_proof = generate_vote_equivocation_proof(
-			(block_num, payload1, set_id + 1, &equivocation_keyring),
-			(block_num, payload2, set_id + 1, &equivocation_keyring),
+		let equivocation_proof = generate_fork_equivocation_proof_vote(
+			(block_num, payload, set_id+1, &equivocation_keyring),
+			header,
 		);
 
 		// the call for reporting the equivocation should error
-		assert_err!(
-			Beefy::report_vote_equivocation_unsigned(
-				RuntimeOrigin::none(),
-				Box::new(equivocation_proof),
-				key_owner_proof,
-			),
-			Error::<Test>::InvalidVoteEquivocationProof,
+		assert_err!(Beefy::report_fork_equivocation_unsigned(
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof),
+			vec![key_owner_proof],
+		),
+			Error::<Test>::InvalidEquivocationProofSession,
 		);
 	});
 }
@@ -1000,9 +1015,14 @@ fn report_fork_equivocation_invalid_session() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		start_era(1);
-
+		let mut era = 1;
+		start_era(era);
 		let block_num = System::block_number();
+		let header = System::finalize();
+
+		era += 1;
+		start_era(era);
+
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 
@@ -1013,27 +1033,26 @@ fn report_fork_equivocation_invalid_session() {
 		// generate a key ownership proof at current era set id
 		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
 
-		start_era(2);
+		era += 1;
+		start_era(era);
 
 		let set_id = Beefy::validator_set().unwrap().id();
 
-		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
-		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
-		// generate an equivocation proof at following era set id = 2
-		let equivocation_proof = generate_vote_equivocation_proof(
-			(block_num, payload1, set_id, &equivocation_keyring),
-			(block_num, payload2, set_id, &equivocation_keyring),
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		// generate an equivocation proof at following era set id = 3
+		let equivocation_proof = generate_fork_equivocation_proof_vote(
+			(block_num, payload, set_id, &equivocation_keyring),
+			header,
 		);
 
 		// report an equivocation for the current set using an key ownership
 		// proof from the previous set, the session should be invalid.
-		assert_err!(
-			Beefy::report_vote_equivocation_unsigned(
-				RuntimeOrigin::none(),
-				Box::new(equivocation_proof),
-				key_owner_proof,
-			),
-			Error::<Test>::InvalidVoteEquivocationProof,
+		assert_err!(Beefy::report_fork_equivocation_unsigned(
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof),
+			vec![key_owner_proof],
+		),
+			Error::<Test>::InvalidEquivocationProofSession,
 		);
 	});
 }
@@ -1043,9 +1062,14 @@ fn report_fork_equivocation_invalid_key_owner_proof() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		start_era(1);
-
+		let mut era = 1;
+		start_era(era);
 		let block_num = System::block_number();
+		let header = System::finalize();
+
+		era += 1;
+		start_era(era);
+
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
@@ -1061,26 +1085,25 @@ fn report_fork_equivocation_invalid_key_owner_proof() {
 		let equivocation_key = &authorities[equivocation_authority_index];
 		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
 
-		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
-		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
-		// generate an equivocation proof for the authority at index 0
-		let equivocation_proof = generate_vote_equivocation_proof(
-			(block_num, payload1, set_id + 1, &equivocation_keyring),
-			(block_num, payload2, set_id + 1, &equivocation_keyring),
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		// generate an equivocation for a future set
+		let equivocation_proof = generate_fork_equivocation_proof_vote(
+			(block_num, payload, set_id+1, &equivocation_keyring),
+			header,
 		);
 
 		// we need to start a new era otherwise the key ownership proof won't be
 		// checked since the authorities are part of the current session
-		start_era(2);
+		era += 1;
+		start_era(era);
 
 		// report an equivocation for the current set using a key ownership
 		// proof for a different key than the one in the equivocation proof.
-		assert_err!(
-			Beefy::report_vote_equivocation_unsigned(
-				RuntimeOrigin::none(),
-				Box::new(equivocation_proof),
-				invalid_key_owner_proof,
-			),
+		assert_err!(Beefy::report_fork_equivocation_unsigned(
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof),
+			vec![invalid_key_owner_proof],
+		),
 			Error::<Test>::InvalidKeyOwnershipProof,
 		);
 	});
@@ -1094,6 +1117,7 @@ fn report_fork_equivocation_invalid_equivocation_proof() {
 		start_era(1);
 
 		let block_num = System::block_number();
+		let header = System::finalize();
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
@@ -1105,55 +1129,55 @@ fn report_fork_equivocation_invalid_equivocation_proof() {
 		// generate a key ownership proof at set id in era 1
 		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
 
-		let assert_invalid_equivocation_proof = |equivocation_proof| {
-			assert_err!(
-				Beefy::report_vote_equivocation_unsigned(
-					RuntimeOrigin::none(),
-					Box::new(equivocation_proof),
-					key_owner_proof.clone(),
-				),
-				Error::<Test>::InvalidVoteEquivocationProof,
-			);
-		};
-
 		start_era(2);
 
-		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
-		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 
-		// both votes target the same block number and payload,
-		// there is no equivocation.
-		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
-			(block_num, payload1.clone(), set_id, &equivocation_keyring),
-			(block_num, payload1.clone(), set_id, &equivocation_keyring),
-		));
-
-		// votes targeting different rounds, there is no equivocation.
-		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
-			(block_num, payload1.clone(), set_id, &equivocation_keyring),
-			(block_num + 1, payload2.clone(), set_id, &equivocation_keyring),
-		));
-
-		// votes signed with different authority keys
-		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
-			(block_num, payload1.clone(), set_id, &equivocation_keyring),
-			(block_num, payload1.clone(), set_id, &BeefyKeyring::Charlie),
-		));
+		// vote targets different round than finalized payload, there is no equivocation.
+		let equivocation_proof = generate_fork_equivocation_proof_vote(
+			(block_num+1, payload.clone(), set_id, &equivocation_keyring),
+			header.clone(),
+		);
+		assert_err!(
+			Beefy::report_fork_equivocation_unsigned(
+				RuntimeOrigin::none(),
+				Box::new(equivocation_proof),
+				vec![key_owner_proof.clone()],
+			),
+			Error::<Test>::InvalidForkEquivocationProof,
+		);
 
-		// votes signed with a key that isn't part of the authority set
-		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
-			(block_num, payload1.clone(), set_id, &equivocation_keyring),
-			(block_num, payload1.clone(), set_id, &BeefyKeyring::Dave),
-		));
+		// vote signed with a key that isn't part of the authority set
+		let equivocation_proof = generate_fork_equivocation_proof_vote(
+			(block_num, payload.clone(), set_id, &BeefyKeyring::Dave),
+			header.clone(),
+		);
+		assert_err!(
+			Beefy::report_fork_equivocation_unsigned(
+				RuntimeOrigin::none(),
+				Box::new(equivocation_proof),
+				vec![key_owner_proof.clone()],
+			),
+			Error::<Test>::InvalidKeyOwnershipProof,
+		);
 
-		// votes targeting different set ids
-		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
-			(block_num, payload1, set_id, &equivocation_keyring),
-			(block_num, payload2, set_id + 1, &equivocation_keyring),
-		));
+		// vote targets future set id
+		let equivocation_proof = generate_fork_equivocation_proof_vote(
+			(block_num, payload.clone(), set_id+1, &equivocation_keyring),
+			header.clone(),
+		);
+		assert_err!(
+			Beefy::report_fork_equivocation_unsigned(
+				RuntimeOrigin::none(),
+				Box::new(equivocation_proof),
+				vec![key_owner_proof.clone()],
+			),
+			Error::<Test>::InvalidEquivocationProofSession,
+		);
 	});
 }
 
+#[ignore]
 #[test]
 fn report_fork_equivocation_validate_unsigned_prevents_duplicates() {
 	use sp_runtime::transaction_validity::{
@@ -1164,9 +1188,14 @@ fn report_fork_equivocation_validate_unsigned_prevents_duplicates() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		start_era(1);
-
+		let mut era = 1;
+		start_era(era);
 		let block_num = System::block_number();
+		let header = System::finalize();
+
+		era += 1;
+		start_era(era);
+
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
@@ -1176,18 +1205,18 @@ fn report_fork_equivocation_validate_unsigned_prevents_duplicates() {
 		let equivocation_key = &authorities[equivocation_authority_index];
 		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
 
-		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
-		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
-		let equivocation_proof = generate_vote_equivocation_proof(
-			(block_num, payload1, set_id, &equivocation_keyring),
-			(block_num, payload2, set_id, &equivocation_keyring),
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		// generate an equivocation for a future set
+		let equivocation_proof = generate_fork_equivocation_proof_vote(
+			(block_num, payload, set_id+1, &equivocation_keyring),
+			header,
 		);
 
 		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
 
-		let call = Call::report_vote_equivocation_unsigned {
+		let call = Call::report_fork_equivocation_unsigned {
 			equivocation_proof: Box::new(equivocation_proof.clone()),
-			key_owner_proof: key_owner_proof.clone(),
+			key_owner_proofs: vec![key_owner_proof.clone()],
 		};
 
 		// only local/inblock reports are allowed
@@ -1220,12 +1249,12 @@ fn report_fork_equivocation_validate_unsigned_prevents_duplicates() {
 		assert_ok!(<Beefy as sp_runtime::traits::ValidateUnsigned>::pre_dispatch(&call));
 
 		// we submit the report
-		Beefy::report_vote_equivocation_unsigned(
+		Beefy::report_fork_equivocation_unsigned(
 			RuntimeOrigin::none(),
 			Box::new(equivocation_proof),
-			key_owner_proof,
+			vec![key_owner_proof],
 		)
-		.unwrap();
+			.unwrap();
 
 		// the report should now be considered stale and the transaction is invalid
 		// the check for staleness should be done on both `validate_unsigned` and on `pre_dispatch`
@@ -1244,6 +1273,7 @@ fn report_fork_equivocation_validate_unsigned_prevents_duplicates() {
 	});
 }
 
+// TODO: adjust
 #[test]
 fn report_fork_equivocation_has_valid_weight() {
 	// the weight depends on the size of the validator set,
@@ -1268,9 +1298,14 @@ fn valid_fork_equivocation_reports_dont_pay_fees() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		start_era(1);
-
+		let mut era = 1;
+		start_era(era);
 		let block_num = System::block_number();
+		let header = System::finalize();
+
+		era += 1;
+		start_era(era);
+
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
@@ -1280,11 +1315,10 @@ fn valid_fork_equivocation_reports_dont_pay_fees() {
 		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
 
 		// generate equivocation proof
-		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
-		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
-		let equivocation_proof = generate_vote_equivocation_proof(
-			(block_num, payload1, set_id, &equivocation_keyring),
-			(block_num, payload2, set_id, &equivocation_keyring),
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let equivocation_proof = generate_fork_equivocation_proof_vote(
+			(block_num, payload, set_id, &equivocation_keyring),
+			header,
 		);
 
 		// create the key ownership proof.
@@ -1302,10 +1336,10 @@ fn valid_fork_equivocation_reports_dont_pay_fees() {
 		assert_eq!(info.pays_fee, Pays::Yes);
 
 		// report the equivocation.
-		let post_info = Beefy::report_vote_equivocation_unsigned(
+		let post_info = Beefy::report_fork_equivocation_unsigned (
 			RuntimeOrigin::none(),
 			Box::new(equivocation_proof.clone()),
-			key_owner_proof.clone(),
+			vec![key_owner_proof.clone()],
 		)
 		.unwrap();
 
@@ -1316,10 +1350,10 @@ fn valid_fork_equivocation_reports_dont_pay_fees() {
 
 		// report the equivocation again which is invalid now since it is
 		// duplicate.
-		let post_info = Beefy::report_vote_equivocation_unsigned(
+		let post_info = Beefy::report_fork_equivocation_unsigned (
 			RuntimeOrigin::none(),
-			Box::new(equivocation_proof),
-			key_owner_proof,
+			Box::new(equivocation_proof.clone()),
+			vec![key_owner_proof.clone()],
 		)
 		.err()
 		.unwrap()

From bd856acb02e59c31b70f47ab62b7a3cbbf8e0ad9 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 31 Aug 2023 13:33:42 +0200
Subject: [PATCH 048/188] handle fork equivocations in validate_unsigned &
 pre_dispatch

---
 substrate/frame/beefy/src/equivocation.rs  | 134 ++++++++++++++-------
 substrate/frame/beefy/src/lib.rs           |   1 +
 substrate/frame/beefy/src/tests.rs         |  24 ++--
 substrate/primitives/runtime/src/traits.rs |   2 +-
 4 files changed, 101 insertions(+), 60 deletions(-)

diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index bae1f282a8b8..15da5894dcc9 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -325,59 +325,101 @@ where
 /// include unsigned equivocation reports.
 impl<T: Config> Pallet<T> {
 	pub fn validate_unsigned(source: TransactionSource, call: &Call<T>) -> TransactionValidity {
-		if let Call::report_vote_equivocation_unsigned { equivocation_proof, key_owner_proof } =
-			call
-		{
-			// discard equivocation report not coming from the local node
-			match source {
-				TransactionSource::Local | TransactionSource::InBlock => { /* allowed */ },
-				_ => {
-					log::warn!(
-						target: LOG_TARGET,
-						"rejecting unsigned report equivocation transaction because it is not local/in-block."
-					);
-					return InvalidTransaction::Call.into()
-				},
-			}
-
-			let evidence = EquivocationEvidenceFor::<T>::VoteEquivocationProof(
-				*equivocation_proof.clone(),
-				key_owner_proof.clone(),
-			);
-			T::EquivocationReportSystem::check_evidence(evidence)?;
-
-			let longevity =
-				<T::EquivocationReportSystem as OffenceReportSystem<_, _>>::Longevity::get();
-
-			ValidTransaction::with_tag_prefix("BeefyEquivocation")
+		// discard equivocation report not coming from the local node
+		match source {
+			TransactionSource::Local | TransactionSource::InBlock => { /* allowed */ },
+			_ => {
+				log::warn!(
+					target: LOG_TARGET,
+					"rejecting unsigned report equivocation transaction because it is not local/in-block."
+				);
+				return InvalidTransaction::Call.into()
+			},
+		}
+		match call {
+			Call::report_vote_equivocation_unsigned { equivocation_proof, key_owner_proof } => {
+				let evidence = EquivocationEvidenceFor::<T>::VoteEquivocationProof(
+					*equivocation_proof.clone(),
+					key_owner_proof.clone(),
+				);
+				T::EquivocationReportSystem::check_evidence(evidence)?;
+
+				let longevity =
+					<T::EquivocationReportSystem as OffenceReportSystem<_, _>>::Longevity::get();
+
+				ValidTransaction::with_tag_prefix("BeefyEquivocation")
 				// We assign the maximum priority for any equivocation report.
-				.priority(TransactionPriority::MAX)
+					.priority(TransactionPriority::MAX)
 				// Only one equivocation report for the same offender at the same slot.
-				.and_provides((
-					equivocation_proof.offender_id().clone(),
-					equivocation_proof.set_id(),
-					*equivocation_proof.round_number(),
-				))
-				.longevity(longevity)
+					.and_provides((
+						equivocation_proof.offender_id().clone(),
+						equivocation_proof.set_id(),
+						*equivocation_proof.round_number(),
+					))
+					.longevity(longevity)
 				// We don't propagate this. This can never be included on a remote node.
-				.propagate(false)
-				.build()
-		} else {
-			InvalidTransaction::Call.into()
+					.propagate(false)
+					.build()
+			},
+			Call::report_fork_equivocation_unsigned { equivocation_proof, key_owner_proofs } => {
+				let evidence = EquivocationEvidenceFor::<T>::ForkEquivocationProof(
+					*equivocation_proof.clone(),
+					key_owner_proofs.clone(),
+				);
+				log::error!("checking evidence");
+				T::EquivocationReportSystem::check_evidence(evidence)?;
+
+				log::error!("passed evidence check");
+				let longevity =
+					<T::EquivocationReportSystem as OffenceReportSystem<_, _>>::Longevity::get();
+				log::error!("retrieved longevity");
+
+				ValidTransaction::with_tag_prefix("BeefyEquivocation")
+				// We assign the maximum priority for any equivocation report.
+					.priority(TransactionPriority::MAX)
+				// Only one equivocation report for the same offender at the same slot.
+					.and_provides((
+						equivocation_proof.offender_ids().clone(),
+						equivocation_proof.set_id(),
+						*equivocation_proof.round_number(),
+					))
+					.longevity(longevity)
+				// We don't propagate this. This can never be included on a remote node.
+					.propagate(false)
+					.build()
+			},
+			_ => {
+				log::error!(
+					target: LOG_TARGET,
+					"lolalol"
+				);
+				InvalidTransaction::Call.into()
+			}
+
 		}
 	}
 
 	pub fn pre_dispatch(call: &Call<T>) -> Result<(), TransactionValidityError> {
-		if let Call::report_vote_equivocation_unsigned { equivocation_proof, key_owner_proof } =
-			call
-		{
-			let evidence = EquivocationEvidenceFor::<T>::VoteEquivocationProof(
-				*equivocation_proof.clone(),
-				key_owner_proof.clone(),
-			);
-			T::EquivocationReportSystem::check_evidence(evidence)
-		} else {
-			Err(InvalidTransaction::Call.into())
+		match call {
+			Call::report_vote_equivocation_unsigned { equivocation_proof, key_owner_proof } =>
+			{
+				let evidence = EquivocationEvidenceFor::<T>::VoteEquivocationProof(
+					*equivocation_proof.clone(),
+					key_owner_proof.clone(),
+				);
+				T::EquivocationReportSystem::check_evidence(evidence)
+			},
+			Call::report_fork_equivocation_unsigned { equivocation_proof, key_owner_proofs } =>
+			{
+				let evidence = EquivocationEvidenceFor::<T>::ForkEquivocationProof(
+					*equivocation_proof.clone(),
+					key_owner_proofs.clone(),
+				);
+				T::EquivocationReportSystem::check_evidence(evidence)
+			},
+			_ => {
+				Err(InvalidTransaction::Call.into())
+			}
 		}
 	}
 }
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 2575c8d55487..8d8534f40a47 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -559,5 +559,6 @@ impl<T: Config> IsMember<T::BeefyId> for Pallet<T> {
 }
 
 pub trait WeightInfo {
+	// TODO: distinguish for fork equivocation proofs
 	fn report_equivocation(validator_count: u32, max_nominators_per_validator: u32) -> Weight;
 }
diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index 995ff70abf6f..115ae266c78b 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -1177,7 +1177,6 @@ fn report_fork_equivocation_invalid_equivocation_proof() {
 	});
 }
 
-#[ignore]
 #[test]
 fn report_fork_equivocation_validate_unsigned_prevents_duplicates() {
 	use sp_runtime::transaction_validity::{
@@ -1206,9 +1205,8 @@ fn report_fork_equivocation_validate_unsigned_prevents_duplicates() {
 		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
-		// generate an equivocation for a future set
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
-			(block_num, payload, set_id+1, &equivocation_keyring),
+			(block_num, payload, set_id, &equivocation_keyring),
 			header,
 		);
 
@@ -1229,23 +1227,24 @@ fn report_fork_equivocation_validate_unsigned_prevents_duplicates() {
 		);
 
 		// the transaction is valid when passed as local
-		let tx_tag = (equivocation_key, set_id, 3u64);
+		let tx_tag = (vec![equivocation_key], set_id, 3u64);
+
+		let call_result = <Beefy as sp_runtime::traits::ValidateUnsigned>::validate_unsigned(
+			TransactionSource::Local,
+			&call,
+		);
 
 		assert_eq!(
-			<Beefy as sp_runtime::traits::ValidateUnsigned>::validate_unsigned(
-				TransactionSource::Local,
-				&call,
-			),
+			call_result,
 			TransactionValidity::Ok(ValidTransaction {
 				priority: TransactionPriority::max_value(),
 				requires: vec![],
-				provides: vec![("BeefyEquivocation", tx_tag).encode()],
+				provides: vec![("BeefyEquivocation", tx_tag.clone()).encode()],
 				longevity: ReportLongevity::get(),
 				propagate: false,
 			})
 		);
 
-		// the pre dispatch checks should also pass
 		assert_ok!(<Beefy as sp_runtime::traits::ValidateUnsigned>::pre_dispatch(&call));
 
 		// we submit the report
@@ -1273,7 +1272,6 @@ fn report_fork_equivocation_validate_unsigned_prevents_duplicates() {
 	});
 }
 
-// TODO: adjust
 #[test]
 fn report_fork_equivocation_has_valid_weight() {
 	// the weight depends on the size of the validator set,
@@ -1325,9 +1323,9 @@ fn valid_fork_equivocation_reports_dont_pay_fees() {
 		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
 
 		// check the dispatch info for the call.
-		let info = Call::<Test>::report_vote_equivocation_unsigned {
+		let info = Call::<Test>::report_fork_equivocation_unsigned {
 			equivocation_proof: Box::new(equivocation_proof.clone()),
-			key_owner_proof: key_owner_proof.clone(),
+			key_owner_proofs: vec![key_owner_proof.clone()],
 		}
 		.get_dispatch_info();
 
diff --git a/substrate/primitives/runtime/src/traits.rs b/substrate/primitives/runtime/src/traits.rs
index 17dc7ce50ea8..b8a2b590f20b 100644
--- a/substrate/primitives/runtime/src/traits.rs
+++ b/substrate/primitives/runtime/src/traits.rs
@@ -1765,7 +1765,7 @@ pub trait ValidateUnsigned {
 	/// this code before the unsigned extrinsic enters the transaction pool and also periodically
 	/// afterwards to ensure the validity. To prevent dos-ing a network with unsigned
 	/// extrinsics, these validity checks should include some checks around uniqueness, for example,
-	/// like checking that the unsigned extrinsic was send by an authority in the active set.
+	/// like checking that the unsigned extrinsic was sent by an authority in the active set.
 	///
 	/// Changes made to storage should be discarded by caller.
 	fn validate_unsigned(source: TransactionSource, call: &Self::Call) -> TransactionValidity;

From 9e1e7b1922323769c070f52c4be278f6a0de3ffe Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 31 Aug 2023 22:11:40 +0200
Subject: [PATCH 049/188] clone tests for fork equivocations via commitments

---
 substrate/frame/beefy/src/tests.rs            | 580 +++++++++++++++++-
 .../primitives/consensus/beefy/src/lib.rs     |   4 +-
 2 files changed, 558 insertions(+), 26 deletions(-)

diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index 115ae266c78b..1bd77ed2de16 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -793,9 +793,9 @@ fn valid_vote_equivocation_reports_dont_pay_fees() {
 	})
 }
 
-// fork equivocation report tests
+// fork equivocation (via vote) report tests
 #[test]
-fn report_fork_equivocation_current_set_works() {
+fn report_fork_equivocation_vote_current_set_works() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
@@ -880,7 +880,7 @@ fn report_fork_equivocation_current_set_works() {
 }
 
 #[test]
-fn report_fork_equivocation_old_set_works() {
+fn report_fork_equivocation_vote_old_set_works() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
@@ -970,7 +970,7 @@ fn report_fork_equivocation_old_set_works() {
 }
 
 #[test]
-fn report_fork_equivocation_invalid_set_id() {
+fn report_fork_equivocation_vote_invalid_set_id() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
@@ -1011,7 +1011,7 @@ fn report_fork_equivocation_invalid_set_id() {
 }
 
 #[test]
-fn report_fork_equivocation_invalid_session() {
+fn report_fork_equivocation_vote_invalid_session() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
@@ -1058,7 +1058,7 @@ fn report_fork_equivocation_invalid_session() {
 }
 
 #[test]
-fn report_fork_equivocation_invalid_key_owner_proof() {
+fn report_fork_equivocation_vote_invalid_key_owner_proof() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
@@ -1110,7 +1110,7 @@ fn report_fork_equivocation_invalid_key_owner_proof() {
 }
 
 #[test]
-fn report_fork_equivocation_invalid_equivocation_proof() {
+fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
@@ -1178,7 +1178,7 @@ fn report_fork_equivocation_invalid_equivocation_proof() {
 }
 
 #[test]
-fn report_fork_equivocation_validate_unsigned_prevents_duplicates() {
+fn report_fork_equivocation_vote_validate_unsigned_prevents_duplicates() {
 	use sp_runtime::transaction_validity::{
 		InvalidTransaction, TransactionPriority, TransactionSource, TransactionValidity,
 		ValidTransaction,
@@ -1273,26 +1273,558 @@ fn report_fork_equivocation_validate_unsigned_prevents_duplicates() {
 }
 
 #[test]
-fn report_fork_equivocation_has_valid_weight() {
-	// the weight depends on the size of the validator set,
-	// but there's a lower bound of 100 validators.
-	assert!((1..=100)
-		.map(|validators| <Test as Config>::WeightInfo::report_equivocation(validators, 1000))
-		.collect::<Vec<_>>()
-		.windows(2)
-		.all(|w| w[0] == w[1]));
+fn valid_fork_equivocation_vote_reports_dont_pay_fees() {
+	let authorities = test_authorities();
+
+	new_test_ext_raw_authorities(authorities).execute_with(|| {
+		let mut era = 1;
+		start_era(era);
+		let block_num = System::block_number();
+		let header = System::finalize();
+
+		era += 1;
+		start_era(era);
+
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+		let set_id = validator_set.id();
+
+		let equivocation_authority_index = 0;
+		let equivocation_key = &authorities[equivocation_authority_index];
+		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+
+		// generate equivocation proof
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let equivocation_proof = generate_fork_equivocation_proof_vote(
+			(block_num, payload, set_id, &equivocation_keyring),
+			header,
+		);
+
+		// create the key ownership proof.
+		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+
+		// check the dispatch info for the call.
+		let info = Call::<Test>::report_fork_equivocation_unsigned {
+			equivocation_proof: Box::new(equivocation_proof.clone()),
+			key_owner_proofs: vec![key_owner_proof.clone()],
+		}
+		.get_dispatch_info();
+
+		// it should have non-zero weight and the fee has to be paid.
+		assert!(info.weight.any_gt(Weight::zero()));
+		assert_eq!(info.pays_fee, Pays::Yes);
+
+		// report the equivocation.
+		let post_info = Beefy::report_fork_equivocation_unsigned (
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof.clone()),
+			vec![key_owner_proof.clone()],
+		)
+		.unwrap();
+
+		// the original weight should be kept, but given that the report
+		// is valid the fee is waived.
+		assert!(post_info.actual_weight.is_none());
+		assert_eq!(post_info.pays_fee, Pays::No);
+
+		// report the equivocation again which is invalid now since it is
+		// duplicate.
+		let post_info = Beefy::report_fork_equivocation_unsigned (
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof.clone()),
+			vec![key_owner_proof.clone()],
+		)
+		.err()
+		.unwrap()
+		.post_info;
+
+		// the fee is not waived and the original weight is kept.
+		assert!(post_info.actual_weight.is_none());
+		assert_eq!(post_info.pays_fee, Pays::Yes);
+	})
+}
+
+// fork equivocation (via signed commitment) report tests
+#[test]
+fn report_fork_equivocation_sc_current_set_works() {
+	let authorities = test_authorities();
+
+	new_test_ext_raw_authorities(authorities).execute_with(|| {
+		assert_eq!(Staking::current_era(), Some(0));
+		assert_eq!(Session::current_index(), 0);
+
+		let mut era = 1;
+		start_era(era);
+		let block_num = System::block_number();
+		let header = System::finalize();
+
+		era += 1;
+		start_era(era);
+
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+		let set_id = validator_set.id();
+		let validators = Session::validators();
+
+		// make sure that all validators have the same balance
+		for validator in &validators {
+			assert_eq!(Balances::total_balance(validator), 10_000_000);
+			assert_eq!(Staking::slashable_balance_of(validator), 10_000);
+
+			assert_eq!(
+				Staking::eras_stakers(era, validator),
+				pallet_staking::Exposure { total: 10_000, own: 10_000, others: vec![] },
+			);
+		}
+
+		assert_eq!(authorities.len(), 2);
+		let equivocation_authority_index = 1;
+		let equivocation_key = &authorities[equivocation_authority_index];
+		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		// generate an fork equivocation proof, with a vote in the same round for a
+		// different payload than finalized
+		let equivocation_proof = generate_fork_equivocation_proof_vote(
+			(block_num, payload, set_id, &equivocation_keyring),
+			header,
+		);
+
+		// create the key ownership proof
+		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+
+		// report the equivocation and the tx should be dispatched successfully
+		assert_ok!(Beefy::report_fork_equivocation_unsigned(
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof),
+			vec![key_owner_proof],
+		),);
+
+		era += 1;
+		start_era(era);
+
+		// check that the balance of 0-th validator is slashed 100%.
+		let equivocation_validator_id = validators[equivocation_authority_index];
+
+		assert_eq!(Balances::total_balance(&equivocation_validator_id), 10_000_000 - 10_000);
+		assert_eq!(Staking::slashable_balance_of(&equivocation_validator_id), 0);
+		assert_eq!(
+			Staking::eras_stakers(era, equivocation_validator_id),
+			pallet_staking::Exposure { total: 0, own: 0, others: vec![] },
+		);
+
+		// check that the balances of all other validators are left intact.
+		for validator in &validators {
+			if *validator == equivocation_validator_id {
+				continue
+			}
+
+			assert_eq!(Balances::total_balance(validator), 10_000_000);
+			assert_eq!(Staking::slashable_balance_of(validator), 10_000);
+
+			assert_eq!(
+				Staking::eras_stakers(era, validator),
+				pallet_staking::Exposure { total: 10_000, own: 10_000, others: vec![] },
+			);
+		}
+	});
+}
+
+#[test]
+fn report_fork_equivocation_sc_old_set_works() {
+	let authorities = test_authorities();
+
+	new_test_ext_raw_authorities(authorities).execute_with(|| {
+		let mut era = 1;
+		start_era(era);
+		let block_num = System::block_number();
+		let header = System::finalize();
+
+		era += 1;
+		start_era(era);
+
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+		let validators = Session::validators();
+		let old_set_id = validator_set.id();
+
+		assert_eq!(authorities.len(), 2);
+		let equivocation_authority_index = 0;
+		let equivocation_key = &authorities[equivocation_authority_index];
+
+		// create the key ownership proof in the "old" set
+		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+
+		era += 1;
+		start_era(era);
+
+		// make sure that all authorities have the same balance
+		for validator in &validators {
+			assert_eq!(Balances::total_balance(validator), 10_000_000);
+			assert_eq!(Staking::slashable_balance_of(validator), 10_000);
+
+			assert_eq!(
+				Staking::eras_stakers(2, validator),
+				pallet_staking::Exposure { total: 10_000, own: 10_000, others: vec![] },
+			);
+		}
+
+		let validator_set = Beefy::validator_set().unwrap();
+		let new_set_id = validator_set.id();
+		assert_eq!(old_set_id + 3, new_set_id);
+
+		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		// generate an fork equivocation proof, with a vote in the same round for a
+		// different payload than finalized
+		let equivocation_proof = generate_fork_equivocation_proof_vote(
+			(block_num, payload, old_set_id, &equivocation_keyring),
+			header,
+		);
+
+		// report the equivocation and the tx should be dispatched successfully
+		assert_ok!(Beefy::report_fork_equivocation_unsigned(
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof),
+			vec![key_owner_proof],
+		),);
+
+		era += 1;
+		start_era(era);
+
+		// check that the balance of 0-th validator is slashed 100%.
+		let equivocation_validator_id = validators[equivocation_authority_index];
+
+		assert_eq!(Balances::total_balance(&equivocation_validator_id), 10_000_000 - 10_000);
+		assert_eq!(Staking::slashable_balance_of(&equivocation_validator_id), 0);
+		assert_eq!(
+			Staking::eras_stakers(era, equivocation_validator_id),
+			pallet_staking::Exposure { total: 0, own: 0, others: vec![] },
+		);
+
+		// check that the balances of all other validators are left intact.
+		for validator in &validators {
+			if *validator == equivocation_validator_id {
+				continue
+			}
+
+			assert_eq!(Balances::total_balance(validator), 10_000_000);
+			assert_eq!(Staking::slashable_balance_of(validator), 10_000);
+
+			assert_eq!(
+				Staking::eras_stakers(3, validator),
+				pallet_staking::Exposure { total: 10_000, own: 10_000, others: vec![] },
+			);
+		}
+	});
+}
+
+#[test]
+fn report_fork_equivocation_sc_invalid_set_id() {
+	let authorities = test_authorities();
+
+	new_test_ext_raw_authorities(authorities).execute_with(|| {
+		let mut era = 1;
+		start_era(era);
+		let block_num = System::block_number();
+		let header = System::finalize();
+
+		era += 1;
+		start_era(era);
+
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+		let set_id = validator_set.id();
+
+		let equivocation_authority_index = 0;
+		let equivocation_key = &authorities[equivocation_authority_index];
+		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+
+		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		// generate an equivocation for a future set
+		let equivocation_proof = generate_fork_equivocation_proof_vote(
+			(block_num, payload, set_id+1, &equivocation_keyring),
+			header,
+		);
+
+		// the call for reporting the equivocation should error
+		assert_err!(Beefy::report_fork_equivocation_unsigned(
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof),
+			vec![key_owner_proof],
+		),
+			Error::<Test>::InvalidEquivocationProofSession,
+		);
+	});
+}
+
+#[test]
+fn report_fork_equivocation_sc_invalid_session() {
+	let authorities = test_authorities();
+
+	new_test_ext_raw_authorities(authorities).execute_with(|| {
+		let mut era = 1;
+		start_era(era);
+		let block_num = System::block_number();
+		let header = System::finalize();
 
-	// after 100 validators the weight should keep increasing
-	// with every extra validator.
-	assert!((100..=1000)
-		.map(|validators| <Test as Config>::WeightInfo::report_equivocation(validators, 1000))
-		.collect::<Vec<_>>()
-		.windows(2)
-		.all(|w| w[0].ref_time() < w[1].ref_time()));
+		era += 1;
+		start_era(era);
+
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+
+		let equivocation_authority_index = 0;
+		let equivocation_key = &authorities[equivocation_authority_index];
+		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+
+		// generate a key ownership proof at current era set id
+		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+
+		era += 1;
+		start_era(era);
+
+		let set_id = Beefy::validator_set().unwrap().id();
+
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		// generate an equivocation proof at following era set id = 3
+		let equivocation_proof = generate_fork_equivocation_proof_vote(
+			(block_num, payload, set_id, &equivocation_keyring),
+			header,
+		);
+
+		// report an equivocation for the current set using an key ownership
+		// proof from the previous set, the session should be invalid.
+		assert_err!(Beefy::report_fork_equivocation_unsigned(
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof),
+			vec![key_owner_proof],
+		),
+			Error::<Test>::InvalidEquivocationProofSession,
+		);
+	});
+}
+
+#[test]
+fn report_fork_equivocation_sc_invalid_key_owner_proof() {
+	let authorities = test_authorities();
+
+	new_test_ext_raw_authorities(authorities).execute_with(|| {
+		let mut era = 1;
+		start_era(era);
+		let block_num = System::block_number();
+		let header = System::finalize();
+
+		era += 1;
+		start_era(era);
+
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+		let set_id = validator_set.id();
+
+		let invalid_owner_authority_index = 1;
+		let invalid_owner_key = &authorities[invalid_owner_authority_index];
+
+		// generate a key ownership proof for the authority at index 1
+		let invalid_key_owner_proof =
+			Historical::prove((BEEFY_KEY_TYPE, &invalid_owner_key)).unwrap();
+
+		let equivocation_authority_index = 0;
+		let equivocation_key = &authorities[equivocation_authority_index];
+		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		// generate an equivocation for a future set
+		let equivocation_proof = generate_fork_equivocation_proof_vote(
+			(block_num, payload, set_id+1, &equivocation_keyring),
+			header,
+		);
+
+		// we need to start a new era otherwise the key ownership proof won't be
+		// checked since the authorities are part of the current session
+		era += 1;
+		start_era(era);
+
+		// report an equivocation for the current set using a key ownership
+		// proof for a different key than the one in the equivocation proof.
+		assert_err!(Beefy::report_fork_equivocation_unsigned(
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof),
+			vec![invalid_key_owner_proof],
+		),
+			Error::<Test>::InvalidKeyOwnershipProof,
+		);
+	});
+}
+
+#[test]
+fn report_fork_equivocation_sc_invalid_equivocation_proof() {
+	let authorities = test_authorities();
+
+	new_test_ext_raw_authorities(authorities).execute_with(|| {
+		start_era(1);
+
+		let block_num = System::block_number();
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+		let set_id = validator_set.id();
+
+		let equivocation_authority_index = 0;
+		let equivocation_key = &authorities[equivocation_authority_index];
+		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+
+		// generate a key ownership proof at set id in era 1
+		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+
+		let assert_invalid_equivocation_proof = |equivocation_proof| {
+			assert_err!(
+				Beefy::report_vote_equivocation_unsigned(
+					RuntimeOrigin::none(),
+					Box::new(equivocation_proof),
+					key_owner_proof.clone(),
+				),
+				Error::<Test>::InvalidVoteEquivocationProof,
+			);
+		};
+
+		start_era(2);
+
+		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
+
+		// both votes target the same block number and payload,
+		// there is no equivocation.
+		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
+			(block_num, payload1.clone(), set_id, &equivocation_keyring),
+			(block_num, payload1.clone(), set_id, &equivocation_keyring),
+		));
+
+		// votes targeting different rounds, there is no equivocation.
+		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
+			(block_num, payload1.clone(), set_id, &equivocation_keyring),
+			(block_num + 1, payload2.clone(), set_id, &equivocation_keyring),
+		));
+
+		// votes signed with different authority keys
+		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
+			(block_num, payload1.clone(), set_id, &equivocation_keyring),
+			(block_num, payload1.clone(), set_id, &BeefyKeyring::Charlie),
+		));
+
+		// votes signed with a key that isn't part of the authority set
+		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
+			(block_num, payload1.clone(), set_id, &equivocation_keyring),
+			(block_num, payload1.clone(), set_id, &BeefyKeyring::Dave),
+		));
+
+		// votes targeting different set ids
+		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
+			(block_num, payload1, set_id, &equivocation_keyring),
+			(block_num, payload2, set_id + 1, &equivocation_keyring),
+		));
+	});
+}
+
+#[test]
+fn report_fork_equivocation_sc_validate_unsigned_prevents_duplicates() {
+	use sp_runtime::transaction_validity::{
+		InvalidTransaction, TransactionPriority, TransactionSource, TransactionValidity,
+		ValidTransaction,
+	};
+
+	let authorities = test_authorities();
+
+	new_test_ext_raw_authorities(authorities).execute_with(|| {
+		let mut era = 1;
+		start_era(era);
+		let block_num = System::block_number();
+		let header = System::finalize();
+
+		era += 1;
+		start_era(era);
+
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+		let set_id = validator_set.id();
+
+		// generate and report an equivocation for the validator at index 0
+		let equivocation_authority_index = 0;
+		let equivocation_key = &authorities[equivocation_authority_index];
+		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let equivocation_proof = generate_fork_equivocation_proof_vote(
+			(block_num, payload, set_id, &equivocation_keyring),
+			header,
+		);
+
+		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+
+		let call = Call::report_fork_equivocation_unsigned {
+			equivocation_proof: Box::new(equivocation_proof.clone()),
+			key_owner_proofs: vec![key_owner_proof.clone()],
+		};
+
+		// only local/inblock reports are allowed
+		assert_eq!(
+			<Beefy as sp_runtime::traits::ValidateUnsigned>::validate_unsigned(
+				TransactionSource::External,
+				&call,
+			),
+			InvalidTransaction::Call.into(),
+		);
+
+		// the transaction is valid when passed as local
+		let tx_tag = (vec![equivocation_key], set_id, 3u64);
+
+		let call_result = <Beefy as sp_runtime::traits::ValidateUnsigned>::validate_unsigned(
+			TransactionSource::Local,
+			&call,
+		);
+
+		assert_eq!(
+			call_result,
+			TransactionValidity::Ok(ValidTransaction {
+				priority: TransactionPriority::max_value(),
+				requires: vec![],
+				provides: vec![("BeefyEquivocation", tx_tag.clone()).encode()],
+				longevity: ReportLongevity::get(),
+				propagate: false,
+			})
+		);
+
+		assert_ok!(<Beefy as sp_runtime::traits::ValidateUnsigned>::pre_dispatch(&call));
+
+		// we submit the report
+		Beefy::report_fork_equivocation_unsigned(
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof),
+			vec![key_owner_proof],
+		)
+			.unwrap();
+
+		// the report should now be considered stale and the transaction is invalid
+		// the check for staleness should be done on both `validate_unsigned` and on `pre_dispatch`
+		assert_err!(
+			<Beefy as sp_runtime::traits::ValidateUnsigned>::validate_unsigned(
+				TransactionSource::Local,
+				&call,
+			),
+			InvalidTransaction::Stale,
+		);
+
+		assert_err!(
+			<Beefy as sp_runtime::traits::ValidateUnsigned>::pre_dispatch(&call),
+			InvalidTransaction::Stale,
+		);
+	});
 }
 
 #[test]
-fn valid_fork_equivocation_reports_dont_pay_fees() {
+fn valid_fork_equivocation_sc_reports_dont_pay_fees() {
 	let authorities = test_authorities();
 
 	new_test_ext_raw_authorities(authorities).execute_with(|| {
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 09da46cdf113..f7c3758d50e6 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -34,13 +34,13 @@
 mod commitment;
 pub mod mmr;
 mod payload;
-#[cfg(all(test, feature = "std"))]
+#[cfg(feature = "std")]
 mod test_utils;
 pub mod witness;
 
 pub use commitment::{Commitment, SignedCommitment, VersionedFinalityProof};
 pub use payload::{known_payloads, BeefyPayloadId, Payload, PayloadProvider};
-#[cfg(all(test, feature = "std"))]
+#[cfg(feature = "std")]
 pub use test_utils::*;
 
 use codec::{Codec, Decode, Encode};

From f5b1ec5c9d1d9d0421be7c2c4d6604040c9f07c6 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Fri, 1 Sep 2023 10:40:56 +0200
Subject: [PATCH 050/188] bump number of beefy authorities in tests

else fork equivocation via signed commitment tests will be less
meaningful (no validators that don't equivocate, ergo don't get slashed)
---
 substrate/frame/beefy/src/mock.rs  |  2 +-
 substrate/frame/beefy/src/tests.rs | 22 +++++++++++-----------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/substrate/frame/beefy/src/mock.rs b/substrate/frame/beefy/src/mock.rs
index d0abdf52c388..98cf5f7a3a0e 100644
--- a/substrate/frame/beefy/src/mock.rs
+++ b/substrate/frame/beefy/src/mock.rs
@@ -288,7 +288,7 @@ pub fn new_test_ext_raw_authorities(authorities: Vec<BeefyId>) -> TestExternalit
 
 	let staking_config = pallet_staking::GenesisConfig::<Test> {
 		stakers,
-		validator_count: 2,
+		validator_count: authorities.len() as u32 - 1,
 		force_era: pallet_staking::Forcing::ForceNew,
 		minimum_validator_count: 0,
 		invulnerables: vec![],
diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index 1bd77ed2de16..f483d42e1002 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -88,7 +88,7 @@ fn session_change_updates_authorities() {
 		assert!(2 == Beefy::validator_set_id());
 
 		let want = beefy_log(ConsensusLog::AuthoritiesChange(
-			ValidatorSet::new(vec![mock_beefy_id(2), mock_beefy_id(4)], 2).unwrap(),
+			ValidatorSet::new(vec![mock_beefy_id(2), mock_beefy_id(3), mock_beefy_id(4)], 2).unwrap(),
 		));
 
 		let log = System::digest().logs[1].clone();
@@ -113,9 +113,9 @@ fn session_change_updates_next_authorities() {
 
 		let next_authorities = Beefy::next_authorities();
 
-		assert_eq!(next_authorities.len(), 2);
+		assert_eq!(next_authorities.len(), 3);
 		assert_eq!(want[1], next_authorities[0]);
-		assert_eq!(want[3], next_authorities[1]);
+		assert_eq!(want[3], next_authorities[2]);
 	});
 }
 
@@ -158,7 +158,7 @@ fn validator_set_updates_work() {
 
 		assert_eq!(vs.id(), 2u64);
 		assert_eq!(want[1], vs.validators()[0]);
-		assert_eq!(want[3], vs.validators()[1]);
+		assert_eq!(want[3], vs.validators()[2]);
 	});
 }
 
@@ -198,7 +198,7 @@ fn cleans_up_old_set_id_session_mappings() {
 /// Returns a list with 3 authorities with known keys:
 /// Alice, Bob and Charlie.
 pub fn test_authorities() -> Vec<BeefyId> {
-	let authorities = vec![BeefyKeyring::Alice, BeefyKeyring::Bob, BeefyKeyring::Charlie];
+	let authorities = vec![BeefyKeyring::Alice, BeefyKeyring::Bob, BeefyKeyring::Charlie, BeefyKeyring::Dave];
 	authorities.into_iter().map(|id| id.public()).collect()
 }
 
@@ -283,7 +283,7 @@ fn report_vote_equivocation_current_set_works() {
 			);
 		}
 
-		assert_eq!(authorities.len(), 2);
+		assert_eq!(authorities.len(), 3);
 		let equivocation_authority_index = 1;
 		let equivocation_key = &authorities[equivocation_authority_index];
 		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
@@ -349,7 +349,7 @@ fn report_vote_equivocation_old_set_works() {
 		let validators = Session::validators();
 		let old_set_id = validator_set.id();
 
-		assert_eq!(authorities.len(), 2);
+		assert_eq!(authorities.len(), 3);
 		let equivocation_authority_index = 0;
 		let equivocation_key = &authorities[equivocation_authority_index];
 
@@ -826,7 +826,7 @@ fn report_fork_equivocation_vote_current_set_works() {
 			);
 		}
 
-		assert_eq!(authorities.len(), 2);
+		assert_eq!(authorities.len(), 3);
 		let equivocation_authority_index = 1;
 		let equivocation_key = &authorities[equivocation_authority_index];
 		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
@@ -897,7 +897,7 @@ fn report_fork_equivocation_vote_old_set_works() {
 		let validators = Session::validators();
 		let old_set_id = validator_set.id();
 
-		assert_eq!(authorities.len(), 2);
+		assert_eq!(authorities.len(), 3);
 		let equivocation_authority_index = 0;
 		let equivocation_key = &authorities[equivocation_authority_index];
 
@@ -1377,10 +1377,10 @@ fn report_fork_equivocation_sc_current_set_works() {
 			);
 		}
 
-		assert_eq!(authorities.len(), 2);
 		let equivocation_authority_index = 1;
 		let equivocation_key = &authorities[equivocation_authority_index];
 		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+		assert_eq!(authorities.len(), 3);
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		// generate an fork equivocation proof, with a vote in the same round for a
@@ -1448,7 +1448,7 @@ fn report_fork_equivocation_sc_old_set_works() {
 		let validators = Session::validators();
 		let old_set_id = validator_set.id();
 
-		assert_eq!(authorities.len(), 2);
+		assert_eq!(authorities.len(), 3);
 		let equivocation_authority_index = 0;
 		let equivocation_key = &authorities[equivocation_authority_index];
 

From 7504b96e2e6bafe8aa8e22364980715f579c7235 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Fri, 1 Sep 2023 12:49:38 +0200
Subject: [PATCH 051/188] test fork equivocation (via commitment) reports

---
 substrate/frame/beefy/src/tests.rs            | 228 ++++++++++--------
 .../consensus/beefy/src/test_utils.rs         |  13 +
 2 files changed, 139 insertions(+), 102 deletions(-)

diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index f483d42e1002..c0daf1fbaa80 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -19,7 +19,7 @@ use std::vec;
 
 use codec::Encode;
 use sp_consensus_beefy::{
-	check_vote_equivocation_proof, generate_vote_equivocation_proof, generate_fork_equivocation_proof_vote, known_payloads::MMR_ROOT_ID,
+	Commitment, check_vote_equivocation_proof, generate_vote_equivocation_proof, generate_fork_equivocation_proof_sc, generate_fork_equivocation_proof_vote, known_payloads::MMR_ROOT_ID,
 	Keyring as BeefyKeyring, Payload, ValidatorSet, KEY_TYPE as BEEFY_KEY_TYPE,
 };
 
@@ -1377,45 +1377,49 @@ fn report_fork_equivocation_sc_current_set_works() {
 			);
 		}
 
-		let equivocation_authority_index = 1;
-		let equivocation_key = &authorities[equivocation_authority_index];
-		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
 		assert_eq!(authorities.len(), 3);
+		let equivocation_authority_indices = [0, 2];
+		let equivocation_keys = equivocation_authority_indices.iter().map(|i| &authorities[*i]).collect::<Vec<_>>();
+		let equivocation_keyrings = equivocation_keys.iter().map(|k| BeefyKeyring::from_public(k).unwrap()).collect();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let commitment = Commitment { validator_set_id: set_id, block_number: block_num, payload };
 		// generate an fork equivocation proof, with a vote in the same round for a
 		// different payload than finalized
-		let equivocation_proof = generate_fork_equivocation_proof_vote(
-			(block_num, payload, set_id, &equivocation_keyring),
+		let equivocation_proof = generate_fork_equivocation_proof_sc(
+			commitment,
+			equivocation_keyrings,
 			header,
 		);
 
 		// create the key ownership proof
-		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+		let key_owner_proofs = equivocation_keys.iter().map(|k| Historical::prove((BEEFY_KEY_TYPE, &k)).unwrap()).collect();
 
 		// report the equivocation and the tx should be dispatched successfully
 		assert_ok!(Beefy::report_fork_equivocation_unsigned(
 			RuntimeOrigin::none(),
 			Box::new(equivocation_proof),
-			vec![key_owner_proof],
+			key_owner_proofs,
 		),);
 
 		era += 1;
 		start_era(era);
 
-		// check that the balance of 0-th validator is slashed 100%.
-		let equivocation_validator_id = validators[equivocation_authority_index];
+		// check that the balance of equivocating validators is slashed 100%.
+		let equivocation_validator_ids = equivocation_authority_indices.iter().map(|i| validators[*i]).collect::<Vec<_>>();
 
-		assert_eq!(Balances::total_balance(&equivocation_validator_id), 10_000_000 - 10_000);
-		assert_eq!(Staking::slashable_balance_of(&equivocation_validator_id), 0);
-		assert_eq!(
-			Staking::eras_stakers(era, equivocation_validator_id),
-			pallet_staking::Exposure { total: 0, own: 0, others: vec![] },
-		);
+		for equivocation_validator_id in &equivocation_validator_ids {
+			assert_eq!(Balances::total_balance(&equivocation_validator_id), 10_000_000 - 10_000);
+			assert_eq!(Staking::slashable_balance_of(&equivocation_validator_id), 0);
+			assert_eq!(
+				Staking::eras_stakers(era, equivocation_validator_id),
+				pallet_staking::Exposure { total: 0, own: 0, others: vec![] },
+			);
+		}
 
 		// check that the balances of all other validators are left intact.
 		for validator in &validators {
-			if *validator == equivocation_validator_id {
+			if equivocation_validator_ids.contains(&validator) {
 				continue
 			}
 
@@ -1449,11 +1453,11 @@ fn report_fork_equivocation_sc_old_set_works() {
 		let old_set_id = validator_set.id();
 
 		assert_eq!(authorities.len(), 3);
-		let equivocation_authority_index = 0;
-		let equivocation_key = &authorities[equivocation_authority_index];
+		let equivocation_authority_indices = [0, 2];
+		let equivocation_keys = equivocation_authority_indices.iter().map(|i| &authorities[*i]).collect::<Vec<_>>();
 
-		// create the key ownership proof in the "old" set
-		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+		// create the key ownership proofs in the "old" set
+		let key_owner_proofs = equivocation_keys.iter().map(|k| Historical::prove((BEEFY_KEY_TYPE, &k)).unwrap()).collect();
 
 		era += 1;
 		start_era(era);
@@ -1473,13 +1477,17 @@ fn report_fork_equivocation_sc_old_set_works() {
 		let new_set_id = validator_set.id();
 		assert_eq!(old_set_id + 3, new_set_id);
 
-		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+		let equivocation_keyrings = equivocation_keys.iter().map(|k| BeefyKeyring::from_public(k).unwrap()).collect();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		// generate an fork equivocation proof, with a vote in the same round for a
 		// different payload than finalized
-		let equivocation_proof = generate_fork_equivocation_proof_vote(
-			(block_num, payload, old_set_id, &equivocation_keyring),
+		let commitment = Commitment { validator_set_id: old_set_id, block_number: block_num, payload };
+		// generate an fork equivocation proof, with a vote in the same round for a
+		// different payload than finalized
+		let equivocation_proof = generate_fork_equivocation_proof_sc(
+			commitment,
+			equivocation_keyrings,
 			header,
 		);
 
@@ -1487,25 +1495,27 @@ fn report_fork_equivocation_sc_old_set_works() {
 		assert_ok!(Beefy::report_fork_equivocation_unsigned(
 			RuntimeOrigin::none(),
 			Box::new(equivocation_proof),
-			vec![key_owner_proof],
+			key_owner_proofs,
 		),);
 
 		era += 1;
 		start_era(era);
 
-		// check that the balance of 0-th validator is slashed 100%.
-		let equivocation_validator_id = validators[equivocation_authority_index];
+		// check that the balance of equivocating validators is slashed 100%.
+		let equivocation_validator_ids = equivocation_authority_indices.iter().map(|i| validators[*i]).collect::<Vec<_>>();
 
-		assert_eq!(Balances::total_balance(&equivocation_validator_id), 10_000_000 - 10_000);
-		assert_eq!(Staking::slashable_balance_of(&equivocation_validator_id), 0);
-		assert_eq!(
-			Staking::eras_stakers(era, equivocation_validator_id),
-			pallet_staking::Exposure { total: 0, own: 0, others: vec![] },
-		);
+		for equivocation_validator_id in &equivocation_validator_ids {
+			assert_eq!(Balances::total_balance(&equivocation_validator_id), 10_000_000 - 10_000);
+			assert_eq!(Staking::slashable_balance_of(&equivocation_validator_id), 0);
+			assert_eq!(
+				Staking::eras_stakers(era, equivocation_validator_id),
+				pallet_staking::Exposure { total: 0, own: 0, others: vec![] },
+			);
+		}
 
 		// check that the balances of all other validators are left intact.
 		for validator in &validators {
-			if *validator == equivocation_validator_id {
+			if equivocation_validator_ids.contains(&validator) {
 				continue
 			}
 
@@ -1537,16 +1547,18 @@ fn report_fork_equivocation_sc_invalid_set_id() {
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
 
-		let equivocation_authority_index = 0;
-		let equivocation_key = &authorities[equivocation_authority_index];
-		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+		let equivocation_authority_indices = [0, 2];
+		let equivocation_keys = equivocation_authority_indices.iter().map(|i| &authorities[*i]).collect::<Vec<_>>();
+		let equivocation_keyrings = equivocation_keys.iter().map(|k| BeefyKeyring::from_public(k).unwrap()).collect();
 
-		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+		let key_owner_proofs = equivocation_keys.iter().map(|k| Historical::prove((BEEFY_KEY_TYPE, &k)).unwrap()).collect();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		// generate an equivocation for a future set
-		let equivocation_proof = generate_fork_equivocation_proof_vote(
-			(block_num, payload, set_id+1, &equivocation_keyring),
+		let commitment = Commitment { validator_set_id: set_id+1, block_number: block_num, payload };
+		let equivocation_proof = generate_fork_equivocation_proof_sc(
+			commitment,
+			equivocation_keyrings,
 			header,
 		);
 
@@ -1554,7 +1566,7 @@ fn report_fork_equivocation_sc_invalid_set_id() {
 		assert_err!(Beefy::report_fork_equivocation_unsigned(
 			RuntimeOrigin::none(),
 			Box::new(equivocation_proof),
-			vec![key_owner_proof],
+			key_owner_proofs,
 		),
 			Error::<Test>::InvalidEquivocationProofSession,
 		);
@@ -1577,12 +1589,12 @@ fn report_fork_equivocation_sc_invalid_session() {
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 
-		let equivocation_authority_index = 0;
-		let equivocation_key = &authorities[equivocation_authority_index];
-		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+		let equivocation_authority_indices = [0, 2];
+		let equivocation_keys = equivocation_authority_indices.iter().map(|i| &authorities[*i]).collect::<Vec<_>>();
+		let equivocation_keyrings = equivocation_keys.iter().map(|k| BeefyKeyring::from_public(k).unwrap()).collect();
 
-		// generate a key ownership proof at current era set id
-		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+		// generate key ownership proofs at current era set id
+		let key_owner_proofs = equivocation_keys.iter().map(|k| Historical::prove((BEEFY_KEY_TYPE, &k)).unwrap()).collect();
 
 		era += 1;
 		start_era(era);
@@ -1591,8 +1603,10 @@ fn report_fork_equivocation_sc_invalid_session() {
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		// generate an equivocation proof at following era set id = 3
-		let equivocation_proof = generate_fork_equivocation_proof_vote(
-			(block_num, payload, set_id, &equivocation_keyring),
+		let commitment = Commitment { validator_set_id: set_id, block_number: block_num, payload };
+		let equivocation_proof = generate_fork_equivocation_proof_sc(
+			commitment,
+			equivocation_keyrings,
 			header,
 		);
 
@@ -1601,7 +1615,7 @@ fn report_fork_equivocation_sc_invalid_session() {
 		assert_err!(Beefy::report_fork_equivocation_unsigned(
 			RuntimeOrigin::none(),
 			Box::new(equivocation_proof),
-			vec![key_owner_proof],
+			key_owner_proofs,
 		),
 			Error::<Test>::InvalidEquivocationProofSession,
 		);
@@ -1627,19 +1641,26 @@ fn report_fork_equivocation_sc_invalid_key_owner_proof() {
 
 		let invalid_owner_authority_index = 1;
 		let invalid_owner_key = &authorities[invalid_owner_authority_index];
+		let valid_owner_authority_index = 0;
+		let valid_owner_key = &authorities[valid_owner_authority_index];
 
 		// generate a key ownership proof for the authority at index 1
 		let invalid_key_owner_proof =
 			Historical::prove((BEEFY_KEY_TYPE, &invalid_owner_key)).unwrap();
+		// generate a key ownership proof for the authority at index 1
+		let valid_key_owner_proof =
+			Historical::prove((BEEFY_KEY_TYPE, &valid_owner_key)).unwrap();
 
-		let equivocation_authority_index = 0;
-		let equivocation_key = &authorities[equivocation_authority_index];
-		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+		let equivocation_authority_indices = [0, 2];
+		let equivocation_keys = equivocation_authority_indices.iter().map(|i| &authorities[*i]).collect::<Vec<_>>();
+		let equivocation_keyrings = equivocation_keys.iter().map(|k| BeefyKeyring::from_public(k).unwrap()).collect();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
-		// generate an equivocation for a future set
-		let equivocation_proof = generate_fork_equivocation_proof_vote(
-			(block_num, payload, set_id+1, &equivocation_keyring),
+		// generate an equivocation proof for the authorities at indices [0, 2]
+		let commitment = Commitment { validator_set_id: set_id + 1, block_number: block_num, payload };
+		let equivocation_proof = generate_fork_equivocation_proof_sc(
+			commitment,
+			equivocation_keyrings,
 			header,
 		);
 
@@ -1649,11 +1670,11 @@ fn report_fork_equivocation_sc_invalid_key_owner_proof() {
 		start_era(era);
 
 		// report an equivocation for the current set using a key ownership
-		// proof for a different key than the one in the equivocation proof.
+		// proof for a different key than the ones in the equivocation proof.
 		assert_err!(Beefy::report_fork_equivocation_unsigned(
 			RuntimeOrigin::none(),
 			Box::new(equivocation_proof),
-			vec![invalid_key_owner_proof],
+			vec![valid_key_owner_proof, invalid_key_owner_proof],
 		),
 			Error::<Test>::InvalidKeyOwnershipProof,
 		);
@@ -1668,63 +1689,66 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 		start_era(1);
 
 		let block_num = System::block_number();
+		let header = System::finalize();
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
 
-		let equivocation_authority_index = 0;
-		let equivocation_key = &authorities[equivocation_authority_index];
-		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+		let equivocation_authority_indices = [0, 2];
+		let equivocation_keys = equivocation_authority_indices.iter().map(|i| &authorities[*i]).collect::<Vec<_>>();
+		let equivocation_keyrings: Vec<_> = equivocation_keys.iter().map(|k| BeefyKeyring::from_public(k).unwrap()).collect();
 
 		// generate a key ownership proof at set id in era 1
-		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
-
-		let assert_invalid_equivocation_proof = |equivocation_proof| {
-			assert_err!(
-				Beefy::report_vote_equivocation_unsigned(
-					RuntimeOrigin::none(),
-					Box::new(equivocation_proof),
-					key_owner_proof.clone(),
-				),
-				Error::<Test>::InvalidVoteEquivocationProof,
-			);
-		};
+		let key_owner_proofs: Vec<_> = equivocation_keys.iter().map(|k| Historical::prove((BEEFY_KEY_TYPE, &k)).unwrap()).collect();
 
 		start_era(2);
 
-		let payload1 = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
-		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
-
-		// both votes target the same block number and payload,
-		// there is no equivocation.
-		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
-			(block_num, payload1.clone(), set_id, &equivocation_keyring),
-			(block_num, payload1.clone(), set_id, &equivocation_keyring),
-		));
-
-		// votes targeting different rounds, there is no equivocation.
-		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
-			(block_num, payload1.clone(), set_id, &equivocation_keyring),
-			(block_num + 1, payload2.clone(), set_id, &equivocation_keyring),
-		));
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 
-		// votes signed with different authority keys
-		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
-			(block_num, payload1.clone(), set_id, &equivocation_keyring),
-			(block_num, payload1.clone(), set_id, &BeefyKeyring::Charlie),
-		));
+		// commitment targets different round than finalized payload, there is no equivocation.
+		let equivocation_proof = generate_fork_equivocation_proof_sc(
+			Commitment { validator_set_id: set_id, block_number: block_num+1, payload: payload.clone() },
+			equivocation_keyrings.clone(),
+			header.clone(),
+		);
+		assert_err!(
+			Beefy::report_fork_equivocation_unsigned(
+				RuntimeOrigin::none(),
+				Box::new(equivocation_proof),
+				key_owner_proofs.clone(),
+			),
+			Error::<Test>::InvalidForkEquivocationProof,
+		);
 
-		// votes signed with a key that isn't part of the authority set
-		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
-			(block_num, payload1.clone(), set_id, &equivocation_keyring),
-			(block_num, payload1.clone(), set_id, &BeefyKeyring::Dave),
-		));
+		// commitment signed with a key that isn't part of the authority set
+		let equivocation_proof = generate_fork_equivocation_proof_sc(
+			Commitment { validator_set_id: set_id, block_number: block_num, payload: payload.clone() },
+			vec![BeefyKeyring::Eve],
+			header.clone(),
+		);
+		assert_err!(
+			Beefy::report_fork_equivocation_unsigned(
+				RuntimeOrigin::none(),
+				Box::new(equivocation_proof),
+				key_owner_proofs.clone(),
+			),
+			Error::<Test>::InvalidKeyOwnershipProof,
+		);
 
-		// votes targeting different set ids
-		assert_invalid_equivocation_proof(generate_vote_equivocation_proof(
-			(block_num, payload1, set_id, &equivocation_keyring),
-			(block_num, payload2, set_id + 1, &equivocation_keyring),
-		));
+		// commitment targets future set id
+		let equivocation_proof = generate_fork_equivocation_proof_sc(
+			Commitment { validator_set_id: set_id+1, block_number: block_num, payload: payload.clone() },
+			equivocation_keyrings,
+			header,
+		);
+		assert_err!(
+			Beefy::report_fork_equivocation_unsigned(
+				RuntimeOrigin::none(),
+				Box::new(equivocation_proof),
+				key_owner_proofs,
+			),
+			Error::<Test>::InvalidEquivocationProofSession,
+		);
 	});
 }
 
diff --git a/substrate/primitives/consensus/beefy/src/test_utils.rs b/substrate/primitives/consensus/beefy/src/test_utils.rs
index d52148073efd..2097822b299f 100644
--- a/substrate/primitives/consensus/beefy/src/test_utils.rs
+++ b/substrate/primitives/consensus/beefy/src/test_utils.rs
@@ -125,3 +125,16 @@ pub fn generate_fork_equivocation_proof_vote<Header>(
 	let signatories = vec![(signed_vote.id, signed_vote.signature)];
 	ForkEquivocationProof { commitment: signed_vote.commitment, signatories, correct_header }
 }
+
+/// Create a new `ForkEquivocationProof` based on signed commitment & correct header.
+pub fn generate_fork_equivocation_proof_sc<Header>(
+	commitment: Commitment<u64>,
+	keyrings: Vec<Keyring>,
+	correct_header: Header,
+) -> ForkEquivocationProof<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature, Header> {
+	let signatories = keyrings
+		.into_iter()
+		.map(|k| (k.public(), k.sign(&commitment.encode())))
+		.collect::<Vec<_>>();
+	ForkEquivocationProof { commitment, signatories, correct_header }
+}

From 036fa2f51117d25bd71b1e8d4b6d7d88025d83b9 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Sun, 3 Sep 2023 23:30:56 +0200
Subject: [PATCH 052/188] cleanup & TODOs

---
 substrate/client/consensus/beefy/src/worker.rs | 8 ++------
 substrate/frame/beefy/src/tests.rs             | 3 +++
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index 838a15b91786..717cad840be4 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -1053,7 +1053,7 @@ pub(crate) mod tests {
 	use sp_api::HeaderT;
 	use sp_blockchain::Backend as BlockchainBackendT;
 	use sp_consensus_beefy::{
-		generate_fork_equivocation_proof_vote, generate_vote_equivocation_proof, known_payloads,
+		generate_fork_equivocation_proof_sc, generate_fork_equivocation_proof_vote, generate_vote_equivocation_proof, known_payloads,
 		known_payloads::MMR_ROOT_ID, mmr::MmrRootProvider, ForkEquivocationProof, Keyring, Payload,
 		SignedCommitment,
 	};
@@ -1746,11 +1746,7 @@ pub(crate) mod tests {
 			validator_set_id: validator_set.id(),
 		};
 		// only Bob and Charlie sign
-		let signatories: Vec<_> = vec![Keyring::Bob, Keyring::Charlie]
-			.iter()
-			.map(|k| (k.public(), k.sign(&commitment.encode())))
-			.collect();
-		let proof = ForkEquivocationProof { commitment, signatories, correct_header: header };
+		let proof = generate_fork_equivocation_proof_sc(commitment, vec![Keyring::Bob, Keyring::Charlie], header);
 		{
 			// expect fisher (Alice) to successfully process it
 			assert_eq!(
diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index c0daf1fbaa80..ec3cfd522bad 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -256,6 +256,7 @@ fn should_sign_and_verify() {
 }
 
 // vote equivocation report tests
+// TODO: deduplicate by extracting common test structure of equivocation classes
 #[test]
 fn report_vote_equivocation_current_set_works() {
 	let authorities = test_authorities();
@@ -794,6 +795,7 @@ fn valid_vote_equivocation_reports_dont_pay_fees() {
 }
 
 // fork equivocation (via vote) report tests
+// TODO: deduplicate by extracting common test structure of equivocation classes
 #[test]
 fn report_fork_equivocation_vote_current_set_works() {
 	let authorities = test_authorities();
@@ -1345,6 +1347,7 @@ fn valid_fork_equivocation_vote_reports_dont_pay_fees() {
 }
 
 // fork equivocation (via signed commitment) report tests
+// TODO: deduplicate by extracting common test structure of equivocation classes
 #[test]
 fn report_fork_equivocation_sc_current_set_works() {
 	let authorities = test_authorities();

From 53e8ae0f2d2f6a6a9ae7ac55e0ccf104db76867e Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 4 Sep 2023 00:17:46 +0200
Subject: [PATCH 053/188] test that overlapping reports stack correctly

Already reported equivocators should neither be slashed again, nor
should they cause the report containing a re-report of their
equivocation to be invalidated by their presence.
---
 substrate/frame/beefy/src/tests.rs | 131 +++++++++++++++++++++++++++++
 1 file changed, 131 insertions(+)

diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index ec3cfd522bad..0443254722df 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -1921,3 +1921,134 @@ fn valid_fork_equivocation_sc_reports_dont_pay_fees() {
 		assert_eq!(post_info.pays_fee, Pays::Yes);
 	})
 }
+
+#[test]
+fn report_fork_equivocation_sc_stacked_reports_stack_correctly() {
+	let authorities = test_authorities();
+
+	new_test_ext_raw_authorities(authorities).execute_with(|| {
+		assert_eq!(Staking::current_era(), Some(0));
+		assert_eq!(Session::current_index(), 0);
+
+		let mut era = 1;
+		start_era(era);
+		let block_num = System::block_number();
+		let header = System::finalize();
+
+		era += 1;
+		start_era(era);
+
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+		let set_id = validator_set.id();
+		let validators = Session::validators();
+
+		// make sure that all validators have the same balance
+		for validator in &validators {
+			assert_eq!(Balances::total_balance(validator), 10_000_000);
+			assert_eq!(Staking::slashable_balance_of(validator), 10_000);
+
+			assert_eq!(
+				Staking::eras_stakers(era, validator),
+				pallet_staking::Exposure { total: 10_000, own: 10_000, others: vec![] },
+			);
+		}
+
+		assert_eq!(authorities.len(), 3);
+		let equivocation_authority_indices = [0, 2];
+		let equivocation_keys = equivocation_authority_indices.iter().map(|i| &authorities[*i]).collect::<Vec<_>>();
+		let equivocation_keyrings: Vec<_> = equivocation_keys.iter().map(|k| BeefyKeyring::from_public(k).unwrap()).collect();
+
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let commitment = Commitment { validator_set_id: set_id, block_number: block_num, payload };
+		// generate two fork equivocation proofs with a signed commitment in the same round for a
+		// different payload than finalized
+		// 1. the first equivocation proof is only for Alice
+		// 2. the second equivocation proof is for all equivocators
+		let equivocation_proof_singleton = generate_fork_equivocation_proof_sc(
+			commitment.clone(),
+			vec![equivocation_keyrings[0]],
+			header.clone(),
+		);
+		let equivocation_proof_full = generate_fork_equivocation_proof_sc(
+			commitment,
+			equivocation_keyrings,
+			header,
+		);
+
+		// create the key ownership proof
+		let key_owner_proofs: Vec<_> = equivocation_keys.iter().map(|k| Historical::prove((BEEFY_KEY_TYPE, &k)).unwrap()).collect();
+
+		// only report a single equivocator and the tx should be dispatched successfully
+		assert_ok!(Beefy::report_fork_equivocation_unsigned(
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof_singleton),
+			vec![key_owner_proofs[0].clone()],
+		),);
+
+		era += 1;
+		start_era(era);
+
+		// check that the balance of the reported equivocating validator is slashed 100%.
+		let equivocation_validator_ids = equivocation_authority_indices.iter().map(|i| validators[*i]).collect::<Vec<_>>();
+
+		assert_eq!(Balances::total_balance(&equivocation_validator_ids[0]), 10_000_000 - 10_000);
+		assert_eq!(Staking::slashable_balance_of(&equivocation_validator_ids[0]), 0);
+		assert_eq!(
+			Staking::eras_stakers(era, equivocation_validator_ids[0]),
+			pallet_staking::Exposure { total: 0, own: 0, others: vec![] },
+		);
+
+		// check that the balances of all other validators are left intact.
+		for validator in &validators {
+			if equivocation_validator_ids[0] == *validator {
+				continue
+			}
+
+			assert_eq!(Balances::total_balance(validator), 10_000_000);
+			assert_eq!(Staking::slashable_balance_of(validator), 10_000);
+
+			assert_eq!(
+				Staking::eras_stakers(era, validator),
+				pallet_staking::Exposure { total: 10_000, own: 10_000, others: vec![] },
+			);
+		}
+
+		// report the full equivocation and the tx should be dispatched successfully
+		assert_ok!(Beefy::report_fork_equivocation_unsigned(
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof_full),
+			key_owner_proofs,
+		),);
+
+		era += 1;
+		start_era(era);
+
+		let equivocation_validator_ids = equivocation_authority_indices.iter().map(|i| validators[*i]).collect::<Vec<_>>();
+
+		// check that the balance of equivocating validators is slashed 100%, and the validator already reported isn't slashed again
+		for equivocation_validator_id in &equivocation_validator_ids {
+			assert_eq!(Balances::total_balance(&equivocation_validator_id), 10_000_000 - 10_000);
+			assert_eq!(Staking::slashable_balance_of(&equivocation_validator_id), 0);
+			assert_eq!(
+				Staking::eras_stakers(era, equivocation_validator_id),
+				pallet_staking::Exposure { total: 0, own: 0, others: vec![] },
+			);
+		}
+
+		// check that the balances of all other validators are left intact.
+		for validator in &validators {
+			if equivocation_validator_ids.contains(&validator) {
+				continue
+			}
+
+			assert_eq!(Balances::total_balance(validator), 10_000_000);
+			assert_eq!(Staking::slashable_balance_of(validator), 10_000);
+
+			assert_eq!(
+				Staking::eras_stakers(era, validator),
+				pallet_staking::Exposure { total: 10_000, own: 10_000, others: vec![] },
+			);
+		}
+	});
+}

From 8a14649ace2891f8f54119e504fc9d46bfcfffde Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 4 Sep 2023 15:55:24 +0200
Subject: [PATCH 054/188] note re. temporary equivocation proof structure

---
 substrate/frame/beefy/src/equivocation.rs | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 15da5894dcc9..d6ba247d73b6 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -288,6 +288,18 @@ where
 
 				// Validate equivocation proof (check commitment is to unexpected payload and
 				// signatures are valid).
+				// NOTE: For proving a fork equivocation, equivocation proofs
+				// currently include headers that the verifier calculates hashes
+				// of and compares with on-chain
+				// `<frame_system::Pallet<T>>::block_hash(block_number)`. Since
+				// this mapping has a horizon of 4096 blocks, equivocations
+				// predating this horizon cannot be slashed. While this thwarts
+				// attacks assuming a 2/3rds of validators honestly participate
+				// in BEEFY finalization and at least one honest relayer can
+				// update the beefy light client at least once every 4096
+				// blocks, this is clearly only a temporary solution. This
+				// verification is to be replaced with mmr ancestry/pre-fix
+				// proofs.
 				if !sp_consensus_beefy::check_fork_equivocation_proof(
 					&equivocation_proof,
 					&expected_block_hash,

From e5537dd34a7e85eea79321587935f6e1e2d3f798 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 5 Sep 2023 08:19:49 +0200
Subject: [PATCH 055/188] fmt

---
 substrate/client/consensus/beefy/src/lib.rs   |  14 +-
 .../client/consensus/beefy/src/worker.rs      |  30 +-
 substrate/frame/beefy/src/equivocation.rs     |  25 +-
 substrate/frame/beefy/src/lib.rs              |   5 +-
 substrate/frame/beefy/src/tests.rs            | 295 +++++++++++-------
 .../consensus/beefy/src/test_utils.rs         |   2 +-
 6 files changed, 232 insertions(+), 139 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/lib.rs b/substrate/client/consensus/beefy/src/lib.rs
index 757a4c64aa6e..97a00460b3b3 100644
--- a/substrate/client/consensus/beefy/src/lib.rs
+++ b/substrate/client/consensus/beefy/src/lib.rs
@@ -260,13 +260,13 @@ pub async fn start_beefy_gadget<B, BE, C, N, P, R, S>(
 	let mut block_import_justif = links.from_block_import_justif_stream.subscribe(100_000).fuse();
 
 	let known_peers = Arc::new(Mutex::new(KnownPeers::new()));
-    let fisherman = Fisherman {
-        backend: backend.clone(),
-        key_store: key_store.clone(),
-        runtime: runtime.clone(),
-        payload_provider: payload_provider.clone(),
-        _phantom: PhantomData,
-    };
+	let fisherman = Fisherman {
+		backend: backend.clone(),
+		key_store: key_store.clone(),
+		runtime: runtime.clone(),
+		payload_provider: payload_provider.clone(),
+		_phantom: PhantomData,
+	};
 	// Default votes filter is to discard everything.
 	// Validator is updated later with correct starting round and set id.
 	let (gossip_validator, gossip_report_stream) =
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index 99014490571a..bf4f7b38952a 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -1071,9 +1071,9 @@ pub(crate) mod tests {
 	use sp_api::HeaderT;
 	use sp_blockchain::Backend as BlockchainBackendT;
 	use sp_consensus_beefy::{
-		generate_fork_equivocation_proof_sc, generate_fork_equivocation_proof_vote, generate_vote_equivocation_proof, known_payloads,
-		known_payloads::MMR_ROOT_ID, mmr::MmrRootProvider, ForkEquivocationProof, Keyring, Payload,
-		SignedCommitment,
+		generate_fork_equivocation_proof_sc, generate_fork_equivocation_proof_vote,
+		generate_vote_equivocation_proof, known_payloads, known_payloads::MMR_ROOT_ID,
+		mmr::MmrRootProvider, Keyring, Payload, SignedCommitment,
 	};
 	use sp_runtime::traits::One;
 	use std::marker::PhantomData;
@@ -1735,7 +1735,11 @@ pub(crate) mod tests {
 		{
 			// expect fisher (Alice) to successfully process it
 			assert_eq!(
-				alice_worker.comms.gossip_validator.fisherman.report_fork_equivocation(proof.clone()),
+				alice_worker
+					.comms
+					.gossip_validator
+					.fisherman
+					.report_fork_equivocation(proof.clone()),
 				Ok(())
 			);
 			// verify Alice reports Bob's equivocation to runtime
@@ -1750,7 +1754,11 @@ pub(crate) mod tests {
 		{
 			// expect fisher (Alice) to successfully process it
 			assert_eq!(
-				alice_worker.comms.gossip_validator.fisherman.report_fork_equivocation(proof.clone()),
+				alice_worker
+					.comms
+					.gossip_validator
+					.fisherman
+					.report_fork_equivocation(proof.clone()),
 				Ok(())
 			);
 			// verify Alice does *not* report her own equivocation to runtime
@@ -1767,11 +1775,19 @@ pub(crate) mod tests {
 			validator_set_id: validator_set.id(),
 		};
 		// only Bob and Charlie sign
-		let proof = generate_fork_equivocation_proof_sc(commitment, vec![Keyring::Bob, Keyring::Charlie], header);
+		let proof = generate_fork_equivocation_proof_sc(
+			commitment,
+			vec![Keyring::Bob, Keyring::Charlie],
+			header,
+		);
 		{
 			// expect fisher (Alice) to successfully process it
 			assert_eq!(
-				alice_worker.comms.gossip_validator.fisherman.report_fork_equivocation(proof.clone()),
+				alice_worker
+					.comms
+					.gossip_validator
+					.fisherman
+					.report_fork_equivocation(proof.clone()),
 				Ok(())
 			);
 			// verify Alice report Bob's and Charlie's equivocation to runtime
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index d6ba247d73b6..001b7caccca7 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -360,16 +360,16 @@ impl<T: Config> Pallet<T> {
 					<T::EquivocationReportSystem as OffenceReportSystem<_, _>>::Longevity::get();
 
 				ValidTransaction::with_tag_prefix("BeefyEquivocation")
-				// We assign the maximum priority for any equivocation report.
+					// We assign the maximum priority for any equivocation report.
 					.priority(TransactionPriority::MAX)
-				// Only one equivocation report for the same offender at the same slot.
+					// Only one equivocation report for the same offender at the same slot.
 					.and_provides((
 						equivocation_proof.offender_id().clone(),
 						equivocation_proof.set_id(),
 						*equivocation_proof.round_number(),
 					))
 					.longevity(longevity)
-				// We don't propagate this. This can never be included on a remote node.
+					// We don't propagate this. This can never be included on a remote node.
 					.propagate(false)
 					.build()
 			},
@@ -387,16 +387,16 @@ impl<T: Config> Pallet<T> {
 				log::error!("retrieved longevity");
 
 				ValidTransaction::with_tag_prefix("BeefyEquivocation")
-				// We assign the maximum priority for any equivocation report.
+					// We assign the maximum priority for any equivocation report.
 					.priority(TransactionPriority::MAX)
-				// Only one equivocation report for the same offender at the same slot.
+					// Only one equivocation report for the same offender at the same slot.
 					.and_provides((
 						equivocation_proof.offender_ids().clone(),
 						equivocation_proof.set_id(),
 						*equivocation_proof.round_number(),
 					))
 					.longevity(longevity)
-				// We don't propagate this. This can never be included on a remote node.
+					// We don't propagate this. This can never be included on a remote node.
 					.propagate(false)
 					.build()
 			},
@@ -406,32 +406,27 @@ impl<T: Config> Pallet<T> {
 					"lolalol"
 				);
 				InvalidTransaction::Call.into()
-			}
-
+			},
 		}
 	}
 
 	pub fn pre_dispatch(call: &Call<T>) -> Result<(), TransactionValidityError> {
 		match call {
-			Call::report_vote_equivocation_unsigned { equivocation_proof, key_owner_proof } =>
-			{
+			Call::report_vote_equivocation_unsigned { equivocation_proof, key_owner_proof } => {
 				let evidence = EquivocationEvidenceFor::<T>::VoteEquivocationProof(
 					*equivocation_proof.clone(),
 					key_owner_proof.clone(),
 				);
 				T::EquivocationReportSystem::check_evidence(evidence)
 			},
-			Call::report_fork_equivocation_unsigned { equivocation_proof, key_owner_proofs } =>
-			{
+			Call::report_fork_equivocation_unsigned { equivocation_proof, key_owner_proofs } => {
 				let evidence = EquivocationEvidenceFor::<T>::ForkEquivocationProof(
 					*equivocation_proof.clone(),
 					key_owner_proofs.clone(),
 				);
 				T::EquivocationReportSystem::check_evidence(evidence)
 			},
-			_ => {
-				Err(InvalidTransaction::Call.into())
-			}
+			_ => Err(InvalidTransaction::Call.into()),
 		}
 	}
 }
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 8d8534f40a47..0a9676bbf34f 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -340,7 +340,10 @@ pub mod pallet {
 
 			T::EquivocationReportSystem::process_evidence(
 				None,
-				EquivocationEvidenceFor::ForkEquivocationProof(*equivocation_proof, key_owner_proofs),
+				EquivocationEvidenceFor::ForkEquivocationProof(
+					*equivocation_proof,
+					key_owner_proofs,
+				),
 			)?;
 			// Waive the fee since the report is valid and beneficial
 			Ok(Pays::No.into())
diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index 0443254722df..2cf965b3719a 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -19,8 +19,10 @@ use std::vec;
 
 use codec::Encode;
 use sp_consensus_beefy::{
-	Commitment, check_vote_equivocation_proof, generate_vote_equivocation_proof, generate_fork_equivocation_proof_sc, generate_fork_equivocation_proof_vote, known_payloads::MMR_ROOT_ID,
-	Keyring as BeefyKeyring, Payload, ValidatorSet, KEY_TYPE as BEEFY_KEY_TYPE,
+	check_vote_equivocation_proof, generate_fork_equivocation_proof_sc,
+	generate_fork_equivocation_proof_vote, generate_vote_equivocation_proof,
+	known_payloads::MMR_ROOT_ID, Commitment, Keyring as BeefyKeyring, Payload, ValidatorSet,
+	KEY_TYPE as BEEFY_KEY_TYPE,
 };
 
 use sp_runtime::DigestItem;
@@ -88,7 +90,8 @@ fn session_change_updates_authorities() {
 		assert!(2 == Beefy::validator_set_id());
 
 		let want = beefy_log(ConsensusLog::AuthoritiesChange(
-			ValidatorSet::new(vec![mock_beefy_id(2), mock_beefy_id(3), mock_beefy_id(4)], 2).unwrap(),
+			ValidatorSet::new(vec![mock_beefy_id(2), mock_beefy_id(3), mock_beefy_id(4)], 2)
+				.unwrap(),
 		));
 
 		let log = System::digest().logs[1].clone();
@@ -198,7 +201,8 @@ fn cleans_up_old_set_id_session_mappings() {
 /// Returns a list with 3 authorities with known keys:
 /// Alice, Bob and Charlie.
 pub fn test_authorities() -> Vec<BeefyId> {
-	let authorities = vec![BeefyKeyring::Alice, BeefyKeyring::Bob, BeefyKeyring::Charlie, BeefyKeyring::Dave];
+	let authorities =
+		vec![BeefyKeyring::Alice, BeefyKeyring::Bob, BeefyKeyring::Charlie, BeefyKeyring::Dave];
 	authorities.into_iter().map(|id| id.public()).collect()
 }
 
@@ -997,16 +1001,17 @@ fn report_fork_equivocation_vote_invalid_set_id() {
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		// generate an equivocation for a future set
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
-			(block_num, payload, set_id+1, &equivocation_keyring),
+			(block_num, payload, set_id + 1, &equivocation_keyring),
 			header,
 		);
 
 		// the call for reporting the equivocation should error
-		assert_err!(Beefy::report_fork_equivocation_unsigned(
-			RuntimeOrigin::none(),
-			Box::new(equivocation_proof),
-			vec![key_owner_proof],
-		),
+		assert_err!(
+			Beefy::report_fork_equivocation_unsigned(
+				RuntimeOrigin::none(),
+				Box::new(equivocation_proof),
+				vec![key_owner_proof],
+			),
 			Error::<Test>::InvalidEquivocationProofSession,
 		);
 	});
@@ -1049,11 +1054,12 @@ fn report_fork_equivocation_vote_invalid_session() {
 
 		// report an equivocation for the current set using an key ownership
 		// proof from the previous set, the session should be invalid.
-		assert_err!(Beefy::report_fork_equivocation_unsigned(
-			RuntimeOrigin::none(),
-			Box::new(equivocation_proof),
-			vec![key_owner_proof],
-		),
+		assert_err!(
+			Beefy::report_fork_equivocation_unsigned(
+				RuntimeOrigin::none(),
+				Box::new(equivocation_proof),
+				vec![key_owner_proof],
+			),
 			Error::<Test>::InvalidEquivocationProofSession,
 		);
 	});
@@ -1090,7 +1096,7 @@ fn report_fork_equivocation_vote_invalid_key_owner_proof() {
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		// generate an equivocation for a future set
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
-			(block_num, payload, set_id+1, &equivocation_keyring),
+			(block_num, payload, set_id + 1, &equivocation_keyring),
 			header,
 		);
 
@@ -1101,11 +1107,12 @@ fn report_fork_equivocation_vote_invalid_key_owner_proof() {
 
 		// report an equivocation for the current set using a key ownership
 		// proof for a different key than the one in the equivocation proof.
-		assert_err!(Beefy::report_fork_equivocation_unsigned(
-			RuntimeOrigin::none(),
-			Box::new(equivocation_proof),
-			vec![invalid_key_owner_proof],
-		),
+		assert_err!(
+			Beefy::report_fork_equivocation_unsigned(
+				RuntimeOrigin::none(),
+				Box::new(equivocation_proof),
+				vec![invalid_key_owner_proof],
+			),
 			Error::<Test>::InvalidKeyOwnershipProof,
 		);
 	});
@@ -1137,7 +1144,7 @@ fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 
 		// vote targets different round than finalized payload, there is no equivocation.
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
-			(block_num+1, payload.clone(), set_id, &equivocation_keyring),
+			(block_num + 1, payload.clone(), set_id, &equivocation_keyring),
 			header.clone(),
 		);
 		assert_err!(
@@ -1165,7 +1172,7 @@ fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 
 		// vote targets future set id
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
-			(block_num, payload.clone(), set_id+1, &equivocation_keyring),
+			(block_num, payload.clone(), set_id + 1, &equivocation_keyring),
 			header.clone(),
 		);
 		assert_err!(
@@ -1255,7 +1262,7 @@ fn report_fork_equivocation_vote_validate_unsigned_prevents_duplicates() {
 			Box::new(equivocation_proof),
 			vec![key_owner_proof],
 		)
-			.unwrap();
+		.unwrap();
 
 		// the report should now be considered stale and the transaction is invalid
 		// the check for staleness should be done on both `validate_unsigned` and on `pre_dispatch`
@@ -1317,7 +1324,7 @@ fn valid_fork_equivocation_vote_reports_dont_pay_fees() {
 		assert_eq!(info.pays_fee, Pays::Yes);
 
 		// report the equivocation.
-		let post_info = Beefy::report_fork_equivocation_unsigned (
+		let post_info = Beefy::report_fork_equivocation_unsigned(
 			RuntimeOrigin::none(),
 			Box::new(equivocation_proof.clone()),
 			vec![key_owner_proof.clone()],
@@ -1331,7 +1338,7 @@ fn valid_fork_equivocation_vote_reports_dont_pay_fees() {
 
 		// report the equivocation again which is invalid now since it is
 		// duplicate.
-		let post_info = Beefy::report_fork_equivocation_unsigned (
+		let post_info = Beefy::report_fork_equivocation_unsigned(
 			RuntimeOrigin::none(),
 			Box::new(equivocation_proof.clone()),
 			vec![key_owner_proof.clone()],
@@ -1382,21 +1389,27 @@ fn report_fork_equivocation_sc_current_set_works() {
 
 		assert_eq!(authorities.len(), 3);
 		let equivocation_authority_indices = [0, 2];
-		let equivocation_keys = equivocation_authority_indices.iter().map(|i| &authorities[*i]).collect::<Vec<_>>();
-		let equivocation_keyrings = equivocation_keys.iter().map(|k| BeefyKeyring::from_public(k).unwrap()).collect();
+		let equivocation_keys = equivocation_authority_indices
+			.iter()
+			.map(|i| &authorities[*i])
+			.collect::<Vec<_>>();
+		let equivocation_keyrings = equivocation_keys
+			.iter()
+			.map(|k| BeefyKeyring::from_public(k).unwrap())
+			.collect();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		let commitment = Commitment { validator_set_id: set_id, block_number: block_num, payload };
 		// generate an fork equivocation proof, with a vote in the same round for a
 		// different payload than finalized
-		let equivocation_proof = generate_fork_equivocation_proof_sc(
-			commitment,
-			equivocation_keyrings,
-			header,
-		);
+		let equivocation_proof =
+			generate_fork_equivocation_proof_sc(commitment, equivocation_keyrings, header);
 
 		// create the key ownership proof
-		let key_owner_proofs = equivocation_keys.iter().map(|k| Historical::prove((BEEFY_KEY_TYPE, &k)).unwrap()).collect();
+		let key_owner_proofs = equivocation_keys
+			.iter()
+			.map(|k| Historical::prove((BEEFY_KEY_TYPE, &k)).unwrap())
+			.collect();
 
 		// report the equivocation and the tx should be dispatched successfully
 		assert_ok!(Beefy::report_fork_equivocation_unsigned(
@@ -1409,7 +1422,10 @@ fn report_fork_equivocation_sc_current_set_works() {
 		start_era(era);
 
 		// check that the balance of equivocating validators is slashed 100%.
-		let equivocation_validator_ids = equivocation_authority_indices.iter().map(|i| validators[*i]).collect::<Vec<_>>();
+		let equivocation_validator_ids = equivocation_authority_indices
+			.iter()
+			.map(|i| validators[*i])
+			.collect::<Vec<_>>();
 
 		for equivocation_validator_id in &equivocation_validator_ids {
 			assert_eq!(Balances::total_balance(&equivocation_validator_id), 10_000_000 - 10_000);
@@ -1457,10 +1473,16 @@ fn report_fork_equivocation_sc_old_set_works() {
 
 		assert_eq!(authorities.len(), 3);
 		let equivocation_authority_indices = [0, 2];
-		let equivocation_keys = equivocation_authority_indices.iter().map(|i| &authorities[*i]).collect::<Vec<_>>();
+		let equivocation_keys = equivocation_authority_indices
+			.iter()
+			.map(|i| &authorities[*i])
+			.collect::<Vec<_>>();
 
 		// create the key ownership proofs in the "old" set
-		let key_owner_proofs = equivocation_keys.iter().map(|k| Historical::prove((BEEFY_KEY_TYPE, &k)).unwrap()).collect();
+		let key_owner_proofs = equivocation_keys
+			.iter()
+			.map(|k| Historical::prove((BEEFY_KEY_TYPE, &k)).unwrap())
+			.collect();
 
 		era += 1;
 		start_era(era);
@@ -1480,19 +1502,20 @@ fn report_fork_equivocation_sc_old_set_works() {
 		let new_set_id = validator_set.id();
 		assert_eq!(old_set_id + 3, new_set_id);
 
-		let equivocation_keyrings = equivocation_keys.iter().map(|k| BeefyKeyring::from_public(k).unwrap()).collect();
+		let equivocation_keyrings = equivocation_keys
+			.iter()
+			.map(|k| BeefyKeyring::from_public(k).unwrap())
+			.collect();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		// generate an fork equivocation proof, with a vote in the same round for a
 		// different payload than finalized
-		let commitment = Commitment { validator_set_id: old_set_id, block_number: block_num, payload };
+		let commitment =
+			Commitment { validator_set_id: old_set_id, block_number: block_num, payload };
 		// generate an fork equivocation proof, with a vote in the same round for a
 		// different payload than finalized
-		let equivocation_proof = generate_fork_equivocation_proof_sc(
-			commitment,
-			equivocation_keyrings,
-			header,
-		);
+		let equivocation_proof =
+			generate_fork_equivocation_proof_sc(commitment, equivocation_keyrings, header);
 
 		// report the equivocation and the tx should be dispatched successfully
 		assert_ok!(Beefy::report_fork_equivocation_unsigned(
@@ -1505,7 +1528,10 @@ fn report_fork_equivocation_sc_old_set_works() {
 		start_era(era);
 
 		// check that the balance of equivocating validators is slashed 100%.
-		let equivocation_validator_ids = equivocation_authority_indices.iter().map(|i| validators[*i]).collect::<Vec<_>>();
+		let equivocation_validator_ids = equivocation_authority_indices
+			.iter()
+			.map(|i| validators[*i])
+			.collect::<Vec<_>>();
 
 		for equivocation_validator_id in &equivocation_validator_ids {
 			assert_eq!(Balances::total_balance(&equivocation_validator_id), 10_000_000 - 10_000);
@@ -1551,26 +1577,34 @@ fn report_fork_equivocation_sc_invalid_set_id() {
 		let set_id = validator_set.id();
 
 		let equivocation_authority_indices = [0, 2];
-		let equivocation_keys = equivocation_authority_indices.iter().map(|i| &authorities[*i]).collect::<Vec<_>>();
-		let equivocation_keyrings = equivocation_keys.iter().map(|k| BeefyKeyring::from_public(k).unwrap()).collect();
-
-		let key_owner_proofs = equivocation_keys.iter().map(|k| Historical::prove((BEEFY_KEY_TYPE, &k)).unwrap()).collect();
+		let equivocation_keys = equivocation_authority_indices
+			.iter()
+			.map(|i| &authorities[*i])
+			.collect::<Vec<_>>();
+		let equivocation_keyrings = equivocation_keys
+			.iter()
+			.map(|k| BeefyKeyring::from_public(k).unwrap())
+			.collect();
+
+		let key_owner_proofs = equivocation_keys
+			.iter()
+			.map(|k| Historical::prove((BEEFY_KEY_TYPE, &k)).unwrap())
+			.collect();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		// generate an equivocation for a future set
-		let commitment = Commitment { validator_set_id: set_id+1, block_number: block_num, payload };
-		let equivocation_proof = generate_fork_equivocation_proof_sc(
-			commitment,
-			equivocation_keyrings,
-			header,
-		);
+		let commitment =
+			Commitment { validator_set_id: set_id + 1, block_number: block_num, payload };
+		let equivocation_proof =
+			generate_fork_equivocation_proof_sc(commitment, equivocation_keyrings, header);
 
 		// the call for reporting the equivocation should error
-		assert_err!(Beefy::report_fork_equivocation_unsigned(
-			RuntimeOrigin::none(),
-			Box::new(equivocation_proof),
-			key_owner_proofs,
-		),
+		assert_err!(
+			Beefy::report_fork_equivocation_unsigned(
+				RuntimeOrigin::none(),
+				Box::new(equivocation_proof),
+				key_owner_proofs,
+			),
 			Error::<Test>::InvalidEquivocationProofSession,
 		);
 	});
@@ -1593,11 +1627,20 @@ fn report_fork_equivocation_sc_invalid_session() {
 		let authorities = validator_set.validators();
 
 		let equivocation_authority_indices = [0, 2];
-		let equivocation_keys = equivocation_authority_indices.iter().map(|i| &authorities[*i]).collect::<Vec<_>>();
-		let equivocation_keyrings = equivocation_keys.iter().map(|k| BeefyKeyring::from_public(k).unwrap()).collect();
+		let equivocation_keys = equivocation_authority_indices
+			.iter()
+			.map(|i| &authorities[*i])
+			.collect::<Vec<_>>();
+		let equivocation_keyrings = equivocation_keys
+			.iter()
+			.map(|k| BeefyKeyring::from_public(k).unwrap())
+			.collect();
 
 		// generate key ownership proofs at current era set id
-		let key_owner_proofs = equivocation_keys.iter().map(|k| Historical::prove((BEEFY_KEY_TYPE, &k)).unwrap()).collect();
+		let key_owner_proofs = equivocation_keys
+			.iter()
+			.map(|k| Historical::prove((BEEFY_KEY_TYPE, &k)).unwrap())
+			.collect();
 
 		era += 1;
 		start_era(era);
@@ -1607,19 +1650,17 @@ fn report_fork_equivocation_sc_invalid_session() {
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		// generate an equivocation proof at following era set id = 3
 		let commitment = Commitment { validator_set_id: set_id, block_number: block_num, payload };
-		let equivocation_proof = generate_fork_equivocation_proof_sc(
-			commitment,
-			equivocation_keyrings,
-			header,
-		);
+		let equivocation_proof =
+			generate_fork_equivocation_proof_sc(commitment, equivocation_keyrings, header);
 
 		// report an equivocation for the current set using an key ownership
 		// proof from the previous set, the session should be invalid.
-		assert_err!(Beefy::report_fork_equivocation_unsigned(
-			RuntimeOrigin::none(),
-			Box::new(equivocation_proof),
-			key_owner_proofs,
-		),
+		assert_err!(
+			Beefy::report_fork_equivocation_unsigned(
+				RuntimeOrigin::none(),
+				Box::new(equivocation_proof),
+				key_owner_proofs,
+			),
 			Error::<Test>::InvalidEquivocationProofSession,
 		);
 	});
@@ -1651,21 +1692,24 @@ fn report_fork_equivocation_sc_invalid_key_owner_proof() {
 		let invalid_key_owner_proof =
 			Historical::prove((BEEFY_KEY_TYPE, &invalid_owner_key)).unwrap();
 		// generate a key ownership proof for the authority at index 1
-		let valid_key_owner_proof =
-			Historical::prove((BEEFY_KEY_TYPE, &valid_owner_key)).unwrap();
+		let valid_key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &valid_owner_key)).unwrap();
 
 		let equivocation_authority_indices = [0, 2];
-		let equivocation_keys = equivocation_authority_indices.iter().map(|i| &authorities[*i]).collect::<Vec<_>>();
-		let equivocation_keyrings = equivocation_keys.iter().map(|k| BeefyKeyring::from_public(k).unwrap()).collect();
+		let equivocation_keys = equivocation_authority_indices
+			.iter()
+			.map(|i| &authorities[*i])
+			.collect::<Vec<_>>();
+		let equivocation_keyrings = equivocation_keys
+			.iter()
+			.map(|k| BeefyKeyring::from_public(k).unwrap())
+			.collect();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		// generate an equivocation proof for the authorities at indices [0, 2]
-		let commitment = Commitment { validator_set_id: set_id + 1, block_number: block_num, payload };
-		let equivocation_proof = generate_fork_equivocation_proof_sc(
-			commitment,
-			equivocation_keyrings,
-			header,
-		);
+		let commitment =
+			Commitment { validator_set_id: set_id + 1, block_number: block_num, payload };
+		let equivocation_proof =
+			generate_fork_equivocation_proof_sc(commitment, equivocation_keyrings, header);
 
 		// we need to start a new era otherwise the key ownership proof won't be
 		// checked since the authorities are part of the current session
@@ -1674,11 +1718,12 @@ fn report_fork_equivocation_sc_invalid_key_owner_proof() {
 
 		// report an equivocation for the current set using a key ownership
 		// proof for a different key than the ones in the equivocation proof.
-		assert_err!(Beefy::report_fork_equivocation_unsigned(
-			RuntimeOrigin::none(),
-			Box::new(equivocation_proof),
-			vec![valid_key_owner_proof, invalid_key_owner_proof],
-		),
+		assert_err!(
+			Beefy::report_fork_equivocation_unsigned(
+				RuntimeOrigin::none(),
+				Box::new(equivocation_proof),
+				vec![valid_key_owner_proof, invalid_key_owner_proof],
+			),
 			Error::<Test>::InvalidKeyOwnershipProof,
 		);
 	});
@@ -1698,11 +1743,20 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 		let set_id = validator_set.id();
 
 		let equivocation_authority_indices = [0, 2];
-		let equivocation_keys = equivocation_authority_indices.iter().map(|i| &authorities[*i]).collect::<Vec<_>>();
-		let equivocation_keyrings: Vec<_> = equivocation_keys.iter().map(|k| BeefyKeyring::from_public(k).unwrap()).collect();
+		let equivocation_keys = equivocation_authority_indices
+			.iter()
+			.map(|i| &authorities[*i])
+			.collect::<Vec<_>>();
+		let equivocation_keyrings: Vec<_> = equivocation_keys
+			.iter()
+			.map(|k| BeefyKeyring::from_public(k).unwrap())
+			.collect();
 
 		// generate a key ownership proof at set id in era 1
-		let key_owner_proofs: Vec<_> = equivocation_keys.iter().map(|k| Historical::prove((BEEFY_KEY_TYPE, &k)).unwrap()).collect();
+		let key_owner_proofs: Vec<_> = equivocation_keys
+			.iter()
+			.map(|k| Historical::prove((BEEFY_KEY_TYPE, &k)).unwrap())
+			.collect();
 
 		start_era(2);
 
@@ -1710,7 +1764,11 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 
 		// commitment targets different round than finalized payload, there is no equivocation.
 		let equivocation_proof = generate_fork_equivocation_proof_sc(
-			Commitment { validator_set_id: set_id, block_number: block_num+1, payload: payload.clone() },
+			Commitment {
+				validator_set_id: set_id,
+				block_number: block_num + 1,
+				payload: payload.clone(),
+			},
 			equivocation_keyrings.clone(),
 			header.clone(),
 		);
@@ -1725,7 +1783,11 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 
 		// commitment signed with a key that isn't part of the authority set
 		let equivocation_proof = generate_fork_equivocation_proof_sc(
-			Commitment { validator_set_id: set_id, block_number: block_num, payload: payload.clone() },
+			Commitment {
+				validator_set_id: set_id,
+				block_number: block_num,
+				payload: payload.clone(),
+			},
 			vec![BeefyKeyring::Eve],
 			header.clone(),
 		);
@@ -1740,7 +1802,11 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 
 		// commitment targets future set id
 		let equivocation_proof = generate_fork_equivocation_proof_sc(
-			Commitment { validator_set_id: set_id+1, block_number: block_num, payload: payload.clone() },
+			Commitment {
+				validator_set_id: set_id + 1,
+				block_number: block_num,
+				payload: payload.clone(),
+			},
 			equivocation_keyrings,
 			header,
 		);
@@ -1831,7 +1897,7 @@ fn report_fork_equivocation_sc_validate_unsigned_prevents_duplicates() {
 			Box::new(equivocation_proof),
 			vec![key_owner_proof],
 		)
-			.unwrap();
+		.unwrap();
 
 		// the report should now be considered stale and the transaction is invalid
 		// the check for staleness should be done on both `validate_unsigned` and on `pre_dispatch`
@@ -1893,7 +1959,7 @@ fn valid_fork_equivocation_sc_reports_dont_pay_fees() {
 		assert_eq!(info.pays_fee, Pays::Yes);
 
 		// report the equivocation.
-		let post_info = Beefy::report_fork_equivocation_unsigned (
+		let post_info = Beefy::report_fork_equivocation_unsigned(
 			RuntimeOrigin::none(),
 			Box::new(equivocation_proof.clone()),
 			vec![key_owner_proof.clone()],
@@ -1907,7 +1973,7 @@ fn valid_fork_equivocation_sc_reports_dont_pay_fees() {
 
 		// report the equivocation again which is invalid now since it is
 		// duplicate.
-		let post_info = Beefy::report_fork_equivocation_unsigned (
+		let post_info = Beefy::report_fork_equivocation_unsigned(
 			RuntimeOrigin::none(),
 			Box::new(equivocation_proof.clone()),
 			vec![key_owner_proof.clone()],
@@ -1956,8 +2022,14 @@ fn report_fork_equivocation_sc_stacked_reports_stack_correctly() {
 
 		assert_eq!(authorities.len(), 3);
 		let equivocation_authority_indices = [0, 2];
-		let equivocation_keys = equivocation_authority_indices.iter().map(|i| &authorities[*i]).collect::<Vec<_>>();
-		let equivocation_keyrings: Vec<_> = equivocation_keys.iter().map(|k| BeefyKeyring::from_public(k).unwrap()).collect();
+		let equivocation_keys = equivocation_authority_indices
+			.iter()
+			.map(|i| &authorities[*i])
+			.collect::<Vec<_>>();
+		let equivocation_keyrings: Vec<_> = equivocation_keys
+			.iter()
+			.map(|k| BeefyKeyring::from_public(k).unwrap())
+			.collect();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		let commitment = Commitment { validator_set_id: set_id, block_number: block_num, payload };
@@ -1970,14 +2042,14 @@ fn report_fork_equivocation_sc_stacked_reports_stack_correctly() {
 			vec![equivocation_keyrings[0]],
 			header.clone(),
 		);
-		let equivocation_proof_full = generate_fork_equivocation_proof_sc(
-			commitment,
-			equivocation_keyrings,
-			header,
-		);
+		let equivocation_proof_full =
+			generate_fork_equivocation_proof_sc(commitment, equivocation_keyrings, header);
 
 		// create the key ownership proof
-		let key_owner_proofs: Vec<_> = equivocation_keys.iter().map(|k| Historical::prove((BEEFY_KEY_TYPE, &k)).unwrap()).collect();
+		let key_owner_proofs: Vec<_> = equivocation_keys
+			.iter()
+			.map(|k| Historical::prove((BEEFY_KEY_TYPE, &k)).unwrap())
+			.collect();
 
 		// only report a single equivocator and the tx should be dispatched successfully
 		assert_ok!(Beefy::report_fork_equivocation_unsigned(
@@ -1990,7 +2062,10 @@ fn report_fork_equivocation_sc_stacked_reports_stack_correctly() {
 		start_era(era);
 
 		// check that the balance of the reported equivocating validator is slashed 100%.
-		let equivocation_validator_ids = equivocation_authority_indices.iter().map(|i| validators[*i]).collect::<Vec<_>>();
+		let equivocation_validator_ids = equivocation_authority_indices
+			.iter()
+			.map(|i| validators[*i])
+			.collect::<Vec<_>>();
 
 		assert_eq!(Balances::total_balance(&equivocation_validator_ids[0]), 10_000_000 - 10_000);
 		assert_eq!(Staking::slashable_balance_of(&equivocation_validator_ids[0]), 0);
@@ -2024,9 +2099,13 @@ fn report_fork_equivocation_sc_stacked_reports_stack_correctly() {
 		era += 1;
 		start_era(era);
 
-		let equivocation_validator_ids = equivocation_authority_indices.iter().map(|i| validators[*i]).collect::<Vec<_>>();
+		let equivocation_validator_ids = equivocation_authority_indices
+			.iter()
+			.map(|i| validators[*i])
+			.collect::<Vec<_>>();
 
-		// check that the balance of equivocating validators is slashed 100%, and the validator already reported isn't slashed again
+		// check that the balance of equivocating validators is slashed 100%, and the validator
+		// already reported isn't slashed again
 		for equivocation_validator_id in &equivocation_validator_ids {
 			assert_eq!(Balances::total_balance(&equivocation_validator_id), 10_000_000 - 10_000);
 			assert_eq!(Staking::slashable_balance_of(&equivocation_validator_id), 0);
diff --git a/substrate/primitives/consensus/beefy/src/test_utils.rs b/substrate/primitives/consensus/beefy/src/test_utils.rs
index 2097822b299f..f480788cd373 100644
--- a/substrate/primitives/consensus/beefy/src/test_utils.rs
+++ b/substrate/primitives/consensus/beefy/src/test_utils.rs
@@ -18,7 +18,7 @@
 #![cfg(feature = "std")]
 
 use crate::{
-	ecdsa_crypto, Commitment, ForkEquivocationProof, Payload, SignedCommitment, ValidatorSetId,
+	ecdsa_crypto, Commitment, ForkEquivocationProof, Payload, ValidatorSetId,
 	VoteEquivocationProof, VoteMessage,
 };
 use codec::Encode;

From c815af6ab0132027f65cc86ba2b86bc3d49c171f Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 5 Sep 2023 08:26:31 +0200
Subject: [PATCH 056/188] cleanup & remove TODO

stick with Vec since now in practice, always iterating through entire
signatories & not performing any key lookup.

also leaving ForkEquivocationProof { commitment, signatories, .. } as is
for now - let's change to SignedCommitment if it gets in the way.
---
 .../client/consensus/beefy/src/communication/fisherman.rs   | 6 +++---
 substrate/primitives/consensus/beefy/src/lib.rs             | 4 ----
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 2fa91ce78396..39798aabfee4 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -130,10 +130,9 @@ where
 			return Ok(())
 		}
 
-		let offender_ids =
-			proof.signatories.iter().cloned().map(|(id, _sig)| id).collect::<Vec<_>>();
+		let offender_ids = proof.offender_ids();
 		if let Some(local_id) = self.key_store.authority_id(validator_set.validators()) {
-			if offender_ids.contains(&local_id) {
+			if offender_ids.contains(&&local_id) {
 				debug!(target: LOG_TARGET, "🥩 Skip equivocation report for own equivocation");
 				// TODO: maybe error here instead?
 				return Ok(())
@@ -146,6 +145,7 @@ where
 		// generate key ownership proof at that block
 		let key_owner_proofs = offender_ids
 			.iter()
+			.cloned()
 			.filter_map(|id| {
 				match runtime_api.generate_key_ownership_proof(hash, set_id, id.clone()) {
 					Ok(Some(proof)) => Some(Ok(proof)),
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index f7c3758d50e6..f574b4583eda 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -257,12 +257,8 @@ impl<Number, Id, Signature> VoteEquivocationProof<Number, Id, Signature> {
 pub struct ForkEquivocationProof<Number, Id, Signature, Header> {
 	/// Commitment for a block on different fork than one at the same height in
 	/// this client's chain.
-	/// TODO: maybe replace {commitment, signatories} with SignedCommitment
-	/// (tradeoff: SignedCommitment not ideal since sigs optional, but fewer
-	/// types to juggle around) - check once usage pattern is clear
 	pub commitment: Commitment<Number>,
 	/// Signatures on this block
-	/// TODO: maybe change to HashMap - check once usage pattern is clear
 	pub signatories: Vec<(Id, Signature)>,
 	/// The proof is valid if
 	/// 1. the header is in our chain

From bb2e7917cc7479aa6757c603cb476b3d6de8488a Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 7 Sep 2023 11:26:55 +0200
Subject: [PATCH 057/188] add reference to #1441

---
 substrate/frame/beefy/src/equivocation.rs | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 001b7caccca7..1aee70c11efb 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -288,18 +288,12 @@ where
 
 				// Validate equivocation proof (check commitment is to unexpected payload and
 				// signatures are valid).
-				// NOTE: For proving a fork equivocation, equivocation proofs
-				// currently include headers that the verifier calculates hashes
-				// of and compares with on-chain
-				// `<frame_system::Pallet<T>>::block_hash(block_number)`. Since
-				// this mapping has a horizon of 4096 blocks, equivocations
-				// predating this horizon cannot be slashed. While this thwarts
-				// attacks assuming a 2/3rds of validators honestly participate
-				// in BEEFY finalization and at least one honest relayer can
-				// update the beefy light client at least once every 4096
-				// blocks, this is clearly only a temporary solution. This
-				// verification is to be replaced with mmr ancestry/pre-fix
-				// proofs.
+				// NOTE: Fork equivocation proof currently only prevents attacks
+				// assuming 2/3rds of validators honestly participate in BEEFY
+				// finalization and at least one honest relayer can update the
+				// beefy light client at least once every 4096 blocks. See
+				// https://github.com/paritytech/polkadot-sdk/issues/1441 for
+				// replacement solution.
 				if !sp_consensus_beefy::check_fork_equivocation_proof(
 					&equivocation_proof,
 					&expected_block_hash,

From 8b48fec021731c696e9e60fd100563db1f9ebd60 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 17 Oct 2023 19:47:57 +0200
Subject: [PATCH 058/188] fixup! Merge remote-tracking branch 'origin/master'
 into rhmb/beefy-slashing-fisherman

---
 substrate/frame/beefy/src/lib.rs | 38 ++++++++++++++++----------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 6acaf24ee502..a12bf587a7e0 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -279,12 +279,29 @@ pub mod pallet {
 			Ok(Pays::No.into())
 		}
 
+		/// Reset BEEFY consensus by setting a new BEEFY genesis at `delay_in_blocks` blocks in the
+		/// future.
+		///
+		/// Note: `delay_in_blocks` has to be at least 1.
+		#[pallet::call_index(2)]
+		#[pallet::weight(<T as Config>::WeightInfo::set_new_genesis())]
+		pub fn set_new_genesis(
+			origin: OriginFor<T>,
+			delay_in_blocks: BlockNumberFor<T>,
+		) -> DispatchResult {
+			ensure_root(origin)?;
+			ensure!(delay_in_blocks >= One::one(), Error::<T>::InvalidConfiguration);
+			let genesis_block = frame_system::Pallet::<T>::block_number() + delay_in_blocks;
+			GenesisBlock::<T>::put(Some(genesis_block));
+			Ok(())
+		}
+
 		/// Report voter voting on invalid fork. This method will verify the
 		/// invalid fork proof and validate the given key ownership proof
 		/// against the extracted offender. If both are valid, the offence
 		/// will be reported.
 		// TODO: fix key_owner_proofs[0].validator_count()
-		#[pallet::call_index(2)]
+		#[pallet::call_index(3)]
 		#[pallet::weight(T::WeightInfo::report_equivocation(
 			key_owner_proofs[0].validator_count(),
 			T::MaxNominators::get(),
@@ -324,7 +341,7 @@ pub mod pallet {
 		/// if the block author is defined it will be defined as the equivocation
 		/// reporter.
 		// TODO: fix key_owner_proofs[0].validator_count()
-		#[pallet::call_index(3)]
+		#[pallet::call_index(4)]
 		#[pallet::weight(T::WeightInfo::report_equivocation(key_owner_proofs[0].validator_count(), T::MaxNominators::get(),))]
 		pub fn report_fork_equivocation_unsigned(
 			origin: OriginFor<T>,
@@ -350,23 +367,6 @@ pub mod pallet {
 			// Waive the fee since the report is valid and beneficial
 			Ok(Pays::No.into())
 		}
-
-		/// Reset BEEFY consensus by setting a new BEEFY genesis at `delay_in_blocks` blocks in the
-		/// future.
-		///
-		/// Note: `delay_in_blocks` has to be at least 1.
-		#[pallet::call_index(2)]
-		#[pallet::weight(<T as Config>::WeightInfo::set_new_genesis())]
-		pub fn set_new_genesis(
-			origin: OriginFor<T>,
-			delay_in_blocks: BlockNumberFor<T>,
-		) -> DispatchResult {
-			ensure_root(origin)?;
-			ensure!(delay_in_blocks >= One::one(), Error::<T>::InvalidConfiguration);
-			let genesis_block = frame_system::Pallet::<T>::block_number() + delay_in_blocks;
-			GenesisBlock::<T>::put(Some(genesis_block));
-			Ok(())
-		}
 	}
 
 	#[pallet::validate_unsigned]

From ddd22d2b8849bc21b596554f5dc6b590e91dd6a9 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 12 Sep 2023 14:40:14 +0200
Subject: [PATCH 059/188] use ckb-merkle-mountain-range fork

---
 Cargo.lock                                            | 3 +--
 substrate/primitives/merkle-mountain-range/Cargo.toml | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index f12e16276e78..2eaa1021b89b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2394,8 +2394,7 @@ dependencies = [
 [[package]]
 name = "ckb-merkle-mountain-range"
 version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56ccb671c5921be8a84686e6212ca184cb1d7c51cadcdbfcbd1cc3f042f5dfb8"
+source = "git+https://github.com/Lederstrumpf/merkle-mountain-range.git?branch=linearize-proof-rebase-v0.5.1#f3aed2bfdaa5ec83d89e97b37c89ecaac9f64497"
 dependencies = [
  "cfg-if",
 ]
diff --git a/substrate/primitives/merkle-mountain-range/Cargo.toml b/substrate/primitives/merkle-mountain-range/Cargo.toml
index 747b967dd9ed..b11aaaeda0cf 100644
--- a/substrate/primitives/merkle-mountain-range/Cargo.toml
+++ b/substrate/primitives/merkle-mountain-range/Cargo.toml
@@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false }
 scale-info = { version = "2.5.0", default-features = false, features = ["derive"] }
 log = { version = "0.4.17", default-features = false }
-mmr-lib = { package = "ckb-merkle-mountain-range", version = "0.5.2", default-features = false }
+mmr-lib = { package = "ckb-merkle-mountain-range", git = "https://github.com/Lederstrumpf/merkle-mountain-range.git", branch = "linearize-proof-rebase-v0.5.1", default-features = false }
 serde = { version = "1.0.188", features = ["derive", "alloc"], default-features = false, optional = true }
 sp-api = { path = "../api", default-features = false}
 sp-core = { path = "../core", default-features = false}

From 5a4f840244e297819cb689b072badb96a15662f8 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 14 Sep 2023 11:40:40 +0200
Subject: [PATCH 060/188] integrate new proof data structure into api

---
 Cargo.lock                                           |  2 +-
 polkadot/runtime/rococo/src/lib.rs                   |  6 +++---
 polkadot/runtime/test-runtime/src/lib.rs             |  6 +++---
 polkadot/runtime/westend/src/lib.rs                  |  6 +++---
 substrate/bin/node/runtime/src/lib.rs                |  6 +++---
 .../client/merkle-mountain-range/src/test_utils.rs   |  6 +++---
 substrate/frame/merkle-mountain-range/src/lib.rs     |  6 +++---
 substrate/frame/merkle-mountain-range/src/mmr/mmr.rs | 12 ++++++------
 .../primitives/merkle-mountain-range/src/lib.rs      |  4 ++--
 9 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 2eaa1021b89b..5c8b8f89e43d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2394,7 +2394,7 @@ dependencies = [
 [[package]]
 name = "ckb-merkle-mountain-range"
 version = "0.5.2"
-source = "git+https://github.com/Lederstrumpf/merkle-mountain-range.git?branch=linearize-proof-rebase-v0.5.1#f3aed2bfdaa5ec83d89e97b37c89ecaac9f64497"
+source = "git+https://github.com/Lederstrumpf/merkle-mountain-range.git?branch=linearize-proof-rebase-v0.5.1#b55b6805d35b6ccbd8dc83c30caf1cae19943981"
 dependencies = [
  "cfg-if",
 ]
diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs
index 733fa9603e04..4222abee6b28 100644
--- a/polkadot/runtime/rococo/src/lib.rs
+++ b/polkadot/runtime/rococo/src/lib.rs
@@ -1861,7 +1861,7 @@ sp_api::impl_runtime_apis! {
 		fn generate_proof(
 			block_numbers: Vec<BlockNumber>,
 			best_known_block_number: Option<BlockNumber>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<mmr::Hash>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, mmr::Hash)>), mmr::Error> {
 			Mmr::generate_proof(block_numbers, best_known_block_number).map(
 				|(leaves, proof)| {
 					(
@@ -1875,7 +1875,7 @@ sp_api::impl_runtime_apis! {
 			)
 		}
 
-		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<mmr::Hash>)
+		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<(u64, mmr::Hash)>)
 			-> Result<(), mmr::Error>
 		{
 			let leaves = leaves.into_iter().map(|leaf|
@@ -1888,7 +1888,7 @@ sp_api::impl_runtime_apis! {
 		fn verify_proof_stateless(
 			root: mmr::Hash,
 			leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			proof: mmr::Proof<mmr::Hash>
+			proof: mmr::Proof<(u64, mmr::Hash)>
 		) -> Result<(), mmr::Error> {
 			let nodes = leaves.into_iter().map(|leaf|mmr::DataOrHash::Data(leaf.into_opaque_leaf())).collect();
 			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, nodes, proof)
diff --git a/polkadot/runtime/test-runtime/src/lib.rs b/polkadot/runtime/test-runtime/src/lib.rs
index 22fed1ab4e81..a044b04d8410 100644
--- a/polkadot/runtime/test-runtime/src/lib.rs
+++ b/polkadot/runtime/test-runtime/src/lib.rs
@@ -1002,11 +1002,11 @@ sp_api::impl_runtime_apis! {
 		fn generate_proof(
 			_block_numbers: Vec<BlockNumber>,
 			_best_known_block_number: Option<BlockNumber>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<Hash>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, Hash)>), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
 
-		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::Proof<Hash>)
+		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::Proof<(u64, Hash)>)
 			-> Result<(), mmr::Error>
 		{
 			Err(mmr::Error::PalletNotIncluded)
@@ -1015,7 +1015,7 @@ sp_api::impl_runtime_apis! {
 		fn verify_proof_stateless(
 			_root: Hash,
 			_leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			_proof: mmr::Proof<Hash>
+			_proof: mmr::Proof<(u64, Hash)>
 		) -> Result<(), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs
index 129826a5aba9..f1b1abc80a07 100644
--- a/polkadot/runtime/westend/src/lib.rs
+++ b/polkadot/runtime/westend/src/lib.rs
@@ -1907,7 +1907,7 @@ sp_api::impl_runtime_apis! {
 		fn generate_proof(
 			block_numbers: Vec<BlockNumber>,
 			best_known_block_number: Option<BlockNumber>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<mmr::Hash>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, mmr::Hash)>), mmr::Error> {
 			Mmr::generate_proof(block_numbers, best_known_block_number).map(
 				|(leaves, proof)| {
 					(
@@ -1921,7 +1921,7 @@ sp_api::impl_runtime_apis! {
 			)
 		}
 
-		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<mmr::Hash>)
+		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<(u64, mmr::Hash)>)
 			-> Result<(), mmr::Error>
 		{
 			let leaves = leaves.into_iter().map(|leaf|
@@ -1934,7 +1934,7 @@ sp_api::impl_runtime_apis! {
 		fn verify_proof_stateless(
 			root: mmr::Hash,
 			leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			proof: mmr::Proof<mmr::Hash>
+			proof: mmr::Proof<(u64, mmr::Hash)>
 		) -> Result<(), mmr::Error> {
 			let nodes = leaves.into_iter().map(|leaf|mmr::DataOrHash::Data(leaf.into_opaque_leaf())).collect();
 			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, nodes, proof)
diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs
index 2070e3f12d04..9bb0b8476595 100644
--- a/substrate/bin/node/runtime/src/lib.rs
+++ b/substrate/bin/node/runtime/src/lib.rs
@@ -2654,7 +2654,7 @@ impl_runtime_apis! {
 		fn generate_proof(
 			block_numbers: Vec<BlockNumber>,
 			best_known_block_number: Option<BlockNumber>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<mmr::Hash>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, mmr::Hash)>), mmr::Error> {
 			Mmr::generate_proof(block_numbers, best_known_block_number).map(
 				|(leaves, proof)| {
 					(
@@ -2668,7 +2668,7 @@ impl_runtime_apis! {
 			)
 		}
 
-		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<mmr::Hash>)
+		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<(u64, mmr::Hash)>)
 			-> Result<(), mmr::Error>
 		{
 			let leaves = leaves.into_iter().map(|leaf|
@@ -2681,7 +2681,7 @@ impl_runtime_apis! {
 		fn verify_proof_stateless(
 			root: mmr::Hash,
 			leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			proof: mmr::Proof<mmr::Hash>
+			proof: mmr::Proof<(u64, mmr::Hash)>
 		) -> Result<(), mmr::Error> {
 			let nodes = leaves.into_iter().map(|leaf|mmr::DataOrHash::Data(leaf.into_opaque_leaf())).collect();
 			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, nodes, proof)
diff --git a/substrate/client/merkle-mountain-range/src/test_utils.rs b/substrate/client/merkle-mountain-range/src/test_utils.rs
index 010b48bb3d7d..bf5eb2885451 100644
--- a/substrate/client/merkle-mountain-range/src/test_utils.rs
+++ b/substrate/client/merkle-mountain-range/src/test_utils.rs
@@ -304,11 +304,11 @@ sp_api::mock_impl_runtime_apis! {
 			&self,
 			_block_numbers: Vec<u64>,
 			_best_known_block_number: Option<u64>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<MmrHash>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, MmrHash)>), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
 
-		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::Proof<MmrHash>)
+		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::Proof<(u64, MmrHash)>)
 			-> Result<(), mmr::Error>
 		{
 			Err(mmr::Error::PalletNotIncluded)
@@ -317,7 +317,7 @@ sp_api::mock_impl_runtime_apis! {
 		fn verify_proof_stateless(
 			_root: MmrHash,
 			_leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			_proof: mmr::Proof<MmrHash>
+			_proof: mmr::Proof<(u64, MmrHash)>
 		) -> Result<(), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
diff --git a/substrate/frame/merkle-mountain-range/src/lib.rs b/substrate/frame/merkle-mountain-range/src/lib.rs
index 664f4bc73901..75acbfd76b8c 100644
--- a/substrate/frame/merkle-mountain-range/src/lib.rs
+++ b/substrate/frame/merkle-mountain-range/src/lib.rs
@@ -245,7 +245,7 @@ pub mod pallet {
 pub fn verify_leaves_proof<H, L>(
 	root: H::Output,
 	leaves: Vec<mmr::Node<H, L>>,
-	proof: primitives::Proof<H::Output>,
+	proof: primitives::Proof<(u64, H::Output)>,
 ) -> Result<(), primitives::Error>
 where
 	H: traits::Hash,
@@ -319,7 +319,7 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 	pub fn generate_proof(
 		block_numbers: Vec<BlockNumberFor<T>>,
 		best_known_block_number: Option<BlockNumberFor<T>>,
-	) -> Result<(Vec<LeafOf<T, I>>, primitives::Proof<HashOf<T, I>>), primitives::Error> {
+	) -> Result<(Vec<LeafOf<T, I>>, primitives::Proof<(u64, HashOf<T, I>)>), primitives::Error> {
 		// check whether best_known_block_number provided, else use current best block
 		let best_known_block_number =
 			best_known_block_number.unwrap_or_else(|| <frame_system::Pallet<T>>::block_number());
@@ -352,7 +352,7 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 	/// or the proof is invalid.
 	pub fn verify_leaves(
 		leaves: Vec<LeafOf<T, I>>,
-		proof: primitives::Proof<HashOf<T, I>>,
+		proof: primitives::Proof<(u64, HashOf<T, I>)>,
 	) -> Result<(), primitives::Error> {
 		if proof.leaf_count > Self::mmr_leaves() ||
 			proof.leaf_count == 0 ||
diff --git a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
index aeb3e7ea6641..7053db321e39 100644
--- a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
+++ b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
@@ -33,7 +33,7 @@ use sp_std::prelude::*;
 pub fn verify_leaves_proof<H, L>(
 	root: H::Output,
 	leaves: Vec<Node<H, L>>,
-	proof: primitives::Proof<H::Output>,
+	proof: primitives::Proof<(u64, H::Output)>,
 ) -> Result<bool, Error>
 where
 	H: sp_runtime::traits::Hash,
@@ -54,7 +54,7 @@ where
 
 	let p = mmr_lib::MerkleProof::<Node<H, L>, Hasher<H, L>>::new(
 		size,
-		proof.items.into_iter().map(Node::Hash).collect(),
+		proof.items.into_iter().map(|(index, hash)| (index, Node::Hash(hash))).collect(),
 	);
 	p.verify(Node::Hash(root), leaves_and_position_data)
 		.map_err(|e| Error::Verify.log_debug(e))
@@ -95,11 +95,11 @@ where
 	pub fn verify_leaves_proof(
 		&self,
 		leaves: Vec<L>,
-		proof: primitives::Proof<HashOf<T, I>>,
+		proof: primitives::Proof<(u64, HashOf<T, I>)>,
 	) -> Result<bool, Error> {
 		let p = mmr_lib::MerkleProof::<NodeOf<T, I, L>, Hasher<HashingOf<T, I>, L>>::new(
 			self.mmr.mmr_size(),
-			proof.items.into_iter().map(Node::Hash).collect(),
+			proof.items.into_iter().map(|(index, hash)| (index, Node::Hash(hash))).collect(),
 		);
 
 		if leaves.len() != proof.leaf_indices.len() {
@@ -166,7 +166,7 @@ where
 	pub fn generate_proof(
 		&self,
 		leaf_indices: Vec<NodeIndex>,
-	) -> Result<(Vec<L>, primitives::Proof<HashOf<T, I>>), Error> {
+	) -> Result<(Vec<L>, primitives::Proof<(u64, HashOf<T, I>)>), Error> {
 		let positions = leaf_indices
 			.iter()
 			.map(|index| mmr_lib::leaf_index_to_pos(*index))
@@ -187,7 +187,7 @@ where
 			.map(|p| primitives::Proof {
 				leaf_indices,
 				leaf_count,
-				items: p.proof_items().iter().map(|x| x.hash()).collect(),
+				items: p.proof_items().iter().map(|(index, item)| (*index, item.hash())).collect(),
 			})
 			.map(|p| (leaves, p))
 	}
diff --git a/substrate/primitives/merkle-mountain-range/src/lib.rs b/substrate/primitives/merkle-mountain-range/src/lib.rs
index 6c0e75005ead..07fb993e4b8d 100644
--- a/substrate/primitives/merkle-mountain-range/src/lib.rs
+++ b/substrate/primitives/merkle-mountain-range/src/lib.rs
@@ -442,7 +442,7 @@ sp_api::decl_runtime_apis! {
 		/// Note this function will use on-chain MMR root hash and check if the proof matches the hash.
 		/// Note, the leaves should be sorted such that corresponding leaves and leaf indices have the
 		/// same position in both the `leaves` vector and the `leaf_indices` vector contained in the [Proof]
-		fn verify_proof(leaves: Vec<EncodableOpaqueLeaf>, proof: Proof<Hash>) -> Result<(), Error>;
+		fn verify_proof(leaves: Vec<EncodableOpaqueLeaf>, proof: Proof<(u64, Hash)>) -> Result<(), Error>;
 
 		/// Verify MMR proof against given root hash for a batch of leaves.
 		///
@@ -451,7 +451,7 @@ sp_api::decl_runtime_apis! {
 		///
 		/// Note, the leaves should be sorted such that corresponding leaves and leaf indices have the
 		/// same position in both the `leaves` vector and the `leaf_indices` vector contained in the [Proof]
-		fn verify_proof_stateless(root: Hash, leaves: Vec<EncodableOpaqueLeaf>, proof: Proof<Hash>)
+		fn verify_proof_stateless(root: Hash, leaves: Vec<EncodableOpaqueLeaf>, proof: Proof<(u64, Hash)>)
 			-> Result<(), Error>;
 	}
 }

From c0db0ca5153c56c578cd8408c117ecacf8d9b69d Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 14 Sep 2023 11:41:36 +0200
Subject: [PATCH 061/188] adjust tests for new proof data structure

---
 .../merkle-mountain-range/rpc/src/lib.rs      |  4 +-
 .../frame/merkle-mountain-range/src/tests.rs  | 46 +++++++++----------
 2 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/substrate/client/merkle-mountain-range/rpc/src/lib.rs b/substrate/client/merkle-mountain-range/rpc/src/lib.rs
index 5be82b600d91..872323bc1d7f 100644
--- a/substrate/client/merkle-mountain-range/rpc/src/lib.rs
+++ b/substrate/client/merkle-mountain-range/rpc/src/lib.rs
@@ -309,7 +309,7 @@ mod tests {
 				Proof {
 					leaf_indices: vec![1],
 					leaf_count: 9,
-					items: vec![H256::repeat_byte(1), H256::repeat_byte(2)],
+					items: vec![(1, H256::repeat_byte(1)), (2, H256::repeat_byte(2))],
 				}
 				.encode(),
 			),
@@ -336,7 +336,7 @@ mod tests {
 				Proof {
 					leaf_indices: vec![1, 2],
 					leaf_count: 9,
-					items: vec![H256::repeat_byte(1), H256::repeat_byte(2)],
+					items: vec![(1, H256::repeat_byte(1)), (2, H256::repeat_byte(2))],
 				}
 				.encode(),
 			),
diff --git a/substrate/frame/merkle-mountain-range/src/tests.rs b/substrate/frame/merkle-mountain-range/src/tests.rs
index 429df75182ee..d3ad0cc721df 100644
--- a/substrate/frame/merkle-mountain-range/src/tests.rs
+++ b/substrate/frame/merkle-mountain-range/src/tests.rs
@@ -287,9 +287,9 @@ fn should_generate_proofs_correctly() {
 					leaf_indices: vec![0],
 					leaf_count: 7,
 					items: vec![
-						hex("ad4cbc033833612ccd4626d5f023b9dfc50a35e838514dd1f3c86f8506728705"),
-						hex("cb24f4614ad5b2a5430344c99545b421d9af83c46fd632d70a332200884b4d46"),
-						hex("dca421199bdcc55bb773c6b6967e8d16675de69062b52285ca63685241fdf626"),
+						(1, hex("ad4cbc033833612ccd4626d5f023b9dfc50a35e838514dd1f3c86f8506728705")),
+						(5, hex("cb24f4614ad5b2a5430344c99545b421d9af83c46fd632d70a332200884b4d46")),
+						(9, hex("dca421199bdcc55bb773c6b6967e8d16675de69062b52285ca63685241fdf626")),
 					],
 				}
 			)
@@ -318,9 +318,9 @@ fn should_generate_proofs_correctly() {
 					leaf_indices: vec![2],
 					leaf_count: 7,
 					items: vec![
-						hex("1b14c1dc7d3e4def11acdf31be0584f4b85c3673f1ff72a3af467b69a3b0d9d0"),
-						hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854"),
-						hex("dca421199bdcc55bb773c6b6967e8d16675de69062b52285ca63685241fdf626"),
+						(2, hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854")),
+						(4, hex("1b14c1dc7d3e4def11acdf31be0584f4b85c3673f1ff72a3af467b69a3b0d9d0")),
+						(9, hex("dca421199bdcc55bb773c6b6967e8d16675de69062b52285ca63685241fdf626")),
 					],
 				}
 			)
@@ -337,9 +337,9 @@ fn should_generate_proofs_correctly() {
 				Proof {
 					leaf_indices: vec![2],
 					leaf_count: 3,
-					items: vec![hex(
+					items: vec![(2, hex(
 						"672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854"
-					)],
+					))],
 				}
 			)
 		);
@@ -358,9 +358,9 @@ fn should_generate_proofs_correctly() {
 					leaf_indices: vec![2],
 					leaf_count: 5,
 					items: vec![
-						hex("1b14c1dc7d3e4def11acdf31be0584f4b85c3673f1ff72a3af467b69a3b0d9d0"),
-						hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854"),
-						hex("3b031d22e24f1126c8f7d2f394b663f9b960ed7abbedb7152e17ce16112656d0")
+						(2, hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854")),
+						(4, hex("1b14c1dc7d3e4def11acdf31be0584f4b85c3673f1ff72a3af467b69a3b0d9d0")),
+						(7, hex("3b031d22e24f1126c8f7d2f394b663f9b960ed7abbedb7152e17ce16112656d0")),
 					],
 				}
 			)
@@ -376,9 +376,9 @@ fn should_generate_proofs_correctly() {
 					leaf_indices: vec![4],
 					leaf_count: 7,
 					items: vec![
-						hex("ae88a0825da50e953e7a359c55fe13c8015e48d03d301b8bdfc9193874da9252"),
-						hex("8ed25570209d8f753d02df07c1884ddb36a3d9d4770e4608b188322151c657fe"),
-						hex("611c2174c6164952a66d985cfe1ec1a623794393e3acff96b136d198f37a648c"),
+						(6, hex("ae88a0825da50e953e7a359c55fe13c8015e48d03d301b8bdfc9193874da9252")),
+						(8, hex("8ed25570209d8f753d02df07c1884ddb36a3d9d4770e4608b188322151c657fe")),
+						(10, hex("611c2174c6164952a66d985cfe1ec1a623794393e3acff96b136d198f37a648c")),
 					],
 				}
 			)
@@ -390,9 +390,9 @@ fn should_generate_proofs_correctly() {
 				Proof {
 					leaf_indices: vec![4],
 					leaf_count: 5,
-					items: vec![hex(
+					items: vec![(6, hex(
 						"ae88a0825da50e953e7a359c55fe13c8015e48d03d301b8bdfc9193874da9252"
-					),],
+					)),],
 				}
 			)
 		);
@@ -406,8 +406,8 @@ fn should_generate_proofs_correctly() {
 					leaf_indices: vec![6],
 					leaf_count: 7,
 					items: vec![
-						hex("ae88a0825da50e953e7a359c55fe13c8015e48d03d301b8bdfc9193874da9252"),
-						hex("7e4316ae2ebf7c3b6821cb3a46ca8b7a4f9351a9b40fcf014bb0a4fd8e8f29da"),
+						(6, hex("ae88a0825da50e953e7a359c55fe13c8015e48d03d301b8bdfc9193874da9252")),
+						(9, hex("7e4316ae2ebf7c3b6821cb3a46ca8b7a4f9351a9b40fcf014bb0a4fd8e8f29da")),
 					],
 				}
 			)
@@ -438,9 +438,9 @@ fn should_generate_batch_proof_correctly() {
 				leaf_indices: vec![0, 4, 5],
 				leaf_count: 7,
 				items: vec![
-					hex("ad4cbc033833612ccd4626d5f023b9dfc50a35e838514dd1f3c86f8506728705"),
-					hex("cb24f4614ad5b2a5430344c99545b421d9af83c46fd632d70a332200884b4d46"),
-					hex("611c2174c6164952a66d985cfe1ec1a623794393e3acff96b136d198f37a648c"),
+					(1, hex("ad4cbc033833612ccd4626d5f023b9dfc50a35e838514dd1f3c86f8506728705")),
+					(5, hex("cb24f4614ad5b2a5430344c99545b421d9af83c46fd632d70a332200884b4d46")),
+					(10, hex("611c2174c6164952a66d985cfe1ec1a623794393e3acff96b136d198f37a648c")),
 				],
 			}
 		);
@@ -455,8 +455,8 @@ fn should_generate_batch_proof_correctly() {
 				leaf_indices: vec![0, 4, 5],
 				leaf_count: 6,
 				items: vec![
-					hex("ad4cbc033833612ccd4626d5f023b9dfc50a35e838514dd1f3c86f8506728705"),
-					hex("cb24f4614ad5b2a5430344c99545b421d9af83c46fd632d70a332200884b4d46"),
+					(1, hex("ad4cbc033833612ccd4626d5f023b9dfc50a35e838514dd1f3c86f8506728705")),
+					(5, hex("cb24f4614ad5b2a5430344c99545b421d9af83c46fd632d70a332200884b4d46")),
 				],
 			}
 		);

From a65014d43a4fbd22e0d49e81508f0470c0e4ddf8 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 9 Oct 2023 12:46:29 +0200
Subject: [PATCH 062/188] fix Proof type parameter usage

---
 polkadot/runtime/rococo/src/lib.rs                   |  6 +++---
 polkadot/runtime/test-runtime/src/lib.rs             |  6 +++---
 polkadot/runtime/westend/src/lib.rs                  |  6 +++---
 substrate/bin/node/runtime/src/lib.rs                |  6 +++---
 .../client/merkle-mountain-range/rpc/src/lib.rs      |  4 ++--
 .../client/merkle-mountain-range/src/test_utils.rs   |  6 +++---
 substrate/frame/merkle-mountain-range/src/lib.rs     |  6 +++---
 substrate/frame/merkle-mountain-range/src/mmr/mmr.rs |  6 +++---
 .../primitives/merkle-mountain-range/src/lib.rs      | 12 ++++++------
 9 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs
index 4222abee6b28..733fa9603e04 100644
--- a/polkadot/runtime/rococo/src/lib.rs
+++ b/polkadot/runtime/rococo/src/lib.rs
@@ -1861,7 +1861,7 @@ sp_api::impl_runtime_apis! {
 		fn generate_proof(
 			block_numbers: Vec<BlockNumber>,
 			best_known_block_number: Option<BlockNumber>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, mmr::Hash)>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<mmr::Hash>), mmr::Error> {
 			Mmr::generate_proof(block_numbers, best_known_block_number).map(
 				|(leaves, proof)| {
 					(
@@ -1875,7 +1875,7 @@ sp_api::impl_runtime_apis! {
 			)
 		}
 
-		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<(u64, mmr::Hash)>)
+		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<mmr::Hash>)
 			-> Result<(), mmr::Error>
 		{
 			let leaves = leaves.into_iter().map(|leaf|
@@ -1888,7 +1888,7 @@ sp_api::impl_runtime_apis! {
 		fn verify_proof_stateless(
 			root: mmr::Hash,
 			leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			proof: mmr::Proof<(u64, mmr::Hash)>
+			proof: mmr::Proof<mmr::Hash>
 		) -> Result<(), mmr::Error> {
 			let nodes = leaves.into_iter().map(|leaf|mmr::DataOrHash::Data(leaf.into_opaque_leaf())).collect();
 			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, nodes, proof)
diff --git a/polkadot/runtime/test-runtime/src/lib.rs b/polkadot/runtime/test-runtime/src/lib.rs
index a044b04d8410..22fed1ab4e81 100644
--- a/polkadot/runtime/test-runtime/src/lib.rs
+++ b/polkadot/runtime/test-runtime/src/lib.rs
@@ -1002,11 +1002,11 @@ sp_api::impl_runtime_apis! {
 		fn generate_proof(
 			_block_numbers: Vec<BlockNumber>,
 			_best_known_block_number: Option<BlockNumber>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, Hash)>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<Hash>), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
 
-		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::Proof<(u64, Hash)>)
+		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::Proof<Hash>)
 			-> Result<(), mmr::Error>
 		{
 			Err(mmr::Error::PalletNotIncluded)
@@ -1015,7 +1015,7 @@ sp_api::impl_runtime_apis! {
 		fn verify_proof_stateless(
 			_root: Hash,
 			_leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			_proof: mmr::Proof<(u64, Hash)>
+			_proof: mmr::Proof<Hash>
 		) -> Result<(), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs
index f1b1abc80a07..129826a5aba9 100644
--- a/polkadot/runtime/westend/src/lib.rs
+++ b/polkadot/runtime/westend/src/lib.rs
@@ -1907,7 +1907,7 @@ sp_api::impl_runtime_apis! {
 		fn generate_proof(
 			block_numbers: Vec<BlockNumber>,
 			best_known_block_number: Option<BlockNumber>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, mmr::Hash)>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<mmr::Hash>), mmr::Error> {
 			Mmr::generate_proof(block_numbers, best_known_block_number).map(
 				|(leaves, proof)| {
 					(
@@ -1921,7 +1921,7 @@ sp_api::impl_runtime_apis! {
 			)
 		}
 
-		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<(u64, mmr::Hash)>)
+		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<mmr::Hash>)
 			-> Result<(), mmr::Error>
 		{
 			let leaves = leaves.into_iter().map(|leaf|
@@ -1934,7 +1934,7 @@ sp_api::impl_runtime_apis! {
 		fn verify_proof_stateless(
 			root: mmr::Hash,
 			leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			proof: mmr::Proof<(u64, mmr::Hash)>
+			proof: mmr::Proof<mmr::Hash>
 		) -> Result<(), mmr::Error> {
 			let nodes = leaves.into_iter().map(|leaf|mmr::DataOrHash::Data(leaf.into_opaque_leaf())).collect();
 			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, nodes, proof)
diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs
index 9bb0b8476595..2070e3f12d04 100644
--- a/substrate/bin/node/runtime/src/lib.rs
+++ b/substrate/bin/node/runtime/src/lib.rs
@@ -2654,7 +2654,7 @@ impl_runtime_apis! {
 		fn generate_proof(
 			block_numbers: Vec<BlockNumber>,
 			best_known_block_number: Option<BlockNumber>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, mmr::Hash)>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<mmr::Hash>), mmr::Error> {
 			Mmr::generate_proof(block_numbers, best_known_block_number).map(
 				|(leaves, proof)| {
 					(
@@ -2668,7 +2668,7 @@ impl_runtime_apis! {
 			)
 		}
 
-		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<(u64, mmr::Hash)>)
+		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<mmr::Hash>)
 			-> Result<(), mmr::Error>
 		{
 			let leaves = leaves.into_iter().map(|leaf|
@@ -2681,7 +2681,7 @@ impl_runtime_apis! {
 		fn verify_proof_stateless(
 			root: mmr::Hash,
 			leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			proof: mmr::Proof<(u64, mmr::Hash)>
+			proof: mmr::Proof<mmr::Hash>
 		) -> Result<(), mmr::Error> {
 			let nodes = leaves.into_iter().map(|leaf|mmr::DataOrHash::Data(leaf.into_opaque_leaf())).collect();
 			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, nodes, proof)
diff --git a/substrate/client/merkle-mountain-range/rpc/src/lib.rs b/substrate/client/merkle-mountain-range/rpc/src/lib.rs
index 872323bc1d7f..6f47810bcd0b 100644
--- a/substrate/client/merkle-mountain-range/rpc/src/lib.rs
+++ b/substrate/client/merkle-mountain-range/rpc/src/lib.rs
@@ -261,7 +261,7 @@ mod tests {
 		let proof = Proof {
 			leaf_indices: vec![1],
 			leaf_count: 9,
-			items: vec![H256::repeat_byte(1), H256::repeat_byte(2)],
+			items: vec![(1, H256::repeat_byte(1)), (1, H256::repeat_byte(2))],
 		};
 
 		let leaf_proof = LeavesProof::new(H256::repeat_byte(0), vec![leaf], proof);
@@ -284,7 +284,7 @@ mod tests {
 		let proof = Proof {
 			leaf_indices: vec![1, 2],
 			leaf_count: 9,
-			items: vec![H256::repeat_byte(1), H256::repeat_byte(2)],
+			items: vec![(1, H256::repeat_byte(1)), (1, H256::repeat_byte(2))],
 		};
 
 		let leaf_proof = LeavesProof::new(H256::repeat_byte(0), vec![leaf_a, leaf_b], proof);
diff --git a/substrate/client/merkle-mountain-range/src/test_utils.rs b/substrate/client/merkle-mountain-range/src/test_utils.rs
index bf5eb2885451..010b48bb3d7d 100644
--- a/substrate/client/merkle-mountain-range/src/test_utils.rs
+++ b/substrate/client/merkle-mountain-range/src/test_utils.rs
@@ -304,11 +304,11 @@ sp_api::mock_impl_runtime_apis! {
 			&self,
 			_block_numbers: Vec<u64>,
 			_best_known_block_number: Option<u64>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, MmrHash)>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<MmrHash>), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
 
-		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::Proof<(u64, MmrHash)>)
+		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::Proof<MmrHash>)
 			-> Result<(), mmr::Error>
 		{
 			Err(mmr::Error::PalletNotIncluded)
@@ -317,7 +317,7 @@ sp_api::mock_impl_runtime_apis! {
 		fn verify_proof_stateless(
 			_root: MmrHash,
 			_leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			_proof: mmr::Proof<(u64, MmrHash)>
+			_proof: mmr::Proof<MmrHash>
 		) -> Result<(), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
diff --git a/substrate/frame/merkle-mountain-range/src/lib.rs b/substrate/frame/merkle-mountain-range/src/lib.rs
index 75acbfd76b8c..664f4bc73901 100644
--- a/substrate/frame/merkle-mountain-range/src/lib.rs
+++ b/substrate/frame/merkle-mountain-range/src/lib.rs
@@ -245,7 +245,7 @@ pub mod pallet {
 pub fn verify_leaves_proof<H, L>(
 	root: H::Output,
 	leaves: Vec<mmr::Node<H, L>>,
-	proof: primitives::Proof<(u64, H::Output)>,
+	proof: primitives::Proof<H::Output>,
 ) -> Result<(), primitives::Error>
 where
 	H: traits::Hash,
@@ -319,7 +319,7 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 	pub fn generate_proof(
 		block_numbers: Vec<BlockNumberFor<T>>,
 		best_known_block_number: Option<BlockNumberFor<T>>,
-	) -> Result<(Vec<LeafOf<T, I>>, primitives::Proof<(u64, HashOf<T, I>)>), primitives::Error> {
+	) -> Result<(Vec<LeafOf<T, I>>, primitives::Proof<HashOf<T, I>>), primitives::Error> {
 		// check whether best_known_block_number provided, else use current best block
 		let best_known_block_number =
 			best_known_block_number.unwrap_or_else(|| <frame_system::Pallet<T>>::block_number());
@@ -352,7 +352,7 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 	/// or the proof is invalid.
 	pub fn verify_leaves(
 		leaves: Vec<LeafOf<T, I>>,
-		proof: primitives::Proof<(u64, HashOf<T, I>)>,
+		proof: primitives::Proof<HashOf<T, I>>,
 	) -> Result<(), primitives::Error> {
 		if proof.leaf_count > Self::mmr_leaves() ||
 			proof.leaf_count == 0 ||
diff --git a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
index 7053db321e39..c655aba34831 100644
--- a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
+++ b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
@@ -33,7 +33,7 @@ use sp_std::prelude::*;
 pub fn verify_leaves_proof<H, L>(
 	root: H::Output,
 	leaves: Vec<Node<H, L>>,
-	proof: primitives::Proof<(u64, H::Output)>,
+	proof: primitives::Proof<H::Output>,
 ) -> Result<bool, Error>
 where
 	H: sp_runtime::traits::Hash,
@@ -95,7 +95,7 @@ where
 	pub fn verify_leaves_proof(
 		&self,
 		leaves: Vec<L>,
-		proof: primitives::Proof<(u64, HashOf<T, I>)>,
+		proof: primitives::Proof<HashOf<T, I>>,
 	) -> Result<bool, Error> {
 		let p = mmr_lib::MerkleProof::<NodeOf<T, I, L>, Hasher<HashingOf<T, I>, L>>::new(
 			self.mmr.mmr_size(),
@@ -166,7 +166,7 @@ where
 	pub fn generate_proof(
 		&self,
 		leaf_indices: Vec<NodeIndex>,
-	) -> Result<(Vec<L>, primitives::Proof<(u64, HashOf<T, I>)>), Error> {
+	) -> Result<(Vec<L>, primitives::Proof<HashOf<T, I>>), Error> {
 		let positions = leaf_indices
 			.iter()
 			.map(|index| mmr_lib::leaf_index_to_pos(*index))
diff --git a/substrate/primitives/merkle-mountain-range/src/lib.rs b/substrate/primitives/merkle-mountain-range/src/lib.rs
index 07fb993e4b8d..66473421bd14 100644
--- a/substrate/primitives/merkle-mountain-range/src/lib.rs
+++ b/substrate/primitives/merkle-mountain-range/src/lib.rs
@@ -356,7 +356,7 @@ pub struct Proof<Hash> {
 	/// Number of leaves in MMR, when the proof was generated.
 	pub leaf_count: NodeIndex,
 	/// Proof elements (hashes of siblings of inner nodes on the path to the leaf).
-	pub items: Vec<Hash>,
+	pub items: Vec<(u64, Hash)>,
 }
 
 /// Merkle Mountain Range operation error.
@@ -442,7 +442,7 @@ sp_api::decl_runtime_apis! {
 		/// Note this function will use on-chain MMR root hash and check if the proof matches the hash.
 		/// Note, the leaves should be sorted such that corresponding leaves and leaf indices have the
 		/// same position in both the `leaves` vector and the `leaf_indices` vector contained in the [Proof]
-		fn verify_proof(leaves: Vec<EncodableOpaqueLeaf>, proof: Proof<(u64, Hash)>) -> Result<(), Error>;
+		fn verify_proof(leaves: Vec<EncodableOpaqueLeaf>, proof: Proof<Hash>) -> Result<(), Error>;
 
 		/// Verify MMR proof against given root hash for a batch of leaves.
 		///
@@ -451,7 +451,7 @@ sp_api::decl_runtime_apis! {
 		///
 		/// Note, the leaves should be sorted such that corresponding leaves and leaf indices have the
 		/// same position in both the `leaves` vector and the `leaf_indices` vector contained in the [Proof]
-		fn verify_proof_stateless(root: Hash, leaves: Vec<EncodableOpaqueLeaf>, proof: Proof<(u64, Hash)>)
+		fn verify_proof_stateless(root: Hash, leaves: Vec<EncodableOpaqueLeaf>, proof: Proof<Hash>)
 			-> Result<(), Error>;
 	}
 }
@@ -479,9 +479,9 @@ mod tests {
 			leaf_indices: vec![5],
 			leaf_count: 10,
 			items: vec![
-				hex("c3e7ba6b511162fead58f2c8b5764ce869ed1118011ac37392522ed16720bbcd"),
-				hex("d3e7ba6b511162fead58f2c8b5764ce869ed1118011ac37392522ed16720bbcd"),
-				hex("e3e7ba6b511162fead58f2c8b5764ce869ed1118011ac37392522ed16720bbcd"),
+				(1, hex("c3e7ba6b511162fead58f2c8b5764ce869ed1118011ac37392522ed16720bbcd")),
+				(2, hex("d3e7ba6b511162fead58f2c8b5764ce869ed1118011ac37392522ed16720bbcd")),
+				(3, hex("e3e7ba6b511162fead58f2c8b5764ce869ed1118011ac37392522ed16720bbcd")),
 			],
 		};
 

From e642dc4317324ffe99ee8a3134fe84fae967712a Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 9 Oct 2023 17:39:17 +0200
Subject: [PATCH 063/188] stubs for tests

---
 .../beefy/src/communication/fisherman.rs      |  4 +
 .../client/consensus/beefy/src/worker.rs      |  8 +-
 substrate/frame/beefy/src/tests.rs            | 81 ++++++++++++++++---
 3 files changed, 79 insertions(+), 14 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 39798aabfee4..4b9b98561fe3 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -190,11 +190,13 @@ where
 	) -> Result<(), Error> {
 		let number = vote.commitment.block_number;
 		let (correct_header, expected_payload) = self.expected_header_and_payload(number)?;
+		let ancestry_proof = unimplemented!();
 		if vote.commitment.payload != expected_payload {
 			let proof = ForkEquivocationProof {
 				commitment: vote.commitment,
 				signatories: vec![(vote.id, vote.signature)],
 				correct_header: correct_header.clone(),
+				ancestry_proof,
 			};
 			self.report_fork_equivocation(proof)?;
 		}
@@ -209,6 +211,7 @@ where
 		let SignedCommitment { commitment, signatures } = signed_commitment;
 		let number = commitment.block_number;
 		let (correct_header, expected_payload) = self.expected_header_and_payload(number)?;
+		let ancestry_proof = unimplemented!();
 		if commitment.payload != expected_payload {
 			let validator_set = self.active_validator_set_at(&correct_header)?;
 			if signatures.len() != validator_set.validators().len() {
@@ -228,6 +231,7 @@ where
 				commitment,
 				signatories,
 				correct_header: correct_header.clone(),
+				ancestry_proof,
 			};
 			self.report_fork_equivocation(proof)?;
 		}
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index bda873c09330..fce83c99670c 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -1726,6 +1726,7 @@ pub(crate) mod tests {
 			.header(hashes[block_number as usize])
 			.unwrap()
 			.unwrap();
+		let ancestry_proof = unimplemented!();
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, "amievil".encode());
 		let votes: Vec<_> = peers
 			.iter()
@@ -1736,7 +1737,8 @@ pub(crate) mod tests {
 			.collect();
 
 		// verify: Alice reports Bob
-		let proof = generate_fork_equivocation_proof_vote(votes[1].clone(), header.clone());
+		let proof =
+			generate_fork_equivocation_proof_vote(votes[1].clone(), header.clone(), ancestry_proof);
 		{
 			// expect fisher (Alice) to successfully process it
 			assert_eq!(
@@ -1755,7 +1757,8 @@ pub(crate) mod tests {
 		}
 
 		// verify: Alice does not self-report
-		let proof = generate_fork_equivocation_proof_vote(votes[0].clone(), header.clone());
+		let proof =
+			generate_fork_equivocation_proof_vote(votes[0].clone(), header.clone(), ancestry_proof);
 		{
 			// expect fisher (Alice) to successfully process it
 			assert_eq!(
@@ -1784,6 +1787,7 @@ pub(crate) mod tests {
 			commitment,
 			vec![Keyring::Bob, Keyring::Charlie],
 			header,
+			ancestry_proof,
 		);
 		{
 			// expect fisher (Alice) to successfully process it
diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index d54716287dc8..c07bec7ce38e 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -812,6 +812,7 @@ fn report_fork_equivocation_vote_current_set_works() {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
+		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
@@ -843,6 +844,7 @@ fn report_fork_equivocation_vote_current_set_works() {
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
 			header,
+			ancestry_proof,
 		);
 
 		// create the key ownership proof
@@ -894,6 +896,7 @@ fn report_fork_equivocation_vote_old_set_works() {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
+		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
@@ -936,6 +939,7 @@ fn report_fork_equivocation_vote_old_set_works() {
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, old_set_id, &equivocation_keyring),
 			header,
+			ancestry_proof,
 		);
 
 		// report the equivocation and the tx should be dispatched successfully
@@ -984,6 +988,7 @@ fn report_fork_equivocation_vote_invalid_set_id() {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
+		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
@@ -1003,6 +1008,7 @@ fn report_fork_equivocation_vote_invalid_set_id() {
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id + 1, &equivocation_keyring),
 			header,
+			ancestry_proof,
 		);
 
 		// the call for reporting the equivocation should error
@@ -1026,6 +1032,7 @@ fn report_fork_equivocation_vote_invalid_session() {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
+		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
@@ -1050,6 +1057,7 @@ fn report_fork_equivocation_vote_invalid_session() {
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
 			header,
+			ancestry_proof,
 		);
 
 		// report an equivocation for the current set using an key ownership
@@ -1074,6 +1082,7 @@ fn report_fork_equivocation_vote_invalid_key_owner_proof() {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
+		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
@@ -1098,6 +1107,7 @@ fn report_fork_equivocation_vote_invalid_key_owner_proof() {
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id + 1, &equivocation_keyring),
 			header,
+			ancestry_proof,
 		);
 
 		// we need to start a new era otherwise the key ownership proof won't be
@@ -1127,6 +1137,7 @@ fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 
 		let block_num = System::block_number();
 		let header = System::finalize();
+		let ancestry_proof = unimplemented!();
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
@@ -1146,6 +1157,7 @@ fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num + 1, payload.clone(), set_id, &equivocation_keyring),
 			header.clone(),
+			ancestry_proof,
 		);
 		assert_err!(
 			Beefy::report_fork_equivocation_unsigned(
@@ -1160,6 +1172,7 @@ fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload.clone(), set_id, &BeefyKeyring::Dave),
 			header.clone(),
+			ancestry_proof,
 		);
 		assert_err!(
 			Beefy::report_fork_equivocation_unsigned(
@@ -1174,6 +1187,7 @@ fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload.clone(), set_id + 1, &equivocation_keyring),
 			header.clone(),
+			ancestry_proof,
 		);
 		assert_err!(
 			Beefy::report_fork_equivocation_unsigned(
@@ -1200,6 +1214,7 @@ fn report_fork_equivocation_vote_validate_unsigned_prevents_duplicates() {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
+		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
@@ -1217,6 +1232,7 @@ fn report_fork_equivocation_vote_validate_unsigned_prevents_duplicates() {
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
 			header,
+			ancestry_proof,
 		);
 
 		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
@@ -1290,6 +1306,7 @@ fn valid_fork_equivocation_vote_reports_dont_pay_fees() {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
+		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
@@ -1307,6 +1324,7 @@ fn valid_fork_equivocation_vote_reports_dont_pay_fees() {
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
 			header,
+			ancestry_proof,
 		);
 
 		// create the key ownership proof.
@@ -1367,6 +1385,7 @@ fn report_fork_equivocation_sc_current_set_works() {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
+		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
@@ -1402,8 +1421,12 @@ fn report_fork_equivocation_sc_current_set_works() {
 		let commitment = Commitment { validator_set_id: set_id, block_number: block_num, payload };
 		// generate an fork equivocation proof, with a vote in the same round for a
 		// different payload than finalized
-		let equivocation_proof =
-			generate_fork_equivocation_proof_sc(commitment, equivocation_keyrings, header);
+		let equivocation_proof = generate_fork_equivocation_proof_sc(
+			commitment,
+			equivocation_keyrings,
+			header,
+			ancestry_proof,
+		);
 
 		// create the key ownership proof
 		let key_owner_proofs = equivocation_keys
@@ -1462,6 +1485,7 @@ fn report_fork_equivocation_sc_old_set_works() {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
+		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
@@ -1514,8 +1538,12 @@ fn report_fork_equivocation_sc_old_set_works() {
 			Commitment { validator_set_id: old_set_id, block_number: block_num, payload };
 		// generate an fork equivocation proof, with a vote in the same round for a
 		// different payload than finalized
-		let equivocation_proof =
-			generate_fork_equivocation_proof_sc(commitment, equivocation_keyrings, header);
+		let equivocation_proof = generate_fork_equivocation_proof_sc(
+			commitment,
+			equivocation_keyrings,
+			header,
+			ancestry_proof,
+		);
 
 		// report the equivocation and the tx should be dispatched successfully
 		assert_ok!(Beefy::report_fork_equivocation_unsigned(
@@ -1568,6 +1596,7 @@ fn report_fork_equivocation_sc_invalid_set_id() {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
+		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
@@ -1595,8 +1624,12 @@ fn report_fork_equivocation_sc_invalid_set_id() {
 		// generate an equivocation for a future set
 		let commitment =
 			Commitment { validator_set_id: set_id + 1, block_number: block_num, payload };
-		let equivocation_proof =
-			generate_fork_equivocation_proof_sc(commitment, equivocation_keyrings, header);
+		let equivocation_proof = generate_fork_equivocation_proof_sc(
+			commitment,
+			equivocation_keyrings,
+			header,
+			ancestry_proof,
+		);
 
 		// the call for reporting the equivocation should error
 		assert_err!(
@@ -1619,6 +1652,7 @@ fn report_fork_equivocation_sc_invalid_session() {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
+		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
@@ -1650,8 +1684,12 @@ fn report_fork_equivocation_sc_invalid_session() {
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		// generate an equivocation proof at following era set id = 3
 		let commitment = Commitment { validator_set_id: set_id, block_number: block_num, payload };
-		let equivocation_proof =
-			generate_fork_equivocation_proof_sc(commitment, equivocation_keyrings, header);
+		let equivocation_proof = generate_fork_equivocation_proof_sc(
+			commitment,
+			equivocation_keyrings,
+			header,
+			ancestry_proof,
+		);
 
 		// report an equivocation for the current set using an key ownership
 		// proof from the previous set, the session should be invalid.
@@ -1675,6 +1713,7 @@ fn report_fork_equivocation_sc_invalid_key_owner_proof() {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
+		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
@@ -1708,8 +1747,12 @@ fn report_fork_equivocation_sc_invalid_key_owner_proof() {
 		// generate an equivocation proof for the authorities at indices [0, 2]
 		let commitment =
 			Commitment { validator_set_id: set_id + 1, block_number: block_num, payload };
-		let equivocation_proof =
-			generate_fork_equivocation_proof_sc(commitment, equivocation_keyrings, header);
+		let equivocation_proof = generate_fork_equivocation_proof_sc(
+			commitment,
+			equivocation_keyrings,
+			header,
+			ancestry_proof,
+		);
 
 		// we need to start a new era otherwise the key ownership proof won't be
 		// checked since the authorities are part of the current session
@@ -1738,6 +1781,7 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 
 		let block_num = System::block_number();
 		let header = System::finalize();
+		let ancestry_proof = unimplemented!();
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
@@ -1771,6 +1815,7 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 			},
 			equivocation_keyrings.clone(),
 			header.clone(),
+			ancestry_proof,
 		);
 		assert_err!(
 			Beefy::report_fork_equivocation_unsigned(
@@ -1790,6 +1835,7 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 			},
 			vec![BeefyKeyring::Eve],
 			header.clone(),
+			ancestry_proof,
 		);
 		assert_err!(
 			Beefy::report_fork_equivocation_unsigned(
@@ -1809,6 +1855,7 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 			},
 			equivocation_keyrings,
 			header,
+			ancestry_proof,
 		);
 		assert_err!(
 			Beefy::report_fork_equivocation_unsigned(
@@ -1835,6 +1882,7 @@ fn report_fork_equivocation_sc_validate_unsigned_prevents_duplicates() {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
+		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
@@ -1852,6 +1900,7 @@ fn report_fork_equivocation_sc_validate_unsigned_prevents_duplicates() {
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
 			header,
+			ancestry_proof,
 		);
 
 		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
@@ -1925,6 +1974,7 @@ fn valid_fork_equivocation_sc_reports_dont_pay_fees() {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
+		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
@@ -1942,6 +1992,7 @@ fn valid_fork_equivocation_sc_reports_dont_pay_fees() {
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
 			header,
+			ancestry_proof,
 		);
 
 		// create the key ownership proof.
@@ -2000,6 +2051,7 @@ fn report_fork_equivocation_sc_stacked_reports_stack_correctly() {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
+		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
@@ -2041,9 +2093,14 @@ fn report_fork_equivocation_sc_stacked_reports_stack_correctly() {
 			commitment.clone(),
 			vec![equivocation_keyrings[0]],
 			header.clone(),
+			ancestry_proof,
+		);
+		let equivocation_proof_full = generate_fork_equivocation_proof_sc(
+			commitment,
+			equivocation_keyrings,
+			header,
+			ancestry_proof,
 		);
-		let equivocation_proof_full =
-			generate_fork_equivocation_proof_sc(commitment, equivocation_keyrings, header);
 
 		// create the key ownership proof
 		let key_owner_proofs: Vec<_> = equivocation_keys

From e882e01ec85673fa70e7854bd95c57ae75fc9c86 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 10 Oct 2023 03:22:14 +0200
Subject: [PATCH 064/188] integrate ancestry proof into equivocation proofs

---
 polkadot/node/service/src/fake_runtime_api.rs |  4 ++--
 polkadot/runtime/rococo/src/lib.rs            |  4 ++--
 polkadot/runtime/test-runtime/src/lib.rs      |  4 ++--
 polkadot/runtime/westend/src/lib.rs           |  4 ++--
 .../beefy/src/communication/fisherman.rs      | 11 +++++----
 .../client/consensus/beefy/src/import.rs      |  6 ++---
 substrate/client/consensus/beefy/src/lib.rs   | 12 +++++-----
 substrate/client/consensus/beefy/src/tests.rs | 21 ++++++++++++----
 .../client/consensus/beefy/src/worker.rs      |  6 ++---
 substrate/frame/beefy/src/equivocation.rs     |  1 +
 substrate/frame/beefy/src/lib.rs              |  3 +++
 .../primitives/consensus/beefy/src/lib.rs     | 24 +++++++++++++------
 .../consensus/beefy/src/test_utils.rs         | 20 +++++++++++-----
 .../merkle-mountain-range/src/lib.rs          |  8 +++++++
 14 files changed, 86 insertions(+), 42 deletions(-)

diff --git a/polkadot/node/service/src/fake_runtime_api.rs b/polkadot/node/service/src/fake_runtime_api.rs
index 418217997385..630ed0e34759 100644
--- a/polkadot/node/service/src/fake_runtime_api.rs
+++ b/polkadot/node/service/src/fake_runtime_api.rs
@@ -231,7 +231,7 @@ sp_api::impl_runtime_apis! {
 		}
 	}
 
-	impl beefy_primitives::BeefyApi<Block, BeefyId> for Runtime {
+	impl beefy_primitives::BeefyApi<Block, BeefyId, Hash> for Runtime {
 		fn beefy_genesis() -> Option<BlockNumber> {
 			unimplemented!()
 		}
@@ -252,7 +252,7 @@ sp_api::impl_runtime_apis! {
 		}
 
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
-			_: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, <Block as BlockT>::Header>,
+			_: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, <Block as BlockT>::Header, Hash>,
 			_: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
 			unimplemented!()
diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs
index 733fa9603e04..361574460212 100644
--- a/polkadot/runtime/rococo/src/lib.rs
+++ b/polkadot/runtime/rococo/src/lib.rs
@@ -1799,7 +1799,7 @@ sp_api::impl_runtime_apis! {
 	}
 
 	#[api_version(3)]
-	impl beefy_primitives::BeefyApi<Block, BeefyId> for Runtime {
+	impl beefy_primitives::BeefyApi<Block, BeefyId, Hash> for Runtime {
 		fn beefy_genesis() -> Option<BlockNumber> {
 			Beefy::genesis_block()
 		}
@@ -1825,7 +1825,7 @@ sp_api::impl_runtime_apis! {
 		}
 
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
-			fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, Header>,
+			fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, Header, Hash>,
 			key_owner_proofs: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
 			let key_owner_proofs = key_owner_proofs.iter().cloned().map(|p| p.decode()).collect::<Option<Vec<_>>>()?;
diff --git a/polkadot/runtime/test-runtime/src/lib.rs b/polkadot/runtime/test-runtime/src/lib.rs
index 22fed1ab4e81..73c9afc85bc5 100644
--- a/polkadot/runtime/test-runtime/src/lib.rs
+++ b/polkadot/runtime/test-runtime/src/lib.rs
@@ -953,7 +953,7 @@ sp_api::impl_runtime_apis! {
 		}
 	}
 
-	impl beefy_primitives::BeefyApi<Block, BeefyId> for Runtime {
+	impl beefy_primitives::BeefyApi<Block, BeefyId, Hash> for Runtime {
 		fn beefy_genesis() -> Option<BlockNumber> {
 			// dummy implementation due to lack of BEEFY pallet.
 			None
@@ -976,7 +976,7 @@ sp_api::impl_runtime_apis! {
 		}
 
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
-			_fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, Header>,
+			_fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, Header, Hash>,
 			_key_owner_proofs: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
 			None
diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs
index 129826a5aba9..5c0087bed4a2 100644
--- a/polkadot/runtime/westend/src/lib.rs
+++ b/polkadot/runtime/westend/src/lib.rs
@@ -1846,7 +1846,7 @@ sp_api::impl_runtime_apis! {
 		}
 	}
 
-	impl beefy_primitives::BeefyApi<Block, BeefyId> for Runtime {
+	impl beefy_primitives::BeefyApi<Block, BeefyId, Hash> for Runtime {
 		fn beefy_genesis() -> Option<BlockNumber> {
 			Beefy::genesis_block()
 		}
@@ -1872,7 +1872,7 @@ sp_api::impl_runtime_apis! {
 		}
 
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
-			fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, Header>,
+			fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, Header, Hash>,
 			key_owner_proofs: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
 			let key_owner_proofs = key_owner_proofs.iter().cloned().map(|p| p.decode()).collect::<Option<Vec<_>>>()?;
diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 4b9b98561fe3..8a6a03c8ff47 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -29,8 +29,8 @@ use sp_blockchain::HeaderBackend;
 use sp_consensus_beefy::{
 	check_fork_equivocation_proof,
 	ecdsa_crypto::{AuthorityId, Signature},
-	BeefyApi, ForkEquivocationProof, Payload, PayloadProvider, SignedCommitment, ValidatorSet,
-	VoteMessage,
+	BeefyApi, ForkEquivocationProof, MmrRootHash, Payload, PayloadProvider, SignedCommitment,
+	ValidatorSet, VoteMessage,
 };
 use sp_runtime::{
 	generic::BlockId,
@@ -71,7 +71,7 @@ where
 	BE: Backend<B>,
 	P: PayloadProvider<B>,
 	R: ProvideRuntimeApi<B> + Send + Sync,
-	R::Api: BeefyApi<B, AuthorityId>,
+	R::Api: BeefyApi<B, AuthorityId, MmrRootHash>,
 {
 	fn expected_header_and_payload(
 		&self,
@@ -107,7 +107,7 @@ where
 
 	pub(crate) fn report_fork_equivocation(
 		&self,
-		proof: ForkEquivocationProof<NumberFor<B>, AuthorityId, Signature, B::Header>,
+		proof: ForkEquivocationProof<NumberFor<B>, AuthorityId, Signature, B::Header, MmrRootHash>,
 	) -> Result<(), Error> {
 		let validator_set = self.active_validator_set_at(&proof.correct_header)?;
 		let set_id = validator_set.id();
@@ -124,6 +124,7 @@ where
 				AuthorityId,
 				BeefySignatureHasher,
 				B::Header,
+				MmrRootHash,
 			>(&proof, &expected_header_hash)
 		{
 			debug!(target: LOG_TARGET, "🥩 Skip report for bad invalid fork proof {:?}", proof);
@@ -181,7 +182,7 @@ where
 	BE: Backend<B>,
 	P: PayloadProvider<B>,
 	R: ProvideRuntimeApi<B> + Send + Sync,
-	R::Api: BeefyApi<B, AuthorityId>,
+	R::Api: BeefyApi<B, AuthorityId, MmrRootHash>,
 {
 	/// Check `vote` for contained block against expected payload.
 	fn check_vote(
diff --git a/substrate/client/consensus/beefy/src/import.rs b/substrate/client/consensus/beefy/src/import.rs
index 5b2abb20aced..09fe0ab4c50a 100644
--- a/substrate/client/consensus/beefy/src/import.rs
+++ b/substrate/client/consensus/beefy/src/import.rs
@@ -22,7 +22,7 @@ use log::debug;
 
 use sp_api::ProvideRuntimeApi;
 use sp_consensus::Error as ConsensusError;
-use sp_consensus_beefy::{ecdsa_crypto::AuthorityId, BeefyApi, BEEFY_ENGINE_ID};
+use sp_consensus_beefy::{ecdsa_crypto::AuthorityId, BeefyApi, MmrRootHash, BEEFY_ENGINE_ID};
 use sp_runtime::{
 	traits::{Block as BlockT, Header as HeaderT, NumberFor},
 	EncodedJustification,
@@ -83,7 +83,7 @@ where
 	Block: BlockT,
 	BE: Backend<Block>,
 	Runtime: ProvideRuntimeApi<Block>,
-	Runtime::Api: BeefyApi<Block, AuthorityId> + Send,
+	Runtime::Api: BeefyApi<Block, AuthorityId, MmrRootHash> + Send,
 {
 	fn decode_and_verify(
 		&self,
@@ -120,7 +120,7 @@ where
 	BE: Backend<Block>,
 	I: BlockImport<Block, Error = ConsensusError> + Send + Sync,
 	Runtime: ProvideRuntimeApi<Block> + Send + Sync,
-	Runtime::Api: BeefyApi<Block, AuthorityId>,
+	Runtime::Api: BeefyApi<Block, AuthorityId, MmrRootHash>,
 {
 	type Error = ConsensusError;
 
diff --git a/substrate/client/consensus/beefy/src/lib.rs b/substrate/client/consensus/beefy/src/lib.rs
index c28ef266afb7..048caf28d1f6 100644
--- a/substrate/client/consensus/beefy/src/lib.rs
+++ b/substrate/client/consensus/beefy/src/lib.rs
@@ -143,7 +143,7 @@ where
 	BE: Backend<B>,
 	I: BlockImport<B, Error = ConsensusError> + Send + Sync,
 	RuntimeApi: ProvideRuntimeApi<B> + Send + Sync,
-	RuntimeApi::Api: BeefyApi<B, AuthorityId>,
+	RuntimeApi::Api: BeefyApi<B, AuthorityId, MmrRootHash>,
 {
 	// Voter -> RPC links
 	let (to_rpc_justif_sender, from_voter_justif_stream) =
@@ -225,7 +225,7 @@ pub async fn start_beefy_gadget<B, BE, C, N, P, R, S>(
 	C: Client<B, BE> + BlockBackend<B>,
 	P: PayloadProvider<B> + Clone,
 	R: ProvideRuntimeApi<B> + Send + Sync + 'static,
-	R::Api: BeefyApi<B, AuthorityId> + MmrApi<B, MmrRootHash, NumberFor<B>>,
+	R::Api: BeefyApi<B, AuthorityId, MmrRootHash> + MmrApi<B, MmrRootHash, NumberFor<B>>,
 	N: GossipNetwork<B> + NetworkRequest + Send + Sync + 'static,
 	S: GossipSyncing<B> + SyncOracle + 'static,
 {
@@ -375,7 +375,7 @@ where
 	B: Block,
 	BE: Backend<B>,
 	R: ProvideRuntimeApi<B>,
-	R::Api: BeefyApi<B, AuthorityId>,
+	R::Api: BeefyApi<B, AuthorityId, MmrRootHash>,
 {
 	// Initialize voter state from AUX DB if compatible.
 	crate::aux_schema::load_persistent(backend)?
@@ -410,7 +410,7 @@ where
 	B: Block,
 	BE: Backend<B>,
 	R: ProvideRuntimeApi<B>,
-	R::Api: BeefyApi<B, AuthorityId>,
+	R::Api: BeefyApi<B, AuthorityId, MmrRootHash>,
 {
 	let beefy_genesis = runtime
 		.runtime_api()
@@ -506,7 +506,7 @@ async fn wait_for_runtime_pallet<B, R>(
 where
 	B: Block,
 	R: ProvideRuntimeApi<B>,
-	R::Api: BeefyApi<B, AuthorityId>,
+	R::Api: BeefyApi<B, AuthorityId, MmrRootHash>,
 {
 	info!(target: LOG_TARGET, "🥩 BEEFY gadget waiting for BEEFY pallet to become available...");
 	loop {
@@ -548,7 +548,7 @@ where
 	B: Block,
 	BE: Backend<B>,
 	R: ProvideRuntimeApi<B>,
-	R::Api: BeefyApi<B, AuthorityId>,
+	R::Api: BeefyApi<B, AuthorityId, MmrRootHash>,
 {
 	debug!(target: LOG_TARGET, "🥩 Try to find validator set active at header: {:?}", at_header);
 	runtime
diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index 3ce6e2e4dcc6..dce597a062de 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -276,7 +276,19 @@ pub(crate) struct TestApi {
 	pub reported_vote_equivocations:
 		Option<Arc<Mutex<Vec<VoteEquivocationProof<NumberFor<Block>, AuthorityId, Signature>>>>>,
 	pub reported_fork_equivocations: Option<
-		Arc<Mutex<Vec<ForkEquivocationProof<NumberFor<Block>, AuthorityId, Signature, Header>>>>,
+		Arc<
+			Mutex<
+				Vec<
+					ForkEquivocationProof<
+						NumberFor<Block>,
+						AuthorityId,
+						Signature,
+						Header,
+						MmrRootHash,
+					>,
+				>,
+			>,
+		>,
 	>,
 }
 
@@ -324,7 +336,7 @@ impl ProvideRuntimeApi<Block> for TestApi {
 	}
 }
 sp_api::mock_impl_runtime_apis! {
-	impl BeefyApi<Block, AuthorityId> for RuntimeApi {
+	impl BeefyApi<Block, AuthorityId, MmrRootHash> for RuntimeApi {
 		fn beefy_genesis() -> Option<NumberFor<Block>> {
 			Some(self.inner.beefy_genesis)
 		}
@@ -346,7 +358,7 @@ sp_api::mock_impl_runtime_apis! {
 		}
 
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
-			proof: ForkEquivocationProof<NumberFor<Block>, AuthorityId, Signature, Header>,
+			proof: ForkEquivocationProof<NumberFor<Block>, AuthorityId, Signature, Header, MmrRootHash>,
 			_dummy: Vec<OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
 			if let Some(equivocations_buf) = self.inner.reported_fork_equivocations.as_ref() {
@@ -430,7 +442,8 @@ fn initialize_beefy<API>(
 ) -> impl Future<Output = ()>
 where
 	API: ProvideRuntimeApi<Block> + Sync + Send + 'static,
-	API::Api: BeefyApi<Block, AuthorityId> + MmrApi<Block, MmrRootHash, NumberFor<Block>>,
+	API::Api:
+		BeefyApi<Block, AuthorityId, MmrRootHash> + MmrApi<Block, MmrRootHash, NumberFor<Block>>,
 {
 	let tasks = FuturesUnordered::new();
 
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index fce83c99670c..8dcadd380e51 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -43,8 +43,8 @@ use sp_consensus::SyncOracle;
 use sp_consensus_beefy::{
 	check_vote_equivocation_proof,
 	ecdsa_crypto::{AuthorityId, Signature},
-	BeefyApi, Commitment, ConsensusLog, PayloadProvider, ValidatorSet, VersionedFinalityProof,
-	VoteEquivocationProof, VoteMessage, BEEFY_ENGINE_ID,
+	BeefyApi, Commitment, ConsensusLog, MmrRootHash, PayloadProvider, ValidatorSet,
+	VersionedFinalityProof, VoteEquivocationProof, VoteMessage, BEEFY_ENGINE_ID,
 };
 use sp_runtime::{
 	generic::OpaqueDigestItemId,
@@ -356,7 +356,7 @@ where
 	P: PayloadProvider<B>,
 	S: SyncOracle,
 	R: ProvideRuntimeApi<B>,
-	R::Api: BeefyApi<B, AuthorityId>,
+	R::Api: BeefyApi<B, AuthorityId, MmrRootHash>,
 	F: BeefyFisherman<B>,
 {
 	fn best_grandpa_block(&self) -> NumberFor<B> {
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 1aee70c11efb..7ba150a07ce0 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -139,6 +139,7 @@ pub enum EquivocationEvidenceFor<T: Config> {
 			<T as Config>::BeefyId,
 			<<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
 			HeaderFor<T>,
+			<T as frame_system::Config>::Hash,
 		>,
 		Vec<<T as Config>::KeyOwnerProof>,
 	),
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index a12bf587a7e0..3593f6ec7dfa 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -314,6 +314,7 @@ pub mod pallet {
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
 					HeaderFor<T>,
+					<T as frame_system::Config>::Hash,
 				>,
 			>,
 			key_owner_proofs: Vec<T::KeyOwnerProof>,
@@ -351,6 +352,7 @@ pub mod pallet {
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
 					HeaderFor<T>,
+					<T as frame_system::Config>::Hash,
 				>,
 			>,
 			key_owner_proofs: Vec<T::KeyOwnerProof>,
@@ -421,6 +423,7 @@ impl<T: Config> Pallet<T> {
 			T::BeefyId,
 			<T::BeefyId as RuntimeAppPublic>::Signature,
 			HeaderFor<T>,
+			<T as frame_system::Config>::Hash,
 		>,
 		key_owner_proofs: Vec<T::KeyOwnerProof>,
 	) -> Option<()> {
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index f574b4583eda..bd61db59ce88 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -40,6 +40,7 @@ pub mod witness;
 
 pub use commitment::{Commitment, SignedCommitment, VersionedFinalityProof};
 pub use payload::{known_payloads, BeefyPayloadId, Payload, PayloadProvider};
+use sp_mmr_primitives::AncestryProof;
 #[cfg(feature = "std")]
 pub use test_utils::*;
 
@@ -254,7 +255,7 @@ impl<Number, Id, Signature> VoteEquivocationProof<Number, Id, Signature> {
 /// Proof of authority misbehavior on a given set id.
 /// This proof shows commitment signed on a different fork.
 #[derive(Clone, Debug, Decode, Encode, PartialEq, TypeInfo)]
-pub struct ForkEquivocationProof<Number, Id, Signature, Header> {
+pub struct ForkEquivocationProof<Number, Id, Signature, Header, Hash> {
 	/// Commitment for a block on different fork than one at the same height in
 	/// this client's chain.
 	pub commitment: Commitment<Number>,
@@ -265,9 +266,11 @@ pub struct ForkEquivocationProof<Number, Id, Signature, Header> {
 	/// 2. its digest's payload != commitment.payload
 	/// 3. commitment is signed by signatories
 	pub correct_header: Header,
+	/// ancestry proof showing mmr root
+	pub ancestry_proof: AncestryProof<Hash>,
 }
 
-impl<Number, Id, Signature, H: Header> ForkEquivocationProof<Number, Id, Signature, H> {
+impl<Number, Id, Signature, H: Header, Hash> ForkEquivocationProof<Number, Id, Signature, H, Hash> {
 	/// Returns the authority id of the misbehaving voter.
 	pub fn offender_ids(&self) -> Vec<&Id> {
 		self.signatories.iter().map(|(id, _)| id).collect()
@@ -345,8 +348,14 @@ where
 /// finalized by GRANDPA. This is fine too, since the slashing risk of committing to
 /// an incorrect block implies validators will only sign blocks they *know* will be
 /// finalized by GRANDPA.
-pub fn check_fork_equivocation_proof<Number, Id, MsgHash, Header>(
-	proof: &ForkEquivocationProof<Number, Id, <Id as RuntimeAppPublic>::Signature, Header>,
+pub fn check_fork_equivocation_proof<Number, Id, MsgHash, Header, NodeHash>(
+	proof: &ForkEquivocationProof<
+		Number,
+		Id,
+		<Id as RuntimeAppPublic>::Signature,
+		Header,
+		NodeHash,
+	>,
 	expected_header_hash: &Header::Hash,
 ) -> bool
 where
@@ -355,7 +364,7 @@ where
 	MsgHash: Hash,
 	Header: sp_api::HeaderT,
 {
-	let ForkEquivocationProof { commitment, signatories, correct_header } = proof;
+	let ForkEquivocationProof { commitment, signatories, correct_header, ancestry_proof } = proof;
 
 	if correct_header.hash() != *expected_header_hash {
 		return false
@@ -422,8 +431,9 @@ sp_api::decl_runtime_apis! {
 	/// API necessary for BEEFY voters. Due to the significant conceptual
 	/// overlap, in large part, this is lifted from the GRANDPA API.
 	#[api_version(3)]
-	pub trait BeefyApi<AuthorityId> where
+	pub trait BeefyApi<AuthorityId, Hash> where
 		AuthorityId : Codec + RuntimeAppPublic,
+		Hash: Codec,
 	{
 		/// Return the block number where BEEFY consensus is enabled/started
 		fn beefy_genesis() -> Option<NumberFor<Block>>;
@@ -455,7 +465,7 @@ sp_api::decl_runtime_apis! {
 		/// hardcoded to return `None`). Only useful in an offchain context.
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
 			fork_equivocation_proof:
-				ForkEquivocationProof<NumberFor<Block>, AuthorityId, <AuthorityId as RuntimeAppPublic>::Signature, Block::Header>,
+				ForkEquivocationProof<NumberFor<Block>, AuthorityId, <AuthorityId as RuntimeAppPublic>::Signature, Block::Header, Hash>,
 			key_owner_proofs: Vec<OpaqueKeyOwnershipProof>,
 		) -> Option<()>;
 
diff --git a/substrate/primitives/consensus/beefy/src/test_utils.rs b/substrate/primitives/consensus/beefy/src/test_utils.rs
index f480788cd373..4682186d4415 100644
--- a/substrate/primitives/consensus/beefy/src/test_utils.rs
+++ b/substrate/primitives/consensus/beefy/src/test_utils.rs
@@ -23,6 +23,7 @@ use crate::{
 };
 use codec::Encode;
 use sp_core::{ecdsa, keccak_256, Pair};
+use sp_mmr_primitives::AncestryProof;
 use std::collections::HashMap;
 use strum::IntoEnumIterator;
 
@@ -117,24 +118,31 @@ pub fn generate_vote_equivocation_proof(
 }
 
 /// Create a new `ForkEquivocationProof` based on vote & correct header.
-pub fn generate_fork_equivocation_proof_vote<Header>(
+pub fn generate_fork_equivocation_proof_vote<Header, Hash>(
 	vote: (u64, Payload, ValidatorSetId, &Keyring),
 	correct_header: Header,
-) -> ForkEquivocationProof<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature, Header> {
+	ancestry_proof: AncestryProof<Hash>,
+) -> ForkEquivocationProof<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature, Header, Hash> {
 	let signed_vote = signed_vote(vote.0, vote.1, vote.2, vote.3);
 	let signatories = vec![(signed_vote.id, signed_vote.signature)];
-	ForkEquivocationProof { commitment: signed_vote.commitment, signatories, correct_header }
+	ForkEquivocationProof {
+		commitment: signed_vote.commitment,
+		signatories,
+		correct_header,
+		ancestry_proof,
+	}
 }
 
 /// Create a new `ForkEquivocationProof` based on signed commitment & correct header.
-pub fn generate_fork_equivocation_proof_sc<Header>(
+pub fn generate_fork_equivocation_proof_sc<Header, Hash>(
 	commitment: Commitment<u64>,
 	keyrings: Vec<Keyring>,
 	correct_header: Header,
-) -> ForkEquivocationProof<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature, Header> {
+	ancestry_proof: AncestryProof<Hash>,
+) -> ForkEquivocationProof<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature, Header, Hash> {
 	let signatories = keyrings
 		.into_iter()
 		.map(|k| (k.public(), k.sign(&commitment.encode())))
 		.collect::<Vec<_>>();
-	ForkEquivocationProof { commitment, signatories, correct_header }
+	ForkEquivocationProof { commitment, signatories, correct_header, ancestry_proof }
 }
diff --git a/substrate/primitives/merkle-mountain-range/src/lib.rs b/substrate/primitives/merkle-mountain-range/src/lib.rs
index 66473421bd14..a16f06816f62 100644
--- a/substrate/primitives/merkle-mountain-range/src/lib.rs
+++ b/substrate/primitives/merkle-mountain-range/src/lib.rs
@@ -359,6 +359,14 @@ pub struct Proof<Hash> {
 	pub items: Vec<(u64, Hash)>,
 }
 
+/// An MMR ancestry proof for a prior mmr root.
+#[derive(codec::Encode, codec::Decode, RuntimeDebug, Clone, PartialEq, Eq, TypeInfo)]
+pub struct AncestryProof<Hash> {
+	prev_peaks: Vec<Hash>,
+	prev_size: u64,
+	proof: Proof<Hash>,
+}
+
 /// Merkle Mountain Range operation error.
 #[cfg_attr(feature = "std", derive(thiserror::Error))]
 #[derive(RuntimeDebug, codec::Encode, codec::Decode, PartialEq, Eq, TypeInfo)]

From f348a862aa4cc37ebe9e8dfab3cccb1f5ca301c2 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 10 Oct 2023 03:46:37 +0200
Subject: [PATCH 065/188] fix proof serde tests

---
 .../client/merkle-mountain-range/rpc/src/lib.rs      | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/substrate/client/merkle-mountain-range/rpc/src/lib.rs b/substrate/client/merkle-mountain-range/rpc/src/lib.rs
index 6f47810bcd0b..c97da8681314 100644
--- a/substrate/client/merkle-mountain-range/rpc/src/lib.rs
+++ b/substrate/client/merkle-mountain-range/rpc/src/lib.rs
@@ -261,7 +261,7 @@ mod tests {
 		let proof = Proof {
 			leaf_indices: vec![1],
 			leaf_count: 9,
-			items: vec![(1, H256::repeat_byte(1)), (1, H256::repeat_byte(2))],
+			items: vec![(1, H256::repeat_byte(1)), (2, H256::repeat_byte(2))],
 		};
 
 		let leaf_proof = LeavesProof::new(H256::repeat_byte(0), vec![leaf], proof);
@@ -272,7 +272,7 @@ mod tests {
 		// then
 		assert_eq!(
 			actual,
-			r#"{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","leaves":"0x041001020304","proof":"0x04010000000000000009000000000000000801010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202"}"#
+			r#"{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","leaves":"0x041001020304","proof":"0x0401000000000000000900000000000000080100000000000000010101010101010101010101010101010101010101010101010101010101010102000000000000000202020202020202020202020202020202020202020202020202020202020202"}"#
 		);
 	}
 
@@ -284,7 +284,7 @@ mod tests {
 		let proof = Proof {
 			leaf_indices: vec![1, 2],
 			leaf_count: 9,
-			items: vec![(1, H256::repeat_byte(1)), (1, H256::repeat_byte(2))],
+			items: vec![(1, H256::repeat_byte(1)), (2, H256::repeat_byte(2))],
 		};
 
 		let leaf_proof = LeavesProof::new(H256::repeat_byte(0), vec![leaf_a, leaf_b], proof);
@@ -295,7 +295,7 @@ mod tests {
 		// then
 		assert_eq!(
 			actual,
-			r#"{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","leaves":"0x0810010203041002020304","proof":"0x080100000000000000020000000000000009000000000000000801010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202"}"#
+			r#"{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","leaves":"0x0810010203041002020304","proof":"0x08010000000000000002000000000000000900000000000000080100000000000000010101010101010101010101010101010101010101010101010101010101010102000000000000000202020202020202020202020202020202020202020202020202020202020202"}"#
 		);
 	}
 
@@ -319,7 +319,7 @@ mod tests {
 		let actual: LeavesProof<H256> = serde_json::from_str(r#"{
 			"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000",
 			"leaves":"0x041001020304",
-			"proof":"0x04010000000000000009000000000000000801010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202"
+			"proof":"0x0401000000000000000900000000000000080100000000000000010101010101010101010101010101010101010101010101010101010101010102000000000000000202020202020202020202020202020202020202020202020202020202020202"
 		}"#).unwrap();
 
 		// then
@@ -346,7 +346,7 @@ mod tests {
 		let actual: LeavesProof<H256> = serde_json::from_str(r#"{
 			"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000",
 			"leaves":"0x0810010203041002020304",
-			"proof":"0x080100000000000000020000000000000009000000000000000801010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202"
+			"proof":"0x08010000000000000002000000000000000900000000000000080100000000000000010101010101010101010101010101010101010101010101010101010101010102000000000000000202020202020202020202020202020202020202020202020202020202020202"
 		}"#).unwrap();
 
 		// then

From 05b82faa16e3eea35a40eb7189342d420ce979f9 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 12 Oct 2023 11:37:28 +0200
Subject: [PATCH 066/188] add generate_ancestry_proof to pallet-mmr

---
 substrate/bin/node/runtime/src/lib.rs         |  4 +++
 .../merkle-mountain-range/src/mmr/mmr.rs      | 28 +++++++++++++++++++
 .../merkle-mountain-range/src/lib.rs          | 19 ++++++++++---
 3 files changed, 47 insertions(+), 4 deletions(-)

diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs
index 2070e3f12d04..ce57703060b3 100644
--- a/substrate/bin/node/runtime/src/lib.rs
+++ b/substrate/bin/node/runtime/src/lib.rs
@@ -2668,6 +2668,10 @@ impl_runtime_apis! {
 			)
 		}
 
+		fn generate_ancestry_proof(prev_best_block: BlockNumber) -> Result<AncestryProof<Hash>, mmr::Error> -> Result<AncestryProof<Hash>> {
+			Mmr::generate_ancestry_proof(prev_best_block)
+		}
+
 		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<mmr::Hash>)
 			-> Result<(), mmr::Error>
 		{
diff --git a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
index c655aba34831..3b6a96978257 100644
--- a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
+++ b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
@@ -191,4 +191,32 @@ where
 			})
 			.map(|p| (leaves, p))
 	}
+
+	pub fn generate_ancestry_proof(
+		&self,
+		prev_mmr_size: NodeIndex,
+	) -> Result<primitives::AncestryProof<HashOf<T, I>>, Error>
+	{
+		let leaf_count = self.leaves;
+		self.mmr
+			.gen_prefix_proof(prev_mmr_size)
+			.map_err(|e| Error::GenerateProof.log_error(e))
+			.map(|p| {
+				let proof = primitives::Proof {
+					leaf_indices: (p.prev_size..leaf_count).collect(),
+					leaf_count,
+					items: p
+						.proof
+						.proof_items()
+						.iter()
+						.map(|(index, item)| (*index, item.hash()))
+						.collect(),
+				};
+				primitives::AncestryProof {
+					prev_peaks: p.prev_peaks.into_iter().map(|p| p.hash()).collect(),
+					prev_size: p.prev_size,
+					proof,
+				}
+			})
+	}
 }
diff --git a/substrate/primitives/merkle-mountain-range/src/lib.rs b/substrate/primitives/merkle-mountain-range/src/lib.rs
index a16f06816f62..30007640070f 100644
--- a/substrate/primitives/merkle-mountain-range/src/lib.rs
+++ b/substrate/primitives/merkle-mountain-range/src/lib.rs
@@ -362,9 +362,12 @@ pub struct Proof<Hash> {
 /// An MMR ancestry proof for a prior mmr root.
 #[derive(codec::Encode, codec::Decode, RuntimeDebug, Clone, PartialEq, Eq, TypeInfo)]
 pub struct AncestryProof<Hash> {
-	prev_peaks: Vec<Hash>,
-	prev_size: u64,
-	proof: Proof<Hash>,
+	/// Peaks of the ancestor's mmr
+	pub prev_peaks: Vec<Hash>,
+	/// Size of the ancestor's mmr
+	pub prev_size: u64,
+	/// Proof of the ancestry
+	pub proof: Proof<Hash>,
 }
 
 /// Merkle Mountain Range operation error.
@@ -430,7 +433,7 @@ impl Error {
 
 sp_api::decl_runtime_apis! {
 	/// API to interact with MMR pallet.
-	#[api_version(2)]
+	#[api_version(3)]
 	pub trait MmrApi<Hash: codec::Codec, BlockNumber: codec::Codec> {
 		/// Return the on-chain MMR root hash.
 		fn mmr_root() -> Result<Hash, Error>;
@@ -461,6 +464,14 @@ sp_api::decl_runtime_apis! {
 		/// same position in both the `leaves` vector and the `leaf_indices` vector contained in the [Proof]
 		fn verify_proof_stateless(root: Hash, leaves: Vec<EncodableOpaqueLeaf>, proof: Proof<Hash>)
 			-> Result<(), Error>;
+
+		/// Generate MMR ancestry proof for prior mmr size
+		fn generate_ancestry_proof(
+			prev_best_block: BlockNumber,
+		) -> Result<AncestryProof<Hash>, Error>;
+
+		/// Verifies that a claimed prev_root is in fact an ancestor of the provided mmr root
+		fn verify_ancestry_proof(ancestry_proof: AncestryProof<Hash>, root: Hash, prev_root: Hash) -> Result<(), Error>;
 	}
 }
 

From 9e2ec2a2c49e08abde4b93570ce7f8cc9807d9fe Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 16 Oct 2023 12:56:10 +0200
Subject: [PATCH 067/188] pallet beefy: ancestry proof gen. & reporting

---
 Cargo.lock                                    |  4 +-
 substrate/frame/beefy/Cargo.toml              |  5 ++
 substrate/frame/beefy/src/equivocation.rs     | 24 ++++++---
 substrate/frame/beefy/src/lib.rs              | 16 +++---
 substrate/frame/beefy/src/mock.rs             | 22 +++++++-
 substrate/frame/beefy/src/tests.rs            | 21 ++++++--
 .../frame/merkle-mountain-range/src/lib.rs    | 34 +++++++++++++
 .../merkle-mountain-range/src/mmr/mmr.rs      | 38 +++++++++++++-
 .../primitives/consensus/beefy/src/lib.rs     | 18 ++++++-
 .../merkle-mountain-range/src/utils.rs        | 51 ++++++++++++++++++-
 10 files changed, 205 insertions(+), 28 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 5c8b8f89e43d..84fb486907d6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2394,7 +2394,7 @@ dependencies = [
 [[package]]
 name = "ckb-merkle-mountain-range"
 version = "0.5.2"
-source = "git+https://github.com/Lederstrumpf/merkle-mountain-range.git?branch=linearize-proof-rebase-v0.5.1#b55b6805d35b6ccbd8dc83c30caf1cae19943981"
+source = "git+https://github.com/Lederstrumpf/merkle-mountain-range.git?branch=linearize-proof-rebase-v0.5.1#a74c5561cd64dcc777ccdc7dfc21499d200e6da9"
 dependencies = [
  "cfg-if",
 ]
@@ -9075,6 +9075,7 @@ dependencies = [
  "log",
  "pallet-authorship",
  "pallet-balances",
+ "pallet-mmr",
  "pallet-offences",
  "pallet-session",
  "pallet-staking",
@@ -9086,6 +9087,7 @@ dependencies = [
  "sp-consensus-beefy",
  "sp-core",
  "sp-io",
+ "sp-mmr-primitives",
  "sp-runtime",
  "sp-session",
  "sp-staking",
diff --git a/substrate/frame/beefy/Cargo.toml b/substrate/frame/beefy/Cargo.toml
index 1445658bafb5..3aa525a868ab 100644
--- a/substrate/frame/beefy/Cargo.toml
+++ b/substrate/frame/beefy/Cargo.toml
@@ -13,10 +13,12 @@ codec = { package = "parity-scale-codec", version = "3.6.1", default-features =
 log = { version = "0.4.17", default-features = false }
 scale-info = { version = "2.5.0", default-features = false, features = ["derive", "serde"] }
 serde = { version = "1.0.188", optional = true }
+pallet-mmr = { path = "../merkle-mountain-range", default-features = false }
 frame-support = { path = "../support", default-features = false}
 frame-system = { path = "../system", default-features = false}
 pallet-authorship = { path = "../authorship", default-features = false}
 pallet-session = { path = "../session", default-features = false}
+sp-mmr-primitives = { path = "../../primitives/merkle-mountain-range", default-features = false, features = ["serde"] }
 sp-consensus-beefy = { path = "../../primitives/consensus/beefy", default-features = false, features = ["serde"] }
 sp-runtime = { path = "../../primitives/runtime", default-features = false, features = ["serde"] }
 sp-session = { path = "../../primitives/session", default-features = false}
@@ -24,6 +26,7 @@ sp-staking = { path = "../../primitives/staking", default-features = false, feat
 sp-std = { path = "../../primitives/std", default-features = false}
 
 [dev-dependencies]
+sp-mmr-primitives = { path = "../../primitives/merkle-mountain-range" }
 frame-election-provider-support = { path = "../election-provider-support" }
 pallet-balances = { path = "../balances" }
 pallet-offences = { path = "../offences" }
@@ -45,6 +48,7 @@ std = [
 	"log/std",
 	"pallet-authorship/std",
 	"pallet-balances/std",
+	"pallet-mmr/std",
 	"pallet-offences/std",
 	"pallet-session/std",
 	"pallet-staking/std",
@@ -66,6 +70,7 @@ try-runtime = [
 	"frame-system/try-runtime",
 	"pallet-authorship/try-runtime",
 	"pallet-balances/try-runtime",
+	"pallet-mmr/try-runtime",
 	"pallet-offences/try-runtime",
 	"pallet-session/try-runtime",
 	"pallet-staking/try-runtime",
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 7ba150a07ce0..6e12e2671053 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -124,7 +124,7 @@ where
 pub struct EquivocationReportSystem<T, R, P, L>(sp_std::marker::PhantomData<(T, R, P, L)>);
 
 /// Equivocation evidence convenience alias.
-pub enum EquivocationEvidenceFor<T: Config> {
+pub enum EquivocationEvidenceFor<T: Config + pallet_mmr::Config> {
 	VoteEquivocationProof(
 		VoteEquivocationProof<
 			BlockNumberFor<T>,
@@ -139,7 +139,7 @@ pub enum EquivocationEvidenceFor<T: Config> {
 			<T as Config>::BeefyId,
 			<<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
 			HeaderFor<T>,
-			<T as frame_system::Config>::Hash,
+			<<T as pallet_mmr::Config>::Hashing as sp_runtime::traits::Hash>::Output,
 		>,
 		Vec<<T as Config>::KeyOwnerProof>,
 	),
@@ -148,7 +148,10 @@ pub enum EquivocationEvidenceFor<T: Config> {
 impl<T, R, P, L> OffenceReportSystem<Option<T::AccountId>, EquivocationEvidenceFor<T>>
 	for EquivocationReportSystem<T, R, P, L>
 where
-	T: Config + pallet_authorship::Config + frame_system::offchain::SendTransactionTypes<Call<T>>,
+	T: Config
+		+ pallet_authorship::Config
+		+ pallet_mmr::Config
+		+ frame_system::offchain::SendTransactionTypes<Call<T>>,
 	R: ReportOffence<
 		T::AccountId,
 		P::IdentificationTuple,
@@ -286,6 +289,8 @@ where
 			EquivocationEvidenceFor::ForkEquivocationProof(equivocation_proof, _) => {
 				let block_number = equivocation_proof.commitment.block_number;
 				let expected_block_hash = <frame_system::Pallet<T>>::block_hash(block_number);
+				let mmr_size = <pallet_mmr::Pallet<T>>::mmr_size();
+				let expected_mmr_root = <pallet_mmr::Pallet<T>>::mmr_root();
 
 				// Validate equivocation proof (check commitment is to unexpected payload and
 				// signatures are valid).
@@ -295,10 +300,15 @@ where
 				// beefy light client at least once every 4096 blocks. See
 				// https://github.com/paritytech/polkadot-sdk/issues/1441 for
 				// replacement solution.
-				if !sp_consensus_beefy::check_fork_equivocation_proof(
-					&equivocation_proof,
-					&expected_block_hash,
-				) {
+				if !sp_consensus_beefy::check_fork_equivocation_proof::<
+					_,
+					_,
+					_,
+					_,
+					<<T as pallet_mmr::Config>::Hashing as sp_runtime::traits::Hash>::Output,
+					sp_mmr_primitives::utils::AncestryHasher<<T as pallet_mmr::Config>::Hashing>,
+				>(equivocation_proof, expected_mmr_root, mmr_size, &expected_block_hash)
+				{
 					return Err(Error::<T>::InvalidForkEquivocationProof.into())
 				}
 			},
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 3593f6ec7dfa..e5b087a5b373 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -66,7 +66,7 @@ pub mod pallet {
 	use sp_consensus_beefy::ForkEquivocationProof;
 
 	#[pallet::config]
-	pub trait Config: frame_system::Config {
+	pub trait Config: frame_system::Config + pallet_mmr::Config {
 		/// Authority identifier type
 		type BeefyId: Member
 			+ Parameter
@@ -214,7 +214,7 @@ pub mod pallet {
 		/// against the extracted offender. If both are valid, the offence
 		/// will be reported.
 		#[pallet::call_index(0)]
-		#[pallet::weight(T::WeightInfo::report_equivocation(
+		#[pallet::weight(<T as pallet::Config>::WeightInfo::report_equivocation(
 			key_owner_proof.validator_count(),
 			T::MaxNominators::get(),
 		))]
@@ -252,7 +252,7 @@ pub mod pallet {
 		/// if the block author is defined it will be defined as the equivocation
 		/// reporter.
 		#[pallet::call_index(1)]
-		#[pallet::weight(T::WeightInfo::report_equivocation(
+		#[pallet::weight(<T as pallet::Config>::WeightInfo::report_equivocation(
 			key_owner_proof.validator_count(),
 			T::MaxNominators::get(),
 		))]
@@ -302,7 +302,7 @@ pub mod pallet {
 		/// will be reported.
 		// TODO: fix key_owner_proofs[0].validator_count()
 		#[pallet::call_index(3)]
-		#[pallet::weight(T::WeightInfo::report_equivocation(
+		#[pallet::weight(<T as pallet::Config>::WeightInfo::report_equivocation(
 			key_owner_proofs[0].validator_count(),
 			T::MaxNominators::get(),
 		))]
@@ -314,7 +314,7 @@ pub mod pallet {
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
 					HeaderFor<T>,
-					<T as frame_system::Config>::Hash,
+					<<T as pallet_mmr::Config>::Hashing as sp_runtime::traits::Hash>::Output,
 				>,
 			>,
 			key_owner_proofs: Vec<T::KeyOwnerProof>,
@@ -343,7 +343,7 @@ pub mod pallet {
 		/// reporter.
 		// TODO: fix key_owner_proofs[0].validator_count()
 		#[pallet::call_index(4)]
-		#[pallet::weight(T::WeightInfo::report_equivocation(key_owner_proofs[0].validator_count(), T::MaxNominators::get(),))]
+		#[pallet::weight(<T as pallet::Config>::WeightInfo::report_equivocation(key_owner_proofs[0].validator_count(), T::MaxNominators::get(),))]
 		pub fn report_fork_equivocation_unsigned(
 			origin: OriginFor<T>,
 			equivocation_proof: Box<
@@ -352,7 +352,7 @@ pub mod pallet {
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
 					HeaderFor<T>,
-					<T as frame_system::Config>::Hash,
+					<<T as pallet_mmr::Config>::Hashing as sp_runtime::traits::Hash>::Output,
 				>,
 			>,
 			key_owner_proofs: Vec<T::KeyOwnerProof>,
@@ -423,7 +423,7 @@ impl<T: Config> Pallet<T> {
 			T::BeefyId,
 			<T::BeefyId as RuntimeAppPublic>::Signature,
 			HeaderFor<T>,
-			<T as frame_system::Config>::Hash,
+			<<T as pallet_mmr::Config>::Hashing as sp_runtime::traits::Hash>::Output,
 		>,
 		key_owner_proofs: Vec<T::KeyOwnerProof>,
 	) -> Option<()> {
diff --git a/substrate/frame/beefy/src/mock.rs b/substrate/frame/beefy/src/mock.rs
index e311a0ecedef..dfd6eec3d68a 100644
--- a/substrate/frame/beefy/src/mock.rs
+++ b/substrate/frame/beefy/src/mock.rs
@@ -29,8 +29,12 @@ use pallet_session::historical as pallet_session_historical;
 use sp_core::{crypto::KeyTypeId, ConstU128};
 use sp_io::TestExternalities;
 use sp_runtime::{
-	app_crypto::ecdsa::Public, curve::PiecewiseLinear, impl_opaque_keys, testing::TestXt,
-	traits::OpaqueKeys, BuildStorage, Perbill,
+	app_crypto::ecdsa::Public,
+	curve::PiecewiseLinear,
+	impl_opaque_keys,
+	testing::TestXt,
+	traits::{Keccak256, OpaqueKeys},
+	BuildStorage, Perbill,
 };
 use sp_staking::{EraIndex, SessionIndex};
 use sp_state_machine::BasicExternalities;
@@ -58,6 +62,7 @@ construct_runtime!(
 		Timestamp: pallet_timestamp,
 		Balances: pallet_balances,
 		Beefy: pallet_beefy,
+		Mmr: pallet_mmr,
 		Staking: pallet_staking,
 		Session: pallet_session,
 		Offences: pallet_offences,
@@ -98,6 +103,18 @@ impl pallet_beefy::Config for Test {
 		super::EquivocationReportSystem<Self, Offences, Historical, ReportLongevity>;
 }
 
+impl pallet_mmr::Config for Test {
+	const INDEXING_PREFIX: &'static [u8] = b"mmr";
+
+	type Hashing = Keccak256;
+
+	type LeafData = pallet_mmr::ParentNumberAndHash<Self>;
+
+	type OnNewRoot = ();
+
+	type WeightInfo = ();
+}
+
 parameter_types! {
 	pub const DisabledValidatorsThreshold: Perbill = Perbill::from_percent(33);
 }
@@ -299,6 +316,7 @@ pub fn start_session(session_index: SessionIndex) {
 		Session::on_initialize(System::block_number());
 		Staking::on_initialize(System::block_number());
 		Beefy::on_initialize(System::block_number());
+		Mmr::on_initialize(System::block_number());
 	}
 
 	assert_eq!(Session::current_index(), session_index);
diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index c07bec7ce38e..46b6477b5370 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -24,6 +24,7 @@ use sp_consensus_beefy::{
 	known_payloads::MMR_ROOT_ID, Commitment, Keyring as BeefyKeyring, Payload, ValidatorSet,
 	KEY_TYPE as BEEFY_KEY_TYPE,
 };
+use sp_core::offchain::{testing::TestOffchainExt, OffchainDbExt, OffchainWorkerExt};
 
 use sp_runtime::DigestItem;
 
@@ -804,18 +805,26 @@ fn valid_vote_equivocation_reports_dont_pay_fees() {
 fn report_fork_equivocation_vote_current_set_works() {
 	let authorities = test_authorities();
 
-	new_test_ext_raw_authorities(authorities).execute_with(|| {
+	let mut ext = new_test_ext_raw_authorities(authorities);
+	let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+	ext.register_extension(OffchainWorkerExt::new(offchain));
+
+	let mut era = 1;
+
+	let (block_num, header) = ext.execute_with(|| {
 		assert_eq!(Staking::current_era(), Some(0));
 		assert_eq!(Session::current_index(), 0);
-
-		let mut era = 1;
 		start_era(era);
+
 		let block_num = System::block_number();
 		let header = System::finalize();
-		let ancestry_proof = unimplemented!();
-
 		era += 1;
 		start_era(era);
+		(block_num, header)
+	});
+	ext.persist_offchain_overlay();
+	ext.execute_with(|| {
 
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
@@ -839,6 +848,8 @@ fn report_fork_equivocation_vote_current_set_works() {
 		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
+
 		// generate an fork equivocation proof, with a vote in the same round for a
 		// different payload than finalized
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
diff --git a/substrate/frame/merkle-mountain-range/src/lib.rs b/substrate/frame/merkle-mountain-range/src/lib.rs
index 664f4bc73901..94e126576ce1 100644
--- a/substrate/frame/merkle-mountain-range/src/lib.rs
+++ b/substrate/frame/merkle-mountain-range/src/lib.rs
@@ -339,11 +339,32 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 		mmr.generate_proof(leaf_indices)
 	}
 
+	pub fn generate_ancestry_proof(
+		prev_best_block: BlockNumberFor<T>,
+		best_known_block_number: Option<BlockNumberFor<T>>,
+	) -> Result<primitives::AncestryProof<HashOf<T, I>>, primitives::Error> {
+		// check whether best_known_block_number provided, else use current best block
+		let best_known_block_number =
+			best_known_block_number.unwrap_or_else(|| <frame_system::Pallet<T>>::block_number());
+
+		let leaves_count =
+			Self::block_num_to_leaf_index(best_known_block_number)?.saturating_add(1);
+		let prev_leaves_count = Self::block_num_to_leaf_index(prev_best_block)?.saturating_add(1);
+
+		let mmr: ModuleMmr<mmr::storage::OffchainStorage, T, I> = mmr::Mmr::new(leaves_count);
+		mmr.generate_ancestry_proof(prev_leaves_count)
+	}
+
 	/// Return the on-chain MMR root hash.
 	pub fn mmr_root() -> HashOf<T, I> {
 		Self::mmr_root_hash()
 	}
 
+	/// Return the on-chain MMR root hash.
+	pub fn mmr_size() -> NodeIndex {
+		Self::mmr_leaves()
+	}
+
 	/// Verify MMR proof for given `leaves`.
 	///
 	/// This method is safe to use within the runtime code.
@@ -370,4 +391,17 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 			Err(primitives::Error::Verify.log_debug("The proof is incorrect."))
 		}
 	}
+
+	pub fn verify_ancestry_proof(
+		ancestry_proof: primitives::AncestryProof<HashOf<T, I>>,
+	) -> Result<(), primitives::Error> {
+		let mmr: ModuleMmr<mmr::storage::OffchainStorage, T, I> =
+			mmr::Mmr::new(ancestry_proof.proof.leaf_count);
+		let is_valid = mmr.verify_ancestry_proof(ancestry_proof)?;
+		if is_valid {
+			Ok(())
+		} else {
+			Err(primitives::Error::Verify.log_debug("The ancestry proof is incorrect."))
+		}
+	}
 }
diff --git a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
index 3b6a96978257..9566c1981ca1 100644
--- a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
+++ b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
@@ -117,6 +117,41 @@ where
 			.map_err(|e| Error::Verify.log_debug(e))
 	}
 
+	pub fn verify_ancestry_proof(
+		&self,
+		ancestry_proof: primitives::AncestryProof<HashOf<T, I>>,
+	) -> Result<bool, Error> {
+		let p = mmr_lib::MerkleProof::<NodeOf<T, I, L>, Hasher<HashingOf<T, I>, L>>::new(
+			self.mmr.mmr_size(),
+			ancestry_proof
+				.proof
+				.items
+				.into_iter()
+				.map(|(index, hash)| (index, Node::Hash(hash)))
+				.collect(),
+		);
+
+		let ancestry_proof = mmr_lib::AncestryProof::<NodeOf<T, I, L>, Hasher<HashingOf<T, I>, L>> {
+			prev_peaks: ancestry_proof
+				.prev_peaks
+				.into_iter()
+				.map(|hash| Node::Hash(hash))
+				.collect(),
+			prev_size: ancestry_proof.prev_size,
+			proof: p,
+		};
+
+		let prev_root =
+			mmr_lib::bagging_peaks_hashes::<NodeOf<T, I, L>, Hasher<HashingOf<T, I>, L>>(
+				ancestry_proof.prev_peaks.clone(),
+			)
+			.map_err(|e| Error::Verify.log_debug(e))?;
+		let root = self.mmr.get_root().map_err(|e| Error::GetRoot.log_error(e))?;
+		ancestry_proof
+			.verify_ancestor(root, prev_root)
+			.map_err(|e| Error::Verify.log_debug(e))
+	}
+
 	/// Return the internal size of the MMR (number of nodes).
 	#[cfg(test)]
 	pub fn size(&self) -> NodeIndex {
@@ -195,8 +230,7 @@ where
 	pub fn generate_ancestry_proof(
 		&self,
 		prev_mmr_size: NodeIndex,
-	) -> Result<primitives::AncestryProof<HashOf<T, I>>, Error>
-	{
+	) -> Result<primitives::AncestryProof<HashOf<T, I>>, Error> {
 		let leaf_count = self.leaves;
 		self.mmr
 			.gen_prefix_proof(prev_mmr_size)
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index bd61db59ce88..21719f91c726 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -37,10 +37,11 @@ mod payload;
 #[cfg(feature = "std")]
 mod test_utils;
 pub mod witness;
+use core::fmt::Debug;
 
 pub use commitment::{Commitment, SignedCommitment, VersionedFinalityProof};
 pub use payload::{known_payloads, BeefyPayloadId, Payload, PayloadProvider};
-use sp_mmr_primitives::AncestryProof;
+use sp_mmr_primitives::{mmr_lib, AncestryProof};
 #[cfg(feature = "std")]
 pub use test_utils::*;
 
@@ -348,7 +349,7 @@ where
 /// finalized by GRANDPA. This is fine too, since the slashing risk of committing to
 /// an incorrect block implies validators will only sign blocks they *know* will be
 /// finalized by GRANDPA.
-pub fn check_fork_equivocation_proof<Number, Id, MsgHash, Header, NodeHash>(
+pub fn check_fork_equivocation_proof<Number, Id, MsgHash, Header, NodeHash, Hasher>(
 	proof: &ForkEquivocationProof<
 		Number,
 		Id,
@@ -356,6 +357,8 @@ pub fn check_fork_equivocation_proof<Number, Id, MsgHash, Header, NodeHash>(
 		Header,
 		NodeHash,
 	>,
+	expected_root: Hasher::Item,
+	mmr_size: u64,
 	expected_header_hash: &Header::Hash,
 ) -> bool
 where
@@ -363,9 +366,20 @@ where
 	Number: Clone + Encode + PartialEq,
 	MsgHash: Hash,
 	Header: sp_api::HeaderT,
+	NodeHash: Clone + Debug + PartialEq + Encode,
+	Hasher: mmr_lib::Merge<Item = NodeHash>,
 {
 	let ForkEquivocationProof { commitment, signatories, correct_header, ancestry_proof } = proof;
 
+	if Ok(false) ==
+		sp_mmr_primitives::utils::verify_ancestry_proof::<NodeHash, Hasher>(
+			expected_root,
+			mmr_size,
+			ancestry_proof.clone(),
+		) {
+		return false
+	}
+
 	if correct_header.hash() != *expected_header_hash {
 		return false
 	}
diff --git a/substrate/primitives/merkle-mountain-range/src/utils.rs b/substrate/primitives/merkle-mountain-range/src/utils.rs
index b9171c96a620..05457333e438 100644
--- a/substrate/primitives/merkle-mountain-range/src/utils.rs
+++ b/substrate/primitives/merkle-mountain-range/src/utils.rs
@@ -20,11 +20,60 @@
 use codec::Encode;
 use mmr_lib::helper;
 
+use core::fmt::Debug;
 use sp_runtime::traits::{CheckedAdd, CheckedSub, Header, One};
 #[cfg(not(feature = "std"))]
 use sp_std::prelude::Vec;
 
-use crate::{Error, LeafIndex, NodeIndex};
+use crate::{AncestryProof, Error, LeafIndex, NodeIndex};
+
+/// Merging & Hashing behavior specific to ancestry proofs.
+pub struct AncestryHasher<H>(sp_std::marker::PhantomData<H>);
+
+impl<H: sp_runtime::traits::Hash> mmr_lib::Merge for AncestryHasher<H> {
+	type Item = H::Output;
+
+	fn merge(left: &Self::Item, right: &Self::Item) -> mmr_lib::Result<Self::Item> {
+		let mut concat = left.as_ref().to_vec();
+		concat.extend_from_slice(right.as_ref());
+
+		Ok(<H as sp_runtime::traits::Hash>::hash(&concat))
+	}
+}
+
+/// Stateless verification of the ancestry proof of a previous root.
+pub fn verify_ancestry_proof<H, M>(
+	root: H,
+	mmr_size: NodeIndex,
+	ancestry_proof: AncestryProof<H>,
+) -> Result<bool, Error>
+where
+	// H: sp_runtime::traits::Hash::Output,
+	H: Clone + Debug + PartialEq + Encode,
+	M: mmr_lib::Merge<Item = H>,
+{
+	let p: mmr_lib::MerkleProof<H, _> = mmr_lib::MerkleProof::<H, M>::new(
+		mmr_size,
+		ancestry_proof
+			.proof
+			.items
+			.into_iter()
+			.map(|(index, hash)| (index, hash))
+			.collect(),
+	);
+
+	let ancestry_proof = mmr_lib::AncestryProof::<H, _> {
+		prev_peaks: ancestry_proof.prev_peaks,
+		prev_size: ancestry_proof.prev_size,
+		proof: p,
+	};
+
+	let prev_root = mmr_lib::bagging_peaks_hashes::<H, M>(ancestry_proof.prev_peaks.clone())
+		.map_err(|e| Error::Verify.log_debug(e))?;
+	ancestry_proof
+		.verify_ancestor(root, prev_root)
+		.map_err(|e| Error::Verify.log_debug(e))
+}
 
 /// Get the first block with MMR.
 pub fn first_mmr_block_num<H: Header>(

From a6927af234e06954a88eb98b020d0f2da2d7150b Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 16 Oct 2023 16:15:34 +0200
Subject: [PATCH 068/188] fisher & worker: ancestry proof gen. & reporting

---
 .../beefy/src/communication/fisherman.rs      | 37 +++++++++++++++----
 substrate/client/consensus/beefy/src/tests.rs | 17 ++++++++-
 .../client/consensus/beefy/src/worker.rs      | 24 +++++++++---
 3 files changed, 64 insertions(+), 14 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 8a6a03c8ff47..56c8f18b186d 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -29,9 +29,10 @@ use sp_blockchain::HeaderBackend;
 use sp_consensus_beefy::{
 	check_fork_equivocation_proof,
 	ecdsa_crypto::{AuthorityId, Signature},
-	BeefyApi, ForkEquivocationProof, MmrRootHash, Payload, PayloadProvider, SignedCommitment,
-	ValidatorSet, VoteMessage,
+	BeefyApi, ForkEquivocationProof, MmrHashing, MmrRootHash, Payload, PayloadProvider,
+	SignedCommitment, ValidatorSet, VoteMessage,
 };
+use sp_mmr_primitives::MmrApi;
 use sp_runtime::{
 	generic::BlockId,
 	traits::{Block, Header, NumberFor},
@@ -71,7 +72,7 @@ where
 	BE: Backend<B>,
 	P: PayloadProvider<B>,
 	R: ProvideRuntimeApi<B> + Send + Sync,
-	R::Api: BeefyApi<B, AuthorityId, MmrRootHash>,
+	R::Api: BeefyApi<B, AuthorityId, MmrRootHash> + MmrApi<B, MmrRootHash, NumberFor<B>>,
 {
 	fn expected_header_and_payload(
 		&self,
@@ -118,6 +119,17 @@ where
 			.expect_block_hash_from_id(&BlockId::Number(proof.commitment.block_number))
 			.map_err(|e| Error::Backend(e.to_string()))?;
 
+		let best_hash = self.backend.blockchain().info().best_hash;
+
+		let expected_mmr_root =
+			self.runtime.runtime_api().mmr_root(best_hash).map_err(Error::RuntimeApi)?;
+
+		let mmr_size = self
+			.runtime
+			.runtime_api()
+			.mmr_leaf_count(best_hash)
+			.map_err(Error::RuntimeApi)?;
+
 		if proof.commitment.validator_set_id != set_id ||
 			!check_fork_equivocation_proof::<
 				NumberFor<B>,
@@ -125,7 +137,8 @@ where
 				BeefySignatureHasher,
 				B::Header,
 				MmrRootHash,
-			>(&proof, &expected_header_hash)
+				sp_mmr_primitives::utils::AncestryHasher<MmrHashing>,
+			>(&proof, expected_mmr_root.unwrap(), mmr_size.unwrap(), &expected_header_hash)
 		{
 			debug!(target: LOG_TARGET, "🥩 Skip report for bad invalid fork proof {:?}", proof);
 			return Ok(())
@@ -182,7 +195,7 @@ where
 	BE: Backend<B>,
 	P: PayloadProvider<B>,
 	R: ProvideRuntimeApi<B> + Send + Sync,
-	R::Api: BeefyApi<B, AuthorityId, MmrRootHash>,
+	R::Api: BeefyApi<B, AuthorityId, MmrRootHash> + MmrApi<B, MmrRootHash, NumberFor<B>>,
 {
 	/// Check `vote` for contained block against expected payload.
 	fn check_vote(
@@ -191,8 +204,13 @@ where
 	) -> Result<(), Error> {
 		let number = vote.commitment.block_number;
 		let (correct_header, expected_payload) = self.expected_header_and_payload(number)?;
-		let ancestry_proof = unimplemented!();
 		if vote.commitment.payload != expected_payload {
+			let ancestry_proof = self
+				.runtime
+				.runtime_api()
+				.generate_ancestry_proof(correct_header.hash(), number)
+				.unwrap()
+				.unwrap();
 			let proof = ForkEquivocationProof {
 				commitment: vote.commitment,
 				signatories: vec![(vote.id, vote.signature)],
@@ -212,8 +230,13 @@ where
 		let SignedCommitment { commitment, signatures } = signed_commitment;
 		let number = commitment.block_number;
 		let (correct_header, expected_payload) = self.expected_header_and_payload(number)?;
-		let ancestry_proof = unimplemented!();
 		if commitment.payload != expected_payload {
+			let ancestry_proof = self
+				.runtime
+				.runtime_api()
+				.generate_ancestry_proof(correct_header.hash(), number)
+				.unwrap()
+				.unwrap();
 			let validator_set = self.active_validator_set_at(&correct_header)?;
 			if signatures.len() != validator_set.validators().len() {
 				// invalid proof
diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index dce597a062de..2fc0339038c2 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -62,7 +62,7 @@ use sp_consensus_beefy::{
 };
 use sp_core::H256;
 use sp_keystore::{testing::MemoryKeystore, Keystore, KeystorePtr};
-use sp_mmr_primitives::{Error as MmrError, MmrApi};
+use sp_mmr_primitives::{AncestryProof, Error as MmrError, LeafIndex, MmrApi, Proof};
 use sp_runtime::{
 	codec::{Decode, Encode},
 	traits::{Header as HeaderT, NumberFor},
@@ -379,6 +379,21 @@ sp_api::mock_impl_runtime_apis! {
 		fn mmr_root() -> Result<MmrRootHash, MmrError> {
 			Ok(self.inner.mmr_root_hash)
 		}
+
+		fn mmr_leaf_count() -> Result<LeafIndex, MmrError> {
+			Ok(0)
+		}
+
+		fn generate_ancestry_proof(
+			_prev_best_block: NumberFor<Block>,
+			_best_known_number: Option<NumberFor<Block>>,
+		) -> Result<AncestryProof<MmrRootHash>, MmrError> {
+			Ok(AncestryProof {
+				prev_peaks: vec![],
+				prev_size: 0,
+				proof: Proof { leaf_indices: vec![], leaf_count: 0, items: vec![] },
+			})
+		}
 	}
 }
 
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index 8dcadd380e51..737df435b758 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -46,6 +46,7 @@ use sp_consensus_beefy::{
 	BeefyApi, Commitment, ConsensusLog, MmrRootHash, PayloadProvider, ValidatorSet,
 	VersionedFinalityProof, VoteEquivocationProof, VoteMessage, BEEFY_ENGINE_ID,
 };
+use sp_mmr_primitives::MmrApi;
 use sp_runtime::{
 	generic::OpaqueDigestItemId,
 	traits::{Block, Header, NumberFor, Zero},
@@ -356,7 +357,7 @@ where
 	P: PayloadProvider<B>,
 	S: SyncOracle,
 	R: ProvideRuntimeApi<B>,
-	R::Api: BeefyApi<B, AuthorityId, MmrRootHash>,
+	R::Api: BeefyApi<B, AuthorityId, MmrRootHash> + MmrApi<B, MmrRootHash, NumberFor<B>>,
 	F: BeefyFisherman<B>,
 {
 	fn best_grandpa_block(&self) -> NumberFor<B> {
@@ -1726,7 +1727,6 @@ pub(crate) mod tests {
 			.header(hashes[block_number as usize])
 			.unwrap()
 			.unwrap();
-		let ancestry_proof = unimplemented!();
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, "amievil".encode());
 		let votes: Vec<_> = peers
 			.iter()
@@ -1737,8 +1737,17 @@ pub(crate) mod tests {
 			.collect();
 
 		// verify: Alice reports Bob
-		let proof =
-			generate_fork_equivocation_proof_vote(votes[1].clone(), header.clone(), ancestry_proof);
+		let ancestry_proof = alice_worker
+			.runtime
+			.runtime_api()
+			.generate_ancestry_proof(*hashes.last().unwrap(), block_number)
+			.unwrap()
+			.unwrap();
+		let proof = generate_fork_equivocation_proof_vote(
+			votes[1].clone(),
+			header.clone(),
+			ancestry_proof.clone(),
+		);
 		{
 			// expect fisher (Alice) to successfully process it
 			assert_eq!(
@@ -1757,8 +1766,11 @@ pub(crate) mod tests {
 		}
 
 		// verify: Alice does not self-report
-		let proof =
-			generate_fork_equivocation_proof_vote(votes[0].clone(), header.clone(), ancestry_proof);
+		let proof = generate_fork_equivocation_proof_vote(
+			votes[0].clone(),
+			header.clone(),
+			ancestry_proof.clone(),
+		);
 		{
 			// expect fisher (Alice) to successfully process it
 			assert_eq!(

From 03208f1164b61e3dee458d4beaffeed032c3382a Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 16 Oct 2023 17:00:18 +0200
Subject: [PATCH 069/188] tests: report_fork_equivocation_vote_old_set_works

---
 substrate/frame/beefy/src/tests.rs | 41 ++++++++++++++++++++++++------
 1 file changed, 33 insertions(+), 8 deletions(-)

diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index 46b6477b5370..72036c476f89 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -811,7 +811,6 @@ fn report_fork_equivocation_vote_current_set_works() {
 	ext.register_extension(OffchainWorkerExt::new(offchain));
 
 	let mut era = 1;
-
 	let (block_num, header) = ext.execute_with(|| {
 		assert_eq!(Staking::current_era(), Some(0));
 		assert_eq!(Session::current_index(), 0);
@@ -824,6 +823,7 @@ fn report_fork_equivocation_vote_current_set_works() {
 		(block_num, header)
 	});
 	ext.persist_offchain_overlay();
+
 	ext.execute_with(|| {
 
 		let validator_set = Beefy::validator_set().unwrap();
@@ -902,13 +902,24 @@ fn report_fork_equivocation_vote_current_set_works() {
 fn report_fork_equivocation_vote_old_set_works() {
 	let authorities = test_authorities();
 
-	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		let mut era = 1;
+	let mut ext = new_test_ext_raw_authorities(authorities);
+	let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+	ext.register_extension(OffchainWorkerExt::new(offchain));
+
+	let mut era = 1;
+	let (
+		block_num,
+		header,
+		validators,
+		old_set_id,
+		equivocation_authority_index,
+		equivocation_key,
+		key_owner_proof,
+	) = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
-		let ancestry_proof = unimplemented!();
-
 		era += 1;
 		start_era(era);
 
@@ -919,14 +930,26 @@ fn report_fork_equivocation_vote_old_set_works() {
 
 		assert_eq!(authorities.len(), 3);
 		let equivocation_authority_index = 0;
-		let equivocation_key = &authorities[equivocation_authority_index];
+		let equivocation_key = authorities[equivocation_authority_index].clone();
 
 		// create the key ownership proof in the "old" set
-		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &&equivocation_key)).unwrap();
 
 		era += 1;
 		start_era(era);
+		(
+			block_num,
+			header,
+			validators,
+			old_set_id,
+			equivocation_authority_index,
+			equivocation_key,
+			key_owner_proof,
+		)
+	});
+	ext.persist_offchain_overlay();
 
+	ext.execute_with(|| {
 		// make sure that all authorities have the same balance
 		for validator in &validators {
 			assert_eq!(Balances::total_balance(validator), 10_000_000);
@@ -942,9 +965,11 @@ fn report_fork_equivocation_vote_old_set_works() {
 		let new_set_id = validator_set.id();
 		assert_eq!(old_set_id + 3, new_set_id);
 
-		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+		let equivocation_keyring = BeefyKeyring::from_public(&equivocation_key).unwrap();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
+
 		// generate an fork equivocation proof, with a vote in the same round for a
 		// different payload than finalized
 		let equivocation_proof = generate_fork_equivocation_proof_vote(

From 920873414968e7a13c01c8315f5c8c2754d28f35 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 17 Oct 2023 09:47:20 +0200
Subject: [PATCH 070/188] verify prev_root is at correct block number

---
 .../beefy/src/communication/fisherman.rs      | 12 +++-
 substrate/frame/beefy/src/equivocation.rs     | 25 +++++--
 .../merkle-mountain-range/src/mmr/mmr.rs      |  5 +-
 .../primitives/consensus/beefy/src/lib.rs     | 68 +++++++++++++++----
 .../merkle-mountain-range/Cargo.toml          |  2 +-
 5 files changed, 87 insertions(+), 25 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 56c8f18b186d..67f961e6e2c9 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -130,16 +130,22 @@ where
 			.mmr_leaf_count(best_hash)
 			.map_err(Error::RuntimeApi)?;
 
+		let first_mmr_block_num = unimplemented!();
+
 		if proof.commitment.validator_set_id != set_id ||
 			!check_fork_equivocation_proof::<
-				NumberFor<B>,
 				AuthorityId,
 				BeefySignatureHasher,
 				B::Header,
 				MmrRootHash,
 				sp_mmr_primitives::utils::AncestryHasher<MmrHashing>,
-			>(&proof, expected_mmr_root.unwrap(), mmr_size.unwrap(), &expected_header_hash)
-		{
+			>(
+				&proof,
+				expected_mmr_root.unwrap(),
+				mmr_size.unwrap(),
+				&expected_header_hash,
+				first_mmr_block_num,
+			) {
 			debug!(target: LOG_TARGET, "🥩 Skip report for bad invalid fork proof {:?}", proof);
 			return Ok(())
 		}
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 6e12e2671053..df80d933ec93 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -289,8 +289,21 @@ where
 			EquivocationEvidenceFor::ForkEquivocationProof(equivocation_proof, _) => {
 				let block_number = equivocation_proof.commitment.block_number;
 				let expected_block_hash = <frame_system::Pallet<T>>::block_hash(block_number);
-				let mmr_size = <pallet_mmr::Pallet<T>>::mmr_size();
+				let mmr_size =
+					sp_mmr_primitives::utils::NodesUtils::new(<pallet_mmr::Pallet<T>>::mmr_size())
+						.size();
 				let expected_mmr_root = <pallet_mmr::Pallet<T>>::mmr_root();
+				// if first_mmr_block_num is invalid, then presumably beefy is not active.
+				// TODO: should we slash in this case?
+				let first_mmr_block_num = {
+					let best_block_num = <frame_system::Pallet<T>>::block_number();
+					let mmr_leaf_count = <pallet_mmr::Pallet<T>>::mmr_size();
+					sp_mmr_primitives::utils::first_mmr_block_num::<HeaderFor<T>>(
+						best_block_num,
+						mmr_leaf_count,
+					)
+					.map_err(|_| Error::<T>::InvalidForkEquivocationProof)?
+				};
 
 				// Validate equivocation proof (check commitment is to unexpected payload and
 				// signatures are valid).
@@ -304,11 +317,15 @@ where
 					_,
 					_,
 					_,
-					_,
 					<<T as pallet_mmr::Config>::Hashing as sp_runtime::traits::Hash>::Output,
 					sp_mmr_primitives::utils::AncestryHasher<<T as pallet_mmr::Config>::Hashing>,
-				>(equivocation_proof, expected_mmr_root, mmr_size, &expected_block_hash)
-				{
+				>(
+					equivocation_proof,
+					expected_mmr_root,
+					mmr_size,
+					&expected_block_hash,
+					first_mmr_block_num,
+				) {
 					return Err(Error::<T>::InvalidForkEquivocationProof.into())
 				}
 			},
diff --git a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
index 9566c1981ca1..0d8d457e1ef5 100644
--- a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
+++ b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
@@ -20,7 +20,7 @@ use crate::{
 		storage::{OffchainStorage, RuntimeStorage, Storage},
 		Hasher, Node, NodeOf,
 	},
-	primitives::{self, Error, NodeIndex},
+	primitives::{self, Error, LeafIndex, NodeIndex},
 	Config, HashOf, HashingOf,
 };
 use sp_mmr_primitives::{mmr_lib, utils::NodesUtils};
@@ -229,9 +229,10 @@ where
 
 	pub fn generate_ancestry_proof(
 		&self,
-		prev_mmr_size: NodeIndex,
+		prev_leaf_count: LeafIndex,
 	) -> Result<primitives::AncestryProof<HashOf<T, I>>, Error> {
 		let leaf_count = self.leaves;
+		let prev_mmr_size = NodesUtils::new(prev_leaf_count).size();
 		self.mmr
 			.gen_prefix_proof(prev_mmr_size)
 			.map_err(|e| Error::GenerateProof.log_error(e))
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 21719f91c726..c16e5d635753 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -349,9 +349,9 @@ where
 /// finalized by GRANDPA. This is fine too, since the slashing risk of committing to
 /// an incorrect block implies validators will only sign blocks they *know* will be
 /// finalized by GRANDPA.
-pub fn check_fork_equivocation_proof<Number, Id, MsgHash, Header, NodeHash, Hasher>(
+pub fn check_fork_equivocation_proof<Id, MsgHash, Header, NodeHash, Hasher>(
 	proof: &ForkEquivocationProof<
-		Number,
+		Header::Number,
 		Id,
 		<Id as RuntimeAppPublic>::Signature,
 		Header,
@@ -360,26 +360,51 @@ pub fn check_fork_equivocation_proof<Number, Id, MsgHash, Header, NodeHash, Hash
 	expected_root: Hasher::Item,
 	mmr_size: u64,
 	expected_header_hash: &Header::Hash,
+	first_mmr_block_num: Header::Number,
 ) -> bool
 where
 	Id: BeefyAuthorityId<MsgHash> + PartialEq,
-	Number: Clone + Encode + PartialEq,
 	MsgHash: Hash,
 	Header: sp_api::HeaderT,
-	NodeHash: Clone + Debug + PartialEq + Encode,
+	NodeHash: Clone + Debug + PartialEq + Encode + Decode,
 	Hasher: mmr_lib::Merge<Item = NodeHash>,
 {
 	let ForkEquivocationProof { commitment, signatories, correct_header, ancestry_proof } = proof;
-
-	if Ok(false) ==
-		sp_mmr_primitives::utils::verify_ancestry_proof::<NodeHash, Hasher>(
-			expected_root,
-			mmr_size,
-			ancestry_proof.clone(),
-		) {
-		return false
+	// verify that the prev_root is at the correct block number
+	// this can be inferred from the leaf_count / mmr_size of the prev_root:
+	// convert the commitment.block_number to an mmr size and compare with the value in the ancestry
+	// proof
+	{
+		let expected_leaf_count = sp_mmr_primitives::utils::block_num_to_leaf_index::<Header>(
+			commitment.block_number,
+			first_mmr_block_num,
+		)
+		.and_then(|leaf_index| {
+			leaf_index.checked_add(1).ok_or_else(|| {
+				sp_mmr_primitives::Error::InvalidNumericOp.log_debug("leaf_index + 1 overflowed")
+			})
+		});
+		// if the block number either under- or overflowed, the commitment.block_number was not
+		// valid and the commitment should not have been signed, hence we can skip the ancestry
+		// proof and slash the signatories
+		if let Ok(expected_leaf_count) = expected_leaf_count {
+			let expected_mmr_size =
+				sp_mmr_primitives::utils::NodesUtils::new(expected_leaf_count).size();
+			if expected_mmr_size != ancestry_proof.prev_size {
+				return false
+			}
+			if Ok(true) !=
+				sp_mmr_primitives::utils::verify_ancestry_proof::<NodeHash, Hasher>(
+					expected_root,
+					mmr_size,
+					ancestry_proof.clone(),
+				) {
+				return false
+			}
+		}
 	}
 
+
 	if correct_header.hash() != *expected_header_hash {
 		return false
 	}
@@ -388,13 +413,26 @@ where
 	let expected_payload = expected_mmr_root_digest
 		.map(|mmr_root| Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode()));
 
+	let ancestry_prev_root =
+		mmr_lib::bagging_peaks_hashes::<NodeHash, Hasher>(ancestry_proof.prev_peaks.clone());
+
+	// if the commitment payload does not commit to an MMR root, then this commitment may have
+	// another purpose and should not be slashed
+	// TODO: what if we can nonetheless show that there's another payload at the same block number?
+	// if we're keeping both header & mmr root slashing, then we may proceed in this case
+	// nonetheless
+	let commitment_prev_root =
+		commitment.payload.get_decoded::<NodeHash>(&known_payloads::MMR_ROOT_ID);
 	// cheap failfasts:
 	// 1. check that `payload` on the `vote` is different that the `expected_payload`
 	// 2. if the signatories signed a payload when there should be none (for
 	// instance for a block prior to BEEFY activation), then expected_payload =
-	// None, and they will likewise be slashed
-	if Some(&commitment.payload) != expected_payload.as_ref() {
-		// check check each signatory's signature on the commitment.
+	// None, and they will likewise be slashed (note we can only check this if a valid header has
+	// been provided - we cannot slash for this with an ancestry proof - by necessity)
+	if Some(&commitment.payload) != expected_payload.as_ref() ||
+		(ancestry_prev_root.is_ok() && commitment_prev_root != ancestry_prev_root.ok())
+	{
+		// check each signatory's signature on the commitment.
 		// if any are invalid, equivocation report is invalid
 		// TODO: refactor check_commitment_signature to take a slice of signatories
 		return signatories.iter().all(|(authority_id, signature)| {
diff --git a/substrate/primitives/merkle-mountain-range/Cargo.toml b/substrate/primitives/merkle-mountain-range/Cargo.toml
index b11aaaeda0cf..0b1c0058af13 100644
--- a/substrate/primitives/merkle-mountain-range/Cargo.toml
+++ b/substrate/primitives/merkle-mountain-range/Cargo.toml
@@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false }
 scale-info = { version = "2.5.0", default-features = false, features = ["derive"] }
 log = { version = "0.4.17", default-features = false }
-mmr-lib = { package = "ckb-merkle-mountain-range", git = "https://github.com/Lederstrumpf/merkle-mountain-range.git", branch = "linearize-proof-rebase-v0.5.1", default-features = false }
+mmr-lib = { package = "ckb-merkle-mountain-range", git = "https://github.com/Lederstrumpf/merkle-mountain-range.git", branch = "linearize-proof-rebase-v0.5.1", features = ["nodeproofs"] }
 serde = { version = "1.0.188", features = ["derive", "alloc"], default-features = false, optional = true }
 sp-api = { path = "../api", default-features = false}
 sp-core = { path = "../core", default-features = false}

From 852d110aec3c400542cac718bb4a1b754321f96a Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 17 Oct 2023 10:26:54 +0200
Subject: [PATCH 071/188] fisherman: retrieve first_mmr_block_num

---
 .../beefy/src/communication/fisherman.rs          | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 67f961e6e2c9..f606a66647e1 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -124,13 +124,22 @@ where
 		let expected_mmr_root =
 			self.runtime.runtime_api().mmr_root(best_hash).map_err(Error::RuntimeApi)?;
 
-		let mmr_size = self
+		let leaf_count = self
 			.runtime
 			.runtime_api()
 			.mmr_leaf_count(best_hash)
 			.map_err(Error::RuntimeApi)?;
 
-		let first_mmr_block_num = unimplemented!();
+		// TODO: if ancestry proof can't be constructed, report equivocation nonetheless if valid
+		// header proof can be provided
+		let first_mmr_block_num = {
+			let best_block_num = self.backend.blockchain().info().best_number;
+			sp_mmr_primitives::utils::first_mmr_block_num::<B::Header>(
+				best_block_num,
+				*leaf_count.as_ref().unwrap(),
+			)
+			.map_err(|e| Error::Backend(e.to_string()))?
+		};
 
 		if proof.commitment.validator_set_id != set_id ||
 			!check_fork_equivocation_proof::<
@@ -142,7 +151,7 @@ where
 			>(
 				&proof,
 				expected_mmr_root.unwrap(),
-				mmr_size.unwrap(),
+				leaf_count.unwrap(),
 				&expected_header_hash,
 				first_mmr_block_num,
 			) {

From 9e91cf311fe32912e0d8f4f3367f82a7fe8305fc Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 17 Oct 2023 13:54:58 +0200
Subject: [PATCH 072/188] impl ancestry in all applicable beefy pallet tests

---
 substrate/frame/beefy/src/tests.rs | 322 ++++++++++++++++++++++-------
 1 file changed, 247 insertions(+), 75 deletions(-)

diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index 72036c476f89..3cc6f8d817f3 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -1019,16 +1019,24 @@ fn report_fork_equivocation_vote_old_set_works() {
 fn report_fork_equivocation_vote_invalid_set_id() {
 	let authorities = test_authorities();
 
-	new_test_ext_raw_authorities(authorities).execute_with(|| {
+	let mut ext = new_test_ext_raw_authorities(authorities);
+	let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+	ext.register_extension(OffchainWorkerExt::new(offchain));
+
+	let (block_num, header) = ext.execute_with(|| {
 		let mut era = 1;
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
-		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
+		(block_num, header)
+	});
+	ext.persist_offchain_overlay();
 
+	ext.execute_with(|| {
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
@@ -1040,6 +1048,7 @@ fn report_fork_equivocation_vote_invalid_set_id() {
 		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		// generate an equivocation for a future set
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id + 1, &equivocation_keyring),
@@ -1063,12 +1072,16 @@ fn report_fork_equivocation_vote_invalid_set_id() {
 fn report_fork_equivocation_vote_invalid_session() {
 	let authorities = test_authorities();
 
-	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		let mut era = 1;
+	let mut ext = new_test_ext_raw_authorities(authorities);
+	let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+	ext.register_extension(OffchainWorkerExt::new(offchain));
+
+	let mut era = 1;
+	let (block_num, header, equivocation_keyring, key_owner_proof) = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
-		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
@@ -1085,10 +1098,15 @@ fn report_fork_equivocation_vote_invalid_session() {
 
 		era += 1;
 		start_era(era);
+		(block_num, header, equivocation_keyring, key_owner_proof)
+	});
+	ext.persist_offchain_overlay();
 
+	ext.execute_with(|| {
 		let set_id = Beefy::validator_set().unwrap().id();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		// generate an equivocation proof at following era set id = 3
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
@@ -1113,16 +1131,24 @@ fn report_fork_equivocation_vote_invalid_session() {
 fn report_fork_equivocation_vote_invalid_key_owner_proof() {
 	let authorities = test_authorities();
 
-	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		let mut era = 1;
+	let mut ext = new_test_ext_raw_authorities(authorities);
+	let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+	ext.register_extension(OffchainWorkerExt::new(offchain));
+
+	let mut era = 1;
+	let (block_num, header) = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
-		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
+		(block_num, header)
+	});
+	ext.persist_offchain_overlay();
 
+	ext.execute_with(|| {
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
@@ -1139,6 +1165,7 @@ fn report_fork_equivocation_vote_invalid_key_owner_proof() {
 		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		// generate an equivocation for a future set
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id + 1, &equivocation_keyring),
@@ -1168,32 +1195,44 @@ fn report_fork_equivocation_vote_invalid_key_owner_proof() {
 fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 	let authorities = test_authorities();
 
-	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		start_era(1);
-
-		let block_num = System::block_number();
-		let header = System::finalize();
-		let ancestry_proof = unimplemented!();
-		let validator_set = Beefy::validator_set().unwrap();
-		let authorities = validator_set.validators();
-		let set_id = validator_set.id();
-
-		let equivocation_authority_index = 0;
-		let equivocation_key = &authorities[equivocation_authority_index];
-		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
-
-		// generate a key ownership proof at set id in era 1
-		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+	let mut ext = new_test_ext_raw_authorities(authorities);
+	let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+	ext.register_extension(OffchainWorkerExt::new(offchain));
 
-		start_era(2);
+	let mut era = 1;
+	let (block_num, header, set_id, equivocation_keyring, key_owner_proof) =
+		ext.execute_with(|| {
+			start_era(era);
+			let block_num = System::block_number();
+			let header = System::finalize();
+
+			let validator_set = Beefy::validator_set().unwrap();
+			let authorities = validator_set.validators();
+			let set_id = validator_set.id();
+
+			let equivocation_authority_index = 0;
+			let equivocation_key = &authorities[equivocation_authority_index];
+			let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+
+			// generate a key ownership proof at set id in era 1
+			let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+
+			era += 1;
+			start_era(era);
+			(block_num, header, set_id, equivocation_keyring, key_owner_proof)
+		});
+	ext.persist_offchain_overlay();
 
+	ext.execute_with(|| {
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 
 		// vote targets different round than finalized payload, there is no equivocation.
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num + 1, payload.clone(), set_id, &equivocation_keyring),
 			header.clone(),
-			ancestry_proof,
+			ancestry_proof.clone(),
 		);
 		assert_err!(
 			Beefy::report_fork_equivocation_unsigned(
@@ -1208,7 +1247,7 @@ fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload.clone(), set_id, &BeefyKeyring::Dave),
 			header.clone(),
-			ancestry_proof,
+			ancestry_proof.clone(),
 		);
 		assert_err!(
 			Beefy::report_fork_equivocation_unsigned(
@@ -1245,16 +1284,24 @@ fn report_fork_equivocation_vote_validate_unsigned_prevents_duplicates() {
 
 	let authorities = test_authorities();
 
-	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		let mut era = 1;
+	let mut ext = new_test_ext_raw_authorities(authorities);
+	let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+	ext.register_extension(OffchainWorkerExt::new(offchain));
+
+	let mut era = 1;
+	let (block_num, header) = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
-		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
+		(block_num, header)
+	});
+	ext.persist_offchain_overlay();
 
+	ext.execute_with(|| {
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
@@ -1265,6 +1312,7 @@ fn report_fork_equivocation_vote_validate_unsigned_prevents_duplicates() {
 		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
 			header,
@@ -1337,16 +1385,24 @@ fn report_fork_equivocation_vote_validate_unsigned_prevents_duplicates() {
 fn valid_fork_equivocation_vote_reports_dont_pay_fees() {
 	let authorities = test_authorities();
 
-	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		let mut era = 1;
+	let mut ext = new_test_ext_raw_authorities(authorities);
+	let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+	ext.register_extension(OffchainWorkerExt::new(offchain));
+
+	let mut era = 1;
+	let (block_num, header) = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
-		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
+		(block_num, header)
+	});
+	ext.persist_offchain_overlay();
 
+	ext.execute_with(|| {
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
@@ -1357,6 +1413,7 @@ fn valid_fork_equivocation_vote_reports_dont_pay_fees() {
 
 		// generate equivocation proof
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
 			header,
@@ -1413,19 +1470,24 @@ fn valid_fork_equivocation_vote_reports_dont_pay_fees() {
 fn report_fork_equivocation_sc_current_set_works() {
 	let authorities = test_authorities();
 
-	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		assert_eq!(Staking::current_era(), Some(0));
-		assert_eq!(Session::current_index(), 0);
+	let mut ext = new_test_ext_raw_authorities(authorities);
+	let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+	ext.register_extension(OffchainWorkerExt::new(offchain));
 
-		let mut era = 1;
+	let mut era = 1;
+	let (block_num, header) = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
-		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
+		(block_num, header)
+	});
+	ext.persist_offchain_overlay();
 
+	ext.execute_with(|| {
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
@@ -1454,6 +1516,7 @@ fn report_fork_equivocation_sc_current_set_works() {
 			.collect();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		let commitment = Commitment { validator_set_id: set_id, block_number: block_num, payload };
 		// generate an fork equivocation proof, with a vote in the same round for a
 		// different payload than finalized
@@ -1516,12 +1579,24 @@ fn report_fork_equivocation_sc_current_set_works() {
 fn report_fork_equivocation_sc_old_set_works() {
 	let authorities = test_authorities();
 
-	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		let mut era = 1;
+	let mut ext = new_test_ext_raw_authorities(authorities);
+	let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+	ext.register_extension(OffchainWorkerExt::new(offchain));
+
+	let mut era = 1;
+	let (
+		block_num,
+		header,
+		validators,
+		old_set_id,
+		equivocation_authority_indices,
+		equivocation_keys,
+		key_owner_proofs,
+	) = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
-		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
@@ -1535,7 +1610,7 @@ fn report_fork_equivocation_sc_old_set_works() {
 		let equivocation_authority_indices = [0, 2];
 		let equivocation_keys = equivocation_authority_indices
 			.iter()
-			.map(|i| &authorities[*i])
+			.map(|i| authorities[*i].clone())
 			.collect::<Vec<_>>();
 
 		// create the key ownership proofs in the "old" set
@@ -1547,6 +1622,19 @@ fn report_fork_equivocation_sc_old_set_works() {
 		era += 1;
 		start_era(era);
 
+		(
+			block_num,
+			header,
+			validators,
+			old_set_id,
+			equivocation_authority_indices,
+			equivocation_keys,
+			key_owner_proofs,
+		)
+	});
+	ext.persist_offchain_overlay();
+
+	ext.execute_with(|| {
 		// make sure that all authorities have the same balance
 		for validator in &validators {
 			assert_eq!(Balances::total_balance(validator), 10_000_000);
@@ -1568,6 +1656,7 @@ fn report_fork_equivocation_sc_old_set_works() {
 			.collect();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		// generate an fork equivocation proof, with a vote in the same round for a
 		// different payload than finalized
 		let commitment =
@@ -1627,16 +1716,24 @@ fn report_fork_equivocation_sc_old_set_works() {
 fn report_fork_equivocation_sc_invalid_set_id() {
 	let authorities = test_authorities();
 
-	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		let mut era = 1;
+	let mut ext = new_test_ext_raw_authorities(authorities);
+	let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+	ext.register_extension(OffchainWorkerExt::new(offchain));
+
+	let mut era = 1;
+	let (block_num, header) = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
-		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
+		(block_num, header)
+	});
+	ext.persist_offchain_overlay();
 
+	ext.execute_with(|| {
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
@@ -1657,6 +1754,7 @@ fn report_fork_equivocation_sc_invalid_set_id() {
 			.collect();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		// generate an equivocation for a future set
 		let commitment =
 			Commitment { validator_set_id: set_id + 1, block_number: block_num, payload };
@@ -1683,12 +1781,16 @@ fn report_fork_equivocation_sc_invalid_set_id() {
 fn report_fork_equivocation_sc_invalid_session() {
 	let authorities = test_authorities();
 
-	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		let mut era = 1;
+	let mut ext = new_test_ext_raw_authorities(authorities);
+	let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+	ext.register_extension(OffchainWorkerExt::new(offchain));
+
+	let mut era = 1;
+	let (block_num, header, equivocation_keyrings, key_owner_proofs) = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
-		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
@@ -1699,7 +1801,7 @@ fn report_fork_equivocation_sc_invalid_session() {
 		let equivocation_authority_indices = [0, 2];
 		let equivocation_keys = equivocation_authority_indices
 			.iter()
-			.map(|i| &authorities[*i])
+			.map(|i| authorities[*i].clone())
 			.collect::<Vec<_>>();
 		let equivocation_keyrings = equivocation_keys
 			.iter()
@@ -1714,10 +1816,15 @@ fn report_fork_equivocation_sc_invalid_session() {
 
 		era += 1;
 		start_era(era);
+		(block_num, header, equivocation_keyrings, key_owner_proofs)
+	});
+	ext.persist_offchain_overlay();
 
+	ext.execute_with(|| {
 		let set_id = Beefy::validator_set().unwrap().id();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		// generate an equivocation proof at following era set id = 3
 		let commitment = Commitment { validator_set_id: set_id, block_number: block_num, payload };
 		let equivocation_proof = generate_fork_equivocation_proof_sc(
@@ -1744,16 +1851,24 @@ fn report_fork_equivocation_sc_invalid_session() {
 fn report_fork_equivocation_sc_invalid_key_owner_proof() {
 	let authorities = test_authorities();
 
-	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		let mut era = 1;
+	let mut ext = new_test_ext_raw_authorities(authorities);
+	let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+	ext.register_extension(OffchainWorkerExt::new(offchain));
+
+	let mut era = 1;
+	let (block_num, header) = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
-		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
+		(block_num, header)
+	});
+	ext.persist_offchain_overlay();
 
+	ext.execute_with(|| {
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
@@ -1780,6 +1895,7 @@ fn report_fork_equivocation_sc_invalid_key_owner_proof() {
 			.collect();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		// generate an equivocation proof for the authorities at indices [0, 2]
 		let commitment =
 			Commitment { validator_set_id: set_id + 1, block_number: block_num, payload };
@@ -1812,12 +1928,24 @@ fn report_fork_equivocation_sc_invalid_key_owner_proof() {
 fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 	let authorities = test_authorities();
 
-	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		start_era(1);
+	let mut ext = new_test_ext_raw_authorities(authorities);
+	let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+	ext.register_extension(OffchainWorkerExt::new(offchain));
 
+	let mut era = 1;
+	let (block_num, header) = ext.execute_with(|| {
+		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
-		let ancestry_proof = unimplemented!();
+
+		era += 1;
+		start_era(era);
+		(block_num, header)
+	});
+	ext.persist_offchain_overlay();
+
+	ext.execute_with(|| {
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
@@ -1841,6 +1969,7 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 		start_era(2);
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 
 		// commitment targets different round than finalized payload, there is no equivocation.
 		let equivocation_proof = generate_fork_equivocation_proof_sc(
@@ -1851,7 +1980,7 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 			},
 			equivocation_keyrings.clone(),
 			header.clone(),
-			ancestry_proof,
+			ancestry_proof.clone(),
 		);
 		assert_err!(
 			Beefy::report_fork_equivocation_unsigned(
@@ -1871,7 +2000,7 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 			},
 			vec![BeefyKeyring::Eve],
 			header.clone(),
-			ancestry_proof,
+			ancestry_proof.clone(),
 		);
 		assert_err!(
 			Beefy::report_fork_equivocation_unsigned(
@@ -1913,16 +2042,24 @@ fn report_fork_equivocation_sc_validate_unsigned_prevents_duplicates() {
 
 	let authorities = test_authorities();
 
-	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		let mut era = 1;
+	let mut ext = new_test_ext_raw_authorities(authorities);
+	let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+	ext.register_extension(OffchainWorkerExt::new(offchain));
+
+	let mut era = 1;
+	let (block_num, header) = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
-		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
+		(block_num, header)
+	});
+	ext.persist_offchain_overlay();
 
+	ext.execute_with(|| {
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
@@ -1933,6 +2070,7 @@ fn report_fork_equivocation_sc_validate_unsigned_prevents_duplicates() {
 		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
 			header,
@@ -2005,16 +2143,24 @@ fn report_fork_equivocation_sc_validate_unsigned_prevents_duplicates() {
 fn valid_fork_equivocation_sc_reports_dont_pay_fees() {
 	let authorities = test_authorities();
 
-	new_test_ext_raw_authorities(authorities).execute_with(|| {
-		let mut era = 1;
+	let mut ext = new_test_ext_raw_authorities(authorities);
+	let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+	ext.register_extension(OffchainWorkerExt::new(offchain));
+
+	let mut era = 1;
+	let (block_num, header) = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
-		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
+		(block_num, header)
+	});
+	ext.persist_offchain_overlay();
 
+	ext.execute_with(|| {
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
@@ -2025,6 +2171,7 @@ fn valid_fork_equivocation_sc_reports_dont_pay_fees() {
 
 		// generate equivocation proof
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
 			header,
@@ -2079,19 +2226,32 @@ fn valid_fork_equivocation_sc_reports_dont_pay_fees() {
 fn report_fork_equivocation_sc_stacked_reports_stack_correctly() {
 	let authorities = test_authorities();
 
-	new_test_ext_raw_authorities(authorities).execute_with(|| {
+	let mut ext = new_test_ext_raw_authorities(authorities);
+	let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+	ext.register_extension(OffchainWorkerExt::new(offchain));
+
+	let mut era = 1;
+	let (block_num, header) = ext.execute_with(|| {
 		assert_eq!(Staking::current_era(), Some(0));
 		assert_eq!(Session::current_index(), 0);
-
-		let mut era = 1;
 		start_era(era);
 		let block_num = System::block_number();
 		let header = System::finalize();
-		let ancestry_proof = unimplemented!();
 
 		era += 1;
 		start_era(era);
+		(block_num, header)
+	});
+	ext.persist_offchain_overlay();
 
+	let (
+		commitment,
+		validators,
+		equivocation_keyrings,
+		equivocation_authority_indices,
+		key_owner_proofs,
+	) = ext.execute_with(|| {
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
@@ -2120,6 +2280,7 @@ fn report_fork_equivocation_sc_stacked_reports_stack_correctly() {
 			.collect();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		let commitment = Commitment { validator_set_id: set_id, block_number: block_num, payload };
 		// generate two fork equivocation proofs with a signed commitment in the same round for a
 		// different payload than finalized
@@ -2129,13 +2290,7 @@ fn report_fork_equivocation_sc_stacked_reports_stack_correctly() {
 			commitment.clone(),
 			vec![equivocation_keyrings[0]],
 			header.clone(),
-			ancestry_proof,
-		);
-		let equivocation_proof_full = generate_fork_equivocation_proof_sc(
-			commitment,
-			equivocation_keyrings,
-			header,
-			ancestry_proof,
+			ancestry_proof.clone(),
 		);
 
 		// create the key ownership proof
@@ -2150,9 +2305,26 @@ fn report_fork_equivocation_sc_stacked_reports_stack_correctly() {
 			Box::new(equivocation_proof_singleton),
 			vec![key_owner_proofs[0].clone()],
 		),);
-
 		era += 1;
 		start_era(era);
+		(
+			commitment,
+			validators,
+			equivocation_keyrings,
+			equivocation_authority_indices,
+			key_owner_proofs,
+		)
+	});
+	ext.persist_offchain_overlay();
+
+	ext.execute_with(|| {
+		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
+		let equivocation_proof_full = generate_fork_equivocation_proof_sc(
+			commitment,
+			equivocation_keyrings,
+			header,
+			ancestry_proof,
+		);
 
 		// check that the balance of the reported equivocating validator is slashed 100%.
 		let equivocation_validator_ids = equivocation_authority_indices

From 36e8acf6897ad7618e3580d639f6a6125e497660 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Wed, 25 Oct 2023 12:37:40 +0200
Subject: [PATCH 073/188] ckb: disable default-features

---
 substrate/primitives/merkle-mountain-range/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/substrate/primitives/merkle-mountain-range/Cargo.toml b/substrate/primitives/merkle-mountain-range/Cargo.toml
index 0b1c0058af13..a29362dea10c 100644
--- a/substrate/primitives/merkle-mountain-range/Cargo.toml
+++ b/substrate/primitives/merkle-mountain-range/Cargo.toml
@@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false }
 scale-info = { version = "2.5.0", default-features = false, features = ["derive"] }
 log = { version = "0.4.17", default-features = false }
-mmr-lib = { package = "ckb-merkle-mountain-range", git = "https://github.com/Lederstrumpf/merkle-mountain-range.git", branch = "linearize-proof-rebase-v0.5.1", features = ["nodeproofs"] }
+mmr-lib = { package = "ckb-merkle-mountain-range", git = "https://github.com/Lederstrumpf/merkle-mountain-range.git", branch = "linearize-proof-rebase-v0.5.1", default-features = false, features = ["nodeproofs"] }
 serde = { version = "1.0.188", features = ["derive", "alloc"], default-features = false, optional = true }
 sp-api = { path = "../api", default-features = false}
 sp-core = { path = "../core", default-features = false}

From 3198cb723b71be36582de282e64eb18b6ddcc563 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Wed, 25 Oct 2023 13:05:40 +0200
Subject: [PATCH 074/188] implement mmr_api v3 on rococo/westend

---
 polkadot/node/service/src/fake_runtime_api.rs     | 12 ++++++++++++
 polkadot/runtime/rococo/src/lib.rs                | 15 ++++++++++++++-
 polkadot/runtime/westend/src/lib.rs               | 13 +++++++++++++
 .../beefy/src/communication/fisherman.rs          |  4 ++--
 substrate/client/consensus/beefy/src/worker.rs    |  2 +-
 .../primitives/merkle-mountain-range/src/lib.rs   |  3 ++-
 6 files changed, 44 insertions(+), 5 deletions(-)

diff --git a/polkadot/node/service/src/fake_runtime_api.rs b/polkadot/node/service/src/fake_runtime_api.rs
index 630ed0e34759..65c56efefe55 100644
--- a/polkadot/node/service/src/fake_runtime_api.rs
+++ b/polkadot/node/service/src/fake_runtime_api.rs
@@ -295,6 +295,18 @@ sp_api::impl_runtime_apis! {
 		) -> Result<(), sp_mmr_primitives::Error> {
 			unimplemented!()
 		}
+
+		fn generate_ancestry_proof(
+			_: u32,
+			_: Option<BlockNumber>,
+		) -> Result<sp_mmr_primitives::AncestryProof<Hash>, sp_mmr_primitives::Error> {
+			unimplemented!()
+		}
+		fn verify_ancestry_proof(
+			_: sp_mmr_primitives::AncestryProof<Hash>,
+		) -> Result<(), sp_mmr_primitives::Error> {
+			unimplemented!()
+		}
 	}
 
 	impl grandpa_primitives::GrandpaApi<Block> for Runtime {
diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs
index 361574460212..e4bfb2e77c89 100644
--- a/polkadot/runtime/rococo/src/lib.rs
+++ b/polkadot/runtime/rococo/src/lib.rs
@@ -1848,7 +1848,7 @@ sp_api::impl_runtime_apis! {
 		}
 	}
 
-	#[api_version(2)]
+	#[api_version(3)]
 	impl mmr::MmrApi<Block, mmr::Hash, BlockNumber> for Runtime {
 		fn mmr_root() -> Result<mmr::Hash, mmr::Error> {
 			Ok(Mmr::mmr_root())
@@ -1893,6 +1893,19 @@ sp_api::impl_runtime_apis! {
 			let nodes = leaves.into_iter().map(|leaf|mmr::DataOrHash::Data(leaf.into_opaque_leaf())).collect();
 			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, nodes, proof)
 		}
+
+		fn generate_ancestry_proof(
+			prev_best_block: BlockNumber,
+			best_known_block_number: Option<BlockNumber>
+		) -> Result<mmr::AncestryProof<mmr::Hash>, mmr::Error> {
+			Mmr::generate_ancestry_proof(prev_best_block, best_known_block_number)
+		}
+
+		fn verify_ancestry_proof(
+			ancestry_proof: sp_mmr_primitives::AncestryProof<mmr::Hash>,
+		) -> Result<(), mmr::Error> {
+			Mmr::verify_ancestry_proof(ancestry_proof)
+		}
 	}
 
 	impl fg_primitives::GrandpaApi<Block> for Runtime {
diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs
index 5c0087bed4a2..408c14793488 100644
--- a/polkadot/runtime/westend/src/lib.rs
+++ b/polkadot/runtime/westend/src/lib.rs
@@ -1939,6 +1939,19 @@ sp_api::impl_runtime_apis! {
 			let nodes = leaves.into_iter().map(|leaf|mmr::DataOrHash::Data(leaf.into_opaque_leaf())).collect();
 			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, nodes, proof)
 		}
+
+		fn generate_ancestry_proof(
+			prev_best_block: BlockNumber,
+			best_known_block_number: Option<BlockNumber>
+		) -> Result<mmr::AncestryProof<mmr::Hash>, mmr::Error> {
+			Mmr::generate_ancestry_proof(prev_best_block, best_known_block_number)
+		}
+
+		fn verify_ancestry_proof(
+			ancestry_proof: sp_mmr_primitives::AncestryProof<mmr::Hash>,
+		) -> Result<(), mmr::Error> {
+			Mmr::verify_ancestry_proof(ancestry_proof)
+		}
 	}
 
 	impl pallet_beefy_mmr::BeefyMmrApi<Block, Hash> for RuntimeApi {
diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index f606a66647e1..8d8f0f55e008 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -223,7 +223,7 @@ where
 			let ancestry_proof = self
 				.runtime
 				.runtime_api()
-				.generate_ancestry_proof(correct_header.hash(), number)
+				.generate_ancestry_proof(correct_header.hash(), number, None)
 				.unwrap()
 				.unwrap();
 			let proof = ForkEquivocationProof {
@@ -249,7 +249,7 @@ where
 			let ancestry_proof = self
 				.runtime
 				.runtime_api()
-				.generate_ancestry_proof(correct_header.hash(), number)
+				.generate_ancestry_proof(correct_header.hash(), number, None)
 				.unwrap()
 				.unwrap();
 			let validator_set = self.active_validator_set_at(&correct_header)?;
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index 737df435b758..77a0d7e72376 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -1740,7 +1740,7 @@ pub(crate) mod tests {
 		let ancestry_proof = alice_worker
 			.runtime
 			.runtime_api()
-			.generate_ancestry_proof(*hashes.last().unwrap(), block_number)
+			.generate_ancestry_proof(*hashes.last().unwrap(), block_number, None)
 			.unwrap()
 			.unwrap();
 		let proof = generate_fork_equivocation_proof_vote(
diff --git a/substrate/primitives/merkle-mountain-range/src/lib.rs b/substrate/primitives/merkle-mountain-range/src/lib.rs
index 30007640070f..3fd2d83192ec 100644
--- a/substrate/primitives/merkle-mountain-range/src/lib.rs
+++ b/substrate/primitives/merkle-mountain-range/src/lib.rs
@@ -468,10 +468,11 @@ sp_api::decl_runtime_apis! {
 		/// Generate MMR ancestry proof for prior mmr size
 		fn generate_ancestry_proof(
 			prev_best_block: BlockNumber,
+			best_known_block_number: Option<BlockNumber>
 		) -> Result<AncestryProof<Hash>, Error>;
 
 		/// Verifies that a claimed prev_root is in fact an ancestor of the provided mmr root
-		fn verify_ancestry_proof(ancestry_proof: AncestryProof<Hash>, root: Hash, prev_root: Hash) -> Result<(), Error>;
+		fn verify_ancestry_proof(ancestry_proof: AncestryProof<Hash>) -> Result<(), Error>;
 	}
 }
 

From 16a80f863351f189bb5152978e54010a27fb33bf Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Wed, 25 Oct 2023 13:07:22 +0200
Subject: [PATCH 075/188] fmt

---
 substrate/frame/beefy/src/tests.rs            |  1 -
 .../frame/merkle-mountain-range/src/tests.rs  | 84 ++++++++++++++-----
 .../primitives/consensus/beefy/src/lib.rs     |  1 -
 3 files changed, 64 insertions(+), 22 deletions(-)

diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index 3cc6f8d817f3..e495dc874d0d 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -825,7 +825,6 @@ fn report_fork_equivocation_vote_current_set_works() {
 	ext.persist_offchain_overlay();
 
 	ext.execute_with(|| {
-
 		let validator_set = Beefy::validator_set().unwrap();
 		let authorities = validator_set.validators();
 		let set_id = validator_set.id();
diff --git a/substrate/frame/merkle-mountain-range/src/tests.rs b/substrate/frame/merkle-mountain-range/src/tests.rs
index d3ad0cc721df..af0f33ea3d51 100644
--- a/substrate/frame/merkle-mountain-range/src/tests.rs
+++ b/substrate/frame/merkle-mountain-range/src/tests.rs
@@ -287,9 +287,18 @@ fn should_generate_proofs_correctly() {
 					leaf_indices: vec![0],
 					leaf_count: 7,
 					items: vec![
-						(1, hex("ad4cbc033833612ccd4626d5f023b9dfc50a35e838514dd1f3c86f8506728705")),
-						(5, hex("cb24f4614ad5b2a5430344c99545b421d9af83c46fd632d70a332200884b4d46")),
-						(9, hex("dca421199bdcc55bb773c6b6967e8d16675de69062b52285ca63685241fdf626")),
+						(
+							1,
+							hex("ad4cbc033833612ccd4626d5f023b9dfc50a35e838514dd1f3c86f8506728705")
+						),
+						(
+							5,
+							hex("cb24f4614ad5b2a5430344c99545b421d9af83c46fd632d70a332200884b4d46")
+						),
+						(
+							9,
+							hex("dca421199bdcc55bb773c6b6967e8d16675de69062b52285ca63685241fdf626")
+						),
 					],
 				}
 			)
@@ -318,9 +327,18 @@ fn should_generate_proofs_correctly() {
 					leaf_indices: vec![2],
 					leaf_count: 7,
 					items: vec![
-						(2, hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854")),
-						(4, hex("1b14c1dc7d3e4def11acdf31be0584f4b85c3673f1ff72a3af467b69a3b0d9d0")),
-						(9, hex("dca421199bdcc55bb773c6b6967e8d16675de69062b52285ca63685241fdf626")),
+						(
+							2,
+							hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854")
+						),
+						(
+							4,
+							hex("1b14c1dc7d3e4def11acdf31be0584f4b85c3673f1ff72a3af467b69a3b0d9d0")
+						),
+						(
+							9,
+							hex("dca421199bdcc55bb773c6b6967e8d16675de69062b52285ca63685241fdf626")
+						),
 					],
 				}
 			)
@@ -337,9 +355,10 @@ fn should_generate_proofs_correctly() {
 				Proof {
 					leaf_indices: vec![2],
 					leaf_count: 3,
-					items: vec![(2, hex(
-						"672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854"
-					))],
+					items: vec![(
+						2,
+						hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854")
+					)],
 				}
 			)
 		);
@@ -358,9 +377,18 @@ fn should_generate_proofs_correctly() {
 					leaf_indices: vec![2],
 					leaf_count: 5,
 					items: vec![
-						(2, hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854")),
-						(4, hex("1b14c1dc7d3e4def11acdf31be0584f4b85c3673f1ff72a3af467b69a3b0d9d0")),
-						(7, hex("3b031d22e24f1126c8f7d2f394b663f9b960ed7abbedb7152e17ce16112656d0")),
+						(
+							2,
+							hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854")
+						),
+						(
+							4,
+							hex("1b14c1dc7d3e4def11acdf31be0584f4b85c3673f1ff72a3af467b69a3b0d9d0")
+						),
+						(
+							7,
+							hex("3b031d22e24f1126c8f7d2f394b663f9b960ed7abbedb7152e17ce16112656d0")
+						),
 					],
 				}
 			)
@@ -376,9 +404,18 @@ fn should_generate_proofs_correctly() {
 					leaf_indices: vec![4],
 					leaf_count: 7,
 					items: vec![
-						(6, hex("ae88a0825da50e953e7a359c55fe13c8015e48d03d301b8bdfc9193874da9252")),
-						(8, hex("8ed25570209d8f753d02df07c1884ddb36a3d9d4770e4608b188322151c657fe")),
-						(10, hex("611c2174c6164952a66d985cfe1ec1a623794393e3acff96b136d198f37a648c")),
+						(
+							6,
+							hex("ae88a0825da50e953e7a359c55fe13c8015e48d03d301b8bdfc9193874da9252")
+						),
+						(
+							8,
+							hex("8ed25570209d8f753d02df07c1884ddb36a3d9d4770e4608b188322151c657fe")
+						),
+						(
+							10,
+							hex("611c2174c6164952a66d985cfe1ec1a623794393e3acff96b136d198f37a648c")
+						),
 					],
 				}
 			)
@@ -390,9 +427,10 @@ fn should_generate_proofs_correctly() {
 				Proof {
 					leaf_indices: vec![4],
 					leaf_count: 5,
-					items: vec![(6, hex(
-						"ae88a0825da50e953e7a359c55fe13c8015e48d03d301b8bdfc9193874da9252"
-					)),],
+					items: vec![(
+						6,
+						hex("ae88a0825da50e953e7a359c55fe13c8015e48d03d301b8bdfc9193874da9252")
+					),],
 				}
 			)
 		);
@@ -406,8 +444,14 @@ fn should_generate_proofs_correctly() {
 					leaf_indices: vec![6],
 					leaf_count: 7,
 					items: vec![
-						(6, hex("ae88a0825da50e953e7a359c55fe13c8015e48d03d301b8bdfc9193874da9252")),
-						(9, hex("7e4316ae2ebf7c3b6821cb3a46ca8b7a4f9351a9b40fcf014bb0a4fd8e8f29da")),
+						(
+							6,
+							hex("ae88a0825da50e953e7a359c55fe13c8015e48d03d301b8bdfc9193874da9252")
+						),
+						(
+							9,
+							hex("7e4316ae2ebf7c3b6821cb3a46ca8b7a4f9351a9b40fcf014bb0a4fd8e8f29da")
+						),
 					],
 				}
 			)
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index c16e5d635753..b53b7628b6fc 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -404,7 +404,6 @@ where
 		}
 	}
 
-
 	if correct_header.hash() != *expected_header_hash {
 		return false
 	}

From 70b970badad2d8cfaff7a570b86182bea2417e5d Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 2 Nov 2023 11:16:22 +0100
Subject: [PATCH 076/188] require at least header OR ancestry proof, not both

---
 .../beefy/src/communication/fisherman.rs      | 33 ++++---
 .../client/consensus/beefy/src/worker.rs      | 12 +--
 substrate/frame/beefy/src/tests.rs            | 88 +++++++++----------
 .../primitives/consensus/beefy/src/lib.rs     | 37 +++++---
 .../consensus/beefy/src/test_utils.rs         |  8 +-
 5 files changed, 102 insertions(+), 76 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 8d8f0f55e008..2b332efc5381 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -97,11 +97,11 @@ where
 
 	fn active_validator_set_at(
 		&self,
-		header: &B::Header,
+		block_hash: <<B as Block>::Header as Header>::Hash,
 	) -> Result<ValidatorSet<AuthorityId>, Error> {
 		self.runtime
 			.runtime_api()
-			.validator_set(header.hash())
+			.validator_set(block_hash)
 			.map_err(Error::RuntimeApi)?
 			.ok_or_else(|| Error::Backend("could not get BEEFY validator set".into()))
 	}
@@ -110,7 +110,12 @@ where
 		&self,
 		proof: ForkEquivocationProof<NumberFor<B>, AuthorityId, Signature, B::Header, MmrRootHash>,
 	) -> Result<(), Error> {
-		let validator_set = self.active_validator_set_at(&proof.correct_header)?;
+		let prev_hash = self
+			.backend
+			.blockchain()
+			.hash(proof.commitment.block_number)
+			.map_err(|e| Error::Backend(e.to_string()))?;
+		let validator_set = self.active_validator_set_at(prev_hash.unwrap())?;
 		let set_id = validator_set.id();
 
 		let expected_header_hash = self
@@ -168,7 +173,6 @@ where
 			}
 		}
 
-		let hash = proof.correct_header.hash();
 		let runtime_api = self.runtime.runtime_api();
 
 		// generate key ownership proof at that block
@@ -176,7 +180,11 @@ where
 			.iter()
 			.cloned()
 			.filter_map(|id| {
-				match runtime_api.generate_key_ownership_proof(hash, set_id, id.clone()) {
+				match runtime_api.generate_key_ownership_proof(
+					prev_hash.unwrap(),
+					set_id,
+					id.clone(),
+				) {
 					Ok(Some(proof)) => Some(Ok(proof)),
 					Ok(None) => {
 						debug!(
@@ -229,8 +237,8 @@ where
 			let proof = ForkEquivocationProof {
 				commitment: vote.commitment,
 				signatories: vec![(vote.id, vote.signature)],
-				correct_header: correct_header.clone(),
-				ancestry_proof,
+				correct_header: Some(correct_header.clone()),
+				ancestry_proof: Some(ancestry_proof),
 			};
 			self.report_fork_equivocation(proof)?;
 		}
@@ -244,6 +252,11 @@ where
 	) -> Result<(), Error> {
 		let SignedCommitment { commitment, signatures } = signed_commitment;
 		let number = commitment.block_number;
+		let prev_hash = self
+			.backend
+			.blockchain()
+			.hash(number)
+			.map_err(|e| Error::Backend(e.to_string()))?;
 		let (correct_header, expected_payload) = self.expected_header_and_payload(number)?;
 		if commitment.payload != expected_payload {
 			let ancestry_proof = self
@@ -252,7 +265,7 @@ where
 				.generate_ancestry_proof(correct_header.hash(), number, None)
 				.unwrap()
 				.unwrap();
-			let validator_set = self.active_validator_set_at(&correct_header)?;
+			let validator_set = self.active_validator_set_at(prev_hash.unwrap())?;
 			if signatures.len() != validator_set.validators().len() {
 				// invalid proof
 				return Ok(())
@@ -269,8 +282,8 @@ where
 			let proof = ForkEquivocationProof {
 				commitment,
 				signatories,
-				correct_header: correct_header.clone(),
-				ancestry_proof,
+				correct_header: Some(correct_header.clone()),
+				ancestry_proof: Some(ancestry_proof),
 			};
 			self.report_fork_equivocation(proof)?;
 		}
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index 77a0d7e72376..083c5688bdaf 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -1745,8 +1745,8 @@ pub(crate) mod tests {
 			.unwrap();
 		let proof = generate_fork_equivocation_proof_vote(
 			votes[1].clone(),
-			header.clone(),
-			ancestry_proof.clone(),
+			Some(header.clone()),
+			Some(ancestry_proof.clone()),
 		);
 		{
 			// expect fisher (Alice) to successfully process it
@@ -1768,8 +1768,8 @@ pub(crate) mod tests {
 		// verify: Alice does not self-report
 		let proof = generate_fork_equivocation_proof_vote(
 			votes[0].clone(),
-			header.clone(),
-			ancestry_proof.clone(),
+			Some(header.clone()),
+			Some(ancestry_proof.clone()),
 		);
 		{
 			// expect fisher (Alice) to successfully process it
@@ -1798,8 +1798,8 @@ pub(crate) mod tests {
 		let proof = generate_fork_equivocation_proof_sc(
 			commitment,
 			vec![Keyring::Bob, Keyring::Charlie],
-			header,
-			ancestry_proof,
+			Some(header),
+			Some(ancestry_proof),
 		);
 		{
 			// expect fisher (Alice) to successfully process it
diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index e495dc874d0d..a1cbd0fd9183 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -853,8 +853,8 @@ fn report_fork_equivocation_vote_current_set_works() {
 		// different payload than finalized
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
-			header,
-			ancestry_proof,
+			Some(header),
+			Some(ancestry_proof),
 		);
 
 		// create the key ownership proof
@@ -973,8 +973,8 @@ fn report_fork_equivocation_vote_old_set_works() {
 		// different payload than finalized
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, old_set_id, &equivocation_keyring),
-			header,
-			ancestry_proof,
+			Some(header),
+			Some(ancestry_proof),
 		);
 
 		// report the equivocation and the tx should be dispatched successfully
@@ -1051,8 +1051,8 @@ fn report_fork_equivocation_vote_invalid_set_id() {
 		// generate an equivocation for a future set
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id + 1, &equivocation_keyring),
-			header,
-			ancestry_proof,
+			Some(header),
+			Some(ancestry_proof),
 		);
 
 		// the call for reporting the equivocation should error
@@ -1109,8 +1109,8 @@ fn report_fork_equivocation_vote_invalid_session() {
 		// generate an equivocation proof at following era set id = 3
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
-			header,
-			ancestry_proof,
+			Some(header),
+			Some(ancestry_proof),
 		);
 
 		// report an equivocation for the current set using an key ownership
@@ -1168,8 +1168,8 @@ fn report_fork_equivocation_vote_invalid_key_owner_proof() {
 		// generate an equivocation for a future set
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id + 1, &equivocation_keyring),
-			header,
-			ancestry_proof,
+			Some(header),
+			Some(ancestry_proof),
 		);
 
 		// we need to start a new era otherwise the key ownership proof won't be
@@ -1230,8 +1230,8 @@ fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 		// vote targets different round than finalized payload, there is no equivocation.
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num + 1, payload.clone(), set_id, &equivocation_keyring),
-			header.clone(),
-			ancestry_proof.clone(),
+			Some(header.clone()),
+			Some(ancestry_proof.clone()),
 		);
 		assert_err!(
 			Beefy::report_fork_equivocation_unsigned(
@@ -1245,8 +1245,8 @@ fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 		// vote signed with a key that isn't part of the authority set
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload.clone(), set_id, &BeefyKeyring::Dave),
-			header.clone(),
-			ancestry_proof.clone(),
+			Some(header.clone()),
+			Some(ancestry_proof.clone()),
 		);
 		assert_err!(
 			Beefy::report_fork_equivocation_unsigned(
@@ -1260,8 +1260,8 @@ fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 		// vote targets future set id
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload.clone(), set_id + 1, &equivocation_keyring),
-			header.clone(),
-			ancestry_proof,
+			Some(header.clone()),
+			Some(ancestry_proof),
 		);
 		assert_err!(
 			Beefy::report_fork_equivocation_unsigned(
@@ -1314,8 +1314,8 @@ fn report_fork_equivocation_vote_validate_unsigned_prevents_duplicates() {
 		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
-			header,
-			ancestry_proof,
+			Some(header),
+			Some(ancestry_proof),
 		);
 
 		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
@@ -1415,8 +1415,8 @@ fn valid_fork_equivocation_vote_reports_dont_pay_fees() {
 		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
-			header,
-			ancestry_proof,
+			Some(header),
+			Some(ancestry_proof),
 		);
 
 		// create the key ownership proof.
@@ -1522,8 +1522,8 @@ fn report_fork_equivocation_sc_current_set_works() {
 		let equivocation_proof = generate_fork_equivocation_proof_sc(
 			commitment,
 			equivocation_keyrings,
-			header,
-			ancestry_proof,
+			Some(header),
+			Some(ancestry_proof),
 		);
 
 		// create the key ownership proof
@@ -1665,8 +1665,8 @@ fn report_fork_equivocation_sc_old_set_works() {
 		let equivocation_proof = generate_fork_equivocation_proof_sc(
 			commitment,
 			equivocation_keyrings,
-			header,
-			ancestry_proof,
+			Some(header),
+			Some(ancestry_proof),
 		);
 
 		// report the equivocation and the tx should be dispatched successfully
@@ -1760,8 +1760,8 @@ fn report_fork_equivocation_sc_invalid_set_id() {
 		let equivocation_proof = generate_fork_equivocation_proof_sc(
 			commitment,
 			equivocation_keyrings,
-			header,
-			ancestry_proof,
+			Some(header),
+			Some(ancestry_proof),
 		);
 
 		// the call for reporting the equivocation should error
@@ -1829,8 +1829,8 @@ fn report_fork_equivocation_sc_invalid_session() {
 		let equivocation_proof = generate_fork_equivocation_proof_sc(
 			commitment,
 			equivocation_keyrings,
-			header,
-			ancestry_proof,
+			Some(header),
+			Some(ancestry_proof),
 		);
 
 		// report an equivocation for the current set using an key ownership
@@ -1901,8 +1901,8 @@ fn report_fork_equivocation_sc_invalid_key_owner_proof() {
 		let equivocation_proof = generate_fork_equivocation_proof_sc(
 			commitment,
 			equivocation_keyrings,
-			header,
-			ancestry_proof,
+			Some(header),
+			Some(ancestry_proof),
 		);
 
 		// we need to start a new era otherwise the key ownership proof won't be
@@ -1978,8 +1978,8 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 				payload: payload.clone(),
 			},
 			equivocation_keyrings.clone(),
-			header.clone(),
-			ancestry_proof.clone(),
+			Some(header.clone()),
+			Some(ancestry_proof.clone()),
 		);
 		assert_err!(
 			Beefy::report_fork_equivocation_unsigned(
@@ -1998,8 +1998,8 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 				payload: payload.clone(),
 			},
 			vec![BeefyKeyring::Eve],
-			header.clone(),
-			ancestry_proof.clone(),
+			Some(header.clone()),
+			Some(ancestry_proof.clone()),
 		);
 		assert_err!(
 			Beefy::report_fork_equivocation_unsigned(
@@ -2018,8 +2018,8 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 				payload: payload.clone(),
 			},
 			equivocation_keyrings,
-			header,
-			ancestry_proof,
+			Some(header),
+			Some(ancestry_proof),
 		);
 		assert_err!(
 			Beefy::report_fork_equivocation_unsigned(
@@ -2072,8 +2072,8 @@ fn report_fork_equivocation_sc_validate_unsigned_prevents_duplicates() {
 		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
-			header,
-			ancestry_proof,
+			Some(header),
+			Some(ancestry_proof),
 		);
 
 		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
@@ -2173,8 +2173,8 @@ fn valid_fork_equivocation_sc_reports_dont_pay_fees() {
 		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
-			header,
-			ancestry_proof,
+			Some(header),
+			Some(ancestry_proof),
 		);
 
 		// create the key ownership proof.
@@ -2288,8 +2288,8 @@ fn report_fork_equivocation_sc_stacked_reports_stack_correctly() {
 		let equivocation_proof_singleton = generate_fork_equivocation_proof_sc(
 			commitment.clone(),
 			vec![equivocation_keyrings[0]],
-			header.clone(),
-			ancestry_proof.clone(),
+			Some(header.clone()),
+			Some(ancestry_proof.clone()),
 		);
 
 		// create the key ownership proof
@@ -2321,8 +2321,8 @@ fn report_fork_equivocation_sc_stacked_reports_stack_correctly() {
 		let equivocation_proof_full = generate_fork_equivocation_proof_sc(
 			commitment,
 			equivocation_keyrings,
-			header,
-			ancestry_proof,
+			Some(header),
+			Some(ancestry_proof),
 		);
 
 		// check that the balance of the reported equivocating validator is slashed 100%.
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index b53b7628b6fc..af40ba612128 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -266,9 +266,9 @@ pub struct ForkEquivocationProof<Number, Id, Signature, Header, Hash> {
 	/// 1. the header is in our chain
 	/// 2. its digest's payload != commitment.payload
 	/// 3. commitment is signed by signatories
-	pub correct_header: Header,
+	pub correct_header: Option<Header>,
 	/// ancestry proof showing mmr root
-	pub ancestry_proof: AncestryProof<Hash>,
+	pub ancestry_proof: Option<AncestryProof<Hash>>,
 }
 
 impl<Number, Id, Signature, H: Header, Hash> ForkEquivocationProof<Number, Id, Signature, H, Hash> {
@@ -374,6 +374,7 @@ where
 	// this can be inferred from the leaf_count / mmr_size of the prev_root:
 	// convert the commitment.block_number to an mmr size and compare with the value in the ancestry
 	// proof
+	let mut ancestry_prev_root = Err(mmr_lib::Error::CorruptedProof);
 	{
 		let expected_leaf_count = sp_mmr_primitives::utils::block_num_to_leaf_index::<Header>(
 			commitment.block_number,
@@ -384,10 +385,18 @@ where
 				sp_mmr_primitives::Error::InvalidNumericOp.log_debug("leaf_index + 1 overflowed")
 			})
 		});
+
+		if (correct_header, ancestry_proof) == (&None, &None) {
+			// at least a header or ancestry proof must be provided
+			return false
+		}
+
 		// if the block number either under- or overflowed, the commitment.block_number was not
 		// valid and the commitment should not have been signed, hence we can skip the ancestry
 		// proof and slash the signatories
-		if let Ok(expected_leaf_count) = expected_leaf_count {
+		if let (Ok(expected_leaf_count), Some(ancestry_proof)) =
+			(expected_leaf_count, ancestry_proof)
+		{
 			let expected_mmr_size =
 				sp_mmr_primitives::utils::NodesUtils::new(expected_leaf_count).size();
 			if expected_mmr_size != ancestry_proof.prev_size {
@@ -401,19 +410,23 @@ where
 				) {
 				return false
 			}
+			ancestry_prev_root = mmr_lib::bagging_peaks_hashes::<NodeHash, Hasher>(
+				ancestry_proof.prev_peaks.clone(),
+			);
 		}
 	}
 
-	if correct_header.hash() != *expected_header_hash {
-		return false
-	}
-
-	let expected_mmr_root_digest = mmr::find_mmr_root_digest::<Header>(correct_header);
-	let expected_payload = expected_mmr_root_digest
-		.map(|mmr_root| Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode()));
+	let mut expected_payload: Option<_> = None;
+	if let Some(correct_header) = correct_header {
+		if correct_header.hash() != *expected_header_hash {
+			return false
+		}
 
-	let ancestry_prev_root =
-		mmr_lib::bagging_peaks_hashes::<NodeHash, Hasher>(ancestry_proof.prev_peaks.clone());
+		let expected_mmr_root_digest = mmr::find_mmr_root_digest::<Header>(correct_header);
+		expected_payload = expected_mmr_root_digest.map(|mmr_root| {
+			Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode())
+		});
+	}
 
 	// if the commitment payload does not commit to an MMR root, then this commitment may have
 	// another purpose and should not be slashed
diff --git a/substrate/primitives/consensus/beefy/src/test_utils.rs b/substrate/primitives/consensus/beefy/src/test_utils.rs
index 4682186d4415..c0989147a601 100644
--- a/substrate/primitives/consensus/beefy/src/test_utils.rs
+++ b/substrate/primitives/consensus/beefy/src/test_utils.rs
@@ -120,8 +120,8 @@ pub fn generate_vote_equivocation_proof(
 /// Create a new `ForkEquivocationProof` based on vote & correct header.
 pub fn generate_fork_equivocation_proof_vote<Header, Hash>(
 	vote: (u64, Payload, ValidatorSetId, &Keyring),
-	correct_header: Header,
-	ancestry_proof: AncestryProof<Hash>,
+	correct_header: Option<Header>,
+	ancestry_proof: Option<AncestryProof<Hash>>,
 ) -> ForkEquivocationProof<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature, Header, Hash> {
 	let signed_vote = signed_vote(vote.0, vote.1, vote.2, vote.3);
 	let signatories = vec![(signed_vote.id, signed_vote.signature)];
@@ -137,8 +137,8 @@ pub fn generate_fork_equivocation_proof_vote<Header, Hash>(
 pub fn generate_fork_equivocation_proof_sc<Header, Hash>(
 	commitment: Commitment<u64>,
 	keyrings: Vec<Keyring>,
-	correct_header: Header,
-	ancestry_proof: AncestryProof<Hash>,
+	correct_header: Option<Header>,
+	ancestry_proof: Option<AncestryProof<Hash>>,
 ) -> ForkEquivocationProof<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature, Header, Hash> {
 	let signatories = keyrings
 		.into_iter()

From 04db083443e71381ff501cd1444fc2f377f09506 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 2 Nov 2023 12:04:31 +0100
Subject: [PATCH 077/188] remove headers from equiv. proofs in frame tests

They're `None` anyway in the mock runtime impl.
---
 substrate/frame/beefy/src/tests.rs | 160 +++++++++++++----------------
 1 file changed, 69 insertions(+), 91 deletions(-)

diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index a1cbd0fd9183..1e7c1051e39b 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -811,16 +811,15 @@ fn report_fork_equivocation_vote_current_set_works() {
 	ext.register_extension(OffchainWorkerExt::new(offchain));
 
 	let mut era = 1;
-	let (block_num, header) = ext.execute_with(|| {
+	let block_num = ext.execute_with(|| {
 		assert_eq!(Staking::current_era(), Some(0));
 		assert_eq!(Session::current_index(), 0);
 		start_era(era);
 
 		let block_num = System::block_number();
-		let header = System::finalize();
 		era += 1;
 		start_era(era);
-		(block_num, header)
+		block_num
 	});
 	ext.persist_offchain_overlay();
 
@@ -853,7 +852,7 @@ fn report_fork_equivocation_vote_current_set_works() {
 		// different payload than finalized
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
-			Some(header),
+			None,
 			Some(ancestry_proof),
 		);
 
@@ -909,7 +908,6 @@ fn report_fork_equivocation_vote_old_set_works() {
 	let mut era = 1;
 	let (
 		block_num,
-		header,
 		validators,
 		old_set_id,
 		equivocation_authority_index,
@@ -918,7 +916,6 @@ fn report_fork_equivocation_vote_old_set_works() {
 	) = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
-		let header = System::finalize();
 		era += 1;
 		start_era(era);
 
@@ -938,7 +935,6 @@ fn report_fork_equivocation_vote_old_set_works() {
 		start_era(era);
 		(
 			block_num,
-			header,
 			validators,
 			old_set_id,
 			equivocation_authority_index,
@@ -973,7 +969,7 @@ fn report_fork_equivocation_vote_old_set_works() {
 		// different payload than finalized
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, old_set_id, &equivocation_keyring),
-			Some(header),
+			None,
 			Some(ancestry_proof),
 		);
 
@@ -1023,15 +1019,14 @@ fn report_fork_equivocation_vote_invalid_set_id() {
 	ext.register_extension(OffchainDbExt::new(offchain.clone()));
 	ext.register_extension(OffchainWorkerExt::new(offchain));
 
-	let (block_num, header) = ext.execute_with(|| {
+	let block_num = ext.execute_with(|| {
 		let mut era = 1;
 		start_era(era);
 		let block_num = System::block_number();
-		let header = System::finalize();
 
 		era += 1;
 		start_era(era);
-		(block_num, header)
+		block_num
 	});
 	ext.persist_offchain_overlay();
 
@@ -1051,7 +1046,7 @@ fn report_fork_equivocation_vote_invalid_set_id() {
 		// generate an equivocation for a future set
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id + 1, &equivocation_keyring),
-			Some(header),
+			None,
 			Some(ancestry_proof),
 		);
 
@@ -1077,10 +1072,9 @@ fn report_fork_equivocation_vote_invalid_session() {
 	ext.register_extension(OffchainWorkerExt::new(offchain));
 
 	let mut era = 1;
-	let (block_num, header, equivocation_keyring, key_owner_proof) = ext.execute_with(|| {
+	let (block_num, equivocation_keyring, key_owner_proof) = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
-		let header = System::finalize();
 
 		era += 1;
 		start_era(era);
@@ -1097,7 +1091,7 @@ fn report_fork_equivocation_vote_invalid_session() {
 
 		era += 1;
 		start_era(era);
-		(block_num, header, equivocation_keyring, key_owner_proof)
+		(block_num, equivocation_keyring, key_owner_proof)
 	});
 	ext.persist_offchain_overlay();
 
@@ -1109,7 +1103,7 @@ fn report_fork_equivocation_vote_invalid_session() {
 		// generate an equivocation proof at following era set id = 3
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
-			Some(header),
+			None,
 			Some(ancestry_proof),
 		);
 
@@ -1136,14 +1130,13 @@ fn report_fork_equivocation_vote_invalid_key_owner_proof() {
 	ext.register_extension(OffchainWorkerExt::new(offchain));
 
 	let mut era = 1;
-	let (block_num, header) = ext.execute_with(|| {
+	let block_num = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
-		let header = System::finalize();
 
 		era += 1;
 		start_era(era);
-		(block_num, header)
+		block_num
 	});
 	ext.persist_offchain_overlay();
 
@@ -1168,7 +1161,7 @@ fn report_fork_equivocation_vote_invalid_key_owner_proof() {
 		// generate an equivocation for a future set
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id + 1, &equivocation_keyring),
-			Some(header),
+			None,
 			Some(ancestry_proof),
 		);
 
@@ -1200,27 +1193,25 @@ fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 	ext.register_extension(OffchainWorkerExt::new(offchain));
 
 	let mut era = 1;
-	let (block_num, header, set_id, equivocation_keyring, key_owner_proof) =
-		ext.execute_with(|| {
-			start_era(era);
-			let block_num = System::block_number();
-			let header = System::finalize();
-
-			let validator_set = Beefy::validator_set().unwrap();
-			let authorities = validator_set.validators();
-			let set_id = validator_set.id();
-
-			let equivocation_authority_index = 0;
-			let equivocation_key = &authorities[equivocation_authority_index];
-			let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
-
-			// generate a key ownership proof at set id in era 1
-			let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
-
-			era += 1;
-			start_era(era);
-			(block_num, header, set_id, equivocation_keyring, key_owner_proof)
-		});
+	let (block_num, set_id, equivocation_keyring, key_owner_proof) = ext.execute_with(|| {
+		start_era(era);
+		let block_num = System::block_number();
+
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+		let set_id = validator_set.id();
+
+		let equivocation_authority_index = 0;
+		let equivocation_key = &authorities[equivocation_authority_index];
+		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+
+		// generate a key ownership proof at set id in era 1
+		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+
+		era += 1;
+		start_era(era);
+		(block_num, set_id, equivocation_keyring, key_owner_proof)
+	});
 	ext.persist_offchain_overlay();
 
 	ext.execute_with(|| {
@@ -1230,7 +1221,7 @@ fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 		// vote targets different round than finalized payload, there is no equivocation.
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num + 1, payload.clone(), set_id, &equivocation_keyring),
-			Some(header.clone()),
+			None,
 			Some(ancestry_proof.clone()),
 		);
 		assert_err!(
@@ -1245,7 +1236,7 @@ fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 		// vote signed with a key that isn't part of the authority set
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload.clone(), set_id, &BeefyKeyring::Dave),
-			Some(header.clone()),
+			None,
 			Some(ancestry_proof.clone()),
 		);
 		assert_err!(
@@ -1260,7 +1251,7 @@ fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 		// vote targets future set id
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload.clone(), set_id + 1, &equivocation_keyring),
-			Some(header.clone()),
+			None,
 			Some(ancestry_proof),
 		);
 		assert_err!(
@@ -1289,14 +1280,13 @@ fn report_fork_equivocation_vote_validate_unsigned_prevents_duplicates() {
 	ext.register_extension(OffchainWorkerExt::new(offchain));
 
 	let mut era = 1;
-	let (block_num, header) = ext.execute_with(|| {
+	let block_num = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
-		let header = System::finalize();
 
 		era += 1;
 		start_era(era);
-		(block_num, header)
+		block_num
 	});
 	ext.persist_offchain_overlay();
 
@@ -1314,7 +1304,7 @@ fn report_fork_equivocation_vote_validate_unsigned_prevents_duplicates() {
 		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
-			Some(header),
+			None,
 			Some(ancestry_proof),
 		);
 
@@ -1390,14 +1380,13 @@ fn valid_fork_equivocation_vote_reports_dont_pay_fees() {
 	ext.register_extension(OffchainWorkerExt::new(offchain));
 
 	let mut era = 1;
-	let (block_num, header) = ext.execute_with(|| {
+	let block_num = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
-		let header = System::finalize();
 
 		era += 1;
 		start_era(era);
-		(block_num, header)
+		block_num
 	});
 	ext.persist_offchain_overlay();
 
@@ -1415,7 +1404,7 @@ fn valid_fork_equivocation_vote_reports_dont_pay_fees() {
 		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
-			Some(header),
+			None,
 			Some(ancestry_proof),
 		);
 
@@ -1475,14 +1464,13 @@ fn report_fork_equivocation_sc_current_set_works() {
 	ext.register_extension(OffchainWorkerExt::new(offchain));
 
 	let mut era = 1;
-	let (block_num, header) = ext.execute_with(|| {
+	let block_num = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
-		let header = System::finalize();
 
 		era += 1;
 		start_era(era);
-		(block_num, header)
+		block_num
 	});
 	ext.persist_offchain_overlay();
 
@@ -1522,7 +1510,7 @@ fn report_fork_equivocation_sc_current_set_works() {
 		let equivocation_proof = generate_fork_equivocation_proof_sc(
 			commitment,
 			equivocation_keyrings,
-			Some(header),
+			None,
 			Some(ancestry_proof),
 		);
 
@@ -1586,7 +1574,6 @@ fn report_fork_equivocation_sc_old_set_works() {
 	let mut era = 1;
 	let (
 		block_num,
-		header,
 		validators,
 		old_set_id,
 		equivocation_authority_indices,
@@ -1595,7 +1582,6 @@ fn report_fork_equivocation_sc_old_set_works() {
 	) = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
-		let header = System::finalize();
 
 		era += 1;
 		start_era(era);
@@ -1623,7 +1609,6 @@ fn report_fork_equivocation_sc_old_set_works() {
 
 		(
 			block_num,
-			header,
 			validators,
 			old_set_id,
 			equivocation_authority_indices,
@@ -1665,7 +1650,7 @@ fn report_fork_equivocation_sc_old_set_works() {
 		let equivocation_proof = generate_fork_equivocation_proof_sc(
 			commitment,
 			equivocation_keyrings,
-			Some(header),
+			None,
 			Some(ancestry_proof),
 		);
 
@@ -1721,14 +1706,13 @@ fn report_fork_equivocation_sc_invalid_set_id() {
 	ext.register_extension(OffchainWorkerExt::new(offchain));
 
 	let mut era = 1;
-	let (block_num, header) = ext.execute_with(|| {
+	let block_num = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
-		let header = System::finalize();
 
 		era += 1;
 		start_era(era);
-		(block_num, header)
+		block_num
 	});
 	ext.persist_offchain_overlay();
 
@@ -1760,7 +1744,7 @@ fn report_fork_equivocation_sc_invalid_set_id() {
 		let equivocation_proof = generate_fork_equivocation_proof_sc(
 			commitment,
 			equivocation_keyrings,
-			Some(header),
+			None,
 			Some(ancestry_proof),
 		);
 
@@ -1786,10 +1770,9 @@ fn report_fork_equivocation_sc_invalid_session() {
 	ext.register_extension(OffchainWorkerExt::new(offchain));
 
 	let mut era = 1;
-	let (block_num, header, equivocation_keyrings, key_owner_proofs) = ext.execute_with(|| {
+	let (block_num, equivocation_keyrings, key_owner_proofs) = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
-		let header = System::finalize();
 
 		era += 1;
 		start_era(era);
@@ -1815,7 +1798,7 @@ fn report_fork_equivocation_sc_invalid_session() {
 
 		era += 1;
 		start_era(era);
-		(block_num, header, equivocation_keyrings, key_owner_proofs)
+		(block_num, equivocation_keyrings, key_owner_proofs)
 	});
 	ext.persist_offchain_overlay();
 
@@ -1829,7 +1812,7 @@ fn report_fork_equivocation_sc_invalid_session() {
 		let equivocation_proof = generate_fork_equivocation_proof_sc(
 			commitment,
 			equivocation_keyrings,
-			Some(header),
+			None,
 			Some(ancestry_proof),
 		);
 
@@ -1856,14 +1839,13 @@ fn report_fork_equivocation_sc_invalid_key_owner_proof() {
 	ext.register_extension(OffchainWorkerExt::new(offchain));
 
 	let mut era = 1;
-	let (block_num, header) = ext.execute_with(|| {
+	let block_num = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
-		let header = System::finalize();
 
 		era += 1;
 		start_era(era);
-		(block_num, header)
+		block_num
 	});
 	ext.persist_offchain_overlay();
 
@@ -1901,7 +1883,7 @@ fn report_fork_equivocation_sc_invalid_key_owner_proof() {
 		let equivocation_proof = generate_fork_equivocation_proof_sc(
 			commitment,
 			equivocation_keyrings,
-			Some(header),
+			None,
 			Some(ancestry_proof),
 		);
 
@@ -1933,14 +1915,13 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 	ext.register_extension(OffchainWorkerExt::new(offchain));
 
 	let mut era = 1;
-	let (block_num, header) = ext.execute_with(|| {
+	let block_num = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
-		let header = System::finalize();
 
 		era += 1;
 		start_era(era);
-		(block_num, header)
+		block_num
 	});
 	ext.persist_offchain_overlay();
 
@@ -1978,7 +1959,7 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 				payload: payload.clone(),
 			},
 			equivocation_keyrings.clone(),
-			Some(header.clone()),
+			None,
 			Some(ancestry_proof.clone()),
 		);
 		assert_err!(
@@ -1998,7 +1979,7 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 				payload: payload.clone(),
 			},
 			vec![BeefyKeyring::Eve],
-			Some(header.clone()),
+			None,
 			Some(ancestry_proof.clone()),
 		);
 		assert_err!(
@@ -2018,7 +1999,7 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 				payload: payload.clone(),
 			},
 			equivocation_keyrings,
-			Some(header),
+			None,
 			Some(ancestry_proof),
 		);
 		assert_err!(
@@ -2047,14 +2028,13 @@ fn report_fork_equivocation_sc_validate_unsigned_prevents_duplicates() {
 	ext.register_extension(OffchainWorkerExt::new(offchain));
 
 	let mut era = 1;
-	let (block_num, header) = ext.execute_with(|| {
+	let block_num = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
-		let header = System::finalize();
 
 		era += 1;
 		start_era(era);
-		(block_num, header)
+		block_num
 	});
 	ext.persist_offchain_overlay();
 
@@ -2072,7 +2052,7 @@ fn report_fork_equivocation_sc_validate_unsigned_prevents_duplicates() {
 		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
-			Some(header),
+			None,
 			Some(ancestry_proof),
 		);
 
@@ -2148,14 +2128,13 @@ fn valid_fork_equivocation_sc_reports_dont_pay_fees() {
 	ext.register_extension(OffchainWorkerExt::new(offchain));
 
 	let mut era = 1;
-	let (block_num, header) = ext.execute_with(|| {
+	let block_num = ext.execute_with(|| {
 		start_era(era);
 		let block_num = System::block_number();
-		let header = System::finalize();
 
 		era += 1;
 		start_era(era);
-		(block_num, header)
+		block_num
 	});
 	ext.persist_offchain_overlay();
 
@@ -2173,7 +2152,7 @@ fn valid_fork_equivocation_sc_reports_dont_pay_fees() {
 		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
 			(block_num, payload, set_id, &equivocation_keyring),
-			Some(header),
+			None,
 			Some(ancestry_proof),
 		);
 
@@ -2231,16 +2210,15 @@ fn report_fork_equivocation_sc_stacked_reports_stack_correctly() {
 	ext.register_extension(OffchainWorkerExt::new(offchain));
 
 	let mut era = 1;
-	let (block_num, header) = ext.execute_with(|| {
+	let block_num = ext.execute_with(|| {
 		assert_eq!(Staking::current_era(), Some(0));
 		assert_eq!(Session::current_index(), 0);
 		start_era(era);
 		let block_num = System::block_number();
-		let header = System::finalize();
 
 		era += 1;
 		start_era(era);
-		(block_num, header)
+		block_num
 	});
 	ext.persist_offchain_overlay();
 
@@ -2288,7 +2266,7 @@ fn report_fork_equivocation_sc_stacked_reports_stack_correctly() {
 		let equivocation_proof_singleton = generate_fork_equivocation_proof_sc(
 			commitment.clone(),
 			vec![equivocation_keyrings[0]],
-			Some(header.clone()),
+			None,
 			Some(ancestry_proof.clone()),
 		);
 
@@ -2321,7 +2299,7 @@ fn report_fork_equivocation_sc_stacked_reports_stack_correctly() {
 		let equivocation_proof_full = generate_fork_equivocation_proof_sc(
 			commitment,
 			equivocation_keyrings,
-			Some(header),
+			None,
 			Some(ancestry_proof),
 		);
 

From 933cb648eec4da4e39a13380ee8d691b7b87b2fa Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Wed, 8 Nov 2023 12:45:06 +0100
Subject: [PATCH 078/188] refactor fisherman: cleanup

---
 .../beefy/src/communication/fisherman.rs      | 110 ++++++++++--------
 1 file changed, 59 insertions(+), 51 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 2b332efc5381..d1937dfa5823 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -74,10 +74,10 @@ where
 	R: ProvideRuntimeApi<B> + Send + Sync,
 	R::Api: BeefyApi<B, AuthorityId, MmrRootHash> + MmrApi<B, MmrRootHash, NumberFor<B>>,
 {
-	fn expected_header_and_payload(
+	fn expected_hash_header_payload_tuple(
 		&self,
 		number: NumberFor<B>,
-	) -> Result<(B::Header, Payload), Error> {
+	) -> Result<(B::Hash, B::Header, Payload), Error> {
 		// This should be un-ambiguous since `number` is finalized.
 		let hash = self
 			.backend
@@ -91,7 +91,7 @@ where
 			.map_err(|e| Error::Backend(e.to_string()))?;
 		self.payload_provider
 			.payload(&header)
-			.map(|payload| (header, payload))
+			.map(|payload| (hash, header, payload))
 			.ok_or_else(|| Error::Backend("BEEFY Payload not found".into()))
 	}
 
@@ -110,40 +110,38 @@ where
 		&self,
 		proof: ForkEquivocationProof<NumberFor<B>, AuthorityId, Signature, B::Header, MmrRootHash>,
 	) -> Result<(), Error> {
-		let prev_hash = self
-			.backend
-			.blockchain()
-			.hash(proof.commitment.block_number)
-			.map_err(|e| Error::Backend(e.to_string()))?;
-		let validator_set = self.active_validator_set_at(prev_hash.unwrap())?;
-		let set_id = validator_set.id();
-
-		let expected_header_hash = self
+		let expected_prev_hash = self
 			.backend
 			.blockchain()
 			.expect_block_hash_from_id(&BlockId::Number(proof.commitment.block_number))
 			.map_err(|e| Error::Backend(e.to_string()))?;
 
-		let best_hash = self.backend.blockchain().info().best_hash;
+		let validator_set = self.active_validator_set_at(expected_prev_hash)?;
+		let set_id = validator_set.id();
 
-		let expected_mmr_root =
-			self.runtime.runtime_api().mmr_root(best_hash).map_err(Error::RuntimeApi)?;
+		let best_hash = self.backend.blockchain().info().best_hash;
+		let best_mmr_root = self
+			.runtime
+			.runtime_api()
+			.mmr_root(best_hash)
+			.map_err(|e| Error::RuntimeApi(e))?
+			.map_err(|e| Error::Backend(e.to_string()))?;
 
+		// if this errors, mmr has not been instantiated yet, hence the pallet is not active yet and
+		// we should not report equivocations
 		let leaf_count = self
 			.runtime
 			.runtime_api()
 			.mmr_leaf_count(best_hash)
-			.map_err(Error::RuntimeApi)?;
+			.map_err(|e| Error::RuntimeApi(e))?
+			.map_err(|e| Error::Backend(e.to_string()))?;
 
 		// TODO: if ancestry proof can't be constructed, report equivocation nonetheless if valid
 		// header proof can be provided
 		let first_mmr_block_num = {
 			let best_block_num = self.backend.blockchain().info().best_number;
-			sp_mmr_primitives::utils::first_mmr_block_num::<B::Header>(
-				best_block_num,
-				*leaf_count.as_ref().unwrap(),
-			)
-			.map_err(|e| Error::Backend(e.to_string()))?
+			sp_mmr_primitives::utils::first_mmr_block_num::<B::Header>(best_block_num, leaf_count)
+				.map_err(|e| Error::Backend(e.to_string()))?
 		};
 
 		if proof.commitment.validator_set_id != set_id ||
@@ -153,13 +151,8 @@ where
 				B::Header,
 				MmrRootHash,
 				sp_mmr_primitives::utils::AncestryHasher<MmrHashing>,
-			>(
-				&proof,
-				expected_mmr_root.unwrap(),
-				leaf_count.unwrap(),
-				&expected_header_hash,
-				first_mmr_block_num,
-			) {
+			>(&proof, best_mmr_root, leaf_count, &expected_prev_hash, first_mmr_block_num)
+		{
 			debug!(target: LOG_TARGET, "🥩 Skip report for bad invalid fork proof {:?}", proof);
 			return Ok(())
 		}
@@ -181,7 +174,7 @@ where
 			.cloned()
 			.filter_map(|id| {
 				match runtime_api.generate_key_ownership_proof(
-					prev_hash.unwrap(),
+					expected_prev_hash,
 					set_id,
 					id.clone(),
 				) {
@@ -226,19 +219,29 @@ where
 		vote: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
 	) -> Result<(), Error> {
 		let number = vote.commitment.block_number;
-		let (correct_header, expected_payload) = self.expected_header_and_payload(number)?;
+		let (correct_hash, correct_header, expected_payload) =
+			self.expected_hash_header_payload_tuple(number)?;
 		if vote.commitment.payload != expected_payload {
-			let ancestry_proof = self
+			let ancestry_proof: Option<_> = match self
 				.runtime
 				.runtime_api()
-				.generate_ancestry_proof(correct_header.hash(), number, None)
-				.unwrap()
-				.unwrap();
+				.generate_ancestry_proof(correct_hash, number, None)
+			{
+				Ok(Ok(ancestry_proof)) => Some(ancestry_proof),
+				Ok(Err(e)) => {
+					debug!(target: LOG_TARGET, "🥩 Failed to generate ancestry proof: {:?}", e);
+					None
+				},
+				Err(e) => {
+					debug!(target: LOG_TARGET, "🥩 Failed to generate ancestry proof: {:?}", e);
+					None
+				},
+			};
 			let proof = ForkEquivocationProof {
 				commitment: vote.commitment,
 				signatories: vec![(vote.id, vote.signature)],
-				correct_header: Some(correct_header.clone()),
-				ancestry_proof: Some(ancestry_proof),
+				correct_header: Some(correct_header),
+				ancestry_proof,
 			};
 			self.report_fork_equivocation(proof)?;
 		}
@@ -252,20 +255,25 @@ where
 	) -> Result<(), Error> {
 		let SignedCommitment { commitment, signatures } = signed_commitment;
 		let number = commitment.block_number;
-		let prev_hash = self
-			.backend
-			.blockchain()
-			.hash(number)
-			.map_err(|e| Error::Backend(e.to_string()))?;
-		let (correct_header, expected_payload) = self.expected_header_and_payload(number)?;
+		let (correct_hash, correct_header, expected_payload) =
+			self.expected_hash_header_payload_tuple(number)?;
 		if commitment.payload != expected_payload {
-			let ancestry_proof = self
-				.runtime
-				.runtime_api()
-				.generate_ancestry_proof(correct_header.hash(), number, None)
-				.unwrap()
-				.unwrap();
-			let validator_set = self.active_validator_set_at(prev_hash.unwrap())?;
+			let ancestry_proof = match self.runtime.runtime_api().generate_ancestry_proof(
+				correct_hash,
+				number,
+				None,
+			) {
+				Ok(Ok(ancestry_proof)) => Some(ancestry_proof),
+				Ok(Err(e)) => {
+					debug!(target: LOG_TARGET, "🥩 Failed to generate ancestry proof: {:?}", e);
+					None
+				},
+				Err(e) => {
+					debug!(target: LOG_TARGET, "🥩 Failed to generate ancestry proof: {:?}", e);
+					None
+				},
+			};
+			let validator_set = self.active_validator_set_at(correct_hash)?;
 			if signatures.len() != validator_set.validators().len() {
 				// invalid proof
 				return Ok(())
@@ -282,8 +290,8 @@ where
 			let proof = ForkEquivocationProof {
 				commitment,
 				signatories,
-				correct_header: Some(correct_header.clone()),
-				ancestry_proof: Some(ancestry_proof),
+				correct_header: Some(correct_header),
+				ancestry_proof,
 			};
 			self.report_fork_equivocation(proof)?;
 		}

From 223370a7b6845a47aeb52d94727047fb41037c2a Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Wed, 8 Nov 2023 17:29:18 +0100
Subject: [PATCH 079/188] handle "equivocations" on future blocks

if validators sign commitments to block numbers higher than the best
known block, they should likewise be reported as an equivocation.
---
 .../beefy/src/communication/fisherman.rs      | 182 +++++++++++-------
 substrate/frame/beefy/src/equivocation.rs     |   3 +-
 .../primitives/consensus/beefy/src/lib.rs     |   9 +-
 3 files changed, 124 insertions(+), 70 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index d1937dfa5823..a272a6ca5bfd 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -110,20 +110,28 @@ where
 		&self,
 		proof: ForkEquivocationProof<NumberFor<B>, AuthorityId, Signature, B::Header, MmrRootHash>,
 	) -> Result<(), Error> {
-		let expected_prev_hash = self
-			.backend
-			.blockchain()
-			.expect_block_hash_from_id(&BlockId::Number(proof.commitment.block_number))
-			.map_err(|e| Error::Backend(e.to_string()))?;
+		let best_block_number = self.backend.blockchain().info().best_number;
+		let best_block_hash = self.backend.blockchain().info().best_hash;
 
-		let validator_set = self.active_validator_set_at(expected_prev_hash)?;
+		// if the commitment is for a block number exceeding our best block number, we assume the
+		// equivocators are part of the current validator set, hence we use the validator set at the
+		// best block
+		let expected_commitment_block_hash = if best_block_number < proof.commitment.block_number {
+			best_block_hash
+		} else {
+			self.backend
+				.blockchain()
+				.expect_block_hash_from_id(&BlockId::Number(proof.commitment.block_number))
+				.map_err(|e| Error::Backend(e.to_string()))?
+		};
+
+		let validator_set = self.active_validator_set_at(expected_commitment_block_hash)?;
 		let set_id = validator_set.id();
 
-		let best_hash = self.backend.blockchain().info().best_hash;
 		let best_mmr_root = self
 			.runtime
 			.runtime_api()
-			.mmr_root(best_hash)
+			.mmr_root(best_block_hash)
 			.map_err(|e| Error::RuntimeApi(e))?
 			.map_err(|e| Error::Backend(e.to_string()))?;
 
@@ -132,17 +140,14 @@ where
 		let leaf_count = self
 			.runtime
 			.runtime_api()
-			.mmr_leaf_count(best_hash)
+			.mmr_leaf_count(best_block_hash)
 			.map_err(|e| Error::RuntimeApi(e))?
 			.map_err(|e| Error::Backend(e.to_string()))?;
-
-		// TODO: if ancestry proof can't be constructed, report equivocation nonetheless if valid
-		// header proof can be provided
-		let first_mmr_block_num = {
-			let best_block_num = self.backend.blockchain().info().best_number;
-			sp_mmr_primitives::utils::first_mmr_block_num::<B::Header>(best_block_num, leaf_count)
-				.map_err(|e| Error::Backend(e.to_string()))?
-		};
+		let first_mmr_block_num = sp_mmr_primitives::utils::first_mmr_block_num::<B::Header>(
+			best_block_number,
+			leaf_count,
+		)
+		.map_err(|e| Error::Backend(e.to_string()))?;
 
 		if proof.commitment.validator_set_id != set_id ||
 			!check_fork_equivocation_proof::<
@@ -151,8 +156,14 @@ where
 				B::Header,
 				MmrRootHash,
 				sp_mmr_primitives::utils::AncestryHasher<MmrHashing>,
-			>(&proof, best_mmr_root, leaf_count, &expected_prev_hash, first_mmr_block_num)
-		{
+			>(
+				&proof,
+				best_mmr_root,
+				leaf_count,
+				&expected_commitment_block_hash,
+				first_mmr_block_num,
+				best_block_number,
+			) {
 			debug!(target: LOG_TARGET, "🥩 Skip report for bad invalid fork proof {:?}", proof);
 			return Ok(())
 		}
@@ -174,7 +185,7 @@ where
 			.cloned()
 			.filter_map(|id| {
 				match runtime_api.generate_key_ownership_proof(
-					expected_prev_hash,
+					expected_commitment_block_hash,
 					set_id,
 					id.clone(),
 				) {
@@ -192,7 +203,6 @@ where
 			.collect::<Result<_, _>>()?;
 
 		// submit invalid fork vote report at **best** block
-		let best_block_hash = self.backend.blockchain().info().best_hash;
 		runtime_api
 			.submit_report_fork_equivocation_unsigned_extrinsic(
 				best_block_hash,
@@ -219,31 +229,43 @@ where
 		vote: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
 	) -> Result<(), Error> {
 		let number = vote.commitment.block_number;
-		let (correct_hash, correct_header, expected_payload) =
-			self.expected_hash_header_payload_tuple(number)?;
-		if vote.commitment.payload != expected_payload {
-			let ancestry_proof: Option<_> = match self
-				.runtime
-				.runtime_api()
-				.generate_ancestry_proof(correct_hash, number, None)
-			{
-				Ok(Ok(ancestry_proof)) => Some(ancestry_proof),
-				Ok(Err(e)) => {
-					debug!(target: LOG_TARGET, "🥩 Failed to generate ancestry proof: {:?}", e);
-					None
-				},
-				Err(e) => {
-					debug!(target: LOG_TARGET, "🥩 Failed to generate ancestry proof: {:?}", e);
-					None
-				},
-			};
+		// if the vote is for a block number exceeding our best block number, there shouldn't even
+		// be a payload to sign yet, hence we assume it is an equivocation and report it
+		if number > self.backend.blockchain().info().best_number {
 			let proof = ForkEquivocationProof {
 				commitment: vote.commitment,
 				signatories: vec![(vote.id, vote.signature)],
-				correct_header: Some(correct_header),
-				ancestry_proof,
+				correct_header: None,
+				ancestry_proof: None,
 			};
 			self.report_fork_equivocation(proof)?;
+		} else {
+			let (correct_hash, correct_header, expected_payload) =
+				self.expected_hash_header_payload_tuple(number)?;
+			if vote.commitment.payload != expected_payload {
+				let ancestry_proof: Option<_> = match self
+					.runtime
+					.runtime_api()
+					.generate_ancestry_proof(correct_hash, number, None)
+				{
+					Ok(Ok(ancestry_proof)) => Some(ancestry_proof),
+					Ok(Err(e)) => {
+						debug!(target: LOG_TARGET, "🥩 Failed to generate ancestry proof: {:?}", e);
+						None
+					},
+					Err(e) => {
+						debug!(target: LOG_TARGET, "🥩 Failed to generate ancestry proof: {:?}", e);
+						None
+					},
+				};
+				let proof = ForkEquivocationProof {
+					commitment: vote.commitment,
+					signatories: vec![(vote.id, vote.signature)],
+					correct_header: Some(correct_header),
+					ancestry_proof,
+				};
+				self.report_fork_equivocation(proof)?;
+			}
 		}
 		Ok(())
 	}
@@ -255,30 +277,14 @@ where
 	) -> Result<(), Error> {
 		let SignedCommitment { commitment, signatures } = signed_commitment;
 		let number = commitment.block_number;
-		let (correct_hash, correct_header, expected_payload) =
-			self.expected_hash_header_payload_tuple(number)?;
-		if commitment.payload != expected_payload {
-			let ancestry_proof = match self.runtime.runtime_api().generate_ancestry_proof(
-				correct_hash,
-				number,
-				None,
-			) {
-				Ok(Ok(ancestry_proof)) => Some(ancestry_proof),
-				Ok(Err(e)) => {
-					debug!(target: LOG_TARGET, "🥩 Failed to generate ancestry proof: {:?}", e);
-					None
-				},
-				Err(e) => {
-					debug!(target: LOG_TARGET, "🥩 Failed to generate ancestry proof: {:?}", e);
-					None
-				},
-			};
-			let validator_set = self.active_validator_set_at(correct_hash)?;
-			if signatures.len() != validator_set.validators().len() {
-				// invalid proof
-				return Ok(())
-			}
-			// report every signer of the bad justification
+		// if the vote is for a block number exceeding our best block number, there shouldn't even
+		// be a payload to sign yet, hence we assume it is an equivocation and report it
+		if number > self.backend.blockchain().info().best_number {
+			// if block number is in the future, we use the latest validator set
+			// as the assumed signatories (note: this assumption is fragile and can possibly be
+			// improved upon)
+			let best_hash = self.backend.blockchain().info().best_hash;
+			let validator_set = self.active_validator_set_at(best_hash)?;
 			let signatories = validator_set
 				.validators()
 				.iter()
@@ -286,14 +292,54 @@ where
 				.zip(signatures.into_iter())
 				.filter_map(|(id, signature)| signature.map(|sig| (id, sig)))
 				.collect();
-
 			let proof = ForkEquivocationProof {
 				commitment,
 				signatories,
-				correct_header: Some(correct_header),
-				ancestry_proof,
+				correct_header: None,
+				ancestry_proof: None,
 			};
 			self.report_fork_equivocation(proof)?;
+		} else {
+			let (correct_hash, correct_header, expected_payload) =
+				self.expected_hash_header_payload_tuple(number)?;
+			if commitment.payload != expected_payload {
+				let ancestry_proof = match self.runtime.runtime_api().generate_ancestry_proof(
+					correct_hash,
+					number,
+					None,
+				) {
+					Ok(Ok(ancestry_proof)) => Some(ancestry_proof),
+					Ok(Err(e)) => {
+						debug!(target: LOG_TARGET, "🥩 Failed to generate ancestry proof: {:?}", e);
+						None
+					},
+					Err(e) => {
+						debug!(target: LOG_TARGET, "🥩 Failed to generate ancestry proof: {:?}", e);
+						None
+					},
+				};
+				let validator_set = self.active_validator_set_at(correct_hash)?;
+				if signatures.len() != validator_set.validators().len() {
+					// invalid proof
+					return Ok(())
+				}
+				// report every signer of the bad justification
+				let signatories = validator_set
+					.validators()
+					.iter()
+					.cloned()
+					.zip(signatures.into_iter())
+					.filter_map(|(id, signature)| signature.map(|sig| (id, sig)))
+					.collect();
+
+				let proof = ForkEquivocationProof {
+					commitment,
+					signatories,
+					correct_header: Some(correct_header),
+					ancestry_proof,
+				};
+				self.report_fork_equivocation(proof)?;
+			}
 		}
 		Ok(())
 	}
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index df80d933ec93..210eab1fed94 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -293,10 +293,10 @@ where
 					sp_mmr_primitives::utils::NodesUtils::new(<pallet_mmr::Pallet<T>>::mmr_size())
 						.size();
 				let expected_mmr_root = <pallet_mmr::Pallet<T>>::mmr_root();
+				let best_block_num = <frame_system::Pallet<T>>::block_number();
 				// if first_mmr_block_num is invalid, then presumably beefy is not active.
 				// TODO: should we slash in this case?
 				let first_mmr_block_num = {
-					let best_block_num = <frame_system::Pallet<T>>::block_number();
 					let mmr_leaf_count = <pallet_mmr::Pallet<T>>::mmr_size();
 					sp_mmr_primitives::utils::first_mmr_block_num::<HeaderFor<T>>(
 						best_block_num,
@@ -325,6 +325,7 @@ where
 					mmr_size,
 					&expected_block_hash,
 					first_mmr_block_num,
+					best_block_num,
 				) {
 					return Err(Error::<T>::InvalidForkEquivocationProof.into())
 				}
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index af40ba612128..c3cb0c656891 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -361,6 +361,7 @@ pub fn check_fork_equivocation_proof<Id, MsgHash, Header, NodeHash, Hasher>(
 	mmr_size: u64,
 	expected_header_hash: &Header::Hash,
 	first_mmr_block_num: Header::Number,
+	best_block_num: Header::Number,
 ) -> bool
 where
 	Id: BeefyAuthorityId<MsgHash> + PartialEq,
@@ -387,7 +388,13 @@ where
 		});
 
 		if (correct_header, ancestry_proof) == (&None, &None) {
-			// at least a header or ancestry proof must be provided
+			if commitment.block_number > best_block_num {
+				return signatories.iter().all(|(authority_id, signature)| {
+					check_commitment_signature(&commitment, authority_id, signature)
+				})
+			}
+			// if commitment isn't to a block number in the future, at least a header or ancestry
+			// proof must be provided
 			return false
 		}
 

From ab891da3c9b375b155feedec851a0fcfcd4c0754 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Wed, 8 Nov 2023 19:19:39 +0100
Subject: [PATCH 080/188] update documentation comment

---
 .../primitives/consensus/beefy/src/lib.rs     | 28 +++++++++++--------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index c3cb0c656891..f7c1a834b540 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -337,18 +337,22 @@ where
 	return valid_first && valid_second
 }
 
-/// Validates [ForkEquivocationProof] by checking:
-/// 1. `commitment` is signed,
-/// 2. `correct_header` is valid and matches `commitment.block_number`.
-/// 2. `commitment.payload` != `expected_payload(correct_header)`.
-/// NOTE: GRANDPA finalization proof is not checked, which leads to slashing on forks.
-/// This is fine since honest validators will not be slashed on the chain finalized
-/// by GRANDPA, which is the only chain that ultimately matters.
-/// The only material difference not checking GRANDPA proofs makes is that validators
-/// are not slashed for signing BEEFY commitments prior to the blocks committed to being
-/// finalized by GRANDPA. This is fine too, since the slashing risk of committing to
-/// an incorrect block implies validators will only sign blocks they *know* will be
-/// finalized by GRANDPA.
+/// Validates [ForkEquivocationProof] with the following checks:
+/// - if the commitment is to a block in our history, then at least a header or an ancestry proof is
+///   provided
+/// - if `correct_header` is provided and matches `commitment.block_number`, commitment.payload` !=
+///   `expected_payload(correct_header)`
+/// - if `ancestry proof` is provided, it proves mmr_root(commitment.block_number) !=
+///   mmr_root(commitment.payload)`
+/// - `commitment` is signed by all claimed signatories
+///
+/// NOTE: GRANDPA finalization proof is not checked, which leads to slashing on forks. This is fine
+/// since honest validators will not be slashed on the chain finalized by GRANDPA, which is the only
+/// chain that ultimately matters. The only material difference not checking GRANDPA proofs makes is
+/// that validators are not slashed for signing BEEFY commitments prior to the blocks committed to
+/// being finalized by GRANDPA. This is fine too, since the slashing risk of committing to an
+/// incorrect block implies validators will only sign blocks they *know* will be finalized by
+/// GRANDPA.
 pub fn check_fork_equivocation_proof<Id, MsgHash, Header, NodeHash, Hasher>(
 	proof: &ForkEquivocationProof<
 		Header::Number,

From ab8349285d4f2eb9009caf8e4e7ba622a8d97941 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Wed, 8 Nov 2023 19:20:32 +0100
Subject: [PATCH 081/188] refactor `check_fork_equivocation_proof`

The logic flow is currently too convoluted. More refactors to follow.
---
 .../primitives/consensus/beefy/src/lib.rs     | 146 +++++++++---------
 1 file changed, 71 insertions(+), 75 deletions(-)

diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index f7c1a834b540..e6f86c0c682c 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -375,95 +375,91 @@ where
 	Hasher: mmr_lib::Merge<Item = NodeHash>,
 {
 	let ForkEquivocationProof { commitment, signatories, correct_header, ancestry_proof } = proof;
-	// verify that the prev_root is at the correct block number
-	// this can be inferred from the leaf_count / mmr_size of the prev_root:
-	// convert the commitment.block_number to an mmr size and compare with the value in the ancestry
-	// proof
-	let mut ancestry_prev_root = Err(mmr_lib::Error::CorruptedProof);
-	{
-		let expected_leaf_count = sp_mmr_primitives::utils::block_num_to_leaf_index::<Header>(
-			commitment.block_number,
-			first_mmr_block_num,
-		)
-		.and_then(|leaf_index| {
-			leaf_index.checked_add(1).ok_or_else(|| {
-				sp_mmr_primitives::Error::InvalidNumericOp.log_debug("leaf_index + 1 overflowed")
-			})
-		});
-
+	if commitment.block_number <= best_block_num {
 		if (correct_header, ancestry_proof) == (&None, &None) {
-			if commitment.block_number > best_block_num {
-				return signatories.iter().all(|(authority_id, signature)| {
-					check_commitment_signature(&commitment, authority_id, signature)
-				})
-			}
 			// if commitment isn't to a block number in the future, at least a header or ancestry
 			// proof must be provided
 			return false
 		}
 
-		// if the block number either under- or overflowed, the commitment.block_number was not
-		// valid and the commitment should not have been signed, hence we can skip the ancestry
-		// proof and slash the signatories
-		if let (Ok(expected_leaf_count), Some(ancestry_proof)) =
-			(expected_leaf_count, ancestry_proof)
-		{
-			let expected_mmr_size =
-				sp_mmr_primitives::utils::NodesUtils::new(expected_leaf_count).size();
-			if expected_mmr_size != ancestry_proof.prev_size {
-				return false
+		// verify that the prev_root is at the correct block number
+		// this can be inferred from the leaf_count / mmr_size of the prev_root:
+		// convert the commitment.block_number to an mmr size and compare with the value in the
+		// ancestry proof
+		let ancestry_prev_root = {
+			let expected_leaf_count = sp_mmr_primitives::utils::block_num_to_leaf_index::<Header>(
+				commitment.block_number,
+				first_mmr_block_num,
+			)
+			.and_then(|leaf_index| {
+				leaf_index.checked_add(1).ok_or_else(|| {
+					sp_mmr_primitives::Error::InvalidNumericOp
+						.log_debug("leaf_index + 1 overflowed")
+				})
+			});
+
+			// if the block number either under- or overflowed, the commitment.block_number was not
+			// valid and the commitment should not have been signed, hence we can skip the ancestry
+			// proof and slash the signatories
+			if let (Ok(expected_leaf_count), Some(ancestry_proof)) =
+				(expected_leaf_count, ancestry_proof)
+			{
+				let expected_mmr_size =
+					sp_mmr_primitives::utils::NodesUtils::new(expected_leaf_count).size();
+				if expected_mmr_size != ancestry_proof.prev_size {
+					return false
+				}
+				if Ok(true) !=
+					sp_mmr_primitives::utils::verify_ancestry_proof::<NodeHash, Hasher>(
+						expected_root,
+						mmr_size,
+						ancestry_proof.clone(),
+					) {
+					return false
+				}
+				mmr_lib::bagging_peaks_hashes::<NodeHash, Hasher>(ancestry_proof.prev_peaks.clone())
+			} else {
+				Err(mmr_lib::Error::CorruptedProof)
 			}
-			if Ok(true) !=
-				sp_mmr_primitives::utils::verify_ancestry_proof::<NodeHash, Hasher>(
-					expected_root,
-					mmr_size,
-					ancestry_proof.clone(),
-				) {
+		};
+
+		let mut expected_payload: Option<_> = None;
+		if let Some(correct_header) = correct_header {
+			if correct_header.hash() != *expected_header_hash {
 				return false
 			}
-			ancestry_prev_root = mmr_lib::bagging_peaks_hashes::<NodeHash, Hasher>(
-				ancestry_proof.prev_peaks.clone(),
-			);
+
+			let expected_mmr_root_digest = mmr::find_mmr_root_digest::<Header>(correct_header);
+			expected_payload = expected_mmr_root_digest.map(|mmr_root| {
+				Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode())
+			});
 		}
-	}
 
-	let mut expected_payload: Option<_> = None;
-	if let Some(correct_header) = correct_header {
-		if correct_header.hash() != *expected_header_hash {
+		// if the commitment payload does not commit to an MMR root, then this commitment may have
+		// another purpose and should not be slashed
+		// TODO: what if we can nonetheless show that there's another payload at the same block
+		// number? if we're keeping both header & mmr root proofs, then we may proceed in this case
+		// nonetheless
+		let commitment_prev_root =
+			commitment.payload.get_decoded::<NodeHash>(&known_payloads::MMR_ROOT_ID);
+		// cheap failfasts:
+		// 1. check that `payload` on the `vote` is different that the `expected_payload`
+		// 2. if the signatories signed a payload when there should be none (for
+		// instance for a block prior to BEEFY activation), then expected_payload =
+		// None, and they will likewise be slashed (note we can only check this if a valid header
+		// has been provided - we cannot slash for this with an ancestry proof - by necessity)
+		if !(Some(&commitment.payload) != expected_payload.as_ref() ||
+			(ancestry_prev_root.is_ok() && commitment_prev_root != ancestry_prev_root.ok()))
+		{
 			return false
 		}
-
-		let expected_mmr_root_digest = mmr::find_mmr_root_digest::<Header>(correct_header);
-		expected_payload = expected_mmr_root_digest.map(|mmr_root| {
-			Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode())
-		});
-	}
-
-	// if the commitment payload does not commit to an MMR root, then this commitment may have
-	// another purpose and should not be slashed
-	// TODO: what if we can nonetheless show that there's another payload at the same block number?
-	// if we're keeping both header & mmr root slashing, then we may proceed in this case
-	// nonetheless
-	let commitment_prev_root =
-		commitment.payload.get_decoded::<NodeHash>(&known_payloads::MMR_ROOT_ID);
-	// cheap failfasts:
-	// 1. check that `payload` on the `vote` is different that the `expected_payload`
-	// 2. if the signatories signed a payload when there should be none (for
-	// instance for a block prior to BEEFY activation), then expected_payload =
-	// None, and they will likewise be slashed (note we can only check this if a valid header has
-	// been provided - we cannot slash for this with an ancestry proof - by necessity)
-	if Some(&commitment.payload) != expected_payload.as_ref() ||
-		(ancestry_prev_root.is_ok() && commitment_prev_root != ancestry_prev_root.ok())
-	{
-		// check each signatory's signature on the commitment.
-		// if any are invalid, equivocation report is invalid
-		// TODO: refactor check_commitment_signature to take a slice of signatories
-		return signatories.iter().all(|(authority_id, signature)| {
-			check_commitment_signature(&commitment, authority_id, signature)
-		})
-	} else {
-		false
 	}
+	// check each signatory's signature on the commitment.
+	// if any are invalid, equivocation report is invalid
+	// TODO: refactor check_commitment_signature to take a slice of signatories
+	signatories.iter().all(|(authority_id, signature)| {
+		check_commitment_signature(&commitment, authority_id, signature)
+	})
 }
 
 /// New BEEFY validator set notification hook.

From 912d5dc0a2579dd1adfa705186e689546ca9b75e Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Wed, 8 Nov 2023 19:47:47 +0100
Subject: [PATCH 082/188] recurse De Morgan's Laws

---
 substrate/primitives/consensus/beefy/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index e6f86c0c682c..e48d5b3e55cc 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -448,8 +448,8 @@ where
 		// instance for a block prior to BEEFY activation), then expected_payload =
 		// None, and they will likewise be slashed (note we can only check this if a valid header
 		// has been provided - we cannot slash for this with an ancestry proof - by necessity)
-		if !(Some(&commitment.payload) != expected_payload.as_ref() ||
-			(ancestry_prev_root.is_ok() && commitment_prev_root != ancestry_prev_root.ok()))
+		if Some(&commitment.payload) == expected_payload.as_ref() &&
+			(!ancestry_prev_root.is_ok() || commitment_prev_root == ancestry_prev_root.ok())
 		{
 			return false
 		}

From 6efed1401c72ec7d963639cd5fcf1050dd835e61 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Wed, 8 Nov 2023 20:21:29 +0100
Subject: [PATCH 083/188] loosen proof failure condition

if commitment is to a block in our history, don't require both the
ancestry and header proof to be valid, only at least one of them should
be. For instance, if an ancestry proof is only checked during a
later block, it would be invalid since it's not proving ancestry of the
current best block. If the header proof is still valid, this still
proves the fork equivocation.
---
 substrate/primitives/consensus/beefy/src/lib.rs | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index e48d5b3e55cc..635068b760cc 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -339,10 +339,10 @@ where
 
 /// Validates [ForkEquivocationProof] with the following checks:
 /// - if the commitment is to a block in our history, then at least a header or an ancestry proof is
-///   provided
-/// - if `correct_header` is provided and matches `commitment.block_number`, commitment.payload` !=
-///   `expected_payload(correct_header)`
-/// - if `ancestry proof` is provided, it proves mmr_root(commitment.block_number) !=
+///   provided:
+///   - a `correct_header` is correct if it's at height `commitment.block_number` and
+///   commitment.payload` != `expected_payload(correct_header)`
+///   - an `ancestry_proof` is correct if it proves mmr_root(commitment.block_number) !=
 ///   mmr_root(commitment.payload)`
 /// - `commitment` is signed by all claimed signatories
 ///
@@ -448,8 +448,9 @@ where
 		// instance for a block prior to BEEFY activation), then expected_payload =
 		// None, and they will likewise be slashed (note we can only check this if a valid header
 		// has been provided - we cannot slash for this with an ancestry proof - by necessity)
-		if Some(&commitment.payload) == expected_payload.as_ref() &&
-			(!ancestry_prev_root.is_ok() || commitment_prev_root == ancestry_prev_root.ok())
+		if Some(&commitment.payload) == expected_payload.as_ref() ||
+			(correct_header.is_none() && !ancestry_prev_root.is_ok()) ||
+			commitment_prev_root == ancestry_prev_root.ok()
 		{
 			return false
 		}

From ccac61f6ce5cf8c900335e415921475dcc5189e1 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Wed, 8 Nov 2023 21:42:42 +0100
Subject: [PATCH 084/188] refactor fork equivocation check

logic easier to follow by compartmentalizing header proof & ancestry
proof checks
---
 .../primitives/consensus/beefy/src/lib.rs     | 136 +++++++++---------
 1 file changed, 70 insertions(+), 66 deletions(-)

diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 635068b760cc..c42b5b69efca 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -382,76 +382,80 @@ where
 			return false
 		}
 
-		// verify that the prev_root is at the correct block number
-		// this can be inferred from the leaf_count / mmr_size of the prev_root:
-		// convert the commitment.block_number to an mmr size and compare with the value in the
-		// ancestry proof
-		let ancestry_prev_root = {
-			let expected_leaf_count = sp_mmr_primitives::utils::block_num_to_leaf_index::<Header>(
-				commitment.block_number,
-				first_mmr_block_num,
-			)
-			.and_then(|leaf_index| {
-				leaf_index.checked_add(1).ok_or_else(|| {
-					sp_mmr_primitives::Error::InvalidNumericOp
-						.log_debug("leaf_index + 1 overflowed")
-				})
-			});
-
-			// if the block number either under- or overflowed, the commitment.block_number was not
-			// valid and the commitment should not have been signed, hence we can skip the ancestry
-			// proof and slash the signatories
-			if let (Ok(expected_leaf_count), Some(ancestry_proof)) =
-				(expected_leaf_count, ancestry_proof)
-			{
-				let expected_mmr_size =
-					sp_mmr_primitives::utils::NodesUtils::new(expected_leaf_count).size();
-				if expected_mmr_size != ancestry_proof.prev_size {
-					return false
-				}
-				if Ok(true) !=
-					sp_mmr_primitives::utils::verify_ancestry_proof::<NodeHash, Hasher>(
-						expected_root,
-						mmr_size,
-						ancestry_proof.clone(),
-					) {
-					return false
-				}
-				mmr_lib::bagging_peaks_hashes::<NodeHash, Hasher>(ancestry_proof.prev_peaks.clone())
-			} else {
-				Err(mmr_lib::Error::CorruptedProof)
+		let header_proof_is_correct = if let Some(correct_header) = correct_header {
+			correct_header.hash() == *expected_header_hash && {
+				let expected_mmr_root_digest = mmr::find_mmr_root_digest::<Header>(correct_header);
+				let expected_payload = expected_mmr_root_digest.map(|mmr_root| {
+					Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode())
+				});
+
+				// check that `payload` of the `commitment` is different that the `expected_payload`
+				// note that if the signatories signed a payload when there should be none (for
+				// instance for a block prior to BEEFY activation), then expected_payload = None,
+				// and they will likewise be slashed.
+				// note that we can only check this if a valid header has been provided - we cannot
+				// slash for this with an ancestry proof - by necessity)
+				Some(&commitment.payload) != expected_payload.as_ref()
 			}
+		} else {
+			// if no header provided, the header proof is also not correct
+			false
 		};
 
-		let mut expected_payload: Option<_> = None;
-		if let Some(correct_header) = correct_header {
-			if correct_header.hash() != *expected_header_hash {
-				return false
-			}
-
-			let expected_mmr_root_digest = mmr::find_mmr_root_digest::<Header>(correct_header);
-			expected_payload = expected_mmr_root_digest.map(|mmr_root| {
-				Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode())
-			});
-		}
+		let ancestry_proof_is_correct = if let Some(ancestry_proof) = ancestry_proof {
+			(|| {
+				let ancestry_prev_root = {
+					let expected_leaf_count = sp_mmr_primitives::utils::block_num_to_leaf_index::<
+						Header,
+					>(commitment.block_number, first_mmr_block_num)
+					.and_then(|leaf_index| {
+						leaf_index.checked_add(1).ok_or_else(|| {
+							sp_mmr_primitives::Error::InvalidNumericOp
+								.log_debug("leaf_index + 1 overflowed")
+						})
+					});
+
+					if let Ok(expected_leaf_count) = expected_leaf_count {
+						let expected_mmr_size =
+							sp_mmr_primitives::utils::NodesUtils::new(expected_leaf_count).size();
+						// verify that the prev_root is at the correct block number
+						// this can be inferred from the leaf_count / mmr_size of the prev_root:
+						// convert the commitment.block_number to an mmr size and compare with the
+						// value in the ancestry proof
+						if expected_mmr_size == ancestry_proof.prev_size {
+							return false
+						}
+						if Ok(true) !=
+							sp_mmr_primitives::utils::verify_ancestry_proof::<NodeHash, Hasher>(
+								expected_root,
+								mmr_size,
+								ancestry_proof.clone(),
+							) {
+							return false
+						}
+						mmr_lib::bagging_peaks_hashes::<NodeHash, Hasher>(
+							ancestry_proof.prev_peaks.clone(),
+						)
+					} else {
+						// if the block number either under- or overflowed, the
+						// commitment.block_number was not valid and the commitment should not have
+						// been signed, hence we can skip the ancestry proof and slash the
+						// signatories
+						return true
+					}
+				};
+				// if the commitment payload does not commit to an MMR root, then this commitment
+				// may have another purpose and should not be slashed
+				let commitment_prev_root =
+					commitment.payload.get_decoded::<NodeHash>(&known_payloads::MMR_ROOT_ID);
+				commitment_prev_root != ancestry_prev_root.ok()
+			})()
+		} else {
+			// if no ancestry proof provided, the proof is also not correct
+			false
+		};
 
-		// if the commitment payload does not commit to an MMR root, then this commitment may have
-		// another purpose and should not be slashed
-		// TODO: what if we can nonetheless show that there's another payload at the same block
-		// number? if we're keeping both header & mmr root proofs, then we may proceed in this case
-		// nonetheless
-		let commitment_prev_root =
-			commitment.payload.get_decoded::<NodeHash>(&known_payloads::MMR_ROOT_ID);
-		// cheap failfasts:
-		// 1. check that `payload` on the `vote` is different that the `expected_payload`
-		// 2. if the signatories signed a payload when there should be none (for
-		// instance for a block prior to BEEFY activation), then expected_payload =
-		// None, and they will likewise be slashed (note we can only check this if a valid header
-		// has been provided - we cannot slash for this with an ancestry proof - by necessity)
-		if Some(&commitment.payload) == expected_payload.as_ref() ||
-			(correct_header.is_none() && !ancestry_prev_root.is_ok()) ||
-			commitment_prev_root == ancestry_prev_root.ok()
-		{
+		if !header_proof_is_correct && !ancestry_proof_is_correct {
 			return false
 		}
 	}

From dc56cab24bbd303f106fc7c420d5f4e93b490883 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 9 Nov 2023 01:43:00 +0100
Subject: [PATCH 085/188] refactor flow

---
 .../primitives/consensus/beefy/src/lib.rs     | 56 ++++++++++---------
 1 file changed, 30 insertions(+), 26 deletions(-)

diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index c42b5b69efca..aaca5a9ab53f 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -378,7 +378,7 @@ where
 	if commitment.block_number <= best_block_num {
 		if (correct_header, ancestry_proof) == (&None, &None) {
 			// if commitment isn't to a block number in the future, at least a header or ancestry
-			// proof must be provided
+			// proof must be provided, otherwise the proof is entirely invalid
 			return false
 		}
 
@@ -402,9 +402,9 @@ where
 			false
 		};
 
-		let ancestry_proof_is_correct = if let Some(ancestry_proof) = ancestry_proof {
-			(|| {
-				let ancestry_prev_root = {
+		let ancestry_proof_is_correct =
+			if let Some(ancestry_proof) = ancestry_proof {
+				(|| {
 					let expected_leaf_count = sp_mmr_primitives::utils::block_num_to_leaf_index::<
 						Header,
 					>(commitment.block_number, first_mmr_block_num)
@@ -420,22 +420,19 @@ where
 							sp_mmr_primitives::utils::NodesUtils::new(expected_leaf_count).size();
 						// verify that the prev_root is at the correct block number
 						// this can be inferred from the leaf_count / mmr_size of the prev_root:
-						// convert the commitment.block_number to an mmr size and compare with the
-						// value in the ancestry proof
+						// we've converted the commitment.block_number to an mmr size and now
+						// compare with the value in the ancestry proof
 						if expected_mmr_size == ancestry_proof.prev_size {
 							return false
 						}
-						if Ok(true) !=
-							sp_mmr_primitives::utils::verify_ancestry_proof::<NodeHash, Hasher>(
-								expected_root,
-								mmr_size,
-								ancestry_proof.clone(),
-							) {
+						if sp_mmr_primitives::utils::verify_ancestry_proof::<NodeHash, Hasher>(
+							expected_root,
+							mmr_size,
+							ancestry_proof.clone(),
+						) != Ok(true)
+						{
 							return false
 						}
-						mmr_lib::bagging_peaks_hashes::<NodeHash, Hasher>(
-							ancestry_proof.prev_peaks.clone(),
-						)
 					} else {
 						// if the block number either under- or overflowed, the
 						// commitment.block_number was not valid and the commitment should not have
@@ -443,19 +440,26 @@ where
 						// signatories
 						return true
 					}
-				};
-				// if the commitment payload does not commit to an MMR root, then this commitment
-				// may have another purpose and should not be slashed
-				let commitment_prev_root =
-					commitment.payload.get_decoded::<NodeHash>(&known_payloads::MMR_ROOT_ID);
-				commitment_prev_root != ancestry_prev_root.ok()
-			})()
-		} else {
-			// if no ancestry proof provided, the proof is also not correct
-			false
-		};
+
+					// once the ancestry proof is verified, calculate the prev_root to compare it
+					// with the commitment's prev_root
+					let ancestry_prev_root = mmr_lib::bagging_peaks_hashes::<NodeHash, Hasher>(
+						ancestry_proof.prev_peaks.clone(),
+					);
+					// if the commitment payload does not commit to an MMR root, then this
+					// commitment may have another purpose and should not be slashed
+					let commitment_prev_root =
+						commitment.payload.get_decoded::<NodeHash>(&known_payloads::MMR_ROOT_ID);
+					commitment_prev_root != ancestry_prev_root.ok()
+				})()
+			} else {
+				// if no ancestry proof provided, the proof is also not correct
+				false
+			};
 
 		if !header_proof_is_correct && !ancestry_proof_is_correct {
+			// if commitment.block_number is in our history, at least the header_proof or the
+			// ancestry_proof must be correct, else the proof is entirely invalid
 			return false
 		}
 	}

From 55f8932c51c4db4c465a21db25450a33ff82df1c Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 9 Nov 2023 02:20:14 +0100
Subject: [PATCH 086/188] split up fork equiv. proof check

also avoid verifying the ancestry proof if the header proof is already valid.
---
 .../primitives/consensus/beefy/src/lib.rs     | 232 +++++++++++-------
 1 file changed, 140 insertions(+), 92 deletions(-)

diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index aaca5a9ab53f..a454ad80c3b4 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -337,6 +337,118 @@ where
 	return valid_first && valid_second
 }
 
+/// Checks wether the provided header's payload differs from the commitment's payload.
+fn check_header_proof<Header>(
+	commitment: &Commitment<Header::Number>,
+	correct_header: &Option<Header>,
+	expected_header_hash: &Header::Hash,
+) -> bool
+where
+	Header: sp_api::HeaderT,
+{
+	if let Some(correct_header) = correct_header {
+		let expected_mmr_root_digest = mmr::find_mmr_root_digest::<Header>(correct_header);
+		let expected_payload = expected_mmr_root_digest.map(|mmr_root| {
+			Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode())
+		});
+		// Check header's hash and that `payload` of the `commitment` is different that the
+		// `expected_payload`. Note that if the signatories signed a payload when there should be
+		// none (for instance for a block prior to BEEFY activation), then expected_payload = None,
+		// and they will likewise be slashed.
+		// Note that we can only check this if a valid header has been provided - we cannot
+		// slash for this with an ancestry proof - by necessity)
+		if correct_header.hash() == *expected_header_hash &&
+			Some(&commitment.payload) != expected_payload.as_ref()
+		{
+			return true
+		}
+	}
+	// if no header provided, the header proof is also not correct
+	false
+}
+
+/// Checks whether an ancestry proof has the correct size and its calculated root differs from the
+/// commitment's payload's.
+fn check_ancestry_proof<Header, NodeHash, Hasher>(
+	commitment: &Commitment<Header::Number>,
+	ancestry_proof: &Option<AncestryProof<Hasher::Item>>,
+	first_mmr_block_num: Header::Number,
+	expected_root: Hasher::Item,
+	mmr_size: u64,
+) -> bool
+where
+	Header: sp_api::HeaderT,
+	NodeHash: Clone + Debug + PartialEq + Encode + Decode,
+	Hasher: mmr_lib::Merge<Item = NodeHash>,
+{
+	if let Some(ancestry_proof) = ancestry_proof {
+		let expected_leaf_count = sp_mmr_primitives::utils::block_num_to_leaf_index::<Header>(
+			commitment.block_number,
+			first_mmr_block_num,
+		)
+		.and_then(|leaf_index| {
+			leaf_index.checked_add(1).ok_or_else(|| {
+				sp_mmr_primitives::Error::InvalidNumericOp.log_debug("leaf_index + 1 overflowed")
+			})
+		});
+
+		if let Ok(expected_leaf_count) = expected_leaf_count {
+			let expected_mmr_size =
+				sp_mmr_primitives::utils::NodesUtils::new(expected_leaf_count).size();
+			// verify that the prev_root is at the correct block number
+			// this can be inferred from the leaf_count / mmr_size of the prev_root:
+			// we've converted the commitment.block_number to an mmr size and now
+			// compare with the value in the ancestry proof
+			if expected_mmr_size != ancestry_proof.prev_size {
+				return false
+			}
+			if sp_mmr_primitives::utils::verify_ancestry_proof::<NodeHash, Hasher>(
+				expected_root,
+				mmr_size,
+				ancestry_proof.clone(),
+			) != Ok(true)
+			{
+				return false
+			}
+		} else {
+			// if the block number either under- or overflowed, the
+			// commitment.block_number was not valid and the commitment should not have
+			// been signed, hence we can skip the ancestry proof and slash the
+			// signatories
+			return true
+		}
+
+		// once the ancestry proof is verified, calculate the prev_root to compare it
+		// with the commitment's prev_root
+		let ancestry_prev_root =
+			mmr_lib::bagging_peaks_hashes::<NodeHash, Hasher>(ancestry_proof.prev_peaks.clone());
+		// if the commitment payload does not commit to an MMR root, then this
+		// commitment may have another purpose and should not be slashed
+		let commitment_prev_root =
+			commitment.payload.get_decoded::<NodeHash>(&known_payloads::MMR_ROOT_ID);
+		return commitment_prev_root != ancestry_prev_root.ok()
+	}
+	// if no ancestry proof provided, the proof is also not correct
+	false
+}
+
+/// Check each signatory's signature on the commitment.
+/// If any are invalid, equivocation report is invalid
+fn check_signatures<Id, MsgHash, Header>(
+	commitment: &Commitment<Header::Number>,
+	signatories: &Vec<(Id, <Id as RuntimeAppPublic>::Signature)>,
+) -> bool
+where
+	Id: BeefyAuthorityId<MsgHash> + PartialEq,
+	MsgHash: Hash,
+	Header: sp_api::HeaderT,
+{
+	signatories.iter().all(|(authority_id, signature)| {
+		// TODO: refactor check_commitment_signature to take a slice of signatories
+		check_commitment_signature(&commitment, authority_id, signature)
+	})
+}
+
 /// Validates [ForkEquivocationProof] with the following checks:
 /// - if the commitment is to a block in our history, then at least a header or an ancestry proof is
 ///   provided:
@@ -375,100 +487,36 @@ where
 	Hasher: mmr_lib::Merge<Item = NodeHash>,
 {
 	let ForkEquivocationProof { commitment, signatories, correct_header, ancestry_proof } = proof;
-	if commitment.block_number <= best_block_num {
-		if (correct_header, ancestry_proof) == (&None, &None) {
-			// if commitment isn't to a block number in the future, at least a header or ancestry
-			// proof must be provided, otherwise the proof is entirely invalid
-			return false
-		}
 
-		let header_proof_is_correct = if let Some(correct_header) = correct_header {
-			correct_header.hash() == *expected_header_hash && {
-				let expected_mmr_root_digest = mmr::find_mmr_root_digest::<Header>(correct_header);
-				let expected_payload = expected_mmr_root_digest.map(|mmr_root| {
-					Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode())
-				});
-
-				// check that `payload` of the `commitment` is different that the `expected_payload`
-				// note that if the signatories signed a payload when there should be none (for
-				// instance for a block prior to BEEFY activation), then expected_payload = None,
-				// and they will likewise be slashed.
-				// note that we can only check this if a valid header has been provided - we cannot
-				// slash for this with an ancestry proof - by necessity)
-				Some(&commitment.payload) != expected_payload.as_ref()
-			}
-		} else {
-			// if no header provided, the header proof is also not correct
-			false
-		};
-
-		let ancestry_proof_is_correct =
-			if let Some(ancestry_proof) = ancestry_proof {
-				(|| {
-					let expected_leaf_count = sp_mmr_primitives::utils::block_num_to_leaf_index::<
-						Header,
-					>(commitment.block_number, first_mmr_block_num)
-					.and_then(|leaf_index| {
-						leaf_index.checked_add(1).ok_or_else(|| {
-							sp_mmr_primitives::Error::InvalidNumericOp
-								.log_debug("leaf_index + 1 overflowed")
-						})
-					});
-
-					if let Ok(expected_leaf_count) = expected_leaf_count {
-						let expected_mmr_size =
-							sp_mmr_primitives::utils::NodesUtils::new(expected_leaf_count).size();
-						// verify that the prev_root is at the correct block number
-						// this can be inferred from the leaf_count / mmr_size of the prev_root:
-						// we've converted the commitment.block_number to an mmr size and now
-						// compare with the value in the ancestry proof
-						if expected_mmr_size == ancestry_proof.prev_size {
-							return false
-						}
-						if sp_mmr_primitives::utils::verify_ancestry_proof::<NodeHash, Hasher>(
-							expected_root,
-							mmr_size,
-							ancestry_proof.clone(),
-						) != Ok(true)
-						{
-							return false
-						}
-					} else {
-						// if the block number either under- or overflowed, the
-						// commitment.block_number was not valid and the commitment should not have
-						// been signed, hence we can skip the ancestry proof and slash the
-						// signatories
-						return true
-					}
-
-					// once the ancestry proof is verified, calculate the prev_root to compare it
-					// with the commitment's prev_root
-					let ancestry_prev_root = mmr_lib::bagging_peaks_hashes::<NodeHash, Hasher>(
-						ancestry_proof.prev_peaks.clone(),
-					);
-					// if the commitment payload does not commit to an MMR root, then this
-					// commitment may have another purpose and should not be slashed
-					let commitment_prev_root =
-						commitment.payload.get_decoded::<NodeHash>(&known_payloads::MMR_ROOT_ID);
-					commitment_prev_root != ancestry_prev_root.ok()
-				})()
-			} else {
-				// if no ancestry proof provided, the proof is also not correct
-				false
-			};
-
-		if !header_proof_is_correct && !ancestry_proof_is_correct {
-			// if commitment.block_number is in our history, at least the header_proof or the
-			// ancestry_proof must be correct, else the proof is entirely invalid
-			return false
-		}
+	if commitment.block_number > best_block_num {
+		return false;
 	}
-	// check each signatory's signature on the commitment.
-	// if any are invalid, equivocation report is invalid
-	// TODO: refactor check_commitment_signature to take a slice of signatories
-	signatories.iter().all(|(authority_id, signature)| {
-		check_commitment_signature(&commitment, authority_id, signature)
-	})
+
+	if (correct_header, ancestry_proof) == (&None, &None) {
+		// if commitment isn't to a block number in the future, at least a header or ancestry
+		// proof must be provided, otherwise the proof is entirely invalid
+		return false;
+	}
+
+	let header_proof_is_correct =
+		check_header_proof(commitment, correct_header, expected_header_hash);
+	if header_proof_is_correct {
+		// avoid verifying the ancestry proof if a valid header proof has been provided
+		return check_signatures::<Id, MsgHash, Header>(commitment, signatories)
+	}
+
+	let ancestry_proof_is_correct = check_ancestry_proof::<Header, NodeHash, Hasher>(
+		commitment,
+		ancestry_proof,
+		first_mmr_block_num,
+		expected_root,
+		mmr_size,
+	);
+	if ancestry_proof_is_correct {
+		return check_signatures::<Id, MsgHash, Header>(commitment, signatories)
+	}
+
+	return false;
 }
 
 /// New BEEFY validator set notification hook.

From f71b3adf3d280ddfd7ea65ef19f7a9a39aee7059 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 9 Nov 2023 02:30:16 +0100
Subject: [PATCH 087/188] refactor: comments & conciseness

---
 .../primitives/consensus/beefy/src/lib.rs     | 49 ++++++-------------
 1 file changed, 16 insertions(+), 33 deletions(-)

diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index a454ad80c3b4..b7b9e51ff55f 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -432,23 +432,6 @@ where
 	false
 }
 
-/// Check each signatory's signature on the commitment.
-/// If any are invalid, equivocation report is invalid
-fn check_signatures<Id, MsgHash, Header>(
-	commitment: &Commitment<Header::Number>,
-	signatories: &Vec<(Id, <Id as RuntimeAppPublic>::Signature)>,
-) -> bool
-where
-	Id: BeefyAuthorityId<MsgHash> + PartialEq,
-	MsgHash: Hash,
-	Header: sp_api::HeaderT,
-{
-	signatories.iter().all(|(authority_id, signature)| {
-		// TODO: refactor check_commitment_signature to take a slice of signatories
-		check_commitment_signature(&commitment, authority_id, signature)
-	})
-}
-
 /// Validates [ForkEquivocationProof] with the following checks:
 /// - if the commitment is to a block in our history, then at least a header or an ancestry proof is
 ///   provided:
@@ -498,24 +481,24 @@ where
 		return false;
 	}
 
-	let header_proof_is_correct =
-		check_header_proof(commitment, correct_header, expected_header_hash);
-	if header_proof_is_correct {
-		// avoid verifying the ancestry proof if a valid header proof has been provided
-		return check_signatures::<Id, MsgHash, Header>(commitment, signatories)
-	}
-
-	let ancestry_proof_is_correct = check_ancestry_proof::<Header, NodeHash, Hasher>(
-		commitment,
-		ancestry_proof,
-		first_mmr_block_num,
-		expected_root,
-		mmr_size,
-	);
-	if ancestry_proof_is_correct {
-		return check_signatures::<Id, MsgHash, Header>(commitment, signatories)
+	// avoid verifying the ancestry proof if a valid header proof has been provided
+	if check_header_proof(commitment, correct_header, expected_header_hash) ||
+		check_ancestry_proof::<Header, NodeHash, Hasher>(
+			commitment,
+			ancestry_proof,
+			first_mmr_block_num,
+			expected_root,
+			mmr_size,
+		) {
+		// if either proof is valid, check the validator's signatures. The proof is verified if they
+		// are all correct.
+		return signatories.iter().all(|(authority_id, signature)| {
+			// TODO: refactor check_commitment_signature to take a slice of signatories
+			check_commitment_signature(&commitment, authority_id, signature)
+		})
 	}
 
+	// if neither the ancestry proof nor the header proof is correct, the proof is invalid
 	return false;
 }
 

From a737d2c843376a9131e77f6d1effcadfee25c791 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 9 Nov 2023 09:03:13 +0100
Subject: [PATCH 088/188] Revert "integrate new proof data structure into api"

This reverts commit 5a4f840244e297819cb689b072badb96a15662f8.
Prepares for temporarily using separate mmr crates.
---
 Cargo.lock                                           | 2 +-
 substrate/frame/merkle-mountain-range/src/mmr/mmr.rs | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 84fb486907d6..5dbecd5698fd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2394,7 +2394,7 @@ dependencies = [
 [[package]]
 name = "ckb-merkle-mountain-range"
 version = "0.5.2"
-source = "git+https://github.com/Lederstrumpf/merkle-mountain-range.git?branch=linearize-proof-rebase-v0.5.1#a74c5561cd64dcc777ccdc7dfc21499d200e6da9"
+source = "git+https://github.com/Lederstrumpf/merkle-mountain-range.git?branch=linearize-proof-rebase-v0.5.1#f3aed2bfdaa5ec83d89e97b37c89ecaac9f64497"
 dependencies = [
  "cfg-if",
 ]
diff --git a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
index 0d8d457e1ef5..ebd5ca249e79 100644
--- a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
+++ b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
@@ -54,7 +54,7 @@ where
 
 	let p = mmr_lib::MerkleProof::<Node<H, L>, Hasher<H, L>>::new(
 		size,
-		proof.items.into_iter().map(|(index, hash)| (index, Node::Hash(hash))).collect(),
+		proof.items.into_iter().map(Node::Hash).collect(),
 	);
 	p.verify(Node::Hash(root), leaves_and_position_data)
 		.map_err(|e| Error::Verify.log_debug(e))
@@ -99,7 +99,7 @@ where
 	) -> Result<bool, Error> {
 		let p = mmr_lib::MerkleProof::<NodeOf<T, I, L>, Hasher<HashingOf<T, I>, L>>::new(
 			self.mmr.mmr_size(),
-			proof.items.into_iter().map(|(index, hash)| (index, Node::Hash(hash))).collect(),
+			proof.items.into_iter().map(Node::Hash).collect(),
 		);
 
 		if leaves.len() != proof.leaf_indices.len() {
@@ -222,7 +222,7 @@ where
 			.map(|p| primitives::Proof {
 				leaf_indices,
 				leaf_count,
-				items: p.proof_items().iter().map(|(index, item)| (*index, item.hash())).collect(),
+				items: p.proof_items().iter().map(|x| x.hash()).collect(),
 			})
 			.map(|p| (leaves, p))
 	}

From ba66fde562f6c920ec8df79e3006f3b21b61d071 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 9 Nov 2023 09:14:49 +0100
Subject: [PATCH 089/188] Revert "fix Proof type parameter usage"

This reverts commit a65014d43a4fbd22e0d49e81508f0470c0e4ddf8.
---
 polkadot/runtime/rococo/src/lib.rs                   |  6 +++---
 polkadot/runtime/test-runtime/src/lib.rs             |  6 +++---
 polkadot/runtime/westend/src/lib.rs                  |  6 +++---
 substrate/bin/node/runtime/src/lib.rs                |  4 ++--
 .../client/merkle-mountain-range/rpc/src/lib.rs      |  4 ++--
 .../client/merkle-mountain-range/src/test_utils.rs   |  6 +++---
 substrate/frame/merkle-mountain-range/src/lib.rs     |  6 +++---
 substrate/frame/merkle-mountain-range/src/mmr/mmr.rs |  6 +++---
 .../primitives/merkle-mountain-range/src/lib.rs      | 12 ++++++------
 9 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs
index e4bfb2e77c89..135329ffa82f 100644
--- a/polkadot/runtime/rococo/src/lib.rs
+++ b/polkadot/runtime/rococo/src/lib.rs
@@ -1861,7 +1861,7 @@ sp_api::impl_runtime_apis! {
 		fn generate_proof(
 			block_numbers: Vec<BlockNumber>,
 			best_known_block_number: Option<BlockNumber>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<mmr::Hash>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, mmr::Hash)>), mmr::Error> {
 			Mmr::generate_proof(block_numbers, best_known_block_number).map(
 				|(leaves, proof)| {
 					(
@@ -1875,7 +1875,7 @@ sp_api::impl_runtime_apis! {
 			)
 		}
 
-		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<mmr::Hash>)
+		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<(u64, mmr::Hash)>)
 			-> Result<(), mmr::Error>
 		{
 			let leaves = leaves.into_iter().map(|leaf|
@@ -1888,7 +1888,7 @@ sp_api::impl_runtime_apis! {
 		fn verify_proof_stateless(
 			root: mmr::Hash,
 			leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			proof: mmr::Proof<mmr::Hash>
+			proof: mmr::Proof<(u64, mmr::Hash)>
 		) -> Result<(), mmr::Error> {
 			let nodes = leaves.into_iter().map(|leaf|mmr::DataOrHash::Data(leaf.into_opaque_leaf())).collect();
 			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, nodes, proof)
diff --git a/polkadot/runtime/test-runtime/src/lib.rs b/polkadot/runtime/test-runtime/src/lib.rs
index 73c9afc85bc5..3cd6bee362fd 100644
--- a/polkadot/runtime/test-runtime/src/lib.rs
+++ b/polkadot/runtime/test-runtime/src/lib.rs
@@ -1002,11 +1002,11 @@ sp_api::impl_runtime_apis! {
 		fn generate_proof(
 			_block_numbers: Vec<BlockNumber>,
 			_best_known_block_number: Option<BlockNumber>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<Hash>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, Hash)>), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
 
-		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::Proof<Hash>)
+		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::Proof<(u64, Hash)>)
 			-> Result<(), mmr::Error>
 		{
 			Err(mmr::Error::PalletNotIncluded)
@@ -1015,7 +1015,7 @@ sp_api::impl_runtime_apis! {
 		fn verify_proof_stateless(
 			_root: Hash,
 			_leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			_proof: mmr::Proof<Hash>
+			_proof: mmr::Proof<(u64, Hash)>
 		) -> Result<(), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs
index 408c14793488..f28b1186314a 100644
--- a/polkadot/runtime/westend/src/lib.rs
+++ b/polkadot/runtime/westend/src/lib.rs
@@ -1907,7 +1907,7 @@ sp_api::impl_runtime_apis! {
 		fn generate_proof(
 			block_numbers: Vec<BlockNumber>,
 			best_known_block_number: Option<BlockNumber>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<mmr::Hash>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, mmr::Hash)>), mmr::Error> {
 			Mmr::generate_proof(block_numbers, best_known_block_number).map(
 				|(leaves, proof)| {
 					(
@@ -1921,7 +1921,7 @@ sp_api::impl_runtime_apis! {
 			)
 		}
 
-		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<mmr::Hash>)
+		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<(u64, mmr::Hash)>)
 			-> Result<(), mmr::Error>
 		{
 			let leaves = leaves.into_iter().map(|leaf|
@@ -1934,7 +1934,7 @@ sp_api::impl_runtime_apis! {
 		fn verify_proof_stateless(
 			root: mmr::Hash,
 			leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			proof: mmr::Proof<mmr::Hash>
+			proof: mmr::Proof<(u64, mmr::Hash)>
 		) -> Result<(), mmr::Error> {
 			let nodes = leaves.into_iter().map(|leaf|mmr::DataOrHash::Data(leaf.into_opaque_leaf())).collect();
 			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, nodes, proof)
diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs
index ce57703060b3..90079167ccda 100644
--- a/substrate/bin/node/runtime/src/lib.rs
+++ b/substrate/bin/node/runtime/src/lib.rs
@@ -2654,7 +2654,7 @@ impl_runtime_apis! {
 		fn generate_proof(
 			block_numbers: Vec<BlockNumber>,
 			best_known_block_number: Option<BlockNumber>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<mmr::Hash>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, mmr::Hash)>), mmr::Error> {
 			Mmr::generate_proof(block_numbers, best_known_block_number).map(
 				|(leaves, proof)| {
 					(
@@ -2685,7 +2685,7 @@ impl_runtime_apis! {
 		fn verify_proof_stateless(
 			root: mmr::Hash,
 			leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			proof: mmr::Proof<mmr::Hash>
+			proof: mmr::Proof<(u64, mmr::Hash)>
 		) -> Result<(), mmr::Error> {
 			let nodes = leaves.into_iter().map(|leaf|mmr::DataOrHash::Data(leaf.into_opaque_leaf())).collect();
 			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, nodes, proof)
diff --git a/substrate/client/merkle-mountain-range/rpc/src/lib.rs b/substrate/client/merkle-mountain-range/rpc/src/lib.rs
index c97da8681314..98a0818724e8 100644
--- a/substrate/client/merkle-mountain-range/rpc/src/lib.rs
+++ b/substrate/client/merkle-mountain-range/rpc/src/lib.rs
@@ -261,7 +261,7 @@ mod tests {
 		let proof = Proof {
 			leaf_indices: vec![1],
 			leaf_count: 9,
-			items: vec![(1, H256::repeat_byte(1)), (2, H256::repeat_byte(2))],
+			items: vec![H256::repeat_byte(1), H256::repeat_byte(2)],
 		};
 
 		let leaf_proof = LeavesProof::new(H256::repeat_byte(0), vec![leaf], proof);
@@ -284,7 +284,7 @@ mod tests {
 		let proof = Proof {
 			leaf_indices: vec![1, 2],
 			leaf_count: 9,
-			items: vec![(1, H256::repeat_byte(1)), (2, H256::repeat_byte(2))],
+			items: vec![H256::repeat_byte(1), H256::repeat_byte(2)],
 		};
 
 		let leaf_proof = LeavesProof::new(H256::repeat_byte(0), vec![leaf_a, leaf_b], proof);
diff --git a/substrate/client/merkle-mountain-range/src/test_utils.rs b/substrate/client/merkle-mountain-range/src/test_utils.rs
index 010b48bb3d7d..bf5eb2885451 100644
--- a/substrate/client/merkle-mountain-range/src/test_utils.rs
+++ b/substrate/client/merkle-mountain-range/src/test_utils.rs
@@ -304,11 +304,11 @@ sp_api::mock_impl_runtime_apis! {
 			&self,
 			_block_numbers: Vec<u64>,
 			_best_known_block_number: Option<u64>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<MmrHash>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, MmrHash)>), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
 
-		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::Proof<MmrHash>)
+		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::Proof<(u64, MmrHash)>)
 			-> Result<(), mmr::Error>
 		{
 			Err(mmr::Error::PalletNotIncluded)
@@ -317,7 +317,7 @@ sp_api::mock_impl_runtime_apis! {
 		fn verify_proof_stateless(
 			_root: MmrHash,
 			_leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			_proof: mmr::Proof<MmrHash>
+			_proof: mmr::Proof<(u64, MmrHash)>
 		) -> Result<(), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
diff --git a/substrate/frame/merkle-mountain-range/src/lib.rs b/substrate/frame/merkle-mountain-range/src/lib.rs
index 94e126576ce1..987f7a6f4c4e 100644
--- a/substrate/frame/merkle-mountain-range/src/lib.rs
+++ b/substrate/frame/merkle-mountain-range/src/lib.rs
@@ -245,7 +245,7 @@ pub mod pallet {
 pub fn verify_leaves_proof<H, L>(
 	root: H::Output,
 	leaves: Vec<mmr::Node<H, L>>,
-	proof: primitives::Proof<H::Output>,
+	proof: primitives::Proof<(u64, H::Output)>,
 ) -> Result<(), primitives::Error>
 where
 	H: traits::Hash,
@@ -319,7 +319,7 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 	pub fn generate_proof(
 		block_numbers: Vec<BlockNumberFor<T>>,
 		best_known_block_number: Option<BlockNumberFor<T>>,
-	) -> Result<(Vec<LeafOf<T, I>>, primitives::Proof<HashOf<T, I>>), primitives::Error> {
+	) -> Result<(Vec<LeafOf<T, I>>, primitives::Proof<(u64, HashOf<T, I>)>), primitives::Error> {
 		// check whether best_known_block_number provided, else use current best block
 		let best_known_block_number =
 			best_known_block_number.unwrap_or_else(|| <frame_system::Pallet<T>>::block_number());
@@ -373,7 +373,7 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 	/// or the proof is invalid.
 	pub fn verify_leaves(
 		leaves: Vec<LeafOf<T, I>>,
-		proof: primitives::Proof<HashOf<T, I>>,
+		proof: primitives::Proof<(u64, HashOf<T, I>)>,
 	) -> Result<(), primitives::Error> {
 		if proof.leaf_count > Self::mmr_leaves() ||
 			proof.leaf_count == 0 ||
diff --git a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
index ebd5ca249e79..dc5c69413083 100644
--- a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
+++ b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
@@ -33,7 +33,7 @@ use sp_std::prelude::*;
 pub fn verify_leaves_proof<H, L>(
 	root: H::Output,
 	leaves: Vec<Node<H, L>>,
-	proof: primitives::Proof<H::Output>,
+	proof: primitives::Proof<(u64, H::Output)>,
 ) -> Result<bool, Error>
 where
 	H: sp_runtime::traits::Hash,
@@ -95,7 +95,7 @@ where
 	pub fn verify_leaves_proof(
 		&self,
 		leaves: Vec<L>,
-		proof: primitives::Proof<HashOf<T, I>>,
+		proof: primitives::Proof<(u64, HashOf<T, I>)>,
 	) -> Result<bool, Error> {
 		let p = mmr_lib::MerkleProof::<NodeOf<T, I, L>, Hasher<HashingOf<T, I>, L>>::new(
 			self.mmr.mmr_size(),
@@ -201,7 +201,7 @@ where
 	pub fn generate_proof(
 		&self,
 		leaf_indices: Vec<NodeIndex>,
-	) -> Result<(Vec<L>, primitives::Proof<HashOf<T, I>>), Error> {
+	) -> Result<(Vec<L>, primitives::Proof<(u64, HashOf<T, I>)>), Error> {
 		let positions = leaf_indices
 			.iter()
 			.map(|index| mmr_lib::leaf_index_to_pos(*index))
diff --git a/substrate/primitives/merkle-mountain-range/src/lib.rs b/substrate/primitives/merkle-mountain-range/src/lib.rs
index 3fd2d83192ec..775875d5c0c1 100644
--- a/substrate/primitives/merkle-mountain-range/src/lib.rs
+++ b/substrate/primitives/merkle-mountain-range/src/lib.rs
@@ -356,7 +356,7 @@ pub struct Proof<Hash> {
 	/// Number of leaves in MMR, when the proof was generated.
 	pub leaf_count: NodeIndex,
 	/// Proof elements (hashes of siblings of inner nodes on the path to the leaf).
-	pub items: Vec<(u64, Hash)>,
+	pub items: Vec<Hash>,
 }
 
 /// An MMR ancestry proof for a prior mmr root.
@@ -453,7 +453,7 @@ sp_api::decl_runtime_apis! {
 		/// Note this function will use on-chain MMR root hash and check if the proof matches the hash.
 		/// Note, the leaves should be sorted such that corresponding leaves and leaf indices have the
 		/// same position in both the `leaves` vector and the `leaf_indices` vector contained in the [Proof]
-		fn verify_proof(leaves: Vec<EncodableOpaqueLeaf>, proof: Proof<Hash>) -> Result<(), Error>;
+		fn verify_proof(leaves: Vec<EncodableOpaqueLeaf>, proof: Proof<(u64, Hash)>) -> Result<(), Error>;
 
 		/// Verify MMR proof against given root hash for a batch of leaves.
 		///
@@ -462,7 +462,7 @@ sp_api::decl_runtime_apis! {
 		///
 		/// Note, the leaves should be sorted such that corresponding leaves and leaf indices have the
 		/// same position in both the `leaves` vector and the `leaf_indices` vector contained in the [Proof]
-		fn verify_proof_stateless(root: Hash, leaves: Vec<EncodableOpaqueLeaf>, proof: Proof<Hash>)
+		fn verify_proof_stateless(root: Hash, leaves: Vec<EncodableOpaqueLeaf>, proof: Proof<(u64, Hash)>)
 			-> Result<(), Error>;
 
 		/// Generate MMR ancestry proof for prior mmr size
@@ -499,9 +499,9 @@ mod tests {
 			leaf_indices: vec![5],
 			leaf_count: 10,
 			items: vec![
-				(1, hex("c3e7ba6b511162fead58f2c8b5764ce869ed1118011ac37392522ed16720bbcd")),
-				(2, hex("d3e7ba6b511162fead58f2c8b5764ce869ed1118011ac37392522ed16720bbcd")),
-				(3, hex("e3e7ba6b511162fead58f2c8b5764ce869ed1118011ac37392522ed16720bbcd")),
+				hex("c3e7ba6b511162fead58f2c8b5764ce869ed1118011ac37392522ed16720bbcd"),
+				hex("d3e7ba6b511162fead58f2c8b5764ce869ed1118011ac37392522ed16720bbcd"),
+				hex("e3e7ba6b511162fead58f2c8b5764ce869ed1118011ac37392522ed16720bbcd"),
 			],
 		};
 

From 5d263a4afe493397bcf3be0fc9204801f9f3cd14 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 9 Nov 2023 09:15:28 +0100
Subject: [PATCH 090/188] Revert "adjust tests for new proof data structure"

This reverts commit c0db0ca5153c56c578cd8408c117ecacf8d9b69d.
---
 .../merkle-mountain-range/rpc/src/lib.rs      |  4 +-
 .../frame/merkle-mountain-range/src/tests.rs  | 90 +++++--------------
 2 files changed, 25 insertions(+), 69 deletions(-)

diff --git a/substrate/client/merkle-mountain-range/rpc/src/lib.rs b/substrate/client/merkle-mountain-range/rpc/src/lib.rs
index 98a0818724e8..1c821e3050fb 100644
--- a/substrate/client/merkle-mountain-range/rpc/src/lib.rs
+++ b/substrate/client/merkle-mountain-range/rpc/src/lib.rs
@@ -309,7 +309,7 @@ mod tests {
 				Proof {
 					leaf_indices: vec![1],
 					leaf_count: 9,
-					items: vec![(1, H256::repeat_byte(1)), (2, H256::repeat_byte(2))],
+					items: vec![H256::repeat_byte(1), H256::repeat_byte(2)],
 				}
 				.encode(),
 			),
@@ -336,7 +336,7 @@ mod tests {
 				Proof {
 					leaf_indices: vec![1, 2],
 					leaf_count: 9,
-					items: vec![(1, H256::repeat_byte(1)), (2, H256::repeat_byte(2))],
+					items: vec![H256::repeat_byte(1), H256::repeat_byte(2)],
 				}
 				.encode(),
 			),
diff --git a/substrate/frame/merkle-mountain-range/src/tests.rs b/substrate/frame/merkle-mountain-range/src/tests.rs
index af0f33ea3d51..429df75182ee 100644
--- a/substrate/frame/merkle-mountain-range/src/tests.rs
+++ b/substrate/frame/merkle-mountain-range/src/tests.rs
@@ -287,18 +287,9 @@ fn should_generate_proofs_correctly() {
 					leaf_indices: vec![0],
 					leaf_count: 7,
 					items: vec![
-						(
-							1,
-							hex("ad4cbc033833612ccd4626d5f023b9dfc50a35e838514dd1f3c86f8506728705")
-						),
-						(
-							5,
-							hex("cb24f4614ad5b2a5430344c99545b421d9af83c46fd632d70a332200884b4d46")
-						),
-						(
-							9,
-							hex("dca421199bdcc55bb773c6b6967e8d16675de69062b52285ca63685241fdf626")
-						),
+						hex("ad4cbc033833612ccd4626d5f023b9dfc50a35e838514dd1f3c86f8506728705"),
+						hex("cb24f4614ad5b2a5430344c99545b421d9af83c46fd632d70a332200884b4d46"),
+						hex("dca421199bdcc55bb773c6b6967e8d16675de69062b52285ca63685241fdf626"),
 					],
 				}
 			)
@@ -327,18 +318,9 @@ fn should_generate_proofs_correctly() {
 					leaf_indices: vec![2],
 					leaf_count: 7,
 					items: vec![
-						(
-							2,
-							hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854")
-						),
-						(
-							4,
-							hex("1b14c1dc7d3e4def11acdf31be0584f4b85c3673f1ff72a3af467b69a3b0d9d0")
-						),
-						(
-							9,
-							hex("dca421199bdcc55bb773c6b6967e8d16675de69062b52285ca63685241fdf626")
-						),
+						hex("1b14c1dc7d3e4def11acdf31be0584f4b85c3673f1ff72a3af467b69a3b0d9d0"),
+						hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854"),
+						hex("dca421199bdcc55bb773c6b6967e8d16675de69062b52285ca63685241fdf626"),
 					],
 				}
 			)
@@ -355,9 +337,8 @@ fn should_generate_proofs_correctly() {
 				Proof {
 					leaf_indices: vec![2],
 					leaf_count: 3,
-					items: vec![(
-						2,
-						hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854")
+					items: vec![hex(
+						"672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854"
 					)],
 				}
 			)
@@ -377,18 +358,9 @@ fn should_generate_proofs_correctly() {
 					leaf_indices: vec![2],
 					leaf_count: 5,
 					items: vec![
-						(
-							2,
-							hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854")
-						),
-						(
-							4,
-							hex("1b14c1dc7d3e4def11acdf31be0584f4b85c3673f1ff72a3af467b69a3b0d9d0")
-						),
-						(
-							7,
-							hex("3b031d22e24f1126c8f7d2f394b663f9b960ed7abbedb7152e17ce16112656d0")
-						),
+						hex("1b14c1dc7d3e4def11acdf31be0584f4b85c3673f1ff72a3af467b69a3b0d9d0"),
+						hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854"),
+						hex("3b031d22e24f1126c8f7d2f394b663f9b960ed7abbedb7152e17ce16112656d0")
 					],
 				}
 			)
@@ -404,18 +376,9 @@ fn should_generate_proofs_correctly() {
 					leaf_indices: vec![4],
 					leaf_count: 7,
 					items: vec![
-						(
-							6,
-							hex("ae88a0825da50e953e7a359c55fe13c8015e48d03d301b8bdfc9193874da9252")
-						),
-						(
-							8,
-							hex("8ed25570209d8f753d02df07c1884ddb36a3d9d4770e4608b188322151c657fe")
-						),
-						(
-							10,
-							hex("611c2174c6164952a66d985cfe1ec1a623794393e3acff96b136d198f37a648c")
-						),
+						hex("ae88a0825da50e953e7a359c55fe13c8015e48d03d301b8bdfc9193874da9252"),
+						hex("8ed25570209d8f753d02df07c1884ddb36a3d9d4770e4608b188322151c657fe"),
+						hex("611c2174c6164952a66d985cfe1ec1a623794393e3acff96b136d198f37a648c"),
 					],
 				}
 			)
@@ -427,9 +390,8 @@ fn should_generate_proofs_correctly() {
 				Proof {
 					leaf_indices: vec![4],
 					leaf_count: 5,
-					items: vec![(
-						6,
-						hex("ae88a0825da50e953e7a359c55fe13c8015e48d03d301b8bdfc9193874da9252")
+					items: vec![hex(
+						"ae88a0825da50e953e7a359c55fe13c8015e48d03d301b8bdfc9193874da9252"
 					),],
 				}
 			)
@@ -444,14 +406,8 @@ fn should_generate_proofs_correctly() {
 					leaf_indices: vec![6],
 					leaf_count: 7,
 					items: vec![
-						(
-							6,
-							hex("ae88a0825da50e953e7a359c55fe13c8015e48d03d301b8bdfc9193874da9252")
-						),
-						(
-							9,
-							hex("7e4316ae2ebf7c3b6821cb3a46ca8b7a4f9351a9b40fcf014bb0a4fd8e8f29da")
-						),
+						hex("ae88a0825da50e953e7a359c55fe13c8015e48d03d301b8bdfc9193874da9252"),
+						hex("7e4316ae2ebf7c3b6821cb3a46ca8b7a4f9351a9b40fcf014bb0a4fd8e8f29da"),
 					],
 				}
 			)
@@ -482,9 +438,9 @@ fn should_generate_batch_proof_correctly() {
 				leaf_indices: vec![0, 4, 5],
 				leaf_count: 7,
 				items: vec![
-					(1, hex("ad4cbc033833612ccd4626d5f023b9dfc50a35e838514dd1f3c86f8506728705")),
-					(5, hex("cb24f4614ad5b2a5430344c99545b421d9af83c46fd632d70a332200884b4d46")),
-					(10, hex("611c2174c6164952a66d985cfe1ec1a623794393e3acff96b136d198f37a648c")),
+					hex("ad4cbc033833612ccd4626d5f023b9dfc50a35e838514dd1f3c86f8506728705"),
+					hex("cb24f4614ad5b2a5430344c99545b421d9af83c46fd632d70a332200884b4d46"),
+					hex("611c2174c6164952a66d985cfe1ec1a623794393e3acff96b136d198f37a648c"),
 				],
 			}
 		);
@@ -499,8 +455,8 @@ fn should_generate_batch_proof_correctly() {
 				leaf_indices: vec![0, 4, 5],
 				leaf_count: 6,
 				items: vec![
-					(1, hex("ad4cbc033833612ccd4626d5f023b9dfc50a35e838514dd1f3c86f8506728705")),
-					(5, hex("cb24f4614ad5b2a5430344c99545b421d9af83c46fd632d70a332200884b4d46")),
+					hex("ad4cbc033833612ccd4626d5f023b9dfc50a35e838514dd1f3c86f8506728705"),
+					hex("cb24f4614ad5b2a5430344c99545b421d9af83c46fd632d70a332200884b4d46"),
 				],
 			}
 		);

From 442d3e9b6f0b9f0732b644980243a0629dd7d1ee Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 21 Nov 2023 10:39:34 +0100
Subject: [PATCH 091/188] cleanup api change reversion

---
 polkadot/runtime/rococo/src/lib.rs                       | 6 +++---
 polkadot/runtime/test-runtime/src/lib.rs                 | 6 +++---
 polkadot/runtime/westend/src/lib.rs                      | 6 +++---
 substrate/client/merkle-mountain-range/src/test_utils.rs | 6 +++---
 substrate/frame/merkle-mountain-range/src/lib.rs         | 6 +++---
 substrate/frame/merkle-mountain-range/src/mmr/mmr.rs     | 6 +++---
 substrate/primitives/merkle-mountain-range/src/lib.rs    | 4 ++--
 7 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs
index 135329ffa82f..e4bfb2e77c89 100644
--- a/polkadot/runtime/rococo/src/lib.rs
+++ b/polkadot/runtime/rococo/src/lib.rs
@@ -1861,7 +1861,7 @@ sp_api::impl_runtime_apis! {
 		fn generate_proof(
 			block_numbers: Vec<BlockNumber>,
 			best_known_block_number: Option<BlockNumber>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, mmr::Hash)>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<mmr::Hash>), mmr::Error> {
 			Mmr::generate_proof(block_numbers, best_known_block_number).map(
 				|(leaves, proof)| {
 					(
@@ -1875,7 +1875,7 @@ sp_api::impl_runtime_apis! {
 			)
 		}
 
-		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<(u64, mmr::Hash)>)
+		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<mmr::Hash>)
 			-> Result<(), mmr::Error>
 		{
 			let leaves = leaves.into_iter().map(|leaf|
@@ -1888,7 +1888,7 @@ sp_api::impl_runtime_apis! {
 		fn verify_proof_stateless(
 			root: mmr::Hash,
 			leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			proof: mmr::Proof<(u64, mmr::Hash)>
+			proof: mmr::Proof<mmr::Hash>
 		) -> Result<(), mmr::Error> {
 			let nodes = leaves.into_iter().map(|leaf|mmr::DataOrHash::Data(leaf.into_opaque_leaf())).collect();
 			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, nodes, proof)
diff --git a/polkadot/runtime/test-runtime/src/lib.rs b/polkadot/runtime/test-runtime/src/lib.rs
index 3cd6bee362fd..73c9afc85bc5 100644
--- a/polkadot/runtime/test-runtime/src/lib.rs
+++ b/polkadot/runtime/test-runtime/src/lib.rs
@@ -1002,11 +1002,11 @@ sp_api::impl_runtime_apis! {
 		fn generate_proof(
 			_block_numbers: Vec<BlockNumber>,
 			_best_known_block_number: Option<BlockNumber>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, Hash)>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<Hash>), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
 
-		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::Proof<(u64, Hash)>)
+		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::Proof<Hash>)
 			-> Result<(), mmr::Error>
 		{
 			Err(mmr::Error::PalletNotIncluded)
@@ -1015,7 +1015,7 @@ sp_api::impl_runtime_apis! {
 		fn verify_proof_stateless(
 			_root: Hash,
 			_leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			_proof: mmr::Proof<(u64, Hash)>
+			_proof: mmr::Proof<Hash>
 		) -> Result<(), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs
index f28b1186314a..408c14793488 100644
--- a/polkadot/runtime/westend/src/lib.rs
+++ b/polkadot/runtime/westend/src/lib.rs
@@ -1907,7 +1907,7 @@ sp_api::impl_runtime_apis! {
 		fn generate_proof(
 			block_numbers: Vec<BlockNumber>,
 			best_known_block_number: Option<BlockNumber>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, mmr::Hash)>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<mmr::Hash>), mmr::Error> {
 			Mmr::generate_proof(block_numbers, best_known_block_number).map(
 				|(leaves, proof)| {
 					(
@@ -1921,7 +1921,7 @@ sp_api::impl_runtime_apis! {
 			)
 		}
 
-		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<(u64, mmr::Hash)>)
+		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<mmr::Hash>)
 			-> Result<(), mmr::Error>
 		{
 			let leaves = leaves.into_iter().map(|leaf|
@@ -1934,7 +1934,7 @@ sp_api::impl_runtime_apis! {
 		fn verify_proof_stateless(
 			root: mmr::Hash,
 			leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			proof: mmr::Proof<(u64, mmr::Hash)>
+			proof: mmr::Proof<mmr::Hash>
 		) -> Result<(), mmr::Error> {
 			let nodes = leaves.into_iter().map(|leaf|mmr::DataOrHash::Data(leaf.into_opaque_leaf())).collect();
 			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, nodes, proof)
diff --git a/substrate/client/merkle-mountain-range/src/test_utils.rs b/substrate/client/merkle-mountain-range/src/test_utils.rs
index bf5eb2885451..010b48bb3d7d 100644
--- a/substrate/client/merkle-mountain-range/src/test_utils.rs
+++ b/substrate/client/merkle-mountain-range/src/test_utils.rs
@@ -304,11 +304,11 @@ sp_api::mock_impl_runtime_apis! {
 			&self,
 			_block_numbers: Vec<u64>,
 			_best_known_block_number: Option<u64>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, MmrHash)>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<MmrHash>), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
 
-		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::Proof<(u64, MmrHash)>)
+		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::Proof<MmrHash>)
 			-> Result<(), mmr::Error>
 		{
 			Err(mmr::Error::PalletNotIncluded)
@@ -317,7 +317,7 @@ sp_api::mock_impl_runtime_apis! {
 		fn verify_proof_stateless(
 			_root: MmrHash,
 			_leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			_proof: mmr::Proof<(u64, MmrHash)>
+			_proof: mmr::Proof<MmrHash>
 		) -> Result<(), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
diff --git a/substrate/frame/merkle-mountain-range/src/lib.rs b/substrate/frame/merkle-mountain-range/src/lib.rs
index 987f7a6f4c4e..94e126576ce1 100644
--- a/substrate/frame/merkle-mountain-range/src/lib.rs
+++ b/substrate/frame/merkle-mountain-range/src/lib.rs
@@ -245,7 +245,7 @@ pub mod pallet {
 pub fn verify_leaves_proof<H, L>(
 	root: H::Output,
 	leaves: Vec<mmr::Node<H, L>>,
-	proof: primitives::Proof<(u64, H::Output)>,
+	proof: primitives::Proof<H::Output>,
 ) -> Result<(), primitives::Error>
 where
 	H: traits::Hash,
@@ -319,7 +319,7 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 	pub fn generate_proof(
 		block_numbers: Vec<BlockNumberFor<T>>,
 		best_known_block_number: Option<BlockNumberFor<T>>,
-	) -> Result<(Vec<LeafOf<T, I>>, primitives::Proof<(u64, HashOf<T, I>)>), primitives::Error> {
+	) -> Result<(Vec<LeafOf<T, I>>, primitives::Proof<HashOf<T, I>>), primitives::Error> {
 		// check whether best_known_block_number provided, else use current best block
 		let best_known_block_number =
 			best_known_block_number.unwrap_or_else(|| <frame_system::Pallet<T>>::block_number());
@@ -373,7 +373,7 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 	/// or the proof is invalid.
 	pub fn verify_leaves(
 		leaves: Vec<LeafOf<T, I>>,
-		proof: primitives::Proof<(u64, HashOf<T, I>)>,
+		proof: primitives::Proof<HashOf<T, I>>,
 	) -> Result<(), primitives::Error> {
 		if proof.leaf_count > Self::mmr_leaves() ||
 			proof.leaf_count == 0 ||
diff --git a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
index dc5c69413083..ebd5ca249e79 100644
--- a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
+++ b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
@@ -33,7 +33,7 @@ use sp_std::prelude::*;
 pub fn verify_leaves_proof<H, L>(
 	root: H::Output,
 	leaves: Vec<Node<H, L>>,
-	proof: primitives::Proof<(u64, H::Output)>,
+	proof: primitives::Proof<H::Output>,
 ) -> Result<bool, Error>
 where
 	H: sp_runtime::traits::Hash,
@@ -95,7 +95,7 @@ where
 	pub fn verify_leaves_proof(
 		&self,
 		leaves: Vec<L>,
-		proof: primitives::Proof<(u64, HashOf<T, I>)>,
+		proof: primitives::Proof<HashOf<T, I>>,
 	) -> Result<bool, Error> {
 		let p = mmr_lib::MerkleProof::<NodeOf<T, I, L>, Hasher<HashingOf<T, I>, L>>::new(
 			self.mmr.mmr_size(),
@@ -201,7 +201,7 @@ where
 	pub fn generate_proof(
 		&self,
 		leaf_indices: Vec<NodeIndex>,
-	) -> Result<(Vec<L>, primitives::Proof<(u64, HashOf<T, I>)>), Error> {
+	) -> Result<(Vec<L>, primitives::Proof<HashOf<T, I>>), Error> {
 		let positions = leaf_indices
 			.iter()
 			.map(|index| mmr_lib::leaf_index_to_pos(*index))
diff --git a/substrate/primitives/merkle-mountain-range/src/lib.rs b/substrate/primitives/merkle-mountain-range/src/lib.rs
index 775875d5c0c1..e4077ea79cbd 100644
--- a/substrate/primitives/merkle-mountain-range/src/lib.rs
+++ b/substrate/primitives/merkle-mountain-range/src/lib.rs
@@ -453,7 +453,7 @@ sp_api::decl_runtime_apis! {
 		/// Note this function will use on-chain MMR root hash and check if the proof matches the hash.
 		/// Note, the leaves should be sorted such that corresponding leaves and leaf indices have the
 		/// same position in both the `leaves` vector and the `leaf_indices` vector contained in the [Proof]
-		fn verify_proof(leaves: Vec<EncodableOpaqueLeaf>, proof: Proof<(u64, Hash)>) -> Result<(), Error>;
+		fn verify_proof(leaves: Vec<EncodableOpaqueLeaf>, proof: Proof<Hash>) -> Result<(), Error>;
 
 		/// Verify MMR proof against given root hash for a batch of leaves.
 		///
@@ -462,7 +462,7 @@ sp_api::decl_runtime_apis! {
 		///
 		/// Note, the leaves should be sorted such that corresponding leaves and leaf indices have the
 		/// same position in both the `leaves` vector and the `leaf_indices` vector contained in the [Proof]
-		fn verify_proof_stateless(root: Hash, leaves: Vec<EncodableOpaqueLeaf>, proof: Proof<(u64, Hash)>)
+		fn verify_proof_stateless(root: Hash, leaves: Vec<EncodableOpaqueLeaf>, proof: Proof<Hash>)
 			-> Result<(), Error>;
 
 		/// Generate MMR ancestry proof for prior mmr size

From 4200f5b18a105a8107e29e82ddfcc8dc67e6e2df Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 21 Nov 2023 10:44:14 +0100
Subject: [PATCH 092/188] bump ckb-merkle-mountain-range 0.5.2 -> 0.6.0

---
 Cargo.lock                                     |  5 +++--
 .../frame/merkle-mountain-range/src/mmr/mmr.rs | 12 +++++++-----
 .../merkle-mountain-range/src/mmr/storage.rs   | 18 ++++++++++++++++--
 .../merkle-mountain-range/Cargo.toml           |  2 +-
 4 files changed, 27 insertions(+), 10 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 5dbecd5698fd..33cec10f78d3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2393,10 +2393,11 @@ dependencies = [
 
 [[package]]
 name = "ckb-merkle-mountain-range"
-version = "0.5.2"
-source = "git+https://github.com/Lederstrumpf/merkle-mountain-range.git?branch=linearize-proof-rebase-v0.5.1#f3aed2bfdaa5ec83d89e97b37c89ecaac9f64497"
+version = "0.6.0"
+source = "git+https://github.com/Lederstrumpf/merkle-mountain-range.git?branch=leaf-node-proof-split#5c2f9744170724483455dc448f8acb7f12ae97c7"
 dependencies = [
  "cfg-if",
+ "itertools 0.10.5",
 ]
 
 [[package]]
diff --git a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
index ebd5ca249e79..f14cdb2f4182 100644
--- a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
+++ b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
@@ -23,7 +23,7 @@ use crate::{
 	primitives::{self, Error, LeafIndex, NodeIndex},
 	Config, HashOf, HashingOf,
 };
-use sp_mmr_primitives::{mmr_lib, utils::NodesUtils};
+use sp_mmr_primitives::{mmr_lib, mmr_lib::MMRStoreReadOps, utils::NodesUtils};
 use sp_std::prelude::*;
 
 /// Stateless verification of the proof for a batch of leaves.
@@ -69,7 +69,8 @@ where
 	T: Config<I>,
 	I: 'static,
 	L: primitives::FullLeaf,
-	Storage<StorageType, T, I, L>: mmr_lib::MMRStore<NodeOf<T, I, L>>,
+	Storage<StorageType, T, I, L>:
+		mmr_lib::MMRStoreReadOps<NodeOf<T, I, L>> + mmr_lib::MMRStoreWriteOps<NodeOf<T, I, L>>,
 {
 	mmr: mmr_lib::MMR<NodeOf<T, I, L>, Hasher<HashingOf<T, I>, L>, Storage<StorageType, T, I, L>>,
 	leaves: NodeIndex,
@@ -80,7 +81,8 @@ where
 	T: Config<I>,
 	I: 'static,
 	L: primitives::FullLeaf,
-	Storage<StorageType, T, I, L>: mmr_lib::MMRStore<NodeOf<T, I, L>>,
+	Storage<StorageType, T, I, L>:
+		mmr_lib::MMRStoreReadOps<NodeOf<T, I, L>> + mmr_lib::MMRStoreWriteOps<NodeOf<T, I, L>>,
 {
 	/// Create a pointer to an existing MMR with given number of leaves.
 	pub fn new(leaves: NodeIndex) -> Self {
@@ -180,7 +182,7 @@ where
 
 	/// Commit the changes to underlying storage, return current number of leaves and
 	/// calculate the new MMR's root hash.
-	pub fn finalize(self) -> Result<(NodeIndex, HashOf<T, I>), Error> {
+	pub fn finalize(&mut self) -> Result<(NodeIndex, HashOf<T, I>), Error> {
 		let root = self.mmr.get_root().map_err(|e| Error::GetRoot.log_error(e))?;
 		self.mmr.commit().map_err(|e| Error::Commit.log_error(e))?;
 		Ok((self.leaves, root.hash()))
@@ -209,7 +211,7 @@ where
 		let store = <Storage<OffchainStorage, T, I, L>>::default();
 		let leaves = positions
 			.iter()
-			.map(|pos| match mmr_lib::MMRStore::get_elem(&store, *pos) {
+			.map(|pos| match store.get_elem(*pos) {
 				Ok(Some(Node::Data(leaf))) => Ok(leaf),
 				e => Err(Error::LeafNotFound.log_debug(e)),
 			})
diff --git a/substrate/frame/merkle-mountain-range/src/mmr/storage.rs b/substrate/frame/merkle-mountain-range/src/mmr/storage.rs
index 03039be83ac1..dd13290992c3 100644
--- a/substrate/frame/merkle-mountain-range/src/mmr/storage.rs
+++ b/substrate/frame/merkle-mountain-range/src/mmr/storage.rs
@@ -60,7 +60,7 @@ impl<StorageType, T, I, L> Default for Storage<StorageType, T, I, L> {
 	}
 }
 
-impl<T, I, L> mmr_lib::MMRStore<NodeOf<T, I, L>> for Storage<OffchainStorage, T, I, L>
+impl<T, I, L> mmr_lib::MMRStoreReadOps<NodeOf<T, I, L>> for Storage<OffchainStorage, T, I, L>
 where
 	T: Config<I>,
 	I: 'static,
@@ -98,13 +98,20 @@ where
 		Ok(sp_io::offchain::local_storage_get(StorageKind::PERSISTENT, &temp_key)
 			.and_then(|v| codec::Decode::decode(&mut &*v).ok()))
 	}
+}
 
+impl<T, I, L> mmr_lib::MMRStoreWriteOps<NodeOf<T, I, L>> for Storage<OffchainStorage, T, I, L>
+where
+	T: Config<I>,
+	I: 'static,
+	L: primitives::FullLeaf + codec::Decode,
+{
 	fn append(&mut self, _: NodeIndex, _: Vec<NodeOf<T, I, L>>) -> mmr_lib::Result<()> {
 		panic!("MMR must not be altered in the off-chain context.")
 	}
 }
 
-impl<T, I, L> mmr_lib::MMRStore<NodeOf<T, I, L>> for Storage<RuntimeStorage, T, I, L>
+impl<T, I, L> mmr_lib::MMRStoreReadOps<NodeOf<T, I, L>> for Storage<RuntimeStorage, T, I, L>
 where
 	T: Config<I>,
 	I: 'static,
@@ -113,7 +120,14 @@ where
 	fn get_elem(&self, pos: NodeIndex) -> mmr_lib::Result<Option<NodeOf<T, I, L>>> {
 		Ok(<Nodes<T, I>>::get(pos).map(Node::Hash))
 	}
+}
 
+impl<T, I, L> mmr_lib::MMRStoreWriteOps<NodeOf<T, I, L>> for Storage<RuntimeStorage, T, I, L>
+where
+	T: Config<I>,
+	I: 'static,
+	L: primitives::FullLeaf,
+{
 	fn append(&mut self, pos: NodeIndex, elems: Vec<NodeOf<T, I, L>>) -> mmr_lib::Result<()> {
 		if elems.is_empty() {
 			return Ok(())
diff --git a/substrate/primitives/merkle-mountain-range/Cargo.toml b/substrate/primitives/merkle-mountain-range/Cargo.toml
index a29362dea10c..c0eacd19e28c 100644
--- a/substrate/primitives/merkle-mountain-range/Cargo.toml
+++ b/substrate/primitives/merkle-mountain-range/Cargo.toml
@@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false }
 scale-info = { version = "2.5.0", default-features = false, features = ["derive"] }
 log = { version = "0.4.17", default-features = false }
-mmr-lib = { package = "ckb-merkle-mountain-range", git = "https://github.com/Lederstrumpf/merkle-mountain-range.git", branch = "linearize-proof-rebase-v0.5.1", default-features = false, features = ["nodeproofs"] }
+mmr-lib = { package = "ckb-merkle-mountain-range", git = "https://github.com/Lederstrumpf/merkle-mountain-range.git", branch = "leaf-node-proof-split", default-features = false }
 serde = { version = "1.0.188", features = ["derive", "alloc"], default-features = false, optional = true }
 sp-api = { path = "../api", default-features = false}
 sp-core = { path = "../core", default-features = false}

From e1269fdb35656dca1773c8ab090d37baee2035b1 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 21 Nov 2023 10:45:16 +0100
Subject: [PATCH 093/188] segregate Proofs and NodeProofs

---
 substrate/client/consensus/beefy/src/tests.rs    |  4 ++--
 .../frame/merkle-mountain-range/src/mmr/mmr.rs   | 16 ++++++++--------
 substrate/primitives/consensus/beefy/src/lib.rs  |  5 +++--
 .../primitives/merkle-mountain-range/src/lib.rs  | 13 ++++++++++++-
 .../merkle-mountain-range/src/utils.rs           |  7 ++++---
 5 files changed, 29 insertions(+), 16 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index 2fc0339038c2..5422fc16d35e 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -62,7 +62,7 @@ use sp_consensus_beefy::{
 };
 use sp_core::H256;
 use sp_keystore::{testing::MemoryKeystore, Keystore, KeystorePtr};
-use sp_mmr_primitives::{AncestryProof, Error as MmrError, LeafIndex, MmrApi, Proof};
+use sp_mmr_primitives::{AncestryProof, Error as MmrError, LeafIndex, MmrApi, NodeProof};
 use sp_runtime::{
 	codec::{Decode, Encode},
 	traits::{Header as HeaderT, NumberFor},
@@ -391,7 +391,7 @@ sp_api::mock_impl_runtime_apis! {
 			Ok(AncestryProof {
 				prev_peaks: vec![],
 				prev_size: 0,
-				proof: Proof { leaf_indices: vec![], leaf_count: 0, items: vec![] },
+				proof: NodeProof { leaf_indices: vec![], leaf_count: 0, items: vec![] },
 			})
 		}
 	}
diff --git a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
index f14cdb2f4182..13e7c5cdbc7b 100644
--- a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
+++ b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
@@ -123,7 +123,7 @@ where
 		&self,
 		ancestry_proof: primitives::AncestryProof<HashOf<T, I>>,
 	) -> Result<bool, Error> {
-		let p = mmr_lib::MerkleProof::<NodeOf<T, I, L>, Hasher<HashingOf<T, I>, L>>::new(
+		let p = mmr_lib::NodeMerkleProof::<NodeOf<T, I, L>, Hasher<HashingOf<T, I>, L>>::new(
 			self.mmr.mmr_size(),
 			ancestry_proof
 				.proof
@@ -143,11 +143,11 @@ where
 			proof: p,
 		};
 
-		let prev_root =
-			mmr_lib::bagging_peaks_hashes::<NodeOf<T, I, L>, Hasher<HashingOf<T, I>, L>>(
-				ancestry_proof.prev_peaks.clone(),
-			)
-			.map_err(|e| Error::Verify.log_debug(e))?;
+		let prev_root = mmr_lib::ancestry_proof::bagging_peaks_hashes::<
+			NodeOf<T, I, L>,
+			Hasher<HashingOf<T, I>, L>,
+		>(ancestry_proof.prev_peaks.clone())
+		.map_err(|e| Error::Verify.log_debug(e))?;
 		let root = self.mmr.get_root().map_err(|e| Error::GetRoot.log_error(e))?;
 		ancestry_proof
 			.verify_ancestor(root, prev_root)
@@ -236,10 +236,10 @@ where
 		let leaf_count = self.leaves;
 		let prev_mmr_size = NodesUtils::new(prev_leaf_count).size();
 		self.mmr
-			.gen_prefix_proof(prev_mmr_size)
+			.gen_ancestry_proof(prev_mmr_size)
 			.map_err(|e| Error::GenerateProof.log_error(e))
 			.map(|p| {
-				let proof = primitives::Proof {
+				let proof = primitives::NodeProof {
 					leaf_indices: (p.prev_size..leaf_count).collect(),
 					leaf_count,
 					items: p
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index b7b9e51ff55f..4dbca31277c4 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -420,8 +420,9 @@ where
 
 		// once the ancestry proof is verified, calculate the prev_root to compare it
 		// with the commitment's prev_root
-		let ancestry_prev_root =
-			mmr_lib::bagging_peaks_hashes::<NodeHash, Hasher>(ancestry_proof.prev_peaks.clone());
+		let ancestry_prev_root = mmr_lib::ancestry_proof::bagging_peaks_hashes::<NodeHash, Hasher>(
+			ancestry_proof.prev_peaks.clone(),
+		);
 		// if the commitment payload does not commit to an MMR root, then this
 		// commitment may have another purpose and should not be slashed
 		let commitment_prev_root =
diff --git a/substrate/primitives/merkle-mountain-range/src/lib.rs b/substrate/primitives/merkle-mountain-range/src/lib.rs
index e4077ea79cbd..ca1446f30f67 100644
--- a/substrate/primitives/merkle-mountain-range/src/lib.rs
+++ b/substrate/primitives/merkle-mountain-range/src/lib.rs
@@ -359,6 +359,17 @@ pub struct Proof<Hash> {
 	pub items: Vec<Hash>,
 }
 
+/// An MMR proof data for a group of leaves.
+#[derive(codec::Encode, codec::Decode, RuntimeDebug, Clone, PartialEq, Eq, TypeInfo)]
+pub struct NodeProof<Hash> {
+	/// The indices of the leaves the proof is for.
+	pub leaf_indices: Vec<LeafIndex>,
+	/// Number of leaves in MMR, when the proof was generated.
+	pub leaf_count: NodeIndex,
+	/// Proof elements (hashes of siblings of inner nodes on the path to the leaf).
+	pub items: Vec<(u64, Hash)>,
+}
+
 /// An MMR ancestry proof for a prior mmr root.
 #[derive(codec::Encode, codec::Decode, RuntimeDebug, Clone, PartialEq, Eq, TypeInfo)]
 pub struct AncestryProof<Hash> {
@@ -367,7 +378,7 @@ pub struct AncestryProof<Hash> {
 	/// Size of the ancestor's mmr
 	pub prev_size: u64,
 	/// Proof of the ancestry
-	pub proof: Proof<Hash>,
+	pub proof: NodeProof<Hash>,
 }
 
 /// Merkle Mountain Range operation error.
diff --git a/substrate/primitives/merkle-mountain-range/src/utils.rs b/substrate/primitives/merkle-mountain-range/src/utils.rs
index 05457333e438..ac806966e104 100644
--- a/substrate/primitives/merkle-mountain-range/src/utils.rs
+++ b/substrate/primitives/merkle-mountain-range/src/utils.rs
@@ -52,7 +52,7 @@ where
 	H: Clone + Debug + PartialEq + Encode,
 	M: mmr_lib::Merge<Item = H>,
 {
-	let p: mmr_lib::MerkleProof<H, _> = mmr_lib::MerkleProof::<H, M>::new(
+	let p: mmr_lib::NodeMerkleProof<H, _> = mmr_lib::NodeMerkleProof::<H, M>::new(
 		mmr_size,
 		ancestry_proof
 			.proof
@@ -68,8 +68,9 @@ where
 		proof: p,
 	};
 
-	let prev_root = mmr_lib::bagging_peaks_hashes::<H, M>(ancestry_proof.prev_peaks.clone())
-		.map_err(|e| Error::Verify.log_debug(e))?;
+	let prev_root =
+		mmr_lib::ancestry_proof::bagging_peaks_hashes::<H, M>(ancestry_proof.prev_peaks.clone())
+			.map_err(|e| Error::Verify.log_debug(e))?;
 	ancestry_proof
 		.verify_ancestor(root, prev_root)
 		.map_err(|e| Error::Verify.log_debug(e))

From 7222fa569e2399313fb66b889e6bc172802dc919 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 21 Nov 2023 10:52:18 +0100
Subject: [PATCH 094/188] primitives::{Proof->LeafProof}

---
 polkadot/node/service/src/fake_runtime_api.rs |  6 ++---
 polkadot/runtime/rococo/src/lib.rs            |  6 ++---
 polkadot/runtime/test-runtime/src/lib.rs      |  6 ++---
 polkadot/runtime/westend/src/lib.rs           |  6 ++---
 substrate/bin/node/runtime/src/lib.rs         |  6 ++---
 .../merkle-mountain-range/rpc/src/lib.rs      | 16 +++++++-------
 .../merkle-mountain-range/src/test_utils.rs   |  6 ++---
 .../frame/merkle-mountain-range/src/lib.rs    | 10 ++++-----
 .../merkle-mountain-range/src/mmr/mmr.rs      | 12 +++++-----
 .../frame/merkle-mountain-range/src/tests.rs  | 22 +++++++++----------
 .../merkle-mountain-range/src/lib.rs          | 12 +++++-----
 11 files changed, 54 insertions(+), 54 deletions(-)

diff --git a/polkadot/node/service/src/fake_runtime_api.rs b/polkadot/node/service/src/fake_runtime_api.rs
index 65c56efefe55..dce8fe2c04f0 100644
--- a/polkadot/node/service/src/fake_runtime_api.rs
+++ b/polkadot/node/service/src/fake_runtime_api.rs
@@ -278,11 +278,11 @@ sp_api::impl_runtime_apis! {
 		fn generate_proof(
 			_: Vec<BlockNumber>,
 			_: Option<BlockNumber>,
-		) -> Result<(Vec<sp_mmr_primitives::EncodableOpaqueLeaf>, sp_mmr_primitives::Proof<Hash>), sp_mmr_primitives::Error> {
+		) -> Result<(Vec<sp_mmr_primitives::EncodableOpaqueLeaf>, sp_mmr_primitives::LeafProof<Hash>), sp_mmr_primitives::Error> {
 			unimplemented!()
 		}
 
-		fn verify_proof(_: Vec<sp_mmr_primitives::EncodableOpaqueLeaf>, _: sp_mmr_primitives::Proof<Hash>)
+		fn verify_proof(_: Vec<sp_mmr_primitives::EncodableOpaqueLeaf>, _: sp_mmr_primitives::LeafProof<Hash>)
 			-> Result<(), sp_mmr_primitives::Error>
 		{
 			unimplemented!()
@@ -291,7 +291,7 @@ sp_api::impl_runtime_apis! {
 		fn verify_proof_stateless(
 			_: Hash,
 			_: Vec<sp_mmr_primitives::EncodableOpaqueLeaf>,
-			_: sp_mmr_primitives::Proof<Hash>
+			_: sp_mmr_primitives::LeafProof<Hash>
 		) -> Result<(), sp_mmr_primitives::Error> {
 			unimplemented!()
 		}
diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs
index e4bfb2e77c89..dd6c5a32209c 100644
--- a/polkadot/runtime/rococo/src/lib.rs
+++ b/polkadot/runtime/rococo/src/lib.rs
@@ -1861,7 +1861,7 @@ sp_api::impl_runtime_apis! {
 		fn generate_proof(
 			block_numbers: Vec<BlockNumber>,
 			best_known_block_number: Option<BlockNumber>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<mmr::Hash>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::LeafProof<mmr::Hash>), mmr::Error> {
 			Mmr::generate_proof(block_numbers, best_known_block_number).map(
 				|(leaves, proof)| {
 					(
@@ -1875,7 +1875,7 @@ sp_api::impl_runtime_apis! {
 			)
 		}
 
-		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<mmr::Hash>)
+		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::LeafProof<mmr::Hash>)
 			-> Result<(), mmr::Error>
 		{
 			let leaves = leaves.into_iter().map(|leaf|
@@ -1888,7 +1888,7 @@ sp_api::impl_runtime_apis! {
 		fn verify_proof_stateless(
 			root: mmr::Hash,
 			leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			proof: mmr::Proof<mmr::Hash>
+			proof: mmr::LeafProof<mmr::Hash>
 		) -> Result<(), mmr::Error> {
 			let nodes = leaves.into_iter().map(|leaf|mmr::DataOrHash::Data(leaf.into_opaque_leaf())).collect();
 			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, nodes, proof)
diff --git a/polkadot/runtime/test-runtime/src/lib.rs b/polkadot/runtime/test-runtime/src/lib.rs
index 73c9afc85bc5..bec34ef4e4db 100644
--- a/polkadot/runtime/test-runtime/src/lib.rs
+++ b/polkadot/runtime/test-runtime/src/lib.rs
@@ -1002,11 +1002,11 @@ sp_api::impl_runtime_apis! {
 		fn generate_proof(
 			_block_numbers: Vec<BlockNumber>,
 			_best_known_block_number: Option<BlockNumber>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<Hash>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::LeafProof<Hash>), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
 
-		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::Proof<Hash>)
+		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::LeafProof<Hash>)
 			-> Result<(), mmr::Error>
 		{
 			Err(mmr::Error::PalletNotIncluded)
@@ -1015,7 +1015,7 @@ sp_api::impl_runtime_apis! {
 		fn verify_proof_stateless(
 			_root: Hash,
 			_leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			_proof: mmr::Proof<Hash>
+			_proof: mmr::LeafProof<Hash>
 		) -> Result<(), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs
index 408c14793488..c1949b857c75 100644
--- a/polkadot/runtime/westend/src/lib.rs
+++ b/polkadot/runtime/westend/src/lib.rs
@@ -1907,7 +1907,7 @@ sp_api::impl_runtime_apis! {
 		fn generate_proof(
 			block_numbers: Vec<BlockNumber>,
 			best_known_block_number: Option<BlockNumber>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<mmr::Hash>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::LeafProof<mmr::Hash>), mmr::Error> {
 			Mmr::generate_proof(block_numbers, best_known_block_number).map(
 				|(leaves, proof)| {
 					(
@@ -1921,7 +1921,7 @@ sp_api::impl_runtime_apis! {
 			)
 		}
 
-		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<mmr::Hash>)
+		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::LeafProof<mmr::Hash>)
 			-> Result<(), mmr::Error>
 		{
 			let leaves = leaves.into_iter().map(|leaf|
@@ -1934,7 +1934,7 @@ sp_api::impl_runtime_apis! {
 		fn verify_proof_stateless(
 			root: mmr::Hash,
 			leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			proof: mmr::Proof<mmr::Hash>
+			proof: mmr::LeafProof<mmr::Hash>
 		) -> Result<(), mmr::Error> {
 			let nodes = leaves.into_iter().map(|leaf|mmr::DataOrHash::Data(leaf.into_opaque_leaf())).collect();
 			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, nodes, proof)
diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs
index 90079167ccda..1063f15c6ed2 100644
--- a/substrate/bin/node/runtime/src/lib.rs
+++ b/substrate/bin/node/runtime/src/lib.rs
@@ -2654,7 +2654,7 @@ impl_runtime_apis! {
 		fn generate_proof(
 			block_numbers: Vec<BlockNumber>,
 			best_known_block_number: Option<BlockNumber>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<(u64, mmr::Hash)>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::LeafProof<mmr::Hash>), mmr::Error> {
 			Mmr::generate_proof(block_numbers, best_known_block_number).map(
 				|(leaves, proof)| {
 					(
@@ -2672,7 +2672,7 @@ impl_runtime_apis! {
 			Mmr::generate_ancestry_proof(prev_best_block)
 		}
 
-		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::Proof<mmr::Hash>)
+		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::LeafProof<mmr::Hash>)
 			-> Result<(), mmr::Error>
 		{
 			let leaves = leaves.into_iter().map(|leaf|
@@ -2685,7 +2685,7 @@ impl_runtime_apis! {
 		fn verify_proof_stateless(
 			root: mmr::Hash,
 			leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			proof: mmr::Proof<(u64, mmr::Hash)>
+			proof: mmr::LeafProof<mmr::Hash>
 		) -> Result<(), mmr::Error> {
 			let nodes = leaves.into_iter().map(|leaf|mmr::DataOrHash::Data(leaf.into_opaque_leaf())).collect();
 			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, nodes, proof)
diff --git a/substrate/client/merkle-mountain-range/rpc/src/lib.rs b/substrate/client/merkle-mountain-range/rpc/src/lib.rs
index 1c821e3050fb..ce3ea791b240 100644
--- a/substrate/client/merkle-mountain-range/rpc/src/lib.rs
+++ b/substrate/client/merkle-mountain-range/rpc/src/lib.rs
@@ -36,7 +36,7 @@ use sp_core::{
 	offchain::{storage::OffchainDb, OffchainDbExt, OffchainStorage},
 	Bytes,
 };
-use sp_mmr_primitives::{Error as MmrError, Proof};
+use sp_mmr_primitives::{Error as MmrError, LeafProof};
 use sp_runtime::traits::Block as BlockT;
 
 pub use sp_mmr_primitives::MmrApi as MmrRuntimeApi;
@@ -52,17 +52,17 @@ pub struct LeavesProof<BlockHash> {
 	pub block_hash: BlockHash,
 	/// SCALE-encoded vector of `LeafData`.
 	pub leaves: Bytes,
-	/// SCALE-encoded proof data. See [sp_mmr_primitives::Proof].
+	/// SCALE-encoded proof data. See [sp_mmr_primitives::LeafProof].
 	pub proof: Bytes,
 }
 
 impl<BlockHash> LeavesProof<BlockHash> {
 	/// Create new `LeavesProof` from a given vector of `Leaf` and a
-	/// [sp_mmr_primitives::Proof].
+	/// [sp_mmr_primitives::LeafProof].
 	pub fn new<Leaf, MmrHash>(
 		block_hash: BlockHash,
 		leaves: Vec<Leaf>,
-		proof: Proof<MmrHash>,
+		proof: LeafProof<MmrHash>,
 	) -> Self
 	where
 		Leaf: Encode,
@@ -258,7 +258,7 @@ mod tests {
 	fn should_serialize_leaf_proof() {
 		// given
 		let leaf = vec![1_u8, 2, 3, 4];
-		let proof = Proof {
+		let proof = LeafProof {
 			leaf_indices: vec![1],
 			leaf_count: 9,
 			items: vec![H256::repeat_byte(1), H256::repeat_byte(2)],
@@ -281,7 +281,7 @@ mod tests {
 		// given
 		let leaf_a = vec![1_u8, 2, 3, 4];
 		let leaf_b = vec![2_u8, 2, 3, 4];
-		let proof = Proof {
+		let proof = LeafProof {
 			leaf_indices: vec![1, 2],
 			leaf_count: 9,
 			items: vec![H256::repeat_byte(1), H256::repeat_byte(2)],
@@ -306,7 +306,7 @@ mod tests {
 			block_hash: H256::repeat_byte(0),
 			leaves: Bytes(vec![vec![1_u8, 2, 3, 4]].encode()),
 			proof: Bytes(
-				Proof {
+				LeafProof {
 					leaf_indices: vec![1],
 					leaf_count: 9,
 					items: vec![H256::repeat_byte(1), H256::repeat_byte(2)],
@@ -333,7 +333,7 @@ mod tests {
 			block_hash: H256::repeat_byte(0),
 			leaves: Bytes(vec![vec![1_u8, 2, 3, 4], vec![2_u8, 2, 3, 4]].encode()),
 			proof: Bytes(
-				Proof {
+				LeafProof {
 					leaf_indices: vec![1, 2],
 					leaf_count: 9,
 					items: vec![H256::repeat_byte(1), H256::repeat_byte(2)],
diff --git a/substrate/client/merkle-mountain-range/src/test_utils.rs b/substrate/client/merkle-mountain-range/src/test_utils.rs
index 010b48bb3d7d..780973c27caa 100644
--- a/substrate/client/merkle-mountain-range/src/test_utils.rs
+++ b/substrate/client/merkle-mountain-range/src/test_utils.rs
@@ -304,11 +304,11 @@ sp_api::mock_impl_runtime_apis! {
 			&self,
 			_block_numbers: Vec<u64>,
 			_best_known_block_number: Option<u64>,
-		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::Proof<MmrHash>), mmr::Error> {
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::LeafProof<MmrHash>), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
 
-		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::Proof<MmrHash>)
+		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::LeafProof<MmrHash>)
 			-> Result<(), mmr::Error>
 		{
 			Err(mmr::Error::PalletNotIncluded)
@@ -317,7 +317,7 @@ sp_api::mock_impl_runtime_apis! {
 		fn verify_proof_stateless(
 			_root: MmrHash,
 			_leaves: Vec<mmr::EncodableOpaqueLeaf>,
-			_proof: mmr::Proof<MmrHash>
+			_proof: mmr::LeafProof<MmrHash>
 		) -> Result<(), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
diff --git a/substrate/frame/merkle-mountain-range/src/lib.rs b/substrate/frame/merkle-mountain-range/src/lib.rs
index 94e126576ce1..04b0199aff94 100644
--- a/substrate/frame/merkle-mountain-range/src/lib.rs
+++ b/substrate/frame/merkle-mountain-range/src/lib.rs
@@ -237,15 +237,15 @@ pub mod pallet {
 
 /// Stateless MMR proof verification for batch of leaves.
 ///
-/// This function can be used to verify received MMR [primitives::Proof] (`proof`)
+/// This function can be used to verify received MMR [primitives::LeafProof] (`proof`)
 /// for given leaves set (`leaves`) against a known MMR root hash (`root`).
 /// Note, the leaves should be sorted such that corresponding leaves and leaf indices have the
 /// same position in both the `leaves` vector and the `leaf_indices` vector contained in the
-/// [primitives::Proof].
+/// [primitives::LeafProof].
 pub fn verify_leaves_proof<H, L>(
 	root: H::Output,
 	leaves: Vec<mmr::Node<H, L>>,
-	proof: primitives::Proof<H::Output>,
+	proof: primitives::LeafProof<H::Output>,
 ) -> Result<(), primitives::Error>
 where
 	H: traits::Hash,
@@ -319,7 +319,7 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 	pub fn generate_proof(
 		block_numbers: Vec<BlockNumberFor<T>>,
 		best_known_block_number: Option<BlockNumberFor<T>>,
-	) -> Result<(Vec<LeafOf<T, I>>, primitives::Proof<HashOf<T, I>>), primitives::Error> {
+	) -> Result<(Vec<LeafOf<T, I>>, primitives::LeafProof<HashOf<T, I>>), primitives::Error> {
 		// check whether best_known_block_number provided, else use current best block
 		let best_known_block_number =
 			best_known_block_number.unwrap_or_else(|| <frame_system::Pallet<T>>::block_number());
@@ -373,7 +373,7 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 	/// or the proof is invalid.
 	pub fn verify_leaves(
 		leaves: Vec<LeafOf<T, I>>,
-		proof: primitives::Proof<HashOf<T, I>>,
+		proof: primitives::LeafProof<HashOf<T, I>>,
 	) -> Result<(), primitives::Error> {
 		if proof.leaf_count > Self::mmr_leaves() ||
 			proof.leaf_count == 0 ||
diff --git a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
index 13e7c5cdbc7b..99167683829e 100644
--- a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
+++ b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
@@ -29,11 +29,11 @@ use sp_std::prelude::*;
 /// Stateless verification of the proof for a batch of leaves.
 /// Note, the leaves should be sorted such that corresponding leaves and leaf indices have the
 /// same position in both the `leaves` vector and the `leaf_indices` vector contained in the
-/// [primitives::Proof]
+/// [primitives::LeafProof]
 pub fn verify_leaves_proof<H, L>(
 	root: H::Output,
 	leaves: Vec<Node<H, L>>,
-	proof: primitives::Proof<H::Output>,
+	proof: primitives::LeafProof<H::Output>,
 ) -> Result<bool, Error>
 where
 	H: sp_runtime::traits::Hash,
@@ -93,11 +93,11 @@ where
 	/// Verify proof for a set of leaves.
 	/// Note, the leaves should be sorted such that corresponding leaves and leaf indices have
 	/// the same position in both the `leaves` vector and the `leaf_indices` vector contained in the
-	/// [primitives::Proof]
+	/// [primitives::LeafProof]
 	pub fn verify_leaves_proof(
 		&self,
 		leaves: Vec<L>,
-		proof: primitives::Proof<HashOf<T, I>>,
+		proof: primitives::LeafProof<HashOf<T, I>>,
 	) -> Result<bool, Error> {
 		let p = mmr_lib::MerkleProof::<NodeOf<T, I, L>, Hasher<HashingOf<T, I>, L>>::new(
 			self.mmr.mmr_size(),
@@ -203,7 +203,7 @@ where
 	pub fn generate_proof(
 		&self,
 		leaf_indices: Vec<NodeIndex>,
-	) -> Result<(Vec<L>, primitives::Proof<HashOf<T, I>>), Error> {
+	) -> Result<(Vec<L>, primitives::LeafProof<HashOf<T, I>>), Error> {
 		let positions = leaf_indices
 			.iter()
 			.map(|index| mmr_lib::leaf_index_to_pos(*index))
@@ -221,7 +221,7 @@ where
 		self.mmr
 			.gen_proof(positions)
 			.map_err(|e| Error::GenerateProof.log_error(e))
-			.map(|p| primitives::Proof {
+			.map(|p| primitives::LeafProof {
 				leaf_indices,
 				leaf_count,
 				items: p.proof_items().iter().map(|x| x.hash()).collect(),
diff --git a/substrate/frame/merkle-mountain-range/src/tests.rs b/substrate/frame/merkle-mountain-range/src/tests.rs
index 429df75182ee..d9d3bd82b76b 100644
--- a/substrate/frame/merkle-mountain-range/src/tests.rs
+++ b/substrate/frame/merkle-mountain-range/src/tests.rs
@@ -22,7 +22,7 @@ use sp_core::{
 	offchain::{testing::TestOffchainExt, OffchainDbExt, OffchainWorkerExt},
 	H256,
 };
-use sp_mmr_primitives::{mmr_lib::helper, utils, Compact, Proof};
+use sp_mmr_primitives::{mmr_lib::helper, utils, Compact, LeafProof};
 use sp_runtime::BuildStorage;
 
 pub(crate) fn new_test_ext() -> sp_io::TestExternalities {
@@ -283,7 +283,7 @@ fn should_generate_proofs_correctly() {
 			proofs[0],
 			(
 				vec![Compact::new(((0, H256::repeat_byte(1)).into(), LeafData::new(1).into(),))],
-				Proof {
+				LeafProof {
 					leaf_indices: vec![0],
 					leaf_count: 7,
 					items: vec![
@@ -298,7 +298,7 @@ fn should_generate_proofs_correctly() {
 			historical_proofs[0][0],
 			(
 				vec![Compact::new(((0, H256::repeat_byte(1)).into(), LeafData::new(1).into(),))],
-				Proof { leaf_indices: vec![0], leaf_count: 1, items: vec![] }
+				LeafProof { leaf_indices: vec![0], leaf_count: 1, items: vec![] }
 			)
 		);
 
@@ -314,7 +314,7 @@ fn should_generate_proofs_correctly() {
 			proofs[2],
 			(
 				vec![Compact::new(((2, H256::repeat_byte(3)).into(), LeafData::new(3).into(),))],
-				Proof {
+				LeafProof {
 					leaf_indices: vec![2],
 					leaf_count: 7,
 					items: vec![
@@ -334,7 +334,7 @@ fn should_generate_proofs_correctly() {
 			historical_proofs[2][0],
 			(
 				vec![Compact::new(((2, H256::repeat_byte(3)).into(), LeafData::new(3).into(),))],
-				Proof {
+				LeafProof {
 					leaf_indices: vec![2],
 					leaf_count: 3,
 					items: vec![hex(
@@ -354,7 +354,7 @@ fn should_generate_proofs_correctly() {
 			historical_proofs[2][2],
 			(
 				vec![Compact::new(((2, H256::repeat_byte(3)).into(), LeafData::new(3).into(),))],
-				Proof {
+				LeafProof {
 					leaf_indices: vec![2],
 					leaf_count: 5,
 					items: vec![
@@ -372,7 +372,7 @@ fn should_generate_proofs_correctly() {
 			(
 				// NOTE: the leaf index is equivalent to the block number(in this case 5) - 1
 				vec![Compact::new(((4, H256::repeat_byte(5)).into(), LeafData::new(5).into(),))],
-				Proof {
+				LeafProof {
 					leaf_indices: vec![4],
 					leaf_count: 7,
 					items: vec![
@@ -387,7 +387,7 @@ fn should_generate_proofs_correctly() {
 			historical_proofs[4][0],
 			(
 				vec![Compact::new(((4, H256::repeat_byte(5)).into(), LeafData::new(5).into(),))],
-				Proof {
+				LeafProof {
 					leaf_indices: vec![4],
 					leaf_count: 5,
 					items: vec![hex(
@@ -402,7 +402,7 @@ fn should_generate_proofs_correctly() {
 			proofs[6],
 			(
 				vec![Compact::new(((6, H256::repeat_byte(7)).into(), LeafData::new(7).into(),))],
-				Proof {
+				LeafProof {
 					leaf_indices: vec![6],
 					leaf_count: 7,
 					items: vec![
@@ -433,7 +433,7 @@ fn should_generate_batch_proof_correctly() {
 		// then
 		assert_eq!(
 			proof,
-			Proof {
+			LeafProof {
 				// the leaf indices are equivalent to the above specified block numbers - 1.
 				leaf_indices: vec![0, 4, 5],
 				leaf_count: 7,
@@ -451,7 +451,7 @@ fn should_generate_batch_proof_correctly() {
 		// then
 		assert_eq!(
 			historical_proof,
-			Proof {
+			LeafProof {
 				leaf_indices: vec![0, 4, 5],
 				leaf_count: 6,
 				items: vec![
diff --git a/substrate/primitives/merkle-mountain-range/src/lib.rs b/substrate/primitives/merkle-mountain-range/src/lib.rs
index ca1446f30f67..78a315622aca 100644
--- a/substrate/primitives/merkle-mountain-range/src/lib.rs
+++ b/substrate/primitives/merkle-mountain-range/src/lib.rs
@@ -350,7 +350,7 @@ impl_leaf_data_for_tuple!(A:0, B:1, C:2, D:3, E:4);
 
 /// An MMR proof data for a group of leaves.
 #[derive(codec::Encode, codec::Decode, RuntimeDebug, Clone, PartialEq, Eq, TypeInfo)]
-pub struct Proof<Hash> {
+pub struct LeafProof<Hash> {
 	/// The indices of the leaves the proof is for.
 	pub leaf_indices: Vec<LeafIndex>,
 	/// Number of leaves in MMR, when the proof was generated.
@@ -457,14 +457,14 @@ sp_api::decl_runtime_apis! {
 		fn generate_proof(
 			block_numbers: Vec<BlockNumber>,
 			best_known_block_number: Option<BlockNumber>
-		) -> Result<(Vec<EncodableOpaqueLeaf>, Proof<Hash>), Error>;
+		) -> Result<(Vec<EncodableOpaqueLeaf>, LeafProof<Hash>), Error>;
 
 		/// Verify MMR proof against on-chain MMR for a batch of leaves.
 		///
 		/// Note this function will use on-chain MMR root hash and check if the proof matches the hash.
 		/// Note, the leaves should be sorted such that corresponding leaves and leaf indices have the
 		/// same position in both the `leaves` vector and the `leaf_indices` vector contained in the [Proof]
-		fn verify_proof(leaves: Vec<EncodableOpaqueLeaf>, proof: Proof<Hash>) -> Result<(), Error>;
+		fn verify_proof(leaves: Vec<EncodableOpaqueLeaf>, proof: LeafProof<Hash>) -> Result<(), Error>;
 
 		/// Verify MMR proof against given root hash for a batch of leaves.
 		///
@@ -473,7 +473,7 @@ sp_api::decl_runtime_apis! {
 		///
 		/// Note, the leaves should be sorted such that corresponding leaves and leaf indices have the
 		/// same position in both the `leaves` vector and the `leaf_indices` vector contained in the [Proof]
-		fn verify_proof_stateless(root: Hash, leaves: Vec<EncodableOpaqueLeaf>, proof: Proof<Hash>)
+		fn verify_proof_stateless(root: Hash, leaves: Vec<EncodableOpaqueLeaf>, proof: LeafProof<Hash>)
 			-> Result<(), Error>;
 
 		/// Generate MMR ancestry proof for prior mmr size
@@ -501,12 +501,12 @@ mod tests {
 
 	type Test = DataOrHash<Keccak256, String>;
 	type TestCompact = Compact<Keccak256, (Test, Test)>;
-	type TestProof = Proof<<Keccak256 as traits::Hash>::Output>;
+	type TestProof = LeafProof<<Keccak256 as traits::Hash>::Output>;
 
 	#[test]
 	fn should_encode_decode_proof() {
 		// given
-		let proof: TestProof = Proof {
+		let proof: TestProof = LeafProof {
 			leaf_indices: vec![5],
 			leaf_count: 10,
 			items: vec![

From 2e3cb9fedc454508fef661929d8426d7d8d36f18 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 21 Nov 2023 17:16:18 +0100
Subject: [PATCH 095/188] bump mmr-lib

---
 Cargo.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index 33cec10f78d3..bc4e9deefc71 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2394,7 +2394,7 @@ dependencies = [
 [[package]]
 name = "ckb-merkle-mountain-range"
 version = "0.6.0"
-source = "git+https://github.com/Lederstrumpf/merkle-mountain-range.git?branch=leaf-node-proof-split#5c2f9744170724483455dc448f8acb7f12ae97c7"
+source = "git+https://github.com/Lederstrumpf/merkle-mountain-range.git?branch=leaf-node-proof-split#7749d1ff5263f9bf1f56ea9193c0a357881695e6"
 dependencies = [
  "cfg-if",
  "itertools 0.10.5",

From 9a32a606400753f067a39a4d02403573ff9ee736 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 21 Nov 2023 18:09:57 +0100
Subject: [PATCH 096/188] fix kitchensink runtime MmrApi impl

---
 substrate/bin/node/runtime/src/lib.rs | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs
index 1063f15c6ed2..ac9b320bf452 100644
--- a/substrate/bin/node/runtime/src/lib.rs
+++ b/substrate/bin/node/runtime/src/lib.rs
@@ -2668,10 +2668,6 @@ impl_runtime_apis! {
 			)
 		}
 
-		fn generate_ancestry_proof(prev_best_block: BlockNumber) -> Result<AncestryProof<Hash>, mmr::Error> -> Result<AncestryProof<Hash>> {
-			Mmr::generate_ancestry_proof(prev_best_block)
-		}
-
 		fn verify_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::LeafProof<mmr::Hash>)
 			-> Result<(), mmr::Error>
 		{
@@ -2690,6 +2686,19 @@ impl_runtime_apis! {
 			let nodes = leaves.into_iter().map(|leaf|mmr::DataOrHash::Data(leaf.into_opaque_leaf())).collect();
 			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, nodes, proof)
 		}
+
+		fn generate_ancestry_proof(
+			prev_best_block: BlockNumber,
+			best_known_block_number: Option<BlockNumber>
+		) -> Result<mmr::AncestryProof<mmr::Hash>, mmr::Error> {
+			Mmr::generate_ancestry_proof(prev_best_block, best_known_block_number)
+		}
+
+		fn verify_ancestry_proof(
+			ancestry_proof: mmr::AncestryProof<mmr::Hash>,
+		) -> Result<(), mmr::Error> {
+			Mmr::verify_ancestry_proof(ancestry_proof)
+		}
 	}
 
 	impl sp_mixnet::runtime_api::MixnetApi<Block> for Runtime {

From 828502c5b93ddacad842f3c7edbe906b1f03720d Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 21 Nov 2023 20:46:07 +0100
Subject: [PATCH 097/188] test-runtime: impl MmrApi

---
 polkadot/runtime/test-runtime/src/lib.rs | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/polkadot/runtime/test-runtime/src/lib.rs b/polkadot/runtime/test-runtime/src/lib.rs
index bec34ef4e4db..7001826e03bf 100644
--- a/polkadot/runtime/test-runtime/src/lib.rs
+++ b/polkadot/runtime/test-runtime/src/lib.rs
@@ -1019,6 +1019,19 @@ sp_api::impl_runtime_apis! {
 		) -> Result<(), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
+
+		fn generate_ancestry_proof(
+			_prev_best_block: BlockNumber,
+			_best_known_block_number: Option<BlockNumber>
+		) -> Result<mmr::AncestryProof<Hash>, mmr::Error> {
+			Err(mmr::Error::PalletNotIncluded)
+		}
+
+		fn verify_ancestry_proof(
+			_ancestry_proof: mmr::AncestryProof<Hash>,
+		) -> Result<(), mmr::Error> {
+			Err(mmr::Error::PalletNotIncluded)
+		}
 	}
 
 	impl fg_primitives::GrandpaApi<Block> for Runtime {

From 9da3ea5aa9b32597517fd0d60f01e523c4a587b0 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Wed, 22 Nov 2023 21:06:10 +0100
Subject: [PATCH 098/188] account for #1189

---
 substrate/frame/beefy/src/tests.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index 404c3b6026a1..0b9b83144393 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -875,7 +875,7 @@ fn report_fork_equivocation_vote_current_set_works() {
 		assert_eq!(Balances::total_balance(&equivocation_validator_id), 10_000_000 - 10_000);
 		assert_eq!(Staking::slashable_balance_of(&equivocation_validator_id), 0);
 		assert_eq!(
-			Staking::eras_stakers(era, equivocation_validator_id),
+			Staking::eras_stakers(era, &equivocation_validator_id),
 			pallet_staking::Exposure { total: 0, own: 0, others: vec![] },
 		);
 
@@ -989,7 +989,7 @@ fn report_fork_equivocation_vote_old_set_works() {
 		assert_eq!(Balances::total_balance(&equivocation_validator_id), 10_000_000 - 10_000);
 		assert_eq!(Staking::slashable_balance_of(&equivocation_validator_id), 0);
 		assert_eq!(
-			Staking::eras_stakers(era, equivocation_validator_id),
+			Staking::eras_stakers(era, &equivocation_validator_id),
 			pallet_staking::Exposure { total: 0, own: 0, others: vec![] },
 		);
 
@@ -2312,7 +2312,7 @@ fn report_fork_equivocation_sc_stacked_reports_stack_correctly() {
 		assert_eq!(Balances::total_balance(&equivocation_validator_ids[0]), 10_000_000 - 10_000);
 		assert_eq!(Staking::slashable_balance_of(&equivocation_validator_ids[0]), 0);
 		assert_eq!(
-			Staking::eras_stakers(era, equivocation_validator_ids[0]),
+			Staking::eras_stakers(era, &equivocation_validator_ids[0]),
 			pallet_staking::Exposure { total: 0, own: 0, others: vec![] },
 		);
 

From 12b8eecd7a5e6019c165a03de1acbd202a0f0901 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Wed, 22 Nov 2023 22:15:35 +0100
Subject: [PATCH 099/188] Revert "fix proof serde tests"

This reverts commit f348a862aa4cc37ebe9e8dfab3cccb1f5ca301c2 since
standard mmr proofs have been reverted to the old format too.
---
 substrate/client/merkle-mountain-range/rpc/src/lib.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/substrate/client/merkle-mountain-range/rpc/src/lib.rs b/substrate/client/merkle-mountain-range/rpc/src/lib.rs
index ce3ea791b240..e6dd2ff99eaa 100644
--- a/substrate/client/merkle-mountain-range/rpc/src/lib.rs
+++ b/substrate/client/merkle-mountain-range/rpc/src/lib.rs
@@ -272,7 +272,7 @@ mod tests {
 		// then
 		assert_eq!(
 			actual,
-			r#"{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","leaves":"0x041001020304","proof":"0x0401000000000000000900000000000000080100000000000000010101010101010101010101010101010101010101010101010101010101010102000000000000000202020202020202020202020202020202020202020202020202020202020202"}"#
+			r#"{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","leaves":"0x041001020304","proof":"0x04010000000000000009000000000000000801010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202"}"#
 		);
 	}
 
@@ -295,7 +295,7 @@ mod tests {
 		// then
 		assert_eq!(
 			actual,
-			r#"{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","leaves":"0x0810010203041002020304","proof":"0x08010000000000000002000000000000000900000000000000080100000000000000010101010101010101010101010101010101010101010101010101010101010102000000000000000202020202020202020202020202020202020202020202020202020202020202"}"#
+			r#"{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","leaves":"0x0810010203041002020304","proof":"0x080100000000000000020000000000000009000000000000000801010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202"}"#
 		);
 	}
 
@@ -319,7 +319,7 @@ mod tests {
 		let actual: LeavesProof<H256> = serde_json::from_str(r#"{
 			"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000",
 			"leaves":"0x041001020304",
-			"proof":"0x0401000000000000000900000000000000080100000000000000010101010101010101010101010101010101010101010101010101010101010102000000000000000202020202020202020202020202020202020202020202020202020202020202"
+			"proof":"0x04010000000000000009000000000000000801010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202"
 		}"#).unwrap();
 
 		// then
@@ -346,7 +346,7 @@ mod tests {
 		let actual: LeavesProof<H256> = serde_json::from_str(r#"{
 			"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000",
 			"leaves":"0x0810010203041002020304",
-			"proof":"0x08010000000000000002000000000000000900000000000000080100000000000000010101010101010101010101010101010101010101010101010101010101010102000000000000000202020202020202020202020202020202020202020202020202020202020202"
+			"proof":"0x080100000000000000020000000000000009000000000000000801010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202"
 		}"#).unwrap();
 
 		// then

From 28cf54a86ea4777d4431b0c3a08f9ef857949d97 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 23 Nov 2023 01:54:33 +0100
Subject: [PATCH 100/188] fix docs

---
 substrate/primitives/merkle-mountain-range/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/substrate/primitives/merkle-mountain-range/src/lib.rs b/substrate/primitives/merkle-mountain-range/src/lib.rs
index 78a315622aca..2dc3082bf854 100644
--- a/substrate/primitives/merkle-mountain-range/src/lib.rs
+++ b/substrate/primitives/merkle-mountain-range/src/lib.rs
@@ -463,7 +463,7 @@ sp_api::decl_runtime_apis! {
 		///
 		/// Note this function will use on-chain MMR root hash and check if the proof matches the hash.
 		/// Note, the leaves should be sorted such that corresponding leaves and leaf indices have the
-		/// same position in both the `leaves` vector and the `leaf_indices` vector contained in the [Proof]
+		/// same position in both the `leaves` vector and the `leaf_indices` vector contained in the [LeafProof]
 		fn verify_proof(leaves: Vec<EncodableOpaqueLeaf>, proof: LeafProof<Hash>) -> Result<(), Error>;
 
 		/// Verify MMR proof against given root hash for a batch of leaves.
@@ -472,7 +472,7 @@ sp_api::decl_runtime_apis! {
 		/// proof is verified against given MMR root hash.
 		///
 		/// Note, the leaves should be sorted such that corresponding leaves and leaf indices have the
-		/// same position in both the `leaves` vector and the `leaf_indices` vector contained in the [Proof]
+		/// same position in both the `leaves` vector and the `leaf_indices` vector contained in the [LeafProof]
 		fn verify_proof_stateless(root: Hash, leaves: Vec<EncodableOpaqueLeaf>, proof: LeafProof<Hash>)
 			-> Result<(), Error>;
 

From f8bf63068f700bd8387a918b693bc0ae210a3553 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 27 Nov 2023 11:54:07 +0100
Subject: [PATCH 101/188] refactor check_fork_equivocation_proof

---
 .../primitives/consensus/beefy/src/lib.rs     | 51 +++++++++----------
 1 file changed, 25 insertions(+), 26 deletions(-)

diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 43bfbdb4cfe1..50ea87b7a6a3 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -522,35 +522,34 @@ where
 {
 	let ForkEquivocationProof { commitment, signatories, correct_header, ancestry_proof } = proof;
 
-	if commitment.block_number > best_block_num {
-		return false;
-	}
-
-	if (correct_header, ancestry_proof) == (&None, &None) {
-		// if commitment isn't to a block number in the future, at least a header or ancestry
-		// proof must be provided, otherwise the proof is entirely invalid
-		return false;
-	}
+	// if commitment is to a block in the future, it's an equivocation as long as it's been signed
+	if commitment.block_number <= best_block_num {
+		if (correct_header, ancestry_proof) == (&None, &None) {
+			// if commitment isn't to a block number in the future, at least a header or ancestry
+			// proof must be provided, otherwise the proof is entirely invalid
+			return false;
+		}
 
-	// avoid verifying the ancestry proof if a valid header proof has been provided
-	if check_header_proof(commitment, correct_header, expected_header_hash) ||
-		check_ancestry_proof::<Header, NodeHash, Hasher>(
-			commitment,
-			ancestry_proof,
-			first_mmr_block_num,
-			expected_root,
-			mmr_size,
-		) {
-		// if either proof is valid, check the validator's signatures. The proof is verified if they
-		// are all correct.
-		return signatories.iter().all(|(authority_id, signature)| {
-			// TODO: refactor check_commitment_signature to take a slice of signatories
-			check_commitment_signature(&commitment, authority_id, signature)
-		})
+		// if neither the ancestry proof nor the header proof is correct, the proof is invalid
+		// avoid verifying the ancestry proof if a valid header proof has been provided
+		if !check_header_proof(commitment, correct_header, expected_header_hash) &&
+			!check_ancestry_proof::<Header, NodeHash, Hasher>(
+				commitment,
+				ancestry_proof,
+				first_mmr_block_num,
+				expected_root,
+				mmr_size,
+			) {
+			return false;
+		}
 	}
 
-	// if neither the ancestry proof nor the header proof is correct, the proof is invalid
-	return false;
+	// if commitment is to future block or either proof is valid, check the validator's signatures.
+	// The proof is verified if they are all correct.
+	return signatories.iter().all(|(authority_id, signature)| {
+		// TODO: refactor check_commitment_signature to take a slice of signatories
+		check_commitment_signature(&commitment, authority_id, signature)
+	})
 }
 
 /// New BEEFY validator set notification hook.

From 87652dcd62e88b12ac9c40fd925f7ab7ad8b43c3 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 27 Nov 2023 12:05:16 +0100
Subject: [PATCH 102/188] test slashing of future block commitments/votes

---
 substrate/frame/beefy/src/tests.rs | 189 +++++++++++++++++++++++++++++
 1 file changed, 189 insertions(+)

diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index 0b9b83144393..5578f0fe4b89 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -1010,6 +1010,95 @@ fn report_fork_equivocation_vote_old_set_works() {
 	});
 }
 
+#[test]
+fn report_fork_equivocation_vote_future_block_works() {
+	let authorities = test_authorities();
+
+	let mut ext = new_test_ext_raw_authorities(authorities);
+	let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+	ext.register_extension(OffchainWorkerExt::new(offchain));
+
+	let mut era = 2;
+	ext.execute_with(|| {
+		assert_eq!(Staking::current_era(), Some(0));
+		assert_eq!(Session::current_index(), 0);
+		start_era(era);
+	});
+	ext.persist_offchain_overlay();
+
+	ext.execute_with(|| {
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+		let set_id = validator_set.id();
+		let validators = Session::validators();
+
+		// make sure that all validators have the same balance
+		for validator in &validators {
+			assert_eq!(Balances::total_balance(validator), 10_000_000);
+			assert_eq!(Staking::slashable_balance_of(validator), 10_000);
+
+			assert_eq!(
+				Staking::eras_stakers(era, validator),
+				pallet_staking::Exposure { total: 10_000, own: 10_000, others: vec![] },
+			);
+		}
+
+		assert_eq!(authorities.len(), 3);
+		let equivocation_authority_index = 1;
+		let equivocation_key = &authorities[equivocation_authority_index];
+		let equivocation_keyring = BeefyKeyring::from_public(equivocation_key).unwrap();
+
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let block_num = System::block_number() + 20;
+
+		// generate an fork equivocation proof, with a vote in the same round for a future block
+		let equivocation_proof = generate_fork_equivocation_proof_vote(
+			(block_num, payload, set_id, &equivocation_keyring),
+			None,
+			None,
+		);
+
+		// create the key ownership proof
+		let key_owner_proof = Historical::prove((BEEFY_KEY_TYPE, &equivocation_key)).unwrap();
+
+		// report the equivocation and the tx should be dispatched successfully
+		assert_ok!(Beefy::report_fork_equivocation_unsigned(
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof),
+			vec![key_owner_proof],
+		),);
+
+		era += 1;
+		start_era(era);
+
+		// check that the balance of 0-th validator is slashed 100%.
+		let equivocation_validator_id = validators[equivocation_authority_index];
+
+		assert_eq!(Balances::total_balance(&equivocation_validator_id), 10_000_000 - 10_000);
+		assert_eq!(Staking::slashable_balance_of(&equivocation_validator_id), 0);
+		assert_eq!(
+			Staking::eras_stakers(era, &equivocation_validator_id),
+			pallet_staking::Exposure { total: 0, own: 0, others: vec![] },
+		);
+
+		// check that the balances of all other validators are left intact.
+		for validator in &validators {
+			if *validator == equivocation_validator_id {
+				continue
+			}
+
+			assert_eq!(Balances::total_balance(validator), 10_000_000);
+			assert_eq!(Staking::slashable_balance_of(validator), 10_000);
+
+			assert_eq!(
+				Staking::eras_stakers(era, validator),
+				pallet_staking::Exposure { total: 10_000, own: 10_000, others: vec![] },
+			);
+		}
+	});
+}
+
 #[test]
 fn report_fork_equivocation_vote_invalid_set_id() {
 	let authorities = test_authorities();
@@ -1696,6 +1785,106 @@ fn report_fork_equivocation_sc_old_set_works() {
 	});
 }
 
+#[test]
+fn report_fork_equivocation_sc_future_block_works() {
+	let authorities = test_authorities();
+
+	let mut ext = new_test_ext_raw_authorities(authorities);
+	let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db());
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+	ext.register_extension(OffchainWorkerExt::new(offchain));
+
+	let mut era = 2;
+	ext.execute_with(|| {
+		start_era(era);
+	});
+	ext.persist_offchain_overlay();
+
+	ext.execute_with(|| {
+		let validator_set = Beefy::validator_set().unwrap();
+		let authorities = validator_set.validators();
+		let set_id = validator_set.id();
+		let validators = Session::validators();
+
+		// make sure that all validators have the same balance
+		for validator in &validators {
+			assert_eq!(Balances::total_balance(validator), 10_000_000);
+			assert_eq!(Staking::slashable_balance_of(validator), 10_000);
+
+			assert_eq!(
+				Staking::eras_stakers(era, validator),
+				pallet_staking::Exposure { total: 10_000, own: 10_000, others: vec![] },
+			);
+		}
+
+		assert_eq!(authorities.len(), 3);
+		let equivocation_authority_indices = [0, 2];
+		let equivocation_keys = equivocation_authority_indices
+			.iter()
+			.map(|i| &authorities[*i])
+			.collect::<Vec<_>>();
+		let equivocation_keyrings = equivocation_keys
+			.iter()
+			.map(|k| BeefyKeyring::from_public(k).unwrap())
+			.collect();
+
+		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
+		let block_num = System::block_number() + 20;
+		// create commitment to a future block
+		let commitment = Commitment { validator_set_id: set_id, block_number: block_num, payload };
+		// generate an fork equivocation proof, with a vote in the same round but for a
+		// future block
+		let equivocation_proof =
+			generate_fork_equivocation_proof_sc(commitment, equivocation_keyrings, None, None);
+
+		// create the key ownership proof
+		let key_owner_proofs = equivocation_keys
+			.iter()
+			.map(|k| Historical::prove((BEEFY_KEY_TYPE, &k)).unwrap())
+			.collect();
+
+		// report the equivocation and the tx should be dispatched successfully
+		assert_ok!(Beefy::report_fork_equivocation_unsigned(
+			RuntimeOrigin::none(),
+			Box::new(equivocation_proof),
+			key_owner_proofs,
+		),);
+
+		era += 1;
+		start_era(era);
+
+		// check that the balance of equivocating validators is slashed 100%.
+		let equivocation_validator_ids = equivocation_authority_indices
+			.iter()
+			.map(|i| validators[*i])
+			.collect::<Vec<_>>();
+
+		for equivocation_validator_id in &equivocation_validator_ids {
+			assert_eq!(Balances::total_balance(&equivocation_validator_id), 10_000_000 - 10_000);
+			assert_eq!(Staking::slashable_balance_of(&equivocation_validator_id), 0);
+			assert_eq!(
+				Staking::eras_stakers(era, equivocation_validator_id),
+				pallet_staking::Exposure { total: 0, own: 0, others: vec![] },
+			);
+		}
+
+		// check that the balances of all other validators are left intact.
+		for validator in &validators {
+			if equivocation_validator_ids.contains(&validator) {
+				continue
+			}
+
+			assert_eq!(Balances::total_balance(validator), 10_000_000);
+			assert_eq!(Staking::slashable_balance_of(validator), 10_000);
+
+			assert_eq!(
+				Staking::eras_stakers(era, validator),
+				pallet_staking::Exposure { total: 10_000, own: 10_000, others: vec![] },
+			);
+		}
+	});
+}
+
 #[test]
 fn report_fork_equivocation_sc_invalid_set_id() {
 	let authorities = test_authorities();

From 5ac0893ab4c6e7e6e9c3029d4e1dda4d7e0a8231 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 27 Nov 2023 19:52:28 +0100
Subject: [PATCH 103/188] fisher: don't fail on key ownership proof errors

just log them
---
 .../client/consensus/beefy/src/communication/fisherman.rs   | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index a272a6ca5bfd..6df1a20dbbfd 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -197,7 +197,11 @@ where
 						);
 						None
 					},
-					Err(e) => Some(Err(Error::RuntimeApi(e))),
+					Err(e) => {
+						debug!(target: LOG_TARGET,
+							   "🥩 Failed to generate key ownership proof for {:?}: {:?}", id, e);
+						None
+					},
 				}
 			})
 			.collect::<Result<_, _>>()?;

From 5f47c9d7b60ab1d3dc6b9df8cc4b0842b6e90574 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 28 Nov 2023 09:05:55 +0100
Subject: [PATCH 104/188] report_fork_equivocation: Result<{()->bool}, ..>

previously, reporting fork equivocations was not differential in whether
or not the equivocation was reported, only whether processing it had errored.
---
 .../client/consensus/beefy/src/communication/fisherman.rs | 8 ++++----
 substrate/client/consensus/beefy/src/worker.rs            | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 6df1a20dbbfd..69d638bb345d 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -109,7 +109,7 @@ where
 	pub(crate) fn report_fork_equivocation(
 		&self,
 		proof: ForkEquivocationProof<NumberFor<B>, AuthorityId, Signature, B::Header, MmrRootHash>,
-	) -> Result<(), Error> {
+	) -> Result<bool, Error> {
 		let best_block_number = self.backend.blockchain().info().best_number;
 		let best_block_hash = self.backend.blockchain().info().best_hash;
 
@@ -165,7 +165,7 @@ where
 				best_block_number,
 			) {
 			debug!(target: LOG_TARGET, "🥩 Skip report for bad invalid fork proof {:?}", proof);
-			return Ok(())
+			return Ok(false)
 		}
 
 		let offender_ids = proof.offender_ids();
@@ -173,7 +173,7 @@ where
 			if offender_ids.contains(&&local_id) {
 				debug!(target: LOG_TARGET, "🥩 Skip equivocation report for own equivocation");
 				// TODO: maybe error here instead?
-				return Ok(())
+				return Ok(false)
 			}
 		}
 
@@ -215,7 +215,7 @@ where
 			)
 			.map_err(Error::RuntimeApi)?;
 
-		Ok(())
+		Ok(true)
 	}
 }
 
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index 87c022d875e3..8831fc406b9f 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -1766,7 +1766,7 @@ pub(crate) mod tests {
 					.gossip_validator
 					.fisherman
 					.report_fork_equivocation(proof.clone()),
-				Ok(())
+				Ok(true)
 			);
 			// verify Alice reports Bob's equivocation to runtime
 			let reported =
@@ -1789,7 +1789,7 @@ pub(crate) mod tests {
 					.gossip_validator
 					.fisherman
 					.report_fork_equivocation(proof.clone()),
-				Ok(())
+				Ok(false)
 			);
 			// verify Alice does *not* report her own equivocation to runtime
 			let reported =
@@ -1819,7 +1819,7 @@ pub(crate) mod tests {
 					.gossip_validator
 					.fisherman
 					.report_fork_equivocation(proof.clone()),
-				Ok(())
+				Ok(true)
 			);
 			// verify Alice report Bob's and Charlie's equivocation to runtime
 			let reported =

From 96de837582ca6e30e88fb7b53115643c263437f0 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 28 Nov 2023 09:08:14 +0100
Subject: [PATCH 105/188] test all combinations of header & ancestry proofs

equivocation should be reported if both or either are correct. if
neither is submitted, should fail if for past block.
---
 .../client/consensus/beefy/src/worker.rs      | 52 +++++++++++++++----
 1 file changed, 41 insertions(+), 11 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index 8831fc406b9f..7724be377eb2 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -1089,7 +1089,7 @@ pub(crate) mod tests {
 	use sp_consensus_beefy::{
 		generate_fork_equivocation_proof_sc, generate_fork_equivocation_proof_vote,
 		generate_vote_equivocation_proof, known_payloads, known_payloads::MMR_ROOT_ID,
-		mmr::MmrRootProvider, Keyring, Payload, SignedCommitment,
+		mmr::MmrRootProvider, ForkEquivocationProof, Keyring, Payload, SignedCommitment,
 	};
 	use sp_runtime::traits::One;
 	use std::marker::PhantomData;
@@ -1758,6 +1758,7 @@ pub(crate) mod tests {
 			Some(header.clone()),
 			Some(ancestry_proof.clone()),
 		);
+
 		{
 			// expect fisher (Alice) to successfully process it
 			assert_eq!(
@@ -1804,14 +1805,26 @@ pub(crate) mod tests {
 			block_number: block_number as u64,
 			validator_set_id: validator_set.id(),
 		};
+
 		// only Bob and Charlie sign
-		let proof = generate_fork_equivocation_proof_sc(
-			commitment,
-			vec![Keyring::Bob, Keyring::Charlie],
-			Some(header),
-			Some(ancestry_proof),
-		);
-		{
+		let signatories = &[Keyring::Bob, Keyring::Charlie]
+			.into_iter()
+			.map(|k| (k.public(), k.sign(&commitment.encode())))
+			.collect::<Vec<_>>();
+
+		// test over all permutations of header and ancestry proof being submitted (proof should
+		// be valid as long as at least one is submitted)
+		for (correct_header, ancestry_proof) in [
+			(Some(header.clone()), Some(ancestry_proof.clone())),
+			(Some(header), None),
+			(None, Some(ancestry_proof)),
+		] {
+			let proof = ForkEquivocationProof {
+				commitment: commitment.clone(),
+				signatories: signatories.clone(),
+				correct_header,
+				ancestry_proof,
+			};
 			// expect fisher (Alice) to successfully process it
 			assert_eq!(
 				alice_worker
@@ -1821,11 +1834,28 @@ pub(crate) mod tests {
 					.report_fork_equivocation(proof.clone()),
 				Ok(true)
 			);
-			// verify Alice report Bob's and Charlie's equivocation to runtime
-			let reported =
+			let mut reported =
 				alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
+			// verify Alice report Bob's and Charlie's equivocation to runtime
 			assert_eq!(reported.len(), 2);
-			assert_eq!(*reported.get(1).unwrap(), proof);
+			assert_eq!(reported.pop(), Some(proof));
 		}
+
+		// test that Alice does not submit invalid proof
+		let proofless_proof = ForkEquivocationProof {
+			commitment,
+			signatories: signatories.clone(),
+			correct_header: None,
+			ancestry_proof: None,
+		};
+
+		assert_eq!(
+			alice_worker
+				.comms
+				.gossip_validator
+				.fisherman
+				.report_fork_equivocation(proofless_proof.clone()),
+			Ok(false)
+		);
 	}
 }

From c46ebe7f9e5f1e38efda47de7f0d2ce409ba52bf Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 28 Nov 2023 09:24:24 +0100
Subject: [PATCH 106/188] slash commitments to future blocks

---
 .../client/consensus/beefy/src/worker.rs      | 28 ++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index 7724be377eb2..9f58241a2715 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -1843,7 +1843,7 @@ pub(crate) mod tests {
 
 		// test that Alice does not submit invalid proof
 		let proofless_proof = ForkEquivocationProof {
-			commitment,
+			commitment: commitment.clone(),
 			signatories: signatories.clone(),
 			correct_header: None,
 			ancestry_proof: None,
@@ -1857,5 +1857,31 @@ pub(crate) mod tests {
 				.report_fork_equivocation(proofless_proof.clone()),
 			Ok(false)
 		);
+
+		let future_commitment = Commitment {
+			payload: commitment.payload,
+			block_number: net.peer(1).client().info().best_number + 10,
+			validator_set_id: commitment.validator_set_id,
+		};
+
+		let future_proof = generate_fork_equivocation_proof_sc(
+			future_commitment,
+			vec![Keyring::Bob, Keyring::Charlie],
+			None,
+			None,
+		);
+
+		assert_eq!(
+			alice_worker
+				.comms
+				.gossip_validator
+				.fisherman
+				.report_fork_equivocation(future_proof.clone()),
+			Ok(true)
+		);
+		let mut reported =
+			alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
+		assert_eq!(reported.len(), 2);
+		assert_eq!(reported.pop(), Some(future_proof));
 	}
 }

From 784aef19e5e1c8b8000fbfbba6e18fe69fb497d2 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 28 Nov 2023 12:24:14 +0100
Subject: [PATCH 107/188] only report equivocation if non-zero signers

---
 .../beefy/src/communication/fisherman.rs      | 61 +++++++++++--------
 1 file changed, 34 insertions(+), 27 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 69d638bb345d..38761e10b101 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -180,7 +180,7 @@ where
 		let runtime_api = self.runtime.runtime_api();
 
 		// generate key ownership proof at that block
-		let key_owner_proofs = offender_ids
+		let key_owner_proofs: Vec<_> = offender_ids
 			.iter()
 			.cloned()
 			.filter_map(|id| {
@@ -206,16 +206,19 @@ where
 			})
 			.collect::<Result<_, _>>()?;
 
-		// submit invalid fork vote report at **best** block
-		runtime_api
-			.submit_report_fork_equivocation_unsigned_extrinsic(
-				best_block_hash,
-				proof,
-				key_owner_proofs,
-			)
-			.map_err(Error::RuntimeApi)?;
-
-		Ok(true)
+		if key_owner_proofs.len() > 0 {
+			// submit invalid fork vote report at **best** block
+			runtime_api
+				.submit_report_fork_equivocation_unsigned_extrinsic(
+					best_block_hash,
+					proof,
+					key_owner_proofs,
+				)
+				.map_err(Error::RuntimeApi)?;
+			Ok(true)
+		} else {
+			Ok(false)
+		}
 	}
 }
 
@@ -289,20 +292,22 @@ where
 			// improved upon)
 			let best_hash = self.backend.blockchain().info().best_hash;
 			let validator_set = self.active_validator_set_at(best_hash)?;
-			let signatories = validator_set
+			let signatories: Vec<_> = validator_set
 				.validators()
 				.iter()
 				.cloned()
 				.zip(signatures.into_iter())
 				.filter_map(|(id, signature)| signature.map(|sig| (id, sig)))
 				.collect();
-			let proof = ForkEquivocationProof {
-				commitment,
-				signatories,
-				correct_header: None,
-				ancestry_proof: None,
-			};
-			self.report_fork_equivocation(proof)?;
+			if signatories.len() > 0 {
+				let proof = ForkEquivocationProof {
+					commitment,
+					signatories,
+					correct_header: None,
+					ancestry_proof: None,
+				};
+				self.report_fork_equivocation(proof)?;
+			}
 		} else {
 			let (correct_hash, correct_header, expected_payload) =
 				self.expected_hash_header_payload_tuple(number)?;
@@ -328,7 +333,7 @@ where
 					return Ok(())
 				}
 				// report every signer of the bad justification
-				let signatories = validator_set
+				let signatories: Vec<_> = validator_set
 					.validators()
 					.iter()
 					.cloned()
@@ -336,13 +341,15 @@ where
 					.filter_map(|(id, signature)| signature.map(|sig| (id, sig)))
 					.collect();
 
-				let proof = ForkEquivocationProof {
-					commitment,
-					signatories,
-					correct_header: Some(correct_header),
-					ancestry_proof,
-				};
-				self.report_fork_equivocation(proof)?;
+				if signatories.len() > 0 {
+					let proof = ForkEquivocationProof {
+						commitment,
+						signatories,
+						correct_header: Some(correct_header),
+						ancestry_proof,
+					};
+					self.report_fork_equivocation(proof)?;
+				}
 			}
 		}
 		Ok(())

From d7c53735f35ea3628e8241589d6915384df93ad8 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 28 Nov 2023 14:17:05 +0100
Subject: [PATCH 108/188] improve docs & naming

---
 substrate/client/consensus/beefy/src/worker.rs  | 2 +-
 substrate/primitives/consensus/beefy/src/lib.rs | 8 +++-----
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index 9f58241a2715..e6d4162fd41c 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -1713,7 +1713,7 @@ pub(crate) mod tests {
 	}
 
 	#[tokio::test]
-	async fn should_report_valid_fork_equivocations() {
+	async fn beefy_reports_fork_equivocations() {
 		let peers = [Keyring::Alice, Keyring::Bob, Keyring::Charlie];
 		let validator_set = ValidatorSet::new(make_beefy_ids(&peers), 0).unwrap();
 		let mut api_alice = TestApi::with_validator_set(&validator_set);
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 50ea87b7a6a3..90bf075eca3b 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -305,6 +305,7 @@ impl<Number, Id, Signature> VoteEquivocationProof<Number, Id, Signature> {
 
 /// Proof of authority misbehavior on a given set id.
 /// This proof shows commitment signed on a different fork.
+/// See [check_fork_equivocation_proof] for proof validity conditions.
 #[derive(Clone, Debug, Decode, Encode, PartialEq, TypeInfo)]
 pub struct ForkEquivocationProof<Number, Id, Signature, Header, Hash> {
 	/// Commitment for a block on different fork than one at the same height in
@@ -312,12 +313,9 @@ pub struct ForkEquivocationProof<Number, Id, Signature, Header, Hash> {
 	pub commitment: Commitment<Number>,
 	/// Signatures on this block
 	pub signatories: Vec<(Id, Signature)>,
-	/// The proof is valid if
-	/// 1. the header is in our chain
-	/// 2. its digest's payload != commitment.payload
-	/// 3. commitment is signed by signatories
+	/// Header at the same height as `commitment.block_number`.
 	pub correct_header: Option<Header>,
-	/// ancestry proof showing mmr root
+	/// Ancestry proof showing mmr root
 	pub ancestry_proof: Option<AncestryProof<Hash>>,
 }
 

From b9e8052e5d0ce1ef4efe5e0d6cdd65c7a5b07f59 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 28 Nov 2023 14:42:31 +0100
Subject: [PATCH 109/188] account for #2446

---
 substrate/client/consensus/beefy/src/tests.rs   |  4 ++--
 substrate/primitives/consensus/beefy/src/lib.rs | 12 +++++++-----
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index 79796a414dee..0177849ba4b9 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -50,7 +50,7 @@ use sc_network_test::{
 };
 use sc_utils::notification::NotificationReceiver;
 use serde::{Deserialize, Serialize};
-use sp_api::{ApiRef, BlockT, ProvideRuntimeApi};
+use sp_api::{ApiRef, ProvideRuntimeApi};
 use sp_application_crypto::key_types::BEEFY as BEEFY_KEY_TYPE;
 use sp_consensus::BlockOrigin;
 use sp_consensus_beefy::{
@@ -66,7 +66,7 @@ use sp_keystore::{testing::MemoryKeystore, Keystore, KeystorePtr};
 use sp_mmr_primitives::{AncestryProof, Error as MmrError, LeafIndex, MmrApi, NodeProof};
 use sp_runtime::{
 	codec::{Decode, Encode},
-	traits::{Header as HeaderT, NumberFor},
+	traits::{Block as BlockT, Header as HeaderT, NumberFor},
 	BuildStorage, DigestItem, EncodedJustification, Justifications, Storage,
 };
 use std::{marker::PhantomData, sync::Arc, task::Poll};
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 90bf075eca3b..a73e3fd70ba0 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -49,7 +49,7 @@ use codec::{Codec, Decode, Encode};
 use scale_info::TypeInfo;
 use sp_application_crypto::RuntimeAppPublic;
 use sp_core::H256;
-use sp_runtime::traits::{Hash, Header, Keccak256, NumberFor};
+use sp_runtime::traits::{Hash, Header as HeaderT, Keccak256, NumberFor};
 use sp_std::prelude::*;
 
 /// Key type for BEEFY module.
@@ -319,7 +319,9 @@ pub struct ForkEquivocationProof<Number, Id, Signature, Header, Hash> {
 	pub ancestry_proof: Option<AncestryProof<Hash>>,
 }
 
-impl<Number, Id, Signature, H: Header, Hash> ForkEquivocationProof<Number, Id, Signature, H, Hash> {
+impl<Number, Id, Signature, H: HeaderT, Hash>
+	ForkEquivocationProof<Number, Id, Signature, H, Hash>
+{
 	/// Returns the authority id of the misbehaving voter.
 	pub fn offender_ids(&self) -> Vec<&Id> {
 		self.signatories.iter().map(|(id, _)| id).collect()
@@ -392,7 +394,7 @@ fn check_header_proof<Header>(
 	expected_header_hash: &Header::Hash,
 ) -> bool
 where
-	Header: sp_api::HeaderT,
+	Header: HeaderT,
 {
 	if let Some(correct_header) = correct_header {
 		let expected_mmr_root_digest = mmr::find_mmr_root_digest::<Header>(correct_header);
@@ -425,7 +427,7 @@ fn check_ancestry_proof<Header, NodeHash, Hasher>(
 	mmr_size: u64,
 ) -> bool
 where
-	Header: sp_api::HeaderT,
+	Header: HeaderT,
 	NodeHash: Clone + Debug + PartialEq + Encode + Decode,
 	Hasher: mmr_lib::Merge<Item = NodeHash>,
 {
@@ -514,7 +516,7 @@ pub fn check_fork_equivocation_proof<Id, MsgHash, Header, NodeHash, Hasher>(
 where
 	Id: BeefyAuthorityId<MsgHash> + PartialEq,
 	MsgHash: Hash,
-	Header: sp_api::HeaderT,
+	Header: HeaderT,
 	NodeHash: Clone + Debug + PartialEq + Encode + Decode,
 	Hasher: mmr_lib::Merge<Item = NodeHash>,
 {

From 9a974f9d6ed7b340a94a222fa1883b3077ce072e Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 28 Nov 2023 18:07:49 +0100
Subject: [PATCH 110/188] bump `BeefyApi` version

---
 polkadot/runtime/rococo/src/lib.rs              | 2 +-
 substrate/primitives/consensus/beefy/src/lib.rs | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs
index c741a481291a..b6c94d5ee28b 100644
--- a/polkadot/runtime/rococo/src/lib.rs
+++ b/polkadot/runtime/rococo/src/lib.rs
@@ -1842,7 +1842,7 @@ sp_api::impl_runtime_apis! {
 		}
 	}
 
-	#[api_version(3)]
+	#[api_version(4)]
 	impl beefy_primitives::BeefyApi<Block, BeefyId, Hash> for Runtime {
 		fn beefy_genesis() -> Option<BlockNumber> {
 			Beefy::genesis_block()
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index a73e3fd70ba0..319009988677 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -591,7 +591,7 @@ impl OpaqueKeyOwnershipProof {
 sp_api::decl_runtime_apis! {
 	/// API necessary for BEEFY voters. Due to the significant conceptual
 	/// overlap, in large part, this is lifted from the GRANDPA API.
-	#[api_version(3)]
+	#[api_version(4)]
 	pub trait BeefyApi<AuthorityId, Hash> where
 		AuthorityId : Codec + RuntimeAppPublic,
 		Hash: Codec,

From 319632751c368687956bd84a7360092cae184237 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Fri, 15 Dec 2023 00:01:27 +0100
Subject: [PATCH 111/188] fix `should_initialize_voter_at_custom_genesis`

---
 substrate/client/consensus/beefy/src/tests.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index d7a8eecdfaa5..fab962cb531e 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -1130,8 +1130,9 @@ async fn should_initialize_voter_at_custom_genesis() {
 	// NOTE: code from `voter_init_setup()` is moved here because the new network event system
 	// doesn't allow creating a new `GossipEngine` as the notification handle is consumed by the
 	// first `GossipEngine`
+	let fisherman = DummyFisherman { _phantom: PhantomData::<Block> };
 	let known_peers = Arc::new(Mutex::new(KnownPeers::new()));
-	let (gossip_validator, _) = GossipValidator::new(known_peers);
+	let (gossip_validator, _) = GossipValidator::new(known_peers, fisherman);
 	let gossip_validator = Arc::new(gossip_validator);
 	let mut gossip_engine = sc_network_gossip::GossipEngine::new(
 		net.peer(0).network_service().clone(),
@@ -1461,7 +1462,7 @@ async fn beefy_reports_vote_equivocations() {
 }
 
 #[tokio::test]
-async fn gossipped_finality_proofs() {
+async fn gossiped_finality_proofs() {
 	sp_tracing::try_init_simple();
 
 	let validators = [BeefyKeyring::Alice, BeefyKeyring::Bob, BeefyKeyring::Charlie];

From ef5ad8487ee83826867bc62a8f970b932802fce9 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Fri, 15 Dec 2023 00:16:05 +0100
Subject: [PATCH 112/188] =?UTF-8?q?forward-port=20`{correct=E2=86=92canoni?=
 =?UTF-8?q?cal}=5Fheader`?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Suggested-by: Serban Iorga <serban300@gmail.com>
---
 .../beefy/src/communication/fisherman.rs      | 12 +++++-----
 .../client/consensus/beefy/src/worker.rs      |  6 ++---
 .../primitives/consensus/beefy/src/lib.rs     | 22 +++++++++----------
 .../consensus/beefy/src/test_utils.rs         | 12 +++++-----
 4 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 38761e10b101..582bb9225e4d 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -242,12 +242,12 @@ where
 			let proof = ForkEquivocationProof {
 				commitment: vote.commitment,
 				signatories: vec![(vote.id, vote.signature)],
-				correct_header: None,
+				canonical_header: None,
 				ancestry_proof: None,
 			};
 			self.report_fork_equivocation(proof)?;
 		} else {
-			let (correct_hash, correct_header, expected_payload) =
+			let (correct_hash, canonical_header, expected_payload) =
 				self.expected_hash_header_payload_tuple(number)?;
 			if vote.commitment.payload != expected_payload {
 				let ancestry_proof: Option<_> = match self
@@ -268,7 +268,7 @@ where
 				let proof = ForkEquivocationProof {
 					commitment: vote.commitment,
 					signatories: vec![(vote.id, vote.signature)],
-					correct_header: Some(correct_header),
+					canonical_header: Some(canonical_header),
 					ancestry_proof,
 				};
 				self.report_fork_equivocation(proof)?;
@@ -303,13 +303,13 @@ where
 				let proof = ForkEquivocationProof {
 					commitment,
 					signatories,
-					correct_header: None,
+					canonical_header: None,
 					ancestry_proof: None,
 				};
 				self.report_fork_equivocation(proof)?;
 			}
 		} else {
-			let (correct_hash, correct_header, expected_payload) =
+			let (correct_hash, canonical_header, expected_payload) =
 				self.expected_hash_header_payload_tuple(number)?;
 			if commitment.payload != expected_payload {
 				let ancestry_proof = match self.runtime.runtime_api().generate_ancestry_proof(
@@ -345,7 +345,7 @@ where
 					let proof = ForkEquivocationProof {
 						commitment,
 						signatories,
-						correct_header: Some(correct_header),
+						canonical_header: Some(canonical_header),
 						ancestry_proof,
 					};
 					self.report_fork_equivocation(proof)?;
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index 354e70e5cf27..e554aac191b3 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -1817,7 +1817,7 @@ pub(crate) mod tests {
 
 		// test over all permutations of header and ancestry proof being submitted (proof should
 		// be valid as long as at least one is submitted)
-		for (correct_header, ancestry_proof) in [
+		for (canonical_header, ancestry_proof) in [
 			(Some(header.clone()), Some(ancestry_proof.clone())),
 			(Some(header), None),
 			(None, Some(ancestry_proof)),
@@ -1825,7 +1825,7 @@ pub(crate) mod tests {
 			let proof = ForkEquivocationProof {
 				commitment: commitment.clone(),
 				signatories: signatories.clone(),
-				correct_header,
+				canonical_header,
 				ancestry_proof,
 			};
 			// expect fisher (Alice) to successfully process it
@@ -1848,7 +1848,7 @@ pub(crate) mod tests {
 		let proofless_proof = ForkEquivocationProof {
 			commitment: commitment.clone(),
 			signatories: signatories.clone(),
-			correct_header: None,
+			canonical_header: None,
 			ancestry_proof: None,
 		};
 
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 319009988677..95376e9638f6 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -313,8 +313,8 @@ pub struct ForkEquivocationProof<Number, Id, Signature, Header, Hash> {
 	pub commitment: Commitment<Number>,
 	/// Signatures on this block
 	pub signatories: Vec<(Id, Signature)>,
-	/// Header at the same height as `commitment.block_number`.
-	pub correct_header: Option<Header>,
+	/// Canonical header at the same height as `commitment.block_number`.
+	pub canonical_header: Option<Header>,
 	/// Ancestry proof showing mmr root
 	pub ancestry_proof: Option<AncestryProof<Hash>>,
 }
@@ -390,14 +390,14 @@ where
 /// Checks wether the provided header's payload differs from the commitment's payload.
 fn check_header_proof<Header>(
 	commitment: &Commitment<Header::Number>,
-	correct_header: &Option<Header>,
+	canonical_header: &Option<Header>,
 	expected_header_hash: &Header::Hash,
 ) -> bool
 where
 	Header: HeaderT,
 {
-	if let Some(correct_header) = correct_header {
-		let expected_mmr_root_digest = mmr::find_mmr_root_digest::<Header>(correct_header);
+	if let Some(canonical_header) = canonical_header {
+		let expected_mmr_root_digest = mmr::find_mmr_root_digest::<Header>(canonical_header);
 		let expected_payload = expected_mmr_root_digest.map(|mmr_root| {
 			Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode())
 		});
@@ -407,7 +407,7 @@ where
 		// and they will likewise be slashed.
 		// Note that we can only check this if a valid header has been provided - we cannot
 		// slash for this with an ancestry proof - by necessity)
-		if correct_header.hash() == *expected_header_hash &&
+		if canonical_header.hash() == *expected_header_hash &&
 			Some(&commitment.payload) != expected_payload.as_ref()
 		{
 			return true
@@ -486,8 +486,8 @@ where
 /// Validates [ForkEquivocationProof] with the following checks:
 /// - if the commitment is to a block in our history, then at least a header or an ancestry proof is
 ///   provided:
-///   - a `correct_header` is correct if it's at height `commitment.block_number` and
-///   commitment.payload` != `expected_payload(correct_header)`
+///   - a `canonical_header` is correct if it's at height `commitment.block_number` and
+///   commitment.payload` != `expected_payload(canonical_header)`
 ///   - an `ancestry_proof` is correct if it proves mmr_root(commitment.block_number) !=
 ///   mmr_root(commitment.payload)`
 /// - `commitment` is signed by all claimed signatories
@@ -520,11 +520,11 @@ where
 	NodeHash: Clone + Debug + PartialEq + Encode + Decode,
 	Hasher: mmr_lib::Merge<Item = NodeHash>,
 {
-	let ForkEquivocationProof { commitment, signatories, correct_header, ancestry_proof } = proof;
+	let ForkEquivocationProof { commitment, signatories, canonical_header, ancestry_proof } = proof;
 
 	// if commitment is to a block in the future, it's an equivocation as long as it's been signed
 	if commitment.block_number <= best_block_num {
-		if (correct_header, ancestry_proof) == (&None, &None) {
+		if (canonical_header, ancestry_proof) == (&None, &None) {
 			// if commitment isn't to a block number in the future, at least a header or ancestry
 			// proof must be provided, otherwise the proof is entirely invalid
 			return false;
@@ -532,7 +532,7 @@ where
 
 		// if neither the ancestry proof nor the header proof is correct, the proof is invalid
 		// avoid verifying the ancestry proof if a valid header proof has been provided
-		if !check_header_proof(commitment, correct_header, expected_header_hash) &&
+		if !check_header_proof(commitment, canonical_header, expected_header_hash) &&
 			!check_ancestry_proof::<Header, NodeHash, Hasher>(
 				commitment,
 				ancestry_proof,
diff --git a/substrate/primitives/consensus/beefy/src/test_utils.rs b/substrate/primitives/consensus/beefy/src/test_utils.rs
index c0989147a601..41eb20494a60 100644
--- a/substrate/primitives/consensus/beefy/src/test_utils.rs
+++ b/substrate/primitives/consensus/beefy/src/test_utils.rs
@@ -117,10 +117,10 @@ pub fn generate_vote_equivocation_proof(
 	VoteEquivocationProof { first, second }
 }
 
-/// Create a new `ForkEquivocationProof` based on vote & correct header.
+/// Create a new `ForkEquivocationProof` based on vote & canonical header.
 pub fn generate_fork_equivocation_proof_vote<Header, Hash>(
 	vote: (u64, Payload, ValidatorSetId, &Keyring),
-	correct_header: Option<Header>,
+	canonical_header: Option<Header>,
 	ancestry_proof: Option<AncestryProof<Hash>>,
 ) -> ForkEquivocationProof<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature, Header, Hash> {
 	let signed_vote = signed_vote(vote.0, vote.1, vote.2, vote.3);
@@ -128,21 +128,21 @@ pub fn generate_fork_equivocation_proof_vote<Header, Hash>(
 	ForkEquivocationProof {
 		commitment: signed_vote.commitment,
 		signatories,
-		correct_header,
+		canonical_header,
 		ancestry_proof,
 	}
 }
 
-/// Create a new `ForkEquivocationProof` based on signed commitment & correct header.
+/// Create a new `ForkEquivocationProof` based on signed commitment & canonical header.
 pub fn generate_fork_equivocation_proof_sc<Header, Hash>(
 	commitment: Commitment<u64>,
 	keyrings: Vec<Keyring>,
-	correct_header: Option<Header>,
+	canonical_header: Option<Header>,
 	ancestry_proof: Option<AncestryProof<Hash>>,
 ) -> ForkEquivocationProof<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature, Header, Hash> {
 	let signatories = keyrings
 		.into_iter()
 		.map(|k| (k.public(), k.sign(&commitment.encode())))
 		.collect::<Vec<_>>();
-	ForkEquivocationProof { commitment, signatories, correct_header, ancestry_proof }
+	ForkEquivocationProof { commitment, signatories, canonical_header, ancestry_proof }
 }

From bebd10859f1f067d0cd52343edcf2fb6f48a1cee Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Fri, 15 Dec 2023 00:37:26 +0100
Subject: [PATCH 113/188] =?UTF-8?q?`{expected=E2=86=92canonical}=5F...`=20?=
 =?UTF-8?q?renames?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Suggested-by: Serban Iorga <serban300@gmail.com>
---
 .../beefy/src/communication/fisherman.rs      | 34 +++++++++----------
 .../primitives/consensus/beefy/src/lib.rs     | 30 ++++++++--------
 2 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 582bb9225e4d..2ae4692d9cd1 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -40,19 +40,19 @@ use sp_runtime::{
 use std::{marker::PhantomData, sync::Arc};
 
 pub(crate) trait BeefyFisherman<B: Block>: Send + Sync {
-	/// Check `vote` for contained block against expected payload.
+	/// Check `vote` for contained block against canonical payload.
 	fn check_vote(
 		&self,
 		vote: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
 	) -> Result<(), Error>;
 
-	/// Check `signed_commitment` for contained block against expected payload.
+	/// Check `signed_commitment` for contained block against canonical payload.
 	fn check_signed_commitment(
 		&self,
 		signed_commitment: SignedCommitment<NumberFor<B>, Signature>,
 	) -> Result<(), Error>;
 
-	/// Check `proof` for contained block against expected payload.
+	/// Check `proof` for contained block against canonical payload.
 	fn check_proof(&self, proof: BeefyVersionedFinalityProof<B>) -> Result<(), Error>;
 }
 
@@ -74,7 +74,7 @@ where
 	R: ProvideRuntimeApi<B> + Send + Sync,
 	R::Api: BeefyApi<B, AuthorityId, MmrRootHash> + MmrApi<B, MmrRootHash, NumberFor<B>>,
 {
-	fn expected_hash_header_payload_tuple(
+	fn canonical_hash_header_payload_tuple(
 		&self,
 		number: NumberFor<B>,
 	) -> Result<(B::Hash, B::Header, Payload), Error> {
@@ -116,7 +116,7 @@ where
 		// if the commitment is for a block number exceeding our best block number, we assume the
 		// equivocators are part of the current validator set, hence we use the validator set at the
 		// best block
-		let expected_commitment_block_hash = if best_block_number < proof.commitment.block_number {
+		let canonical_commitment_block_hash = if best_block_number < proof.commitment.block_number {
 			best_block_hash
 		} else {
 			self.backend
@@ -125,7 +125,7 @@ where
 				.map_err(|e| Error::Backend(e.to_string()))?
 		};
 
-		let validator_set = self.active_validator_set_at(expected_commitment_block_hash)?;
+		let validator_set = self.active_validator_set_at(canonical_commitment_block_hash)?;
 		let set_id = validator_set.id();
 
 		let best_mmr_root = self
@@ -160,7 +160,7 @@ where
 				&proof,
 				best_mmr_root,
 				leaf_count,
-				&expected_commitment_block_hash,
+				&canonical_commitment_block_hash,
 				first_mmr_block_num,
 				best_block_number,
 			) {
@@ -185,7 +185,7 @@ where
 			.cloned()
 			.filter_map(|id| {
 				match runtime_api.generate_key_ownership_proof(
-					expected_commitment_block_hash,
+					canonical_commitment_block_hash,
 					set_id,
 					id.clone(),
 				) {
@@ -230,7 +230,7 @@ where
 	R: ProvideRuntimeApi<B> + Send + Sync,
 	R::Api: BeefyApi<B, AuthorityId, MmrRootHash> + MmrApi<B, MmrRootHash, NumberFor<B>>,
 {
-	/// Check `vote` for contained block against expected payload.
+	/// Check `vote` for contained block against canonical payload.
 	fn check_vote(
 		&self,
 		vote: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
@@ -247,9 +247,9 @@ where
 			};
 			self.report_fork_equivocation(proof)?;
 		} else {
-			let (correct_hash, canonical_header, expected_payload) =
-				self.expected_hash_header_payload_tuple(number)?;
-			if vote.commitment.payload != expected_payload {
+			let (correct_hash, canonical_header, canonical_payload) =
+				self.canonical_hash_header_payload_tuple(number)?;
+			if vote.commitment.payload != canonical_payload {
 				let ancestry_proof: Option<_> = match self
 					.runtime
 					.runtime_api()
@@ -277,7 +277,7 @@ where
 		Ok(())
 	}
 
-	/// Check `signed_commitment` for contained block against expected payload.
+	/// Check `signed_commitment` for contained block against canonical payload.
 	fn check_signed_commitment(
 		&self,
 		signed_commitment: SignedCommitment<NumberFor<B>, Signature>,
@@ -309,9 +309,9 @@ where
 				self.report_fork_equivocation(proof)?;
 			}
 		} else {
-			let (correct_hash, canonical_header, expected_payload) =
-				self.expected_hash_header_payload_tuple(number)?;
-			if commitment.payload != expected_payload {
+			let (correct_hash, canonical_header, canonical_payload) =
+				self.canonical_hash_header_payload_tuple(number)?;
+			if commitment.payload != canonical_payload {
 				let ancestry_proof = match self.runtime.runtime_api().generate_ancestry_proof(
 					correct_hash,
 					number,
@@ -355,7 +355,7 @@ where
 		Ok(())
 	}
 
-	/// Check `proof` for contained block against expected payload.
+	/// Check `proof` for contained block against canonical payload.
 	fn check_proof(&self, proof: BeefyVersionedFinalityProof<B>) -> Result<(), Error> {
 		match proof {
 			BeefyVersionedFinalityProof::<B>::V1(signed_commitment) =>
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 95376e9638f6..2d6da7f14222 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -387,28 +387,28 @@ where
 	return valid_first && valid_second
 }
 
-/// Checks wether the provided header's payload differs from the commitment's payload.
+/// Checks whether the provided header's payload differs from the commitment's payload.
 fn check_header_proof<Header>(
 	commitment: &Commitment<Header::Number>,
 	canonical_header: &Option<Header>,
-	expected_header_hash: &Header::Hash,
+	canonical_header_hash: &Header::Hash,
 ) -> bool
 where
 	Header: HeaderT,
 {
 	if let Some(canonical_header) = canonical_header {
-		let expected_mmr_root_digest = mmr::find_mmr_root_digest::<Header>(canonical_header);
-		let expected_payload = expected_mmr_root_digest.map(|mmr_root| {
+		let canonical_mmr_root_digest = mmr::find_mmr_root_digest::<Header>(canonical_header);
+		let canonical_payload = canonical_mmr_root_digest.map(|mmr_root| {
 			Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode())
 		});
 		// Check header's hash and that `payload` of the `commitment` is different that the
-		// `expected_payload`. Note that if the signatories signed a payload when there should be
-		// none (for instance for a block prior to BEEFY activation), then expected_payload = None,
+		// `canonical_payload`. Note that if the signatories signed a payload when there should be
+		// none (for instance for a block prior to BEEFY activation), then canonical_payload = None,
 		// and they will likewise be slashed.
 		// Note that we can only check this if a valid header has been provided - we cannot
 		// slash for this with an ancestry proof - by necessity)
-		if canonical_header.hash() == *expected_header_hash &&
-			Some(&commitment.payload) != expected_payload.as_ref()
+		if canonical_header.hash() == *canonical_header_hash &&
+			Some(&commitment.payload) != canonical_payload.as_ref()
 		{
 			return true
 		}
@@ -423,7 +423,7 @@ fn check_ancestry_proof<Header, NodeHash, Hasher>(
 	commitment: &Commitment<Header::Number>,
 	ancestry_proof: &Option<AncestryProof<Hasher::Item>>,
 	first_mmr_block_num: Header::Number,
-	expected_root: Hasher::Item,
+	canonical_root: Hasher::Item,
 	mmr_size: u64,
 ) -> bool
 where
@@ -453,7 +453,7 @@ where
 				return false
 			}
 			if sp_mmr_primitives::utils::verify_ancestry_proof::<NodeHash, Hasher>(
-				expected_root,
+				canonical_root,
 				mmr_size,
 				ancestry_proof.clone(),
 			) != Ok(true)
@@ -487,7 +487,7 @@ where
 /// - if the commitment is to a block in our history, then at least a header or an ancestry proof is
 ///   provided:
 ///   - a `canonical_header` is correct if it's at height `commitment.block_number` and
-///   commitment.payload` != `expected_payload(canonical_header)`
+///   commitment.payload` != `canonical_payload(canonical_header)`
 ///   - an `ancestry_proof` is correct if it proves mmr_root(commitment.block_number) !=
 ///   mmr_root(commitment.payload)`
 /// - `commitment` is signed by all claimed signatories
@@ -507,9 +507,9 @@ pub fn check_fork_equivocation_proof<Id, MsgHash, Header, NodeHash, Hasher>(
 		Header,
 		NodeHash,
 	>,
-	expected_root: Hasher::Item,
+	canonical_root: Hasher::Item,
 	mmr_size: u64,
-	expected_header_hash: &Header::Hash,
+	canonical_header_hash: &Header::Hash,
 	first_mmr_block_num: Header::Number,
 	best_block_num: Header::Number,
 ) -> bool
@@ -532,12 +532,12 @@ where
 
 		// if neither the ancestry proof nor the header proof is correct, the proof is invalid
 		// avoid verifying the ancestry proof if a valid header proof has been provided
-		if !check_header_proof(commitment, canonical_header, expected_header_hash) &&
+		if !check_header_proof(commitment, canonical_header, canonical_header_hash) &&
 			!check_ancestry_proof::<Header, NodeHash, Hasher>(
 				commitment,
 				ancestry_proof,
 				first_mmr_block_num,
-				expected_root,
+				canonical_root,
 				mmr_size,
 			) {
 			return false;

From 9d917f9511457f1d52ef8cbed6b2b6261d8c6e32 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 18 Dec 2023 10:24:27 +0100
Subject: [PATCH 114/188] impl serde for `application_crypto::Signature`

---
 Cargo.lock                                    |  9 ++++
 polkadot/primitives/Cargo.toml                |  2 +
 substrate/frame/benchmarking/Cargo.toml       |  3 +-
 substrate/frame/im-online/Cargo.toml          |  2 +
 .../primitives/application-crypto/Cargo.toml  |  4 +-
 .../primitives/application-crypto/src/lib.rs  | 43 +++++++++++++++++++
 .../primitives/authority-discovery/Cargo.toml |  2 +
 .../primitives/consensus/aura/Cargo.toml      |  2 +
 .../primitives/consensus/babe/Cargo.toml      |  2 +
 .../primitives/consensus/beefy/Cargo.toml     |  3 +-
 .../primitives/consensus/grandpa/Cargo.toml   |  2 +
 substrate/primitives/mixnet/Cargo.toml        |  2 +
 .../primitives/statement-store/Cargo.toml     |  2 +
 13 files changed, 75 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 89bc7421f39c..aba2ae99c4ee 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9951,6 +9951,7 @@ dependencies = [
 name = "pallet-im-online"
 version = "4.0.0-dev"
 dependencies = [
+ "array-bytes 6.1.0",
  "frame-benchmarking",
  "frame-support",
  "frame-system",
@@ -12825,6 +12826,7 @@ dependencies = [
 name = "polkadot-primitives"
 version = "1.0.0"
 dependencies = [
+ "array-bytes 6.1.0",
  "bitvec",
  "hex-literal",
  "parity-scale-codec",
@@ -17153,6 +17155,7 @@ dependencies = [
 name = "sp-application-crypto"
 version = "23.0.0"
 dependencies = [
+ "array-bytes 6.1.0",
  "parity-scale-codec",
  "scale-info",
  "serde",
@@ -17222,6 +17225,7 @@ dependencies = [
 name = "sp-authority-discovery"
 version = "4.0.0-dev"
 dependencies = [
+ "array-bytes 6.1.0",
  "parity-scale-codec",
  "scale-info",
  "sp-api",
@@ -17276,6 +17280,7 @@ dependencies = [
 name = "sp-consensus-aura"
 version = "0.10.0-dev"
 dependencies = [
+ "array-bytes 6.1.0",
  "async-trait",
  "parity-scale-codec",
  "scale-info",
@@ -17292,6 +17297,7 @@ dependencies = [
 name = "sp-consensus-babe"
 version = "0.10.0-dev"
 dependencies = [
+ "array-bytes 6.1.0",
  "async-trait",
  "parity-scale-codec",
  "scale-info",
@@ -17330,6 +17336,7 @@ dependencies = [
 name = "sp-consensus-grandpa"
 version = "4.0.0-dev"
 dependencies = [
+ "array-bytes 6.1.0",
  "finality-grandpa",
  "log",
  "parity-scale-codec",
@@ -17642,6 +17649,7 @@ dependencies = [
 name = "sp-mixnet"
 version = "0.1.0-dev"
 dependencies = [
+ "array-bytes 6.1.0",
  "parity-scale-codec",
  "scale-info",
  "sp-api",
@@ -17909,6 +17917,7 @@ name = "sp-statement-store"
 version = "4.0.0-dev"
 dependencies = [
  "aes-gcm 0.10.3",
+ "array-bytes 6.1.0",
  "curve25519-dalek 4.0.0",
  "ed25519-dalek",
  "hkdf",
diff --git a/polkadot/primitives/Cargo.toml b/polkadot/primitives/Cargo.toml
index 5e746c622cf2..d812f9f62d71 100644
--- a/polkadot/primitives/Cargo.toml
+++ b/polkadot/primitives/Cargo.toml
@@ -7,6 +7,7 @@ license.workspace = true
 description = "Shared primitives used by Polkadot runtime"
 
 [dependencies]
+array-bytes = { version = "6.1", optional = true }
 bitvec = { version = "1.0.0", default-features = false, features = ["alloc", "serde"] }
 hex-literal = "0.4.1"
 parity-scale-codec = { version = "3.6.1", default-features = false, features = ["bit-vec", "derive"] }
@@ -32,6 +33,7 @@ polkadot-parachain-primitives = { path = "../parachain", default-features = fals
 [features]
 default = ["std"]
 std = [
+    "array-bytes",
 	"application-crypto/std",
 	"bitvec/std",
 	"inherents/std",
diff --git a/substrate/frame/benchmarking/Cargo.toml b/substrate/frame/benchmarking/Cargo.toml
index 9cfaac1abfd0..c252f506cf6b 100644
--- a/substrate/frame/benchmarking/Cargo.toml
+++ b/substrate/frame/benchmarking/Cargo.toml
@@ -13,6 +13,7 @@ readme = "README.md"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
+array-bytes = { version = "6.1", optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false }
 linregress = { version = "0.5.1", optional = true }
 log = { version = "0.4.17", default-features = false }
@@ -33,13 +34,13 @@ sp-storage = { path = "../../primitives/storage", default-features = false }
 static_assertions = "1.1.0"
 
 [dev-dependencies]
-array-bytes = "6.1"
 rusty-fork = { version = "0.3.0", default-features = false }
 sp-keystore = { path = "../../primitives/keystore" }
 
 [features]
 default = ["std"]
 std = [
+	"array-bytes",
 	"codec/std",
 	"frame-support-procedural/std",
 	"frame-support/std",
diff --git a/substrate/frame/im-online/Cargo.toml b/substrate/frame/im-online/Cargo.toml
index 5ec260c9b5be..6c9fd6b01335 100644
--- a/substrate/frame/im-online/Cargo.toml
+++ b/substrate/frame/im-online/Cargo.toml
@@ -13,6 +13,7 @@ readme = "README.md"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
+array-bytes = { version = "6.1", optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] }
 log = { version = "0.4.17", default-features = false }
 scale-info = { version = "2.10.0", default-features = false, features = ["derive", "serde"] }
@@ -33,6 +34,7 @@ pallet-session = { path = "../session" }
 [features]
 default = ["std"]
 std = [
+	"array-bytes",
 	"codec/std",
 	"frame-benchmarking?/std",
 	"frame-support/std",
diff --git a/substrate/primitives/application-crypto/Cargo.toml b/substrate/primitives/application-crypto/Cargo.toml
index a6c937a3469e..081692f8a1ab 100644
--- a/substrate/primitives/application-crypto/Cargo.toml
+++ b/substrate/primitives/application-crypto/Cargo.toml
@@ -15,6 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 
 [dependencies]
+array-bytes = { version = "6.1", optional = true }
 sp-core = { path = "../core", default-features = false }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] }
 scale-info = { version = "2.10.0", default-features = false, features = ["derive"] }
@@ -35,12 +36,13 @@ std = [
 ]
 
 # Serde support without relying on std features.
-serde = ["dep:serde", "scale-info/serde", "sp-core/serde"]
+serde = ["array-bytes", "dep:serde", "scale-info/serde", "sp-core/serde"]
 
 # This feature enables all crypto primitives for `no_std` builds like microcontrollers
 # or Intel SGX.
 # For the regular wasm runtime builds this should not be used.
 full_crypto = [
+	"array-bytes",
 	"sp-core/full_crypto",
 	"sp-io/disable_oom",
 	# Don't add `panic_handler` and `alloc_error_handler` since they are expected to be provided
diff --git a/substrate/primitives/application-crypto/src/lib.rs b/substrate/primitives/application-crypto/src/lib.rs
index 686b486f3353..d3d36554a814 100644
--- a/substrate/primitives/application-crypto/src/lib.rs
+++ b/substrate/primitives/application-crypto/src/lib.rs
@@ -32,6 +32,8 @@ pub use sp_core::{
 	crypto::{ByteArray, CryptoType, Derive, IsWrappedBy, Public, UncheckedFrom, Wraps},
 	RuntimeDebug,
 };
+#[cfg(all(not(feature = "std"), feature = "serde"))]
+use sp_std::alloc::{format, string::String};
 
 #[doc(hidden)]
 pub use codec;
@@ -419,6 +421,8 @@ macro_rules! app_crypto_signature_full_crypto {
 			pub struct Signature($sig);
 		}
 
+		$crate::app_crypto_signature_if_serde!();
+
 		impl $crate::CryptoType for Signature {
 			type Pair = Pair;
 		}
@@ -452,6 +456,8 @@ macro_rules! app_crypto_signature_not_full_crypto {
 			pub struct Signature($sig);
 		}
 
+		$crate::app_crypto_signature_if_serde!();
+
 		impl $crate::CryptoType for Signature {}
 
 		impl $crate::AppCrypto for Signature {
@@ -463,6 +469,43 @@ macro_rules! app_crypto_signature_not_full_crypto {
 	};
 }
 
+/// Declares serde implementation for Signature type if serde feature is enabled.
+#[cfg(feature = "serde")]
+#[doc(hidden)]
+#[macro_export]
+macro_rules! app_crypto_signature_if_serde {
+	() => {
+		impl $crate::serde::Serialize for Signature {
+			fn serialize<S>(&self, serializer: S) -> core::result::Result<S::Ok, S::Error>
+			where
+				S: $crate::serde::Serializer,
+			{
+				serializer.serialize_str(&array_bytes::bytes2hex("", self))
+			}
+		}
+
+		impl<'de> $crate::serde::Deserialize<'de> for Signature {
+			fn deserialize<D>(deserializer: D) -> core::result::Result<Self, D::Error>
+			where
+				D: $crate::serde::Deserializer<'de>,
+			{
+				let signature_hex = array_bytes::hex2bytes(&String::deserialize(deserializer)?)
+					.map_err(|e| $crate::serde::de::Error::custom(format!("{:?}", e)))?;
+				Signature::try_from(signature_hex.as_ref())
+					.map_err(|e| $crate::serde::de::Error::custom(format!("{:?}", e)))
+			}
+		}
+	};
+}
+
+/// Declares serde implementation for Signature type if serde feature is enabled.
+#[cfg(not(feature = "serde"))]
+#[doc(hidden)]
+#[macro_export]
+macro_rules! app_crypto_signature_if_serde {
+	() => {};
+}
+
 /// Declares `Signature` type which is functionally equivalent to `$sig`, but is new
 /// application-specific type whose identifier is `$key_type`.
 /// For full functionality, app_crypto_signature_(not)_full_crypto! must be called too.
diff --git a/substrate/primitives/authority-discovery/Cargo.toml b/substrate/primitives/authority-discovery/Cargo.toml
index c8a93980be28..cb40803f3758 100644
--- a/substrate/primitives/authority-discovery/Cargo.toml
+++ b/substrate/primitives/authority-discovery/Cargo.toml
@@ -13,6 +13,7 @@ readme = "README.md"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
+array-bytes = { version = "6.1", optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false }
 scale-info = { version = "2.10.0", default-features = false, features = ["derive"] }
 sp-api = { path = "../api", default-features = false }
@@ -31,6 +32,7 @@ std = [
 	"sp-std/std",
 ]
 serde = [
+	"array-bytes",
 	"scale-info/serde",
 	"sp-application-crypto/serde",
 	"sp-runtime/serde",
diff --git a/substrate/primitives/consensus/aura/Cargo.toml b/substrate/primitives/consensus/aura/Cargo.toml
index 4a19999a469a..587ce0489437 100644
--- a/substrate/primitives/consensus/aura/Cargo.toml
+++ b/substrate/primitives/consensus/aura/Cargo.toml
@@ -13,6 +13,7 @@ readme = "README.md"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
+array-bytes = { version = "6.1", optional = true }
 async-trait = { version = "0.1.57", optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false }
 scale-info = { version = "2.10.0", default-features = false, features = ["derive"] }
@@ -27,6 +28,7 @@ sp-timestamp = { path = "../../timestamp", default-features = false }
 [features]
 default = ["std"]
 std = [
+	"array-bytes",
 	"async-trait",
 	"codec/std",
 	"scale-info/std",
diff --git a/substrate/primitives/consensus/babe/Cargo.toml b/substrate/primitives/consensus/babe/Cargo.toml
index 6ec50ea022b7..56cdea24525f 100644
--- a/substrate/primitives/consensus/babe/Cargo.toml
+++ b/substrate/primitives/consensus/babe/Cargo.toml
@@ -13,6 +13,7 @@ readme = "README.md"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
+array-bytes = { version = "6.1", optional = true }
 async-trait = { version = "0.1.57", optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false }
 scale-info = { version = "2.10.0", default-features = false, features = ["derive"] }
@@ -45,6 +46,7 @@ std = [
 
 # Serde support without relying on std features.
 serde = [
+	"array-bytes",
 	"dep:serde",
 	"scale-info/serde",
 	"sp-application-crypto/serde",
diff --git a/substrate/primitives/consensus/beefy/Cargo.toml b/substrate/primitives/consensus/beefy/Cargo.toml
index 916125d783d9..fa540da6e3a0 100644
--- a/substrate/primitives/consensus/beefy/Cargo.toml
+++ b/substrate/primitives/consensus/beefy/Cargo.toml
@@ -12,6 +12,7 @@ description = "Primitives for BEEFY protocol."
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
+array-bytes = { version = "6.1", optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] }
 scale-info = { version = "2.10.0", default-features = false, features = ["derive"] }
 serde = { version = "1.0.193", default-features = false, optional = true, features = ["alloc", "derive"] }
@@ -26,7 +27,6 @@ strum = { version = "0.24.1", features = ["derive"], default-features = false }
 lazy_static = "1.4.0"
 
 [dev-dependencies]
-array-bytes = "6.1"
 w3f-bls = { version = "0.1.3", features = ["std"] }
 
 [features]
@@ -47,6 +47,7 @@ std = [
 
 # Serde support without relying on std features.
 serde = [
+	"array-bytes",
 	"dep:serde",
 	"scale-info/serde",
 	"sp-application-crypto/serde",
diff --git a/substrate/primitives/consensus/grandpa/Cargo.toml b/substrate/primitives/consensus/grandpa/Cargo.toml
index 1ddc89df9836..2d5b26c0e968 100644
--- a/substrate/primitives/consensus/grandpa/Cargo.toml
+++ b/substrate/primitives/consensus/grandpa/Cargo.toml
@@ -14,6 +14,7 @@ readme = "README.md"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
+array-bytes = { version = "6.1", optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] }
 grandpa = { package = "finality-grandpa", version = "0.16.2", default-features = false, features = ["derive-codec"] }
 log = { version = "0.4.17", default-features = false }
@@ -44,6 +45,7 @@ std = [
 
 # Serde support without relying on std features.
 serde = [
+	"array-bytes",
 	"dep:serde",
 	"scale-info/serde",
 	"sp-application-crypto/serde",
diff --git a/substrate/primitives/mixnet/Cargo.toml b/substrate/primitives/mixnet/Cargo.toml
index a03fdab8741a..a65ba4f6d029 100644
--- a/substrate/primitives/mixnet/Cargo.toml
+++ b/substrate/primitives/mixnet/Cargo.toml
@@ -13,6 +13,7 @@ readme = "README.md"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
+array-bytes = { version = "6.1", optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] }
 scale-info = { version = "2.10.0", default-features = false, features = ["derive"] }
 sp-api = { default-features = false, path = "../api" }
@@ -22,6 +23,7 @@ sp-std = { default-features = false, path = "../std" }
 [features]
 default = ["std"]
 std = [
+	"array-bytes",
 	"codec/std",
 	"scale-info/std",
 	"sp-api/std",
diff --git a/substrate/primitives/statement-store/Cargo.toml b/substrate/primitives/statement-store/Cargo.toml
index 089af92f0623..70d870b1e9d1 100644
--- a/substrate/primitives/statement-store/Cargo.toml
+++ b/substrate/primitives/statement-store/Cargo.toml
@@ -13,6 +13,7 @@ readme = "README.md"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
+array-bytes = { version = "6.1", optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] }
 scale-info = { version = "2.10.0", default-features = false, features = ["derive"] }
 sp-core = { path = "../core", default-features = false }
@@ -38,6 +39,7 @@ default = ["std"]
 std = [
 	"aes-gcm",
 	"aes-gcm?/std",
+	"array-bytes",
 	"codec/std",
 	"curve25519-dalek",
 	"ed25519-dalek",

From a123e78c914220726291680f43905d5fff1e8c99 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 18 Dec 2023 11:06:55 +0100
Subject: [PATCH 115/188] impl serde for `sp_core::bandersnatch::Signature`

---
 Cargo.lock                                    |  1 +
 .../primitives/consensus/sassafras/Cargo.toml |  2 ++
 substrate/primitives/core/src/bandersnatch.rs | 23 +++++++++++++++++++
 3 files changed, 26 insertions(+)

diff --git a/Cargo.lock b/Cargo.lock
index aba2ae99c4ee..8587c3176b64 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -17365,6 +17365,7 @@ dependencies = [
 name = "sp-consensus-sassafras"
 version = "0.3.4-dev"
 dependencies = [
+ "array-bytes 6.1.0",
  "parity-scale-codec",
  "scale-info",
  "serde",
diff --git a/substrate/primitives/consensus/sassafras/Cargo.toml b/substrate/primitives/consensus/sassafras/Cargo.toml
index e71f82b4382f..cec86147067a 100644
--- a/substrate/primitives/consensus/sassafras/Cargo.toml
+++ b/substrate/primitives/consensus/sassafras/Cargo.toml
@@ -15,6 +15,7 @@ publish = false
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
+array-bytes = { version = "6.1", optional = true }
 scale-codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false }
 scale-info = { version = "2.10.0", default-features = false, features = ["derive"] }
 serde = { version = "1.0.193", default-features = false, features = ["derive"], optional = true }
@@ -41,6 +42,7 @@ std = [
 
 # Serde support without relying on std features.
 serde = [
+    "array-bytes",
 	"dep:serde",
 	"scale-info/serde",
 	"sp-application-crypto/serde",
diff --git a/substrate/primitives/core/src/bandersnatch.rs b/substrate/primitives/core/src/bandersnatch.rs
index 463d49fd8890..e492259f0ab9 100644
--- a/substrate/primitives/core/src/bandersnatch.rs
+++ b/substrate/primitives/core/src/bandersnatch.rs
@@ -193,6 +193,29 @@ impl ByteArray for Signature {
 	const LEN: usize = SIGNATURE_SERIALIZED_SIZE;
 }
 
+#[cfg(feature = "serde")]
+impl Serialize for Signature {
+	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+	where
+		S: Serializer,
+	{
+		serializer.serialize_str(&array_bytes::bytes2hex("", self))
+	}
+}
+
+#[cfg(feature = "serde")]
+impl<'de> Deserialize<'de> for Signature {
+	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+	where
+		D: Deserializer<'de>,
+	{
+		let signature_hex = array_bytes::hex2bytes(&String::deserialize(deserializer)?)
+			.map_err(|e| de::Error::custom(format!("{:?}", e)))?;
+		Signature::try_from(signature_hex.as_ref())
+			.map_err(|e| de::Error::custom(format!("{:?}", e)))
+	}
+}
+
 impl CryptoType for Signature {
 	#[cfg(feature = "full_crypto")]
 	type Pair = Pair;

From 7a28ac32ee71a5a874e8c73af96ece9ccf5dc731 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 18 Dec 2023 12:09:41 +0100
Subject: [PATCH 116/188] stub for `check_versioned_finality_proof`

---
 .../client/consensus/beefy/rpc/src/lib.rs     | 27 ++++++++++++++-----
 .../consensus/beefy/src/commitment.rs         |  8 +++---
 .../primitives/consensus/beefy/src/payload.rs | 16 ++++++++++-
 3 files changed, 41 insertions(+), 10 deletions(-)

diff --git a/substrate/client/consensus/beefy/rpc/src/lib.rs b/substrate/client/consensus/beefy/rpc/src/lib.rs
index f5c0ff32627d..b2e1065b4b5a 100644
--- a/substrate/client/consensus/beefy/rpc/src/lib.rs
+++ b/substrate/client/consensus/beefy/rpc/src/lib.rs
@@ -35,8 +35,9 @@ use jsonrpsee::{
 };
 use log::warn;
 
-use sc_consensus_beefy::communication::notification::{
-	BeefyBestBlockStream, BeefyVersionedFinalityProofStream,
+use sc_consensus_beefy::{
+	communication::notification::{BeefyBestBlockStream, BeefyVersionedFinalityProofStream},
+	justification::BeefyVersionedFinalityProof,
 };
 
 mod notification;
@@ -83,7 +84,7 @@ impl From<Error> for JsonRpseeError {
 
 // Provides RPC methods for interacting with BEEFY.
 #[rpc(client, server)]
-pub trait BeefyApi<Notification, Hash> {
+pub trait BeefyApi<Notification, Block: BlockT> {
 	/// Returns the block most recently finalized by BEEFY, alongside its justification.
 	#[subscription(
 		name = "beefy_subscribeJustifications" => "beefy_justifications",
@@ -98,7 +99,15 @@ pub trait BeefyApi<Notification, Hash> {
 	/// in the network or if the client is still initializing or syncing with the network.
 	/// In such case an error would be returned.
 	#[method(name = "beefy_getFinalizedHead")]
-	async fn latest_finalized(&self) -> RpcResult<Hash>;
+	async fn latest_finalized(&self) -> RpcResult<Block::Hash>;
+
+	/// Checks a versioned finality proof for equivocations against canonical chain.
+	/// If an equivocation is detected, the call also reports the equivocation to the runtime.
+	#[method(name = "beefy_check_versioned_finality_proof")]
+	async fn check_versioned_finality_proof(
+		&self,
+		versioned_finality_proof: BeefyVersionedFinalityProof<Block>,
+	) -> RpcResult<bool>;
 }
 
 /// Implements the BeefyApi RPC trait for interacting with BEEFY.
@@ -133,8 +142,7 @@ where
 }
 
 #[async_trait]
-impl<Block> BeefyApiServer<notification::EncodedVersionedFinalityProof, Block::Hash>
-	for Beefy<Block>
+impl<Block> BeefyApiServer<notification::EncodedVersionedFinalityProof, Block> for Beefy<Block>
 where
 	Block: BlockT,
 {
@@ -160,6 +168,13 @@ where
 			.ok_or(Error::EndpointNotReady)
 			.map_err(Into::into)
 	}
+
+	async fn check_versioned_finality_proof(
+		&self,
+		versioned_finality_proof: BeefyVersionedFinalityProof<Block>,
+	) -> RpcResult<bool> {
+		return Ok(false)
+	}
 }
 
 #[cfg(test)]
diff --git a/substrate/primitives/consensus/beefy/src/commitment.rs b/substrate/primitives/consensus/beefy/src/commitment.rs
index 3c451b70afb0..f7796c5afe1c 100644
--- a/substrate/primitives/consensus/beefy/src/commitment.rs
+++ b/substrate/primitives/consensus/beefy/src/commitment.rs
@@ -17,6 +17,8 @@
 
 use codec::{Decode, Encode, Error, Input};
 use scale_info::TypeInfo;
+#[cfg(feature = "serde")]
+use serde::{Deserialize, Serialize};
 use sp_std::{cmp, prelude::*};
 
 use crate::{Payload, ValidatorSetId};
@@ -27,7 +29,7 @@ use crate::{Payload, ValidatorSetId};
 /// height [block_number](Commitment::block_number).
 /// GRANDPA validators collect signatures on commitments and a stream of such signed commitments
 /// (see [SignedCommitment]) forms the BEEFY protocol.
-#[derive(Clone, Debug, PartialEq, Eq, Encode, Decode, TypeInfo)]
+#[derive(Clone, Debug, PartialEq, Eq, Encode, Decode, TypeInfo, Serialize, Deserialize)]
 pub struct Commitment<TBlockNumber> {
 	///  A collection of payloads to be signed, see [`Payload`] for details.
 	///
@@ -86,7 +88,7 @@ where
 /// Note that SCALE-encoding of the structure is optimized for size efficiency over the wire,
 /// please take a look at custom [`Encode`] and [`Decode`] implementations and
 /// `CompactSignedCommitment` struct.
-#[derive(Clone, Debug, PartialEq, Eq, TypeInfo)]
+#[derive(Clone, Debug, PartialEq, Eq, TypeInfo, Serialize, Deserialize)]
 pub struct SignedCommitment<TBlockNumber, TSignature> {
 	/// The commitment signatures are collected for.
 	pub commitment: Commitment<TBlockNumber>,
@@ -234,7 +236,7 @@ where
 ///
 /// Note that this enum is subject to change in the future with introduction
 /// of additional cryptographic primitives to BEEFY.
-#[derive(Clone, Debug, PartialEq, codec::Encode, codec::Decode)]
+#[derive(Clone, Debug, PartialEq, codec::Encode, codec::Decode, Serialize, Deserialize)]
 pub enum VersionedFinalityProof<N, S> {
 	#[codec(index = 1)]
 	/// Current active version
diff --git a/substrate/primitives/consensus/beefy/src/payload.rs b/substrate/primitives/consensus/beefy/src/payload.rs
index 43b83911dd62..c703fe1bd796 100644
--- a/substrate/primitives/consensus/beefy/src/payload.rs
+++ b/substrate/primitives/consensus/beefy/src/payload.rs
@@ -17,6 +17,7 @@
 
 use codec::{Decode, Encode};
 use scale_info::TypeInfo;
+use serde::{Deserialize, Serialize};
 use sp_runtime::traits::Block;
 use sp_std::prelude::*;
 
@@ -39,7 +40,20 @@ pub mod known_payloads {
 /// Identifiers MUST be sorted by the [`BeefyPayloadId`] to allow efficient lookup of expected
 /// value. Duplicated identifiers are disallowed. It's okay for different implementations to only
 /// support a subset of possible values.
-#[derive(Decode, Encode, Debug, PartialEq, Eq, Clone, Ord, PartialOrd, Hash, TypeInfo)]
+#[derive(
+	Decode,
+	Encode,
+	Debug,
+	PartialEq,
+	Eq,
+	Clone,
+	Ord,
+	PartialOrd,
+	Hash,
+	TypeInfo,
+	Serialize,
+	Deserialize,
+)]
 pub struct Payload(Vec<(BeefyPayloadId, Vec<u8>)>);
 
 impl Payload {

From 940d8463c6b80d0e0a02ef6c2147d4e3122b0fa6 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 4 Jan 2024 09:28:02 +0100
Subject: [PATCH 117/188] update docs

---
 substrate/frame/staking/src/migrations.rs | 7 ++++---
 substrate/frame/staking/src/slashing.rs   | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/substrate/frame/staking/src/migrations.rs b/substrate/frame/staking/src/migrations.rs
index 311e9667cebc..9aec1f72209f 100644
--- a/substrate/frame/staking/src/migrations.rs
+++ b/substrate/frame/staking/src/migrations.rs
@@ -296,11 +296,12 @@ pub mod v10 {
 	#[storage_alias]
 	type EarliestUnappliedSlash<T: Config> = StorageValue<Pallet<T>, EraIndex>;
 
-	/// Apply any pending slashes that where queued.
+	/// Apply any pending slashes that were queued.
 	///
-	/// That means we might slash someone a bit too early, but we will definitely
+	/// That means we might slash someone a bit too early, but we definitely
 	/// won't forget to slash them. The cap of 512 is somewhat randomly taken to
-	/// prevent us from iterating over an arbitrary large number of keys `on_runtime_upgrade`.
+	/// prevent us from iterating over an arbitrary large number of keys
+	/// `on_runtime_upgrade`.
 	pub struct MigrateToV10<T>(sp_std::marker::PhantomData<T>);
 	impl<T: Config> OnRuntimeUpgrade for MigrateToV10<T> {
 		fn on_runtime_upgrade() -> frame_support::weights::Weight {
diff --git a/substrate/frame/staking/src/slashing.rs b/substrate/frame/staking/src/slashing.rs
index 709fd1441ec3..f7797cc18b8d 100644
--- a/substrate/frame/staking/src/slashing.rs
+++ b/substrate/frame/staking/src/slashing.rs
@@ -47,7 +47,7 @@
 //! has multiple misbehaviors. However, accounting for such cases is necessary
 //! to deter a class of "rage-quit" attacks.
 //!
-//! Based on research at <https://research.web3.foundation/en/latest/polkadot/slashing/npos.html>
+//! Based on research at <https://research.web3.foundation/Polkadot/security/slashing/amounts>
 
 use crate::{
 	BalanceOf, Config, Error, Exposure, NegativeImbalanceOf, NominatorSlashInEra,

From e680e6b4002b44bc4de90520742b525353e9aa79 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 4 Jan 2024 11:07:16 +0100
Subject: [PATCH 118/188] Doc: payload checking reports equivocations too

Suggested-by: Svyatoslav Nikolsky <svyatonik@gmail.com>
---
 .../consensus/beefy/src/communication/fisherman.rs       | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 2ae4692d9cd1..f7da111cf1ef 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -230,7 +230,8 @@ where
 	R: ProvideRuntimeApi<B> + Send + Sync,
 	R::Api: BeefyApi<B, AuthorityId, MmrRootHash> + MmrApi<B, MmrRootHash, NumberFor<B>>,
 {
-	/// Check `vote` for contained block against canonical payload.
+	/// Check `vote` for contained block against canonical payload. If an equivocation is detected,
+	/// this also reports it.
 	fn check_vote(
 		&self,
 		vote: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
@@ -277,7 +278,8 @@ where
 		Ok(())
 	}
 
-	/// Check `signed_commitment` for contained block against canonical payload.
+	/// Check `vote` for contained block against canonical payload. If an equivocation is detected,
+	/// this also reports it.
 	fn check_signed_commitment(
 		&self,
 		signed_commitment: SignedCommitment<NumberFor<B>, Signature>,
@@ -355,7 +357,8 @@ where
 		Ok(())
 	}
 
-	/// Check `proof` for contained block against canonical payload.
+	/// Check `vote` for contained block against canonical payload. If an equivocation is detected,
+	/// this also reports it.
 	fn check_proof(&self, proof: BeefyVersionedFinalityProof<B>) -> Result<(), Error> {
 		match proof {
 			BeefyVersionedFinalityProof::<B>::V1(signed_commitment) =>

From 0f8c8bd029aa3aa74e9c19736695c238101ff7e5 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 4 Jan 2024 11:11:50 +0100
Subject: [PATCH 119/188] Revert "impl serde for
 `sp_core::bandersnatch::Signature`"

This reverts commit a123e78c914220726291680f43905d5fff1e8c99.
---
 Cargo.lock                                    |  1 -
 .../primitives/consensus/sassafras/Cargo.toml |  2 --
 substrate/primitives/core/src/bandersnatch.rs | 23 -------------------
 3 files changed, 26 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 8587c3176b64..aba2ae99c4ee 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -17365,7 +17365,6 @@ dependencies = [
 name = "sp-consensus-sassafras"
 version = "0.3.4-dev"
 dependencies = [
- "array-bytes 6.1.0",
  "parity-scale-codec",
  "scale-info",
  "serde",
diff --git a/substrate/primitives/consensus/sassafras/Cargo.toml b/substrate/primitives/consensus/sassafras/Cargo.toml
index cec86147067a..e71f82b4382f 100644
--- a/substrate/primitives/consensus/sassafras/Cargo.toml
+++ b/substrate/primitives/consensus/sassafras/Cargo.toml
@@ -15,7 +15,6 @@ publish = false
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-array-bytes = { version = "6.1", optional = true }
 scale-codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false }
 scale-info = { version = "2.10.0", default-features = false, features = ["derive"] }
 serde = { version = "1.0.193", default-features = false, features = ["derive"], optional = true }
@@ -42,7 +41,6 @@ std = [
 
 # Serde support without relying on std features.
 serde = [
-    "array-bytes",
 	"dep:serde",
 	"scale-info/serde",
 	"sp-application-crypto/serde",
diff --git a/substrate/primitives/core/src/bandersnatch.rs b/substrate/primitives/core/src/bandersnatch.rs
index e492259f0ab9..463d49fd8890 100644
--- a/substrate/primitives/core/src/bandersnatch.rs
+++ b/substrate/primitives/core/src/bandersnatch.rs
@@ -193,29 +193,6 @@ impl ByteArray for Signature {
 	const LEN: usize = SIGNATURE_SERIALIZED_SIZE;
 }
 
-#[cfg(feature = "serde")]
-impl Serialize for Signature {
-	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-	where
-		S: Serializer,
-	{
-		serializer.serialize_str(&array_bytes::bytes2hex("", self))
-	}
-}
-
-#[cfg(feature = "serde")]
-impl<'de> Deserialize<'de> for Signature {
-	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-	where
-		D: Deserializer<'de>,
-	{
-		let signature_hex = array_bytes::hex2bytes(&String::deserialize(deserializer)?)
-			.map_err(|e| de::Error::custom(format!("{:?}", e)))?;
-		Signature::try_from(signature_hex.as_ref())
-			.map_err(|e| de::Error::custom(format!("{:?}", e)))
-	}
-}
-
 impl CryptoType for Signature {
 	#[cfg(feature = "full_crypto")]
 	type Pair = Pair;

From abea602e6288d85c047b50d9ecdae020ca9cfff6 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 4 Jan 2024 11:11:57 +0100
Subject: [PATCH 120/188] Revert "impl serde for
 `application_crypto::Signature`"

This reverts commit 9d917f9511457f1d52ef8cbed6b2b6261d8c6e32.
---
 Cargo.lock                                    |  9 ----
 polkadot/primitives/Cargo.toml                |  2 -
 substrate/frame/benchmarking/Cargo.toml       |  3 +-
 substrate/frame/im-online/Cargo.toml          |  2 -
 .../primitives/application-crypto/Cargo.toml  |  4 +-
 .../primitives/application-crypto/src/lib.rs  | 43 -------------------
 .../primitives/authority-discovery/Cargo.toml |  2 -
 .../primitives/consensus/aura/Cargo.toml      |  2 -
 .../primitives/consensus/babe/Cargo.toml      |  2 -
 .../primitives/consensus/beefy/Cargo.toml     |  3 +-
 .../primitives/consensus/grandpa/Cargo.toml   |  2 -
 substrate/primitives/mixnet/Cargo.toml        |  2 -
 .../primitives/statement-store/Cargo.toml     |  2 -
 13 files changed, 3 insertions(+), 75 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index aba2ae99c4ee..89bc7421f39c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9951,7 +9951,6 @@ dependencies = [
 name = "pallet-im-online"
 version = "4.0.0-dev"
 dependencies = [
- "array-bytes 6.1.0",
  "frame-benchmarking",
  "frame-support",
  "frame-system",
@@ -12826,7 +12825,6 @@ dependencies = [
 name = "polkadot-primitives"
 version = "1.0.0"
 dependencies = [
- "array-bytes 6.1.0",
  "bitvec",
  "hex-literal",
  "parity-scale-codec",
@@ -17155,7 +17153,6 @@ dependencies = [
 name = "sp-application-crypto"
 version = "23.0.0"
 dependencies = [
- "array-bytes 6.1.0",
  "parity-scale-codec",
  "scale-info",
  "serde",
@@ -17225,7 +17222,6 @@ dependencies = [
 name = "sp-authority-discovery"
 version = "4.0.0-dev"
 dependencies = [
- "array-bytes 6.1.0",
  "parity-scale-codec",
  "scale-info",
  "sp-api",
@@ -17280,7 +17276,6 @@ dependencies = [
 name = "sp-consensus-aura"
 version = "0.10.0-dev"
 dependencies = [
- "array-bytes 6.1.0",
  "async-trait",
  "parity-scale-codec",
  "scale-info",
@@ -17297,7 +17292,6 @@ dependencies = [
 name = "sp-consensus-babe"
 version = "0.10.0-dev"
 dependencies = [
- "array-bytes 6.1.0",
  "async-trait",
  "parity-scale-codec",
  "scale-info",
@@ -17336,7 +17330,6 @@ dependencies = [
 name = "sp-consensus-grandpa"
 version = "4.0.0-dev"
 dependencies = [
- "array-bytes 6.1.0",
  "finality-grandpa",
  "log",
  "parity-scale-codec",
@@ -17649,7 +17642,6 @@ dependencies = [
 name = "sp-mixnet"
 version = "0.1.0-dev"
 dependencies = [
- "array-bytes 6.1.0",
  "parity-scale-codec",
  "scale-info",
  "sp-api",
@@ -17917,7 +17909,6 @@ name = "sp-statement-store"
 version = "4.0.0-dev"
 dependencies = [
  "aes-gcm 0.10.3",
- "array-bytes 6.1.0",
  "curve25519-dalek 4.0.0",
  "ed25519-dalek",
  "hkdf",
diff --git a/polkadot/primitives/Cargo.toml b/polkadot/primitives/Cargo.toml
index d812f9f62d71..5e746c622cf2 100644
--- a/polkadot/primitives/Cargo.toml
+++ b/polkadot/primitives/Cargo.toml
@@ -7,7 +7,6 @@ license.workspace = true
 description = "Shared primitives used by Polkadot runtime"
 
 [dependencies]
-array-bytes = { version = "6.1", optional = true }
 bitvec = { version = "1.0.0", default-features = false, features = ["alloc", "serde"] }
 hex-literal = "0.4.1"
 parity-scale-codec = { version = "3.6.1", default-features = false, features = ["bit-vec", "derive"] }
@@ -33,7 +32,6 @@ polkadot-parachain-primitives = { path = "../parachain", default-features = fals
 [features]
 default = ["std"]
 std = [
-    "array-bytes",
 	"application-crypto/std",
 	"bitvec/std",
 	"inherents/std",
diff --git a/substrate/frame/benchmarking/Cargo.toml b/substrate/frame/benchmarking/Cargo.toml
index c252f506cf6b..9cfaac1abfd0 100644
--- a/substrate/frame/benchmarking/Cargo.toml
+++ b/substrate/frame/benchmarking/Cargo.toml
@@ -13,7 +13,6 @@ readme = "README.md"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-array-bytes = { version = "6.1", optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false }
 linregress = { version = "0.5.1", optional = true }
 log = { version = "0.4.17", default-features = false }
@@ -34,13 +33,13 @@ sp-storage = { path = "../../primitives/storage", default-features = false }
 static_assertions = "1.1.0"
 
 [dev-dependencies]
+array-bytes = "6.1"
 rusty-fork = { version = "0.3.0", default-features = false }
 sp-keystore = { path = "../../primitives/keystore" }
 
 [features]
 default = ["std"]
 std = [
-	"array-bytes",
 	"codec/std",
 	"frame-support-procedural/std",
 	"frame-support/std",
diff --git a/substrate/frame/im-online/Cargo.toml b/substrate/frame/im-online/Cargo.toml
index 6c9fd6b01335..5ec260c9b5be 100644
--- a/substrate/frame/im-online/Cargo.toml
+++ b/substrate/frame/im-online/Cargo.toml
@@ -13,7 +13,6 @@ readme = "README.md"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-array-bytes = { version = "6.1", optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] }
 log = { version = "0.4.17", default-features = false }
 scale-info = { version = "2.10.0", default-features = false, features = ["derive", "serde"] }
@@ -34,7 +33,6 @@ pallet-session = { path = "../session" }
 [features]
 default = ["std"]
 std = [
-	"array-bytes",
 	"codec/std",
 	"frame-benchmarking?/std",
 	"frame-support/std",
diff --git a/substrate/primitives/application-crypto/Cargo.toml b/substrate/primitives/application-crypto/Cargo.toml
index 081692f8a1ab..a6c937a3469e 100644
--- a/substrate/primitives/application-crypto/Cargo.toml
+++ b/substrate/primitives/application-crypto/Cargo.toml
@@ -15,7 +15,6 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 
 [dependencies]
-array-bytes = { version = "6.1", optional = true }
 sp-core = { path = "../core", default-features = false }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] }
 scale-info = { version = "2.10.0", default-features = false, features = ["derive"] }
@@ -36,13 +35,12 @@ std = [
 ]
 
 # Serde support without relying on std features.
-serde = ["array-bytes", "dep:serde", "scale-info/serde", "sp-core/serde"]
+serde = ["dep:serde", "scale-info/serde", "sp-core/serde"]
 
 # This feature enables all crypto primitives for `no_std` builds like microcontrollers
 # or Intel SGX.
 # For the regular wasm runtime builds this should not be used.
 full_crypto = [
-	"array-bytes",
 	"sp-core/full_crypto",
 	"sp-io/disable_oom",
 	# Don't add `panic_handler` and `alloc_error_handler` since they are expected to be provided
diff --git a/substrate/primitives/application-crypto/src/lib.rs b/substrate/primitives/application-crypto/src/lib.rs
index d3d36554a814..686b486f3353 100644
--- a/substrate/primitives/application-crypto/src/lib.rs
+++ b/substrate/primitives/application-crypto/src/lib.rs
@@ -32,8 +32,6 @@ pub use sp_core::{
 	crypto::{ByteArray, CryptoType, Derive, IsWrappedBy, Public, UncheckedFrom, Wraps},
 	RuntimeDebug,
 };
-#[cfg(all(not(feature = "std"), feature = "serde"))]
-use sp_std::alloc::{format, string::String};
 
 #[doc(hidden)]
 pub use codec;
@@ -421,8 +419,6 @@ macro_rules! app_crypto_signature_full_crypto {
 			pub struct Signature($sig);
 		}
 
-		$crate::app_crypto_signature_if_serde!();
-
 		impl $crate::CryptoType for Signature {
 			type Pair = Pair;
 		}
@@ -456,8 +452,6 @@ macro_rules! app_crypto_signature_not_full_crypto {
 			pub struct Signature($sig);
 		}
 
-		$crate::app_crypto_signature_if_serde!();
-
 		impl $crate::CryptoType for Signature {}
 
 		impl $crate::AppCrypto for Signature {
@@ -469,43 +463,6 @@ macro_rules! app_crypto_signature_not_full_crypto {
 	};
 }
 
-/// Declares serde implementation for Signature type if serde feature is enabled.
-#[cfg(feature = "serde")]
-#[doc(hidden)]
-#[macro_export]
-macro_rules! app_crypto_signature_if_serde {
-	() => {
-		impl $crate::serde::Serialize for Signature {
-			fn serialize<S>(&self, serializer: S) -> core::result::Result<S::Ok, S::Error>
-			where
-				S: $crate::serde::Serializer,
-			{
-				serializer.serialize_str(&array_bytes::bytes2hex("", self))
-			}
-		}
-
-		impl<'de> $crate::serde::Deserialize<'de> for Signature {
-			fn deserialize<D>(deserializer: D) -> core::result::Result<Self, D::Error>
-			where
-				D: $crate::serde::Deserializer<'de>,
-			{
-				let signature_hex = array_bytes::hex2bytes(&String::deserialize(deserializer)?)
-					.map_err(|e| $crate::serde::de::Error::custom(format!("{:?}", e)))?;
-				Signature::try_from(signature_hex.as_ref())
-					.map_err(|e| $crate::serde::de::Error::custom(format!("{:?}", e)))
-			}
-		}
-	};
-}
-
-/// Declares serde implementation for Signature type if serde feature is enabled.
-#[cfg(not(feature = "serde"))]
-#[doc(hidden)]
-#[macro_export]
-macro_rules! app_crypto_signature_if_serde {
-	() => {};
-}
-
 /// Declares `Signature` type which is functionally equivalent to `$sig`, but is new
 /// application-specific type whose identifier is `$key_type`.
 /// For full functionality, app_crypto_signature_(not)_full_crypto! must be called too.
diff --git a/substrate/primitives/authority-discovery/Cargo.toml b/substrate/primitives/authority-discovery/Cargo.toml
index cb40803f3758..c8a93980be28 100644
--- a/substrate/primitives/authority-discovery/Cargo.toml
+++ b/substrate/primitives/authority-discovery/Cargo.toml
@@ -13,7 +13,6 @@ readme = "README.md"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-array-bytes = { version = "6.1", optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false }
 scale-info = { version = "2.10.0", default-features = false, features = ["derive"] }
 sp-api = { path = "../api", default-features = false }
@@ -32,7 +31,6 @@ std = [
 	"sp-std/std",
 ]
 serde = [
-	"array-bytes",
 	"scale-info/serde",
 	"sp-application-crypto/serde",
 	"sp-runtime/serde",
diff --git a/substrate/primitives/consensus/aura/Cargo.toml b/substrate/primitives/consensus/aura/Cargo.toml
index 587ce0489437..4a19999a469a 100644
--- a/substrate/primitives/consensus/aura/Cargo.toml
+++ b/substrate/primitives/consensus/aura/Cargo.toml
@@ -13,7 +13,6 @@ readme = "README.md"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-array-bytes = { version = "6.1", optional = true }
 async-trait = { version = "0.1.57", optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false }
 scale-info = { version = "2.10.0", default-features = false, features = ["derive"] }
@@ -28,7 +27,6 @@ sp-timestamp = { path = "../../timestamp", default-features = false }
 [features]
 default = ["std"]
 std = [
-	"array-bytes",
 	"async-trait",
 	"codec/std",
 	"scale-info/std",
diff --git a/substrate/primitives/consensus/babe/Cargo.toml b/substrate/primitives/consensus/babe/Cargo.toml
index 56cdea24525f..6ec50ea022b7 100644
--- a/substrate/primitives/consensus/babe/Cargo.toml
+++ b/substrate/primitives/consensus/babe/Cargo.toml
@@ -13,7 +13,6 @@ readme = "README.md"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-array-bytes = { version = "6.1", optional = true }
 async-trait = { version = "0.1.57", optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false }
 scale-info = { version = "2.10.0", default-features = false, features = ["derive"] }
@@ -46,7 +45,6 @@ std = [
 
 # Serde support without relying on std features.
 serde = [
-	"array-bytes",
 	"dep:serde",
 	"scale-info/serde",
 	"sp-application-crypto/serde",
diff --git a/substrate/primitives/consensus/beefy/Cargo.toml b/substrate/primitives/consensus/beefy/Cargo.toml
index fa540da6e3a0..916125d783d9 100644
--- a/substrate/primitives/consensus/beefy/Cargo.toml
+++ b/substrate/primitives/consensus/beefy/Cargo.toml
@@ -12,7 +12,6 @@ description = "Primitives for BEEFY protocol."
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-array-bytes = { version = "6.1", optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] }
 scale-info = { version = "2.10.0", default-features = false, features = ["derive"] }
 serde = { version = "1.0.193", default-features = false, optional = true, features = ["alloc", "derive"] }
@@ -27,6 +26,7 @@ strum = { version = "0.24.1", features = ["derive"], default-features = false }
 lazy_static = "1.4.0"
 
 [dev-dependencies]
+array-bytes = "6.1"
 w3f-bls = { version = "0.1.3", features = ["std"] }
 
 [features]
@@ -47,7 +47,6 @@ std = [
 
 # Serde support without relying on std features.
 serde = [
-	"array-bytes",
 	"dep:serde",
 	"scale-info/serde",
 	"sp-application-crypto/serde",
diff --git a/substrate/primitives/consensus/grandpa/Cargo.toml b/substrate/primitives/consensus/grandpa/Cargo.toml
index 2d5b26c0e968..1ddc89df9836 100644
--- a/substrate/primitives/consensus/grandpa/Cargo.toml
+++ b/substrate/primitives/consensus/grandpa/Cargo.toml
@@ -14,7 +14,6 @@ readme = "README.md"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-array-bytes = { version = "6.1", optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] }
 grandpa = { package = "finality-grandpa", version = "0.16.2", default-features = false, features = ["derive-codec"] }
 log = { version = "0.4.17", default-features = false }
@@ -45,7 +44,6 @@ std = [
 
 # Serde support without relying on std features.
 serde = [
-	"array-bytes",
 	"dep:serde",
 	"scale-info/serde",
 	"sp-application-crypto/serde",
diff --git a/substrate/primitives/mixnet/Cargo.toml b/substrate/primitives/mixnet/Cargo.toml
index a65ba4f6d029..a03fdab8741a 100644
--- a/substrate/primitives/mixnet/Cargo.toml
+++ b/substrate/primitives/mixnet/Cargo.toml
@@ -13,7 +13,6 @@ readme = "README.md"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-array-bytes = { version = "6.1", optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] }
 scale-info = { version = "2.10.0", default-features = false, features = ["derive"] }
 sp-api = { default-features = false, path = "../api" }
@@ -23,7 +22,6 @@ sp-std = { default-features = false, path = "../std" }
 [features]
 default = ["std"]
 std = [
-	"array-bytes",
 	"codec/std",
 	"scale-info/std",
 	"sp-api/std",
diff --git a/substrate/primitives/statement-store/Cargo.toml b/substrate/primitives/statement-store/Cargo.toml
index 70d870b1e9d1..089af92f0623 100644
--- a/substrate/primitives/statement-store/Cargo.toml
+++ b/substrate/primitives/statement-store/Cargo.toml
@@ -13,7 +13,6 @@ readme = "README.md"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-array-bytes = { version = "6.1", optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] }
 scale-info = { version = "2.10.0", default-features = false, features = ["derive"] }
 sp-core = { path = "../core", default-features = false }
@@ -39,7 +38,6 @@ default = ["std"]
 std = [
 	"aes-gcm",
 	"aes-gcm?/std",
-	"array-bytes",
 	"codec/std",
 	"curve25519-dalek",
 	"ed25519-dalek",

From 10d9ed583d63814f6ab94ab2faa0fe53fdc469d1 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 4 Jan 2024 11:12:18 +0100
Subject: [PATCH 121/188] Revert "stub for `check_versioned_finality_proof`"

This reverts commit 7a28ac32ee71a5a874e8c73af96ece9ccf5dc731.
---
 .../client/consensus/beefy/rpc/src/lib.rs     | 27 +++++--------------
 .../consensus/beefy/src/commitment.rs         |  8 +++---
 .../primitives/consensus/beefy/src/payload.rs | 16 +----------
 3 files changed, 10 insertions(+), 41 deletions(-)

diff --git a/substrate/client/consensus/beefy/rpc/src/lib.rs b/substrate/client/consensus/beefy/rpc/src/lib.rs
index b2e1065b4b5a..f5c0ff32627d 100644
--- a/substrate/client/consensus/beefy/rpc/src/lib.rs
+++ b/substrate/client/consensus/beefy/rpc/src/lib.rs
@@ -35,9 +35,8 @@ use jsonrpsee::{
 };
 use log::warn;
 
-use sc_consensus_beefy::{
-	communication::notification::{BeefyBestBlockStream, BeefyVersionedFinalityProofStream},
-	justification::BeefyVersionedFinalityProof,
+use sc_consensus_beefy::communication::notification::{
+	BeefyBestBlockStream, BeefyVersionedFinalityProofStream,
 };
 
 mod notification;
@@ -84,7 +83,7 @@ impl From<Error> for JsonRpseeError {
 
 // Provides RPC methods for interacting with BEEFY.
 #[rpc(client, server)]
-pub trait BeefyApi<Notification, Block: BlockT> {
+pub trait BeefyApi<Notification, Hash> {
 	/// Returns the block most recently finalized by BEEFY, alongside its justification.
 	#[subscription(
 		name = "beefy_subscribeJustifications" => "beefy_justifications",
@@ -99,15 +98,7 @@ pub trait BeefyApi<Notification, Block: BlockT> {
 	/// in the network or if the client is still initializing or syncing with the network.
 	/// In such case an error would be returned.
 	#[method(name = "beefy_getFinalizedHead")]
-	async fn latest_finalized(&self) -> RpcResult<Block::Hash>;
-
-	/// Checks a versioned finality proof for equivocations against canonical chain.
-	/// If an equivocation is detected, the call also reports the equivocation to the runtime.
-	#[method(name = "beefy_check_versioned_finality_proof")]
-	async fn check_versioned_finality_proof(
-		&self,
-		versioned_finality_proof: BeefyVersionedFinalityProof<Block>,
-	) -> RpcResult<bool>;
+	async fn latest_finalized(&self) -> RpcResult<Hash>;
 }
 
 /// Implements the BeefyApi RPC trait for interacting with BEEFY.
@@ -142,7 +133,8 @@ where
 }
 
 #[async_trait]
-impl<Block> BeefyApiServer<notification::EncodedVersionedFinalityProof, Block> for Beefy<Block>
+impl<Block> BeefyApiServer<notification::EncodedVersionedFinalityProof, Block::Hash>
+	for Beefy<Block>
 where
 	Block: BlockT,
 {
@@ -168,13 +160,6 @@ where
 			.ok_or(Error::EndpointNotReady)
 			.map_err(Into::into)
 	}
-
-	async fn check_versioned_finality_proof(
-		&self,
-		versioned_finality_proof: BeefyVersionedFinalityProof<Block>,
-	) -> RpcResult<bool> {
-		return Ok(false)
-	}
 }
 
 #[cfg(test)]
diff --git a/substrate/primitives/consensus/beefy/src/commitment.rs b/substrate/primitives/consensus/beefy/src/commitment.rs
index f7796c5afe1c..3c451b70afb0 100644
--- a/substrate/primitives/consensus/beefy/src/commitment.rs
+++ b/substrate/primitives/consensus/beefy/src/commitment.rs
@@ -17,8 +17,6 @@
 
 use codec::{Decode, Encode, Error, Input};
 use scale_info::TypeInfo;
-#[cfg(feature = "serde")]
-use serde::{Deserialize, Serialize};
 use sp_std::{cmp, prelude::*};
 
 use crate::{Payload, ValidatorSetId};
@@ -29,7 +27,7 @@ use crate::{Payload, ValidatorSetId};
 /// height [block_number](Commitment::block_number).
 /// GRANDPA validators collect signatures on commitments and a stream of such signed commitments
 /// (see [SignedCommitment]) forms the BEEFY protocol.
-#[derive(Clone, Debug, PartialEq, Eq, Encode, Decode, TypeInfo, Serialize, Deserialize)]
+#[derive(Clone, Debug, PartialEq, Eq, Encode, Decode, TypeInfo)]
 pub struct Commitment<TBlockNumber> {
 	///  A collection of payloads to be signed, see [`Payload`] for details.
 	///
@@ -88,7 +86,7 @@ where
 /// Note that SCALE-encoding of the structure is optimized for size efficiency over the wire,
 /// please take a look at custom [`Encode`] and [`Decode`] implementations and
 /// `CompactSignedCommitment` struct.
-#[derive(Clone, Debug, PartialEq, Eq, TypeInfo, Serialize, Deserialize)]
+#[derive(Clone, Debug, PartialEq, Eq, TypeInfo)]
 pub struct SignedCommitment<TBlockNumber, TSignature> {
 	/// The commitment signatures are collected for.
 	pub commitment: Commitment<TBlockNumber>,
@@ -236,7 +234,7 @@ where
 ///
 /// Note that this enum is subject to change in the future with introduction
 /// of additional cryptographic primitives to BEEFY.
-#[derive(Clone, Debug, PartialEq, codec::Encode, codec::Decode, Serialize, Deserialize)]
+#[derive(Clone, Debug, PartialEq, codec::Encode, codec::Decode)]
 pub enum VersionedFinalityProof<N, S> {
 	#[codec(index = 1)]
 	/// Current active version
diff --git a/substrate/primitives/consensus/beefy/src/payload.rs b/substrate/primitives/consensus/beefy/src/payload.rs
index c703fe1bd796..43b83911dd62 100644
--- a/substrate/primitives/consensus/beefy/src/payload.rs
+++ b/substrate/primitives/consensus/beefy/src/payload.rs
@@ -17,7 +17,6 @@
 
 use codec::{Decode, Encode};
 use scale_info::TypeInfo;
-use serde::{Deserialize, Serialize};
 use sp_runtime::traits::Block;
 use sp_std::prelude::*;
 
@@ -40,20 +39,7 @@ pub mod known_payloads {
 /// Identifiers MUST be sorted by the [`BeefyPayloadId`] to allow efficient lookup of expected
 /// value. Duplicated identifiers are disallowed. It's okay for different implementations to only
 /// support a subset of possible values.
-#[derive(
-	Decode,
-	Encode,
-	Debug,
-	PartialEq,
-	Eq,
-	Clone,
-	Ord,
-	PartialOrd,
-	Hash,
-	TypeInfo,
-	Serialize,
-	Deserialize,
-)]
+#[derive(Decode, Encode, Debug, PartialEq, Eq, Clone, Ord, PartialOrd, Hash, TypeInfo)]
 pub struct Payload(Vec<(BeefyPayloadId, Vec<u8>)>);
 
 impl Payload {

From 20968b1dec7e98ded89eda90fb05556a4ec8e158 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 4 Jan 2024 13:00:01 +0100
Subject: [PATCH 122/188] fixup! Doc: payload checking reports equivocations
 too

---
 .../beefy/src/communication/fisherman.rs          | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index f7da111cf1ef..2f562ca3be1e 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -40,19 +40,22 @@ use sp_runtime::{
 use std::{marker::PhantomData, sync::Arc};
 
 pub(crate) trait BeefyFisherman<B: Block>: Send + Sync {
-	/// Check `vote` for contained block against canonical payload.
+	/// Check `vote` for contained block against canonical payload. If an equivocation is detected,
+	/// this should also report it.
 	fn check_vote(
 		&self,
 		vote: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
 	) -> Result<(), Error>;
 
-	/// Check `signed_commitment` for contained block against canonical payload.
+	/// Check `signed_commitment` for contained block against canonical payload. If an equivocation is detected,
+	/// this should also report it.
 	fn check_signed_commitment(
 		&self,
 		signed_commitment: SignedCommitment<NumberFor<B>, Signature>,
 	) -> Result<(), Error>;
 
-	/// Check `proof` for contained block against canonical payload.
+	/// Check `proof` for contained block against canonical payload. If an equivocation is detected,
+	/// this should also report it.
 	fn check_proof(&self, proof: BeefyVersionedFinalityProof<B>) -> Result<(), Error>;
 }
 
@@ -278,8 +281,8 @@ where
 		Ok(())
 	}
 
-	/// Check `vote` for contained block against canonical payload. If an equivocation is detected,
-	/// this also reports it.
+	/// Check `signed_commitment` for contained block against canonical payload. If an equivocation
+	/// is detected, this also reports it.
 	fn check_signed_commitment(
 		&self,
 		signed_commitment: SignedCommitment<NumberFor<B>, Signature>,
@@ -357,7 +360,7 @@ where
 		Ok(())
 	}
 
-	/// Check `vote` for contained block against canonical payload. If an equivocation is detected,
+	/// Check `proof` for contained block against canonical payload. If an equivocation is detected,
 	/// this also reports it.
 	fn check_proof(&self, proof: BeefyVersionedFinalityProof<B>) -> Result<(), Error> {
 		match proof {

From 370f76590af2e788e5d388dc4e25d78b976aa8be Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 4 Jan 2024 13:02:54 +0100
Subject: [PATCH 123/188] filter invalidly signed commitments at call site

only correctly signed commitments that equivocate should be reported. If
the signature is invalid, we should skip the validator from the
equivocation report and only report those who have validly signed the
equivocation (i.e. if signature on vote is invalid not report it at all).
---
 .../beefy/src/communication/fisherman.rs      | 27 ++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 2f562ca3be1e..1d023688c26c 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -47,8 +47,8 @@ pub(crate) trait BeefyFisherman<B: Block>: Send + Sync {
 		vote: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
 	) -> Result<(), Error>;
 
-	/// Check `signed_commitment` for contained block against canonical payload. If an equivocation is detected,
-	/// this should also report it.
+	/// Check `signed_commitment` for contained block against canonical payload. If an equivocation
+	/// is detected, this should also report it.
 	fn check_signed_commitment(
 		&self,
 		signed_commitment: SignedCommitment<NumberFor<B>, Signature>,
@@ -240,6 +240,14 @@ where
 		vote: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
 	) -> Result<(), Error> {
 		let number = vote.commitment.block_number;
+		// if the vote's commitment has not been signed by the purported signer, we ignore it
+		if !sp_consensus_beefy::check_commitment_signature::<_, _, BeefySignatureHasher>(
+			&vote.commitment,
+			&vote.id,
+			&vote.signature,
+		) {
+			return Ok(())
+		};
 		// if the vote is for a block number exceeding our best block number, there shouldn't even
 		// be a payload to sign yet, hence we assume it is an equivocation and report it
 		if number > self.backend.blockchain().info().best_number {
@@ -302,7 +310,20 @@ where
 				.iter()
 				.cloned()
 				.zip(signatures.into_iter())
-				.filter_map(|(id, signature)| signature.map(|sig| (id, sig)))
+				.filter_map(|(id, signature)| match signature {
+					Some(sig) =>
+						if sp_consensus_beefy::check_commitment_signature::<
+							_,
+							_,
+							BeefySignatureHasher,
+						>(&commitment, &id, &sig)
+						{
+							Some((id, sig))
+						} else {
+							None
+						},
+					None => None,
+				})
 				.collect();
 			if signatories.len() > 0 {
 				let proof = ForkEquivocationProof {

From 28502cdc7963a06c53d72af57649d8bc94dba119 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 11 Jan 2024 10:10:17 +0100
Subject: [PATCH 124/188] Apply suggestions from code review

Co-authored-by: Adrian Catangiu <adrian@parity.io>
---
 substrate/primitives/consensus/beefy/src/lib.rs      | 12 ++++--------
 .../primitives/merkle-mountain-range/Cargo.toml      | 12 ++++++------
 .../primitives/merkle-mountain-range/src/lib.rs      |  4 ++--
 .../primitives/merkle-mountain-range/src/utils.rs    |  1 -
 4 files changed, 12 insertions(+), 17 deletions(-)

diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 2d6da7f14222..9e735a77916d 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -308,8 +308,8 @@ impl<Number, Id, Signature> VoteEquivocationProof<Number, Id, Signature> {
 /// See [check_fork_equivocation_proof] for proof validity conditions.
 #[derive(Clone, Debug, Decode, Encode, PartialEq, TypeInfo)]
 pub struct ForkEquivocationProof<Number, Id, Signature, Header, Hash> {
-	/// Commitment for a block on different fork than one at the same height in
-	/// this client's chain.
+	/// Commitment for a block on a different fork than one at the same height in
+	/// the chain where this proof is submitted.
 	pub commitment: Commitment<Number>,
 	/// Signatures on this block
 	pub signatories: Vec<(Id, Signature)>,
@@ -322,7 +322,7 @@ pub struct ForkEquivocationProof<Number, Id, Signature, Header, Hash> {
 impl<Number, Id, Signature, H: HeaderT, Hash>
 	ForkEquivocationProof<Number, Id, Signature, H, Hash>
 {
-	/// Returns the authority id of the misbehaving voter.
+	/// Returns the authority ids of the misbehaving voters.
 	pub fn offender_ids(&self) -> Vec<&Id> {
 		self.signatories.iter().map(|(id, _)| id).collect()
 	}
@@ -407,11 +407,7 @@ where
 		// and they will likewise be slashed.
 		// Note that we can only check this if a valid header has been provided - we cannot
 		// slash for this with an ancestry proof - by necessity)
-		if canonical_header.hash() == *canonical_header_hash &&
-			Some(&commitment.payload) != canonical_payload.as_ref()
-		{
-			return true
-		}
+		return Some(&commitment.payload) != canonical_payload.as_ref()
 	}
 	// if no header provided, the header proof is also not correct
 	false
diff --git a/substrate/primitives/merkle-mountain-range/Cargo.toml b/substrate/primitives/merkle-mountain-range/Cargo.toml
index acdc3c1fa415..b29ebd317dd8 100644
--- a/substrate/primitives/merkle-mountain-range/Cargo.toml
+++ b/substrate/primitives/merkle-mountain-range/Cargo.toml
@@ -15,13 +15,13 @@ targets = ["x86_64-unknown-linux-gnu"]
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false }
 scale-info = { version = "2.10.0", default-features = false, features = ["derive"] }
 log = { version = "0.4.17", default-features = false }
-mmr-lib = { package = "ckb-merkle-mountain-range", git = "https://github.com/Lederstrumpf/merkle-mountain-range.git", branch = "leaf-node-proof-split", default-features = false }
+mmr-lib = { package = "ckb-merkle-mountain-range", git = "https://github.com/paritytech/merkle-mountain-range.git", branch = "leaf-node-proof-split", default-features = false }
 serde = { version = "1.0.193", features = ["alloc", "derive"], default-features = false, optional = true }
-sp-api = { path = "../api", default-features = false}
-sp-core = { path = "../core", default-features = false}
-sp-debug-derive = { path = "../debug-derive", default-features = false}
-sp-runtime = { path = "../runtime", default-features = false}
-sp-std = { path = "../std", default-features = false}
+sp-api = { path = "../api", default-features = false }
+sp-core = { path = "../core", default-features = false }
+sp-debug-derive = { path = "../debug-derive", default-features = false }
+sp-runtime = { path = "../runtime", default-features = false }
+sp-std = { path = "../std", default-features = false }
 thiserror = "1.0"
 
 [dev-dependencies]
diff --git a/substrate/primitives/merkle-mountain-range/src/lib.rs b/substrate/primitives/merkle-mountain-range/src/lib.rs
index 2dc3082bf854..75900c7481ed 100644
--- a/substrate/primitives/merkle-mountain-range/src/lib.rs
+++ b/substrate/primitives/merkle-mountain-range/src/lib.rs
@@ -366,7 +366,7 @@ pub struct NodeProof<Hash> {
 	pub leaf_indices: Vec<LeafIndex>,
 	/// Number of leaves in MMR, when the proof was generated.
 	pub leaf_count: NodeIndex,
-	/// Proof elements (hashes of siblings of inner nodes on the path to the leaf).
+	/// Proof elements (positions and hashes of siblings of inner nodes on the path to the node(s)).
 	pub items: Vec<(u64, Hash)>,
 }
 
@@ -482,7 +482,7 @@ sp_api::decl_runtime_apis! {
 			best_known_block_number: Option<BlockNumber>
 		) -> Result<AncestryProof<Hash>, Error>;
 
-		/// Verifies that a claimed prev_root is in fact an ancestor of the provided mmr root
+		/// Verifies that a claimed prev_root is in fact an ancestor of the current mmr root
 		fn verify_ancestry_proof(ancestry_proof: AncestryProof<Hash>) -> Result<(), Error>;
 	}
 }
diff --git a/substrate/primitives/merkle-mountain-range/src/utils.rs b/substrate/primitives/merkle-mountain-range/src/utils.rs
index ac806966e104..9a960533c802 100644
--- a/substrate/primitives/merkle-mountain-range/src/utils.rs
+++ b/substrate/primitives/merkle-mountain-range/src/utils.rs
@@ -48,7 +48,6 @@ pub fn verify_ancestry_proof<H, M>(
 	ancestry_proof: AncestryProof<H>,
 ) -> Result<bool, Error>
 where
-	// H: sp_runtime::traits::Hash::Output,
 	H: Clone + Debug + PartialEq + Encode,
 	M: mmr_lib::Merge<Item = H>,
 {

From d18897d05c89bbf2c13ea8cd52a862b585d6e49a Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 11 Jan 2024 13:22:23 +0100
Subject: [PATCH 125/188] migrate BeefyFisherman trait fns into Fisherman

Suggested-by: Serban Iorga <serban@parity.io>

Was premature generalization - can reintroduce if needed later.
---
 .../beefy/src/communication/fisherman.rs      | 37 ++-----------------
 .../beefy/src/communication/gossip.rs         | 29 ++++++++++-----
 substrate/client/consensus/beefy/src/tests.rs | 19 ----------
 .../client/consensus/beefy/src/worker.rs      | 19 ++++------
 4 files changed, 31 insertions(+), 73 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 1d023688c26c..15e498704067 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -39,26 +39,6 @@ use sp_runtime::{
 };
 use std::{marker::PhantomData, sync::Arc};
 
-pub(crate) trait BeefyFisherman<B: Block>: Send + Sync {
-	/// Check `vote` for contained block against canonical payload. If an equivocation is detected,
-	/// this should also report it.
-	fn check_vote(
-		&self,
-		vote: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
-	) -> Result<(), Error>;
-
-	/// Check `signed_commitment` for contained block against canonical payload. If an equivocation
-	/// is detected, this should also report it.
-	fn check_signed_commitment(
-		&self,
-		signed_commitment: SignedCommitment<NumberFor<B>, Signature>,
-	) -> Result<(), Error>;
-
-	/// Check `proof` for contained block against canonical payload. If an equivocation is detected,
-	/// this should also report it.
-	fn check_proof(&self, proof: BeefyVersionedFinalityProof<B>) -> Result<(), Error>;
-}
-
 /// Helper wrapper used to check gossiped votes for (historical) equivocations,
 /// and report any such protocol infringements.
 pub(crate) struct Fisherman<B: Block, BE, R, P> {
@@ -72,8 +52,8 @@ pub(crate) struct Fisherman<B: Block, BE, R, P> {
 impl<B, BE, R, P> Fisherman<B, BE, R, P>
 where
 	B: Block,
-	BE: Backend<B>,
-	P: PayloadProvider<B>,
+	BE: Backend<B> + Send + Sync,
+	P: PayloadProvider<B> + Send + Sync,
 	R: ProvideRuntimeApi<B> + Send + Sync,
 	R::Api: BeefyApi<B, AuthorityId, MmrRootHash> + MmrApi<B, MmrRootHash, NumberFor<B>>,
 {
@@ -223,19 +203,10 @@ where
 			Ok(false)
 		}
 	}
-}
 
-impl<B, BE, R, P> BeefyFisherman<B> for Fisherman<B, BE, R, P>
-where
-	B: Block,
-	BE: Backend<B>,
-	P: PayloadProvider<B>,
-	R: ProvideRuntimeApi<B> + Send + Sync,
-	R::Api: BeefyApi<B, AuthorityId, MmrRootHash> + MmrApi<B, MmrRootHash, NumberFor<B>>,
-{
 	/// Check `vote` for contained block against canonical payload. If an equivocation is detected,
 	/// this also reports it.
-	fn check_vote(
+	pub(crate) fn check_vote(
 		&self,
 		vote: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
 	) -> Result<(), Error> {
@@ -383,7 +354,7 @@ where
 
 	/// Check `proof` for contained block against canonical payload. If an equivocation is detected,
 	/// this also reports it.
-	fn check_proof(&self, proof: BeefyVersionedFinalityProof<B>) -> Result<(), Error> {
+	pub(crate) fn check_proof(&self, proof: BeefyVersionedFinalityProof<B>) -> Result<(), Error> {
 		match proof {
 			BeefyVersionedFinalityProof::<B>::V1(signed_commitment) =>
 				self.check_signed_commitment(signed_commitment),
diff --git a/substrate/client/consensus/beefy/src/communication/gossip.rs b/substrate/client/consensus/beefy/src/communication/gossip.rs
index 56b1f561e7a1..b5b7f52496c9 100644
--- a/substrate/client/consensus/beefy/src/communication/gossip.rs
+++ b/substrate/client/consensus/beefy/src/communication/gossip.rs
@@ -25,13 +25,15 @@ use sp_runtime::traits::{Block, Hash, Header, NumberFor};
 use codec::{Decode, DecodeAll, Encode};
 use log::{debug, trace};
 use parking_lot::{Mutex, RwLock};
+use sc_client_api::Backend;
 use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender};
+use sp_api::ProvideRuntimeApi;
 use wasm_timer::Instant;
 
 use crate::{
 	communication::{
 		benefit, cost,
-		fisherman::BeefyFisherman,
+		fisherman::Fisherman,
 		peers::{KnownPeers, PeerReport},
 	},
 	justification::{
@@ -42,8 +44,9 @@ use crate::{
 };
 use sp_consensus_beefy::{
 	ecdsa_crypto::{AuthorityId, Signature},
-	ValidatorSet, ValidatorSetId, VoteMessage,
+	BeefyApi, MmrRootHash, PayloadProvider, ValidatorSet, ValidatorSetId, VoteMessage,
 };
+use sp_mmr_primitives::MmrApi;
 
 // Timeout for rebroadcasting messages.
 #[cfg(not(test))]
@@ -229,25 +232,28 @@ impl<B: Block> Filter<B> {
 /// to create forks before head of GRANDPA are reported.
 ///
 ///All messaging is handled in a single BEEFY global topic.
-pub(crate) struct GossipValidator<B: Block, F> {
+pub(crate) struct GossipValidator<B: Block, BE, R, P> {
 	votes_topic: B::Hash,
 	justifs_topic: B::Hash,
 	gossip_filter: RwLock<Filter<B>>,
 	next_rebroadcast: Mutex<Instant>,
 	known_peers: Arc<Mutex<KnownPeers<B>>>,
 	report_sender: TracingUnboundedSender<PeerReport>,
-	pub(crate) fisherman: F,
+	pub(crate) fisherman: Fisherman<B, BE, R, P>,
 }
 
-impl<B, F> GossipValidator<B, F>
+impl<B, BE, R, P> GossipValidator<B, BE, R, P>
 where
 	B: Block,
-	F: BeefyFisherman<B>,
+	BE: Backend<B> + Send + Sync,
+	P: PayloadProvider<B> + Send + Sync,
+	R: ProvideRuntimeApi<B> + Send + Sync,
+	R::Api: BeefyApi<B, AuthorityId, MmrRootHash> + MmrApi<B, MmrRootHash, NumberFor<B>>,
 {
 	pub(crate) fn new(
 		known_peers: Arc<Mutex<KnownPeers<B>>>,
-		fisherman: F,
-	) -> (GossipValidator<B, F>, TracingUnboundedReceiver<PeerReport>) {
+		fisherman: Fisherman<B, BE, R, P>,
+	) -> (GossipValidator<B, BE, R, P>, TracingUnboundedReceiver<PeerReport>) {
 		let (tx, rx) = tracing_unbounded("mpsc_beefy_gossip_validator", 10_000);
 		let val = GossipValidator {
 			votes_topic: votes_topic::<B>(),
@@ -385,10 +391,13 @@ where
 	}
 }
 
-impl<B, F> Validator<B> for GossipValidator<B, F>
+impl<B, BE, R, P> Validator<B> for GossipValidator<B, BE, R, P>
 where
 	B: Block,
-	F: BeefyFisherman<B>,
+	BE: Backend<B> + Send + Sync,
+	P: PayloadProvider<B> + Send + Sync,
+	R: ProvideRuntimeApi<B> + Send + Sync,
+	R::Api: BeefyApi<B, AuthorityId, MmrRootHash> + MmrApi<B, MmrRootHash, NumberFor<B>>,
 {
 	fn peer_disconnected(&self, _context: &mut dyn ValidatorContext<B>, who: &PeerId) {
 		self.known_peers.lock().remove(who);
diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index fab962cb531e..366e952f7364 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -22,7 +22,6 @@ use crate::{
 	aux_schema::{load_persistent, tests::verify_persisted_version},
 	beefy_block_import_and_links,
 	communication::{
-		fisherman::BeefyFisherman,
 		gossip::{
 			proofs_topic, tests::sign_commitment, votes_topic, GossipFilterCfg, GossipMessage,
 			GossipValidator,
@@ -251,24 +250,6 @@ pub(crate) struct DummyFisherman<B> {
 	pub _phantom: PhantomData<B>,
 }
 
-impl<B: BlockT> BeefyFisherman<B> for DummyFisherman<B> {
-	fn check_proof(&self, _: BeefyVersionedFinalityProof<B>) -> Result<(), Error> {
-		Ok(())
-	}
-	fn check_signed_commitment(
-		&self,
-		_: SignedCommitment<NumberFor<B>, Signature>,
-	) -> Result<(), Error> {
-		Ok(())
-	}
-	fn check_vote(
-		&self,
-		_: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
-	) -> Result<(), Error> {
-		Ok(())
-	}
-}
-
 #[derive(Clone)]
 pub(crate) struct TestApi {
 	pub beefy_genesis: u64,
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index e554aac191b3..1217a3ecf964 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -18,7 +18,6 @@
 
 use crate::{
 	communication::{
-		fisherman::BeefyFisherman,
 		gossip::{proofs_topic, votes_topic, GossipFilterCfg, GossipMessage, GossipValidator},
 		peers::PeerReport,
 		request_response::outgoing_requests_engine::{OnDemandJustificationsEngine, ResponseInfo},
@@ -318,15 +317,15 @@ impl<B: Block> PersistedState<B> {
 /// Helper object holding BEEFY worker communication/gossip components.
 ///
 /// These are created once, but will be reused if worker is restarted/reinitialized.
-pub(crate) struct BeefyComms<B: Block, F: BeefyFisherman<B>> {
+pub(crate) struct BeefyComms<B: Block, BE, P, R> {
 	pub gossip_engine: GossipEngine<B>,
-	pub gossip_validator: Arc<GossipValidator<B, F>>,
+	pub gossip_validator: Arc<GossipValidator<B, BE, R, P>>,
 	pub gossip_report_stream: TracingUnboundedReceiver<PeerReport>,
 	pub on_demand_justifications: OnDemandJustificationsEngine<B>,
 }
 
 /// A BEEFY worker plays the BEEFY protocol
-pub(crate) struct BeefyWorker<B: Block, BE, P, RuntimeApi, S, F: BeefyFisherman<B>> {
+pub(crate) struct BeefyWorker<B: Block, BE, P, RuntimeApi, S> {
 	// utilities
 	pub backend: Arc<BE>,
 	pub payload_provider: P,
@@ -335,7 +334,7 @@ pub(crate) struct BeefyWorker<B: Block, BE, P, RuntimeApi, S, F: BeefyFisherman<
 	pub key_store: Arc<BeefyKeystore>,
 
 	// communication (created once, but returned and reused if worker is restarted/reinitialized)
-	pub comms: BeefyComms<B, F>,
+	pub comms: BeefyComms<B, BE, P, RuntimeApi>,
 
 	// channels
 	/// Links between the block importer, the background voter and the RPC layer.
@@ -350,15 +349,14 @@ pub(crate) struct BeefyWorker<B: Block, BE, P, RuntimeApi, S, F: BeefyFisherman<
 	pub persisted_state: PersistedState<B>,
 }
 
-impl<B, BE, P, R, S, F> BeefyWorker<B, BE, P, R, S, F>
+impl<B, BE, P, R, S> BeefyWorker<B, BE, P, R, S>
 where
 	B: Block + Codec,
 	BE: Backend<B>,
-	P: PayloadProvider<B>,
+	P: PayloadProvider<B> + Send + Sync,
 	S: SyncOracle,
-	R: ProvideRuntimeApi<B>,
+	R: ProvideRuntimeApi<B> + Send + Sync,
 	R::Api: BeefyApi<B, AuthorityId, MmrRootHash> + MmrApi<B, MmrRootHash, NumberFor<B>>,
-	F: BeefyFisherman<B>,
 {
 	fn best_grandpa_block(&self) -> NumberFor<B> {
 		*self.persisted_state.voting_oracle.best_grandpa_block_header.number()
@@ -824,7 +822,7 @@ where
 		mut self,
 		block_import_justif: &mut Fuse<NotificationReceiver<BeefyVersionedFinalityProof<B>>>,
 		finality_notifications: &mut Fuse<FinalityNotifications<B>>,
-	) -> (Error, BeefyComms<B, F>) {
+	) -> (Error, BeefyComms<B, BE, P, R>) {
 		info!(
 			target: LOG_TARGET,
 			"🥩 run BEEFY worker, best grandpa: #{:?}.",
@@ -1133,7 +1131,6 @@ pub(crate) mod tests {
 		MmrRootProvider<Block, TestApi>,
 		TestApi,
 		Arc<SyncingService<Block>>,
-		Fisherman<Block, Backend, TestApi, MmrRootProvider<Block, TestApi>>,
 	> {
 		let key_store: Arc<BeefyKeystore> = Arc::new(Some(create_beefy_keystore(*key)).into());
 

From 8a2a4af1ef24222a32e96912c818579895278509 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 11 Jan 2024 13:29:01 +0100
Subject: [PATCH 126/188] consistent lexical ordering of generic parameters

---
 .../consensus/beefy/src/communication/fisherman.rs   |  4 ++--
 .../consensus/beefy/src/communication/gossip.rs      | 12 ++++++------
 substrate/client/consensus/beefy/src/worker.rs       |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 15e498704067..3510f3f6b785 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -41,7 +41,7 @@ use std::{marker::PhantomData, sync::Arc};
 
 /// Helper wrapper used to check gossiped votes for (historical) equivocations,
 /// and report any such protocol infringements.
-pub(crate) struct Fisherman<B: Block, BE, R, P> {
+pub(crate) struct Fisherman<B: Block, BE, P, R> {
 	pub backend: Arc<BE>,
 	pub runtime: Arc<R>,
 	pub key_store: Arc<BeefyKeystore>,
@@ -49,7 +49,7 @@ pub(crate) struct Fisherman<B: Block, BE, R, P> {
 	pub _phantom: PhantomData<B>,
 }
 
-impl<B, BE, R, P> Fisherman<B, BE, R, P>
+impl<B, BE, R, P> Fisherman<B, BE, P, R>
 where
 	B: Block,
 	BE: Backend<B> + Send + Sync,
diff --git a/substrate/client/consensus/beefy/src/communication/gossip.rs b/substrate/client/consensus/beefy/src/communication/gossip.rs
index b5b7f52496c9..55da209b885b 100644
--- a/substrate/client/consensus/beefy/src/communication/gossip.rs
+++ b/substrate/client/consensus/beefy/src/communication/gossip.rs
@@ -232,17 +232,17 @@ impl<B: Block> Filter<B> {
 /// to create forks before head of GRANDPA are reported.
 ///
 ///All messaging is handled in a single BEEFY global topic.
-pub(crate) struct GossipValidator<B: Block, BE, R, P> {
+pub(crate) struct GossipValidator<B: Block, BE, P, R> {
 	votes_topic: B::Hash,
 	justifs_topic: B::Hash,
 	gossip_filter: RwLock<Filter<B>>,
 	next_rebroadcast: Mutex<Instant>,
 	known_peers: Arc<Mutex<KnownPeers<B>>>,
 	report_sender: TracingUnboundedSender<PeerReport>,
-	pub(crate) fisherman: Fisherman<B, BE, R, P>,
+	pub(crate) fisherman: Fisherman<B, BE, P, R>,
 }
 
-impl<B, BE, R, P> GossipValidator<B, BE, R, P>
+impl<B, BE, P, R> GossipValidator<B, BE, P, R>
 where
 	B: Block,
 	BE: Backend<B> + Send + Sync,
@@ -252,8 +252,8 @@ where
 {
 	pub(crate) fn new(
 		known_peers: Arc<Mutex<KnownPeers<B>>>,
-		fisherman: Fisherman<B, BE, R, P>,
-	) -> (GossipValidator<B, BE, R, P>, TracingUnboundedReceiver<PeerReport>) {
+		fisherman: Fisherman<B, BE, P, R>,
+	) -> (GossipValidator<B, BE, P, R>, TracingUnboundedReceiver<PeerReport>) {
 		let (tx, rx) = tracing_unbounded("mpsc_beefy_gossip_validator", 10_000);
 		let val = GossipValidator {
 			votes_topic: votes_topic::<B>(),
@@ -391,7 +391,7 @@ where
 	}
 }
 
-impl<B, BE, R, P> Validator<B> for GossipValidator<B, BE, R, P>
+impl<B, BE, P, R> Validator<B> for GossipValidator<B, BE, P, R>
 where
 	B: Block,
 	BE: Backend<B> + Send + Sync,
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index 1217a3ecf964..666ebacea670 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -319,7 +319,7 @@ impl<B: Block> PersistedState<B> {
 /// These are created once, but will be reused if worker is restarted/reinitialized.
 pub(crate) struct BeefyComms<B: Block, BE, P, R> {
 	pub gossip_engine: GossipEngine<B>,
-	pub gossip_validator: Arc<GossipValidator<B, BE, R, P>>,
+	pub gossip_validator: Arc<GossipValidator<B, BE, P, R>>,
 	pub gossip_report_stream: TracingUnboundedReceiver<PeerReport>,
 	pub on_demand_justifications: OnDemandJustificationsEngine<B>,
 }

From 35b302da1078a42a240e017f864f270a74368b75 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 11 Jan 2024 14:32:49 +0100
Subject: [PATCH 127/188] account for #2856

---
 substrate/bin/node/runtime/src/lib.rs | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs
index 5fca12c8aeeb..911800061b47 100644
--- a/substrate/bin/node/runtime/src/lib.rs
+++ b/substrate/bin/node/runtime/src/lib.rs
@@ -2702,8 +2702,8 @@ impl_runtime_apis! {
 		}
 	}
 
-	#[api_version(3)]
-	impl sp_consensus_beefy::BeefyApi<Block, BeefyId> for Runtime {
+	#[api_version(4)]
+	impl sp_consensus_beefy::BeefyApi<Block, BeefyId, Hash> for Runtime {
 		fn beefy_genesis() -> Option<BlockNumber> {
 			Beefy::genesis_block()
 		}
@@ -2712,8 +2712,8 @@ impl_runtime_apis! {
 			Beefy::validator_set()
 		}
 
-		fn submit_report_equivocation_unsigned_extrinsic(
-			equivocation_proof: sp_consensus_beefy::EquivocationProof<
+		fn submit_report_vote_equivocation_unsigned_extrinsic(
+			vote_equivocation_proof: sp_consensus_beefy::VoteEquivocationProof<
 				BlockNumber,
 				BeefyId,
 				BeefySignature,
@@ -2722,12 +2722,24 @@ impl_runtime_apis! {
 		) -> Option<()> {
 			let key_owner_proof = key_owner_proof.decode()?;
 
-			Beefy::submit_unsigned_equivocation_report(
-				equivocation_proof,
+			Beefy::submit_unsigned_vote_equivocation_report(
+				vote_equivocation_proof,
 				key_owner_proof,
 			)
 		}
 
+		fn submit_report_fork_equivocation_unsigned_extrinsic(
+			fork_equivocation_proof: sp_consensus_beefy::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, Header, Hash>,
+			key_owner_proofs: Vec<sp_consensus_beefy::OpaqueKeyOwnershipProof>,
+		) -> Option<()> {
+			let key_owner_proofs = key_owner_proofs.iter().cloned().map(|p| p.decode()).collect::<Option<Vec<_>>>()?;
+
+			Beefy::submit_unsigned_fork_equivocation_report(
+				fork_equivocation_proof,
+				key_owner_proofs,
+			)
+		}
+
 		fn generate_key_ownership_proof(
 			_set_id: sp_consensus_beefy::ValidatorSetId,
 			authority_id: BeefyId,

From ddee581d3a439350ec88588b76fdaed93b425f37 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 11 Jan 2024 18:16:10 +0100
Subject: [PATCH 128/188] fix sc_consensus_beefy integration tests

---
 substrate/client/consensus/beefy/src/tests.rs | 39 ++++++++++++++-----
 1 file changed, 30 insertions(+), 9 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index 2ea8e4003fc3..3c961c309a9b 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -22,6 +22,7 @@ use crate::{
 	aux_schema::{load_persistent, tests::verify_persisted_version},
 	beefy_block_import_and_links,
 	communication::{
+		fisherman::Fisherman,
 		gossip::{
 			proofs_topic, tests::sign_commitment, votes_topic, GossipFilterCfg, GossipMessage,
 			GossipValidator,
@@ -31,6 +32,7 @@ use crate::{
 	error::Error,
 	gossip_protocol_name,
 	justification::*,
+	keystore::BeefyKeystore,
 	load_or_init_voter_state, wait_for_runtime_pallet, BeefyRPCLinks, BeefyVoterLinks, KnownPeers,
 	PersistedState,
 };
@@ -57,8 +59,8 @@ use sp_consensus_beefy::{
 	known_payloads,
 	mmr::{find_mmr_root_digest, MmrRootProvider},
 	BeefyApi, Commitment, ConsensusLog, ForkEquivocationProof, Keyring as BeefyKeyring,
-	MmrRootHash, OpaqueKeyOwnershipProof, Payload, SignedCommitment, ValidatorSet, ValidatorSetId,
-	VersionedFinalityProof, VoteEquivocationProof, VoteMessage, BEEFY_ENGINE_ID,
+	MmrRootHash, OpaqueKeyOwnershipProof, Payload, PayloadProvider, SignedCommitment, ValidatorSet,
+	ValidatorSetId, VersionedFinalityProof, VoteEquivocationProof, VoteMessage, BEEFY_ENGINE_ID,
 };
 use sp_core::H256;
 use sp_keystore::{testing::MemoryKeystore, Keystore, KeystorePtr};
@@ -409,13 +411,31 @@ pub(crate) fn create_beefy_keystore(authority: BeefyKeyring) -> KeystorePtr {
 	keystore.into()
 }
 
+fn create_fisherman<BE>(
+	beefy_keyring: BeefyKeyring, // Assuming BeefyKeyring contains necessary keys for the Fisherman
+	api: Arc<TestApi>,
+	backend: Arc<BE>,
+) -> Fisherman<Block, BE, MmrRootProvider<Block, TestApi>, TestApi> {
+	let key_store: Arc<BeefyKeystore> = Arc::new(Some(create_beefy_keystore(beefy_keyring)).into());
+	let payload_provider = MmrRootProvider::new(api.clone());
+
+	Fisherman {
+		backend,
+		key_store,
+		runtime: api.clone().into(),
+		payload_provider,
+		_phantom: PhantomData,
+	}
+}
+
 async fn voter_init_setup(
+	keyring: BeefyKeyring,
 	net: &mut BeefyTestNet,
 	finality: &mut futures::stream::Fuse<FinalityNotifications<Block>>,
 	api: &TestApi,
 ) -> sp_blockchain::Result<PersistedState<Block>> {
 	let backend = net.peer(0).client().as_backend();
-	let fisherman = DummyFisherman { _phantom: PhantomData };
+	let fisherman = create_fisherman(keyring, Arc::new(api.clone()), backend.clone());
 	let known_peers = Arc::new(Mutex::new(KnownPeers::new()));
 	let (gossip_validator, _) = GossipValidator::new(known_peers, fisherman);
 	let gossip_validator = Arc::new(gossip_validator);
@@ -1065,7 +1085,7 @@ async fn should_initialize_voter_at_genesis() {
 
 	let api = TestApi::with_validator_set(&validator_set);
 	// load persistent state - nothing in DB, should init at genesis
-	let persisted_state = voter_init_setup(&mut net, &mut finality, &api).await.unwrap();
+	let persisted_state = voter_init_setup(keys[0], &mut net, &mut finality, &api).await.unwrap();
 
 	// Test initialization at session boundary.
 	// verify voter initialized with two sessions starting at blocks 1 and 10
@@ -1111,7 +1131,7 @@ async fn should_initialize_voter_at_custom_genesis() {
 	// NOTE: code from `voter_init_setup()` is moved here because the new network event system
 	// doesn't allow creating a new `GossipEngine` as the notification handle is consumed by the
 	// first `GossipEngine`
-	let fisherman = DummyFisherman { _phantom: PhantomData::<Block> };
+	let fisherman = create_fisherman(BeefyKeyring::Alice, Arc::new(api.clone()), backend.clone());
 	let known_peers = Arc::new(Mutex::new(KnownPeers::new()));
 	let (gossip_validator, _) = GossipValidator::new(known_peers, fisherman);
 	let gossip_validator = Arc::new(gossip_validator);
@@ -1218,7 +1238,7 @@ async fn should_initialize_voter_when_last_final_is_session_boundary() {
 
 	let api = TestApi::with_validator_set(&validator_set);
 	// load persistent state - nothing in DB, should init at session boundary
-	let persisted_state = voter_init_setup(&mut net, &mut finality, &api).await.unwrap();
+	let persisted_state = voter_init_setup(keys[0], &mut net, &mut finality, &api).await.unwrap();
 
 	// verify voter initialized with single session starting at block 10
 	assert_eq!(persisted_state.voting_oracle().sessions().len(), 1);
@@ -1271,7 +1291,7 @@ async fn should_initialize_voter_at_latest_finalized() {
 
 	let api = TestApi::with_validator_set(&validator_set);
 	// load persistent state - nothing in DB, should init at last BEEFY finalized
-	let persisted_state = voter_init_setup(&mut net, &mut finality, &api).await.unwrap();
+	let persisted_state = voter_init_setup(keys[0], &mut net, &mut finality, &api).await.unwrap();
 
 	// verify voter initialized with single session starting at block 12
 	assert_eq!(persisted_state.voting_oracle().sessions().len(), 1);
@@ -1309,7 +1329,7 @@ async fn should_initialize_voter_at_custom_genesis_when_state_unavailable() {
 	net.peer(0).client().as_client().finalize_block(hashes[30], None).unwrap();
 
 	// load persistent state - nothing in DB, should init at genesis
-	let persisted_state = voter_init_setup(&mut net, &mut finality, &api).await.unwrap();
+	let persisted_state = voter_init_setup(keys[0], &mut net, &mut finality, &api).await.unwrap();
 
 	// Test initialization at session boundary.
 	// verify voter initialized with all sessions pending, first one starting at block 5 (start of
@@ -1457,11 +1477,12 @@ async fn gossiped_finality_proofs() {
 	let min_block_delta = 1;
 
 	let mut net = BeefyTestNet::new(3);
+	let backend = net.peer(0).client().as_backend();
 	let api = Arc::new(TestApi::with_validator_set(&validator_set));
 	let beefy_peers = peers.iter().enumerate().map(|(id, key)| (id, key, api.clone())).collect();
 
 	let charlie = &mut net.peers[2];
-	let fisherman = DummyFisherman { _phantom: PhantomData::<Block> };
+	let fisherman = create_fisherman(BeefyKeyring::Alice, api, backend.clone());
 	let known_peers = Arc::new(Mutex::new(KnownPeers::<Block>::new()));
 	// Charlie will run just the gossip engine and not the full voter.
 	let (gossip_validator, _) = GossipValidator::new(known_peers, fisherman);

From 30ea46ced80cef7079e460ceced4c1a6cf0ed6fc Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 11 Jan 2024 21:18:15 +0100
Subject: [PATCH 129/188] fix sc_consensus_beefy::communication::gossip "unit"
 tests

---
 .../beefy/src/communication/gossip.rs         | 37 +++++++++++++++----
 substrate/client/consensus/beefy/src/tests.rs |  2 +-
 2 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/gossip.rs b/substrate/client/consensus/beefy/src/communication/gossip.rs
index ed416db9341e..be081767ad2a 100644
--- a/substrate/client/consensus/beefy/src/communication/gossip.rs
+++ b/substrate/client/consensus/beefy/src/communication/gossip.rs
@@ -514,8 +514,11 @@ where
 #[cfg(test)]
 pub(crate) mod tests {
 	use super::*;
-	use crate::{keystore::BeefyKeystore, tests::DummyFisherman};
-	use sc_network_test::Block;
+	use crate::{
+		keystore::BeefyKeystore,
+		tests::{create_beefy_keystore, create_fisherman, BeefyTestNet, TestApi},
+	};
+	use sc_network_test::{Block, TestNetFactory};
 	use sp_application_crypto::key_types::BEEFY as BEEFY_KEY_TYPE;
 	use sp_consensus_beefy::{
 		ecdsa_crypto::Signature, known_payloads, Commitment, Keyring, MmrRootHash, Payload,
@@ -581,9 +584,15 @@ pub(crate) mod tests {
 
 	#[test]
 	fn should_validate_messages() {
-		let keys = vec![Keyring::Alice.public()];
+		let keyring = Keyring::Alice;
+		let keys = vec![keyring.public()];
 		let validator_set = ValidatorSet::<AuthorityId>::new(keys.clone(), 0).unwrap();
-		let fisherman = DummyFisherman { _phantom: PhantomData::<Block> };
+
+		let api = TestApi::new(0, &validator_set, MmrRootHash::repeat_byte(0xbf));
+		let mut net = BeefyTestNet::new(1);
+		let backend = net.peer(0).client().as_backend();
+		let fisherman = create_fisherman(keyring, Arc::new(api.clone()), backend.clone());
+
 		let (gv, mut report_stream) =
 			GossipValidator::new(Arc::new(Mutex::new(KnownPeers::new())), fisherman);
 		let sender = PeerId::random();
@@ -696,9 +705,15 @@ pub(crate) mod tests {
 
 	#[test]
 	fn messages_allowed_and_expired() {
-		let keys = vec![Keyring::Alice.public()];
+		let keyring = Keyring::Alice;
+		let keys = vec![keyring.public()];
 		let validator_set = ValidatorSet::<AuthorityId>::new(keys.clone(), 0).unwrap();
-		let fisherman = DummyFisherman { _phantom: PhantomData::<Block> };
+
+		let api = TestApi::new(0, &validator_set, MmrRootHash::repeat_byte(0xbf));
+		let mut net = BeefyTestNet::new(1);
+		let backend = net.peer(0).client().as_backend();
+		let fisherman = create_fisherman(keyring, Arc::new(api.clone()), backend.clone());
+
 		let (gv, _) = GossipValidator::new(Arc::new(Mutex::new(KnownPeers::new())), fisherman);
 		gv.update_filter(GossipFilterCfg { start: 0, end: 10, validator_set: &validator_set });
 		let sender = sc_network::PeerId::random();
@@ -774,9 +789,15 @@ pub(crate) mod tests {
 
 	#[test]
 	fn messages_rebroadcast() {
-		let keys = vec![Keyring::Alice.public()];
+		let keyring = Keyring::Alice;
+		let keys = vec![keyring.public()];
 		let validator_set = ValidatorSet::<AuthorityId>::new(keys.clone(), 0).unwrap();
-		let fisherman = DummyFisherman { _phantom: PhantomData::<Block> };
+
+		let api = TestApi::new(0, &validator_set, MmrRootHash::repeat_byte(0xbf));
+		let mut net = BeefyTestNet::new(1);
+		let backend = net.peer(0).client().as_backend();
+		let fisherman = create_fisherman(keyring, Arc::new(api.clone()), backend.clone());
+
 		let (gv, _) = GossipValidator::new(Arc::new(Mutex::new(KnownPeers::new())), fisherman);
 		gv.update_filter(GossipFilterCfg { start: 0, end: 10, validator_set: &validator_set });
 		let sender = sc_network::PeerId::random();
diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index 3c961c309a9b..0d7159022351 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -411,7 +411,7 @@ pub(crate) fn create_beefy_keystore(authority: BeefyKeyring) -> KeystorePtr {
 	keystore.into()
 }
 
-fn create_fisherman<BE>(
+pub(crate) fn create_fisherman<BE>(
 	beefy_keyring: BeefyKeyring, // Assuming BeefyKeyring contains necessary keys for the Fisherman
 	api: Arc<TestApi>,
 	backend: Arc<BE>,

From 0283226cca69d659b0a217e34fc7ef17cb4cc860 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 11 Jan 2024 21:20:05 +0100
Subject: [PATCH 130/188] fmt

---
 substrate/primitives/merkle-mountain-range/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/substrate/primitives/merkle-mountain-range/src/lib.rs b/substrate/primitives/merkle-mountain-range/src/lib.rs
index 75900c7481ed..26021e719b77 100644
--- a/substrate/primitives/merkle-mountain-range/src/lib.rs
+++ b/substrate/primitives/merkle-mountain-range/src/lib.rs
@@ -366,7 +366,8 @@ pub struct NodeProof<Hash> {
 	pub leaf_indices: Vec<LeafIndex>,
 	/// Number of leaves in MMR, when the proof was generated.
 	pub leaf_count: NodeIndex,
-	/// Proof elements (positions and hashes of siblings of inner nodes on the path to the node(s)).
+	/// Proof elements (positions and hashes of siblings of inner nodes on the path to the
+	/// node(s)).
 	pub items: Vec<(u64, Hash)>,
 }
 

From a639db613311eb80ffb306e2270084a4149b55a2 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 11 Jan 2024 22:52:49 +0100
Subject: [PATCH 131/188] revert removal of header hash verification

---
 substrate/primitives/consensus/beefy/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 9e735a77916d..1ba7638bc1d9 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -407,7 +407,8 @@ where
 		// and they will likewise be slashed.
 		// Note that we can only check this if a valid header has been provided - we cannot
 		// slash for this with an ancestry proof - by necessity)
-		return Some(&commitment.payload) != canonical_payload.as_ref()
+		return canonical_header.hash() == *canonical_header_hash &&
+			Some(&commitment.payload) != canonical_payload.as_ref()
 	}
 	// if no header provided, the header proof is also not correct
 	false

From ba13cf23de8dff778cff59a6a46b6e23fdab2f44 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 15 Jan 2024 21:15:27 +0100
Subject: [PATCH 132/188] fix missed rename

---
 substrate/frame/beefy/src/lib.rs                | 2 +-
 substrate/frame/beefy/src/tests.rs              | 4 ++--
 substrate/primitives/consensus/beefy/src/lib.rs | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index e5b087a5b373..5ccc5eb43beb 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -218,7 +218,7 @@ pub mod pallet {
 			key_owner_proof.validator_count(),
 			T::MaxNominators::get(),
 		))]
-		pub fn report_equivocation(
+		pub fn report_vote_equivocation(
 			origin: OriginFor<T>,
 			equivocation_proof: Box<
 				VoteEquivocationProof<
diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index 5578f0fe4b89..481bbd87a6b9 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -717,7 +717,7 @@ fn report_vote_equivocation_has_valid_weight() {
 	// the weight depends on the size of the validator set,
 	// but there's a lower bound of 100 validators.
 	assert!((1..=100)
-		.map(|validators| <Test as Config>::WeightInfo::report_equivocation(validators, 1000))
+		.map(|validators| <Test as Config>::WeightInfo::report_vote_equivocation(validators, 1000))
 		.collect::<Vec<_>>()
 		.windows(2)
 		.all(|w| w[0] == w[1]));
@@ -725,7 +725,7 @@ fn report_vote_equivocation_has_valid_weight() {
 	// after 100 validators the weight should keep increasing
 	// with every extra validator.
 	assert!((100..=1000)
-		.map(|validators| <Test as Config>::WeightInfo::report_equivocation(validators, 1000))
+		.map(|validators| <Test as Config>::WeightInfo::report_vote_equivocation(validators, 1000))
 		.collect::<Vec<_>>()
 		.windows(2)
 		.all(|w| w[0].ref_time() < w[1].ref_time()));
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 1ba7638bc1d9..5ccbb89a69c6 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -401,7 +401,7 @@ where
 		let canonical_payload = canonical_mmr_root_digest.map(|mmr_root| {
 			Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode())
 		});
-		// Check header's hash and that `payload` of the `commitment` is different that the
+		// Check header's hash and that the `payload` of the `commitment` differs from the
 		// `canonical_payload`. Note that if the signatories signed a payload when there should be
 		// none (for instance for a block prior to BEEFY activation), then canonical_payload = None,
 		// and they will likewise be slashed.

From 70785985ec9f2ed2326a1bce98cdac81a2a695e8 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 15 Jan 2024 21:51:43 +0100
Subject: [PATCH 133/188] distinguish weights for
 report_{fork,vote}_equivocation

---
 substrate/frame/beefy/src/default_weights.rs | 33 +++++++++++++++++++-
 substrate/frame/beefy/src/lib.rs             | 18 ++++++-----
 2 files changed, 43 insertions(+), 8 deletions(-)

diff --git a/substrate/frame/beefy/src/default_weights.rs b/substrate/frame/beefy/src/default_weights.rs
index 8042f0c932eb..de63001aa3db 100644
--- a/substrate/frame/beefy/src/default_weights.rs
+++ b/substrate/frame/beefy/src/default_weights.rs
@@ -24,7 +24,7 @@ use frame_support::weights::{
 };
 
 impl crate::WeightInfo for () {
-	fn report_equivocation(validator_count: u32, max_nominators_per_validator: u32) -> Weight {
+	fn report_vote_equivocation(validator_count: u32, max_nominators_per_validator: u32) -> Weight {
 		// we take the validator set count from the membership proof to
 		// calculate the weight but we set a floor of 100 validators.
 		let validator_count = validator_count.max(100) as u64;
@@ -50,6 +50,37 @@ impl crate::WeightInfo for () {
 			.saturating_add(DbWeight::get().reads(2))
 	}
 
+	fn report_fork_equivocation(
+		validator_count: u32,
+		max_nominators_per_validator: u32,
+		key_owner_proofs_len: usize,
+	) -> Weight {
+		// we take the validator set count from the membership proof to
+		// calculate the weight but we set a floor of 100 validators.
+		let validator_count = validator_count.max(100) as u64;
+
+		// checking membership proof
+		Weight::from_parts(35u64 * WEIGHT_REF_TIME_PER_MICROS, 0)
+			.saturating_add(
+				Weight::from_parts(175u64 * WEIGHT_REF_TIME_PER_NANOS, 0)
+					.saturating_mul(validator_count)
+					.saturating_mul(key_owner_proofs_len as u64),
+			)
+			.saturating_add(DbWeight::get().reads(5))
+			// check equivocation proof
+			.saturating_add(Weight::from_parts(95u64 * WEIGHT_REF_TIME_PER_MICROS, 0))
+			// report offence
+			.saturating_add(Weight::from_parts(110u64 * WEIGHT_REF_TIME_PER_MICROS, 0))
+			.saturating_add(Weight::from_parts(
+				25u64 * WEIGHT_REF_TIME_PER_MICROS * max_nominators_per_validator as u64,
+				0,
+			))
+			.saturating_add(DbWeight::get().reads(14 + 3 * max_nominators_per_validator as u64))
+			.saturating_add(DbWeight::get().writes(10 + 3 * max_nominators_per_validator as u64))
+			// fetching set id -> session index mappings
+			.saturating_add(DbWeight::get().reads(2))
+	}
+
 	fn set_new_genesis() -> Weight {
 		DbWeight::get().writes(1)
 	}
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 5ccc5eb43beb..b83a8273080c 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -214,7 +214,7 @@ pub mod pallet {
 		/// against the extracted offender. If both are valid, the offence
 		/// will be reported.
 		#[pallet::call_index(0)]
-		#[pallet::weight(<T as pallet::Config>::WeightInfo::report_equivocation(
+		#[pallet::weight(<T as pallet::Config>::WeightInfo::report_vote_equivocation(
 			key_owner_proof.validator_count(),
 			T::MaxNominators::get(),
 		))]
@@ -252,7 +252,7 @@ pub mod pallet {
 		/// if the block author is defined it will be defined as the equivocation
 		/// reporter.
 		#[pallet::call_index(1)]
-		#[pallet::weight(<T as pallet::Config>::WeightInfo::report_equivocation(
+		#[pallet::weight(<T as pallet::Config>::WeightInfo::report_vote_equivocation(
 			key_owner_proof.validator_count(),
 			T::MaxNominators::get(),
 		))]
@@ -300,11 +300,11 @@ pub mod pallet {
 		/// invalid fork proof and validate the given key ownership proof
 		/// against the extracted offender. If both are valid, the offence
 		/// will be reported.
-		// TODO: fix key_owner_proofs[0].validator_count()
 		#[pallet::call_index(3)]
-		#[pallet::weight(<T as pallet::Config>::WeightInfo::report_equivocation(
+		#[pallet::weight(<T as pallet::Config>::WeightInfo::report_fork_equivocation(
 			key_owner_proofs[0].validator_count(),
 			T::MaxNominators::get(),
+			key_owner_proofs.len(),
 		))]
 		pub fn report_fork_equivocation(
 			origin: OriginFor<T>,
@@ -343,7 +343,7 @@ pub mod pallet {
 		/// reporter.
 		// TODO: fix key_owner_proofs[0].validator_count()
 		#[pallet::call_index(4)]
-		#[pallet::weight(<T as pallet::Config>::WeightInfo::report_equivocation(key_owner_proofs[0].validator_count(), T::MaxNominators::get(),))]
+		#[pallet::weight(<T as pallet::Config>::WeightInfo::report_fork_equivocation(key_owner_proofs[0].validator_count(), T::MaxNominators::get(), key_owner_proofs.len()))]
 		pub fn report_fork_equivocation_unsigned(
 			origin: OriginFor<T>,
 			equivocation_proof: Box<
@@ -584,7 +584,11 @@ impl<T: Config> IsMember<T::BeefyId> for Pallet<T> {
 }
 
 pub trait WeightInfo {
-	// TODO: distinguish for fork equivocation proofs
-	fn report_equivocation(validator_count: u32, max_nominators_per_validator: u32) -> Weight;
+	fn report_vote_equivocation(validator_count: u32, max_nominators_per_validator: u32) -> Weight;
+	fn report_fork_equivocation(
+		validator_count: u32,
+		max_nominators_per_validator: u32,
+		key_owner_proofs_len: usize,
+	) -> Weight;
 	fn set_new_genesis() -> Weight;
 }

From fa73cb5f0a4a5d69968a8562b38e5c3b4f80ab01 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 15 Jan 2024 22:07:41 +0100
Subject: [PATCH 134/188] clippy lints

---
 .../consensus/beefy/src/communication/gossip.rs       |  3 +--
 substrate/client/consensus/beefy/src/tests.rs         | 11 +++--------
 2 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/gossip.rs b/substrate/client/consensus/beefy/src/communication/gossip.rs
index be081767ad2a..5cfebd7fd2b5 100644
--- a/substrate/client/consensus/beefy/src/communication/gossip.rs
+++ b/substrate/client/consensus/beefy/src/communication/gossip.rs
@@ -516,7 +516,7 @@ pub(crate) mod tests {
 	use super::*;
 	use crate::{
 		keystore::BeefyKeystore,
-		tests::{create_beefy_keystore, create_fisherman, BeefyTestNet, TestApi},
+		tests::{create_fisherman, BeefyTestNet, TestApi},
 	};
 	use sc_network_test::{Block, TestNetFactory};
 	use sp_application_crypto::key_types::BEEFY as BEEFY_KEY_TYPE;
@@ -525,7 +525,6 @@ pub(crate) mod tests {
 		SignedCommitment, VoteMessage,
 	};
 	use sp_keystore::{testing::MemoryKeystore, Keystore};
-	use std::marker::PhantomData;
 
 	struct TestContext;
 	impl<B: sp_runtime::traits::Block> ValidatorContext<B> for TestContext {
diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index 0d7159022351..4ce3f4da8b97 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -29,7 +29,6 @@ use crate::{
 		},
 		request_response::{on_demand_justifications_protocol_config, BeefyJustifsRequestHandler},
 	},
-	error::Error,
 	gossip_protocol_name,
 	justification::*,
 	keystore::BeefyKeystore,
@@ -59,15 +58,15 @@ use sp_consensus_beefy::{
 	known_payloads,
 	mmr::{find_mmr_root_digest, MmrRootProvider},
 	BeefyApi, Commitment, ConsensusLog, ForkEquivocationProof, Keyring as BeefyKeyring,
-	MmrRootHash, OpaqueKeyOwnershipProof, Payload, PayloadProvider, SignedCommitment, ValidatorSet,
-	ValidatorSetId, VersionedFinalityProof, VoteEquivocationProof, VoteMessage, BEEFY_ENGINE_ID,
+	MmrRootHash, OpaqueKeyOwnershipProof, Payload, SignedCommitment, ValidatorSet, ValidatorSetId,
+	VersionedFinalityProof, VoteEquivocationProof, VoteMessage, BEEFY_ENGINE_ID,
 };
 use sp_core::H256;
 use sp_keystore::{testing::MemoryKeystore, Keystore, KeystorePtr};
 use sp_mmr_primitives::{AncestryProof, Error as MmrError, LeafIndex, MmrApi, NodeProof};
 use sp_runtime::{
 	codec::{Decode, Encode},
-	traits::{Block as BlockT, Header as HeaderT, NumberFor},
+	traits::{Header as HeaderT, NumberFor},
 	BuildStorage, DigestItem, EncodedJustification, Justifications, Storage,
 };
 use std::{marker::PhantomData, sync::Arc, task::Poll};
@@ -248,10 +247,6 @@ impl TestNetFactory for BeefyTestNet {
 	}
 }
 
-pub(crate) struct DummyFisherman<B> {
-	pub _phantom: PhantomData<B>,
-}
-
 #[derive(Clone)]
 pub(crate) struct TestApi {
 	pub beefy_genesis: u64,

From c01784c3e97415d880c58dc7736568beb617053a Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 16 Jan 2024 09:02:49 +0100
Subject: [PATCH 135/188] tighten equivocation reporting: future -> unfinalized
 blocks

while the runtime cannot distinguish between unfinalized and finalized
blocks, the client can, and we should take advantage of this.
---
 .../beefy/src/communication/fisherman.rs      | 29 +++++++++++++------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 3510f3f6b785..f11e34726d32 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -219,9 +219,10 @@ where
 		) {
 			return Ok(())
 		};
-		// if the vote is for a block number exceeding our best block number, there shouldn't even
-		// be a payload to sign yet, hence we assume it is an equivocation and report it
-		if number > self.backend.blockchain().info().best_number {
+		// if the vote is for a block number exceeding the finalized tip (from client's
+		// perspective), there shouldn't even be a payload to sign yet (from client's perspective),
+		// hence client assumes it is an equivocation and reports it
+		if number > self.backend.blockchain().info().finalized_number {
 			let proof = ForkEquivocationProof {
 				commitment: vote.commitment,
 				signatories: vec![(vote.id, vote.signature)],
@@ -268,14 +269,24 @@ where
 	) -> Result<(), Error> {
 		let SignedCommitment { commitment, signatures } = signed_commitment;
 		let number = commitment.block_number;
-		// if the vote is for a block number exceeding our best block number, there shouldn't even
-		// be a payload to sign yet, hence we assume it is an equivocation and report it
-		if number > self.backend.blockchain().info().best_number {
-			// if block number is in the future, we use the latest validator set
+		let best_number = self.backend.blockchain().info().best_number;
+		let finalized_number = self.backend.blockchain().info().finalized_number;
+		// if the signed commitment is to a block number exceeding the finalized tip (from client's
+		// perspective), there shouldn't even be a payload to sign yet (from client's perspective),
+		// hence client assumes it is an equivocation and reports it
+		if number > finalized_number {
+			// if block number exceeds our best block, we use the latest validator set available
 			// as the assumed signatories (note: this assumption is fragile and can possibly be
 			// improved upon)
-			let best_hash = self.backend.blockchain().info().best_hash;
-			let validator_set = self.active_validator_set_at(best_hash)?;
+			let block_hash = if number > best_number {
+				self.backend.blockchain().info().best_hash
+			} else {
+				self.backend
+					.blockchain()
+					.expect_block_hash_from_id(&BlockId::Number(number))
+					.map_err(|e| Error::Backend(e.to_string()))?
+			};
+			let validator_set = self.active_validator_set_at(block_hash)?;
 			let signatories: Vec<_> = validator_set
 				.validators()
 				.iter()

From dc8422336f060ffbdafb5963179c143022f5985e Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 16 Jan 2024 09:08:38 +0100
Subject: [PATCH 136/188] Revert "tighten equivocation reporting: future ->
 unfinalized blocks"

This reverts commit c01784c3e97415d880c58dc7736568beb617053a.
Actually, if the block is unfinalized but less or equal to the client's
best_height, the client should only report it if it equivocates against
the block in its history, which matches the prior logic of this code:

If the client on a fork, this will be a false positive, but also
irrelevant if the client's fork will never be finalized. If the client's
history will ultimately be finalized, then already reporting the
equivocation beforehand is useful since it avoids caching this
equivocation on the client until their fork becomes finalized (and
again, if their fork is not finalized, this will be irrelevant anyway).
---
 .../beefy/src/communication/fisherman.rs      | 29 ++++++-------------
 1 file changed, 9 insertions(+), 20 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index f11e34726d32..3510f3f6b785 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -219,10 +219,9 @@ where
 		) {
 			return Ok(())
 		};
-		// if the vote is for a block number exceeding the finalized tip (from client's
-		// perspective), there shouldn't even be a payload to sign yet (from client's perspective),
-		// hence client assumes it is an equivocation and reports it
-		if number > self.backend.blockchain().info().finalized_number {
+		// if the vote is for a block number exceeding our best block number, there shouldn't even
+		// be a payload to sign yet, hence we assume it is an equivocation and report it
+		if number > self.backend.blockchain().info().best_number {
 			let proof = ForkEquivocationProof {
 				commitment: vote.commitment,
 				signatories: vec![(vote.id, vote.signature)],
@@ -269,24 +268,14 @@ where
 	) -> Result<(), Error> {
 		let SignedCommitment { commitment, signatures } = signed_commitment;
 		let number = commitment.block_number;
-		let best_number = self.backend.blockchain().info().best_number;
-		let finalized_number = self.backend.blockchain().info().finalized_number;
-		// if the signed commitment is to a block number exceeding the finalized tip (from client's
-		// perspective), there shouldn't even be a payload to sign yet (from client's perspective),
-		// hence client assumes it is an equivocation and reports it
-		if number > finalized_number {
-			// if block number exceeds our best block, we use the latest validator set available
+		// if the vote is for a block number exceeding our best block number, there shouldn't even
+		// be a payload to sign yet, hence we assume it is an equivocation and report it
+		if number > self.backend.blockchain().info().best_number {
+			// if block number is in the future, we use the latest validator set
 			// as the assumed signatories (note: this assumption is fragile and can possibly be
 			// improved upon)
-			let block_hash = if number > best_number {
-				self.backend.blockchain().info().best_hash
-			} else {
-				self.backend
-					.blockchain()
-					.expect_block_hash_from_id(&BlockId::Number(number))
-					.map_err(|e| Error::Backend(e.to_string()))?
-			};
-			let validator_set = self.active_validator_set_at(block_hash)?;
+			let best_hash = self.backend.blockchain().info().best_hash;
+			let validator_set = self.active_validator_set_at(best_hash)?;
 			let signatories: Vec<_> = validator_set
 				.validators()
 				.iter()

From 816b726c5f593a9d111d74ab0dc4757d7bd63a7f Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 16 Jan 2024 14:17:49 +0100
Subject: [PATCH 137/188] use `expect_validator_set_nonblocking`

---
 .../beefy/src/communication/fisherman.rs      | 13 +++---
 substrate/client/consensus/beefy/src/lib.rs   | 43 +++++++++++++++++++
 2 files changed, 51 insertions(+), 5 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 3510f3f6b785..c3c494352075 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -18,6 +18,7 @@
 
 use crate::{
 	error::Error,
+	expect_validator_set_nonblocking,
 	justification::BeefyVersionedFinalityProof,
 	keystore::{BeefyKeystore, BeefySignatureHasher},
 	LOG_TARGET,
@@ -82,11 +83,13 @@ where
 		&self,
 		block_hash: <<B as Block>::Header as Header>::Hash,
 	) -> Result<ValidatorSet<AuthorityId>, Error> {
-		self.runtime
-			.runtime_api()
-			.validator_set(block_hash)
-			.map_err(Error::RuntimeApi)?
-			.ok_or_else(|| Error::Backend("could not get BEEFY validator set".into()))
+		let header = self
+			.backend
+			.blockchain()
+			.expect_header(block_hash)
+			.map_err(|e| Error::Backend(e.to_string()))?;
+		expect_validator_set_nonblocking(&*self.runtime, &*self.backend, &header)
+			.map_err(|e| Error::Backend(e.to_string()))
 	}
 
 	pub(crate) fn report_fork_equivocation(
diff --git a/substrate/client/consensus/beefy/src/lib.rs b/substrate/client/consensus/beefy/src/lib.rs
index 9c1d29f952c2..137d745480eb 100644
--- a/substrate/client/consensus/beefy/src/lib.rs
+++ b/substrate/client/consensus/beefy/src/lib.rs
@@ -634,3 +634,46 @@ where
 		}
 	}
 }
+
+/// Provides validator set active `at_header`. It tries to get it from state, otherwise falls
+/// back to walk up the chain looking the validator set enactment in header digests.
+///
+/// Note: unlike `expect_validator_set` this function is non-blocking, but will therefore error if
+/// a requested header is not available (yet).
+fn expect_validator_set_nonblocking<B, BE, R>(
+	runtime: &R,
+	backend: &BE,
+	at_header: &B::Header,
+) -> ClientResult<ValidatorSet<AuthorityId>>
+where
+	B: Block,
+	BE: Backend<B>,
+	R: ProvideRuntimeApi<B>,
+	R::Api: BeefyApi<B, AuthorityId, MmrRootHash>,
+{
+	let blockchain = backend.blockchain();
+
+	// Walk up the chain looking for the validator set active at 'at_header'. Process both state and
+	// header digests.
+	debug!(target: LOG_TARGET, "🥩 Trying to find validator set active at header: {:?}", at_header);
+	let mut header = at_header.clone();
+	loop {
+		debug!(target: LOG_TARGET, "🥩 Looking for auth set change at block number: {:?}", *header.number());
+		if let Ok(Some(active)) = runtime.runtime_api().validator_set(header.hash()) {
+			return Ok(active)
+		} else {
+			match worker::find_authorities_change::<B>(&header) {
+				Some(active) => return Ok(active),
+				// Move up the chain. Ultimately we'll get it from chain genesis state, or error out
+				// there.
+				None => match blockchain.header(*header.parent_hash())? {
+					Some(parent) => header = parent,
+					None => {
+						warn!(target: LOG_TARGET, "header {} not found", header.parent_hash());
+						return Err(ClientError::MissingHeader(header.parent_hash().to_string()))
+					},
+				},
+			}
+		}
+	}
+}

From d4902beb59618e79eab4e1b398c744d0bde430df Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 16 Jan 2024 14:08:52 +0100
Subject: [PATCH 138/188] Check equivocations on all votes & proofs for
 non-active rounds

If a vote or proof is for a past round, it might equivocate against
finalized state, so we should check for this.
If a vote or proof is for a future non-active round (from the client's
perspective), then it should not have been created yet (from the
client's perspective) and hence should also be reported for an
equivocation (in case the client was lagging on finality, this
equivocation report would be ineffectual)
---
 .../beefy/src/communication/gossip.rs         | 42 ++++++++++++++-----
 .../consensus/beefy/src/justification.rs      |  4 +-
 2 files changed, 33 insertions(+), 13 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/gossip.rs b/substrate/client/consensus/beefy/src/communication/gossip.rs
index 5cfebd7fd2b5..eccafa23df6d 100644
--- a/substrate/client/consensus/beefy/src/communication/gossip.rs
+++ b/substrate/client/consensus/beefy/src/communication/gossip.rs
@@ -292,17 +292,28 @@ where
 		let set_id = vote.commitment.validator_set_id;
 		self.known_peers.lock().note_vote_for(*sender, round);
 
-		// Verify general usefulness of the message.
-		// We are going to discard old votes right away (without verification).
+		// Verify general utility of the message.
+		// Have fisherman check the vote for equivocations against finalized
+		// state if the vote is rejected since it could either equivocate against finalized state or
+		// be for a future non-active round that should not be voted for yet (from the client's
+		// perspective). The check is best-effort and ignores errors such as state pruned. We do not
+		// check for equivocations on accepted votes here since this check is performed later in
+		// `sc_consensus_beefy::worker::BeefyWorker::handle_vote`, and in case the vote does not
+		// equivocate, we'd otherwise only produce a false-positive report here. We are going to
+		// discard old votes right away (without verification).
 		{
 			let filter = self.gossip_filter.read();
 
 			match filter.consider_vote(round, set_id) {
-				Consider::RejectFuture => return Action::Discard(cost::FUTURE_MESSAGE),
-				Consider::RejectOutOfScope => return Action::Discard(cost::OUT_OF_SCOPE_MESSAGE),
+				Consider::RejectFuture => {
+					let _ = self.fisherman.check_vote(vote);
+					return Action::Discard(cost::FUTURE_MESSAGE)
+				},
+				Consider::RejectOutOfScope => {
+					let _ = self.fisherman.check_vote(vote);
+					return Action::Discard(cost::OUT_OF_SCOPE_MESSAGE)
+				},
 				Consider::RejectPast => {
-					// We know `vote` is for some past (finalized) block. Have fisherman check
-					// for equivocations. Best-effort, ignore errors such as state pruned.
 					let _ = self.fisherman.check_vote(vote);
 					// TODO: maybe raise cost reputation when seeing votes that are intentional
 					// spam: votes that trigger fisherman reports, but don't go through either
@@ -346,11 +357,14 @@ where
 		let action = {
 			let guard = self.gossip_filter.read();
 
-			// Verify general usefulness of the justification.
+			// Verify general utility of the justification.
+			// The justification is only useful if it is for an active round: voters should
+			// broadcast finality proofs once they have seen sufficient affirming votes to build a
+			// valid one. If the proof is not for an active round, the fisherman will check the
+			// proof for equivocations against its state (and report if applicable). The check is
+			// best-effort and ignores errors such as state pruned.
 			match guard.consider_finality_proof(round, set_id) {
 				Consider::RejectPast => {
-					// We know `proof` is for some past (finalized) block. Have fisherman check
-					// for equivocations. Best-effort, ignore errors such as state pruned.
 					let _ = self.fisherman.check_proof(proof);
 					// TODO: maybe raise cost reputation when seeing votes that are intentional
 					// spam: votes that trigger fisherman reports, but don't go through either
@@ -358,8 +372,14 @@ where
 					// The idea is to more quickly disconnect neighbors which are attempting DoS.
 					return Action::Discard(cost::OUTDATED_MESSAGE)
 				},
-				Consider::RejectFuture => return Action::Discard(cost::FUTURE_MESSAGE),
-				Consider::RejectOutOfScope => return Action::Discard(cost::OUT_OF_SCOPE_MESSAGE),
+				Consider::RejectFuture => {
+					let _ = self.fisherman.check_proof(proof);
+					return Action::Discard(cost::FUTURE_MESSAGE)
+				},
+				Consider::RejectOutOfScope => {
+					let _ = self.fisherman.check_proof(proof);
+					return Action::Discard(cost::OUT_OF_SCOPE_MESSAGE)
+				},
 				Consider::Accept => {},
 			}
 
diff --git a/substrate/client/consensus/beefy/src/justification.rs b/substrate/client/consensus/beefy/src/justification.rs
index 483184e2374a..d3f08998d3af 100644
--- a/substrate/client/consensus/beefy/src/justification.rs
+++ b/substrate/client/consensus/beefy/src/justification.rs
@@ -37,7 +37,7 @@ pub(crate) fn proof_block_num_and_set_id<Block: BlockT>(
 	}
 }
 
-/// Decode and verify a Beefy FinalityProof.
+/// Decode and verify a BEEFY FinalityProof.
 pub(crate) fn decode_and_verify_finality_proof<Block: BlockT>(
 	encoded: &[u8],
 	target_number: NumberFor<Block>,
@@ -48,7 +48,7 @@ pub(crate) fn decode_and_verify_finality_proof<Block: BlockT>(
 	verify_with_validator_set::<Block>(target_number, validator_set, &proof).map(|_| proof)
 }
 
-/// Verify the Beefy finality proof against the validator set at the block it was generated.
+/// Verify the BEEFY finality proof against the validator set at the block it was generated.
 pub(crate) fn verify_with_validator_set<Block: BlockT>(
 	target_number: NumberFor<Block>,
 	validator_set: &ValidatorSet<AuthorityId>,

From 3e9a4a74e6abe533c1db21d0183b06339b919cb8 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 18 Jan 2024 10:51:45 +0100
Subject: [PATCH 139/188] Check equivocations on all votes & proofs

Have fisherman check the vote or proof for fork equivocations regardless of the action on it:
1. We check votes/proofs on past rounds *and* active rounds since they might equivocate against grandpa-finalized state.
2. We check votes/proofs on future non-active rounds since these should not have been voted on yet (from the client's perspective). In case the block is not even in the client's history, but is in fact already finalized, the resulting equivocation report will be ineffectual.
---
 .../beefy/src/communication/gossip.rs         | 68 +++++++++----------
 1 file changed, 32 insertions(+), 36 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/gossip.rs b/substrate/client/consensus/beefy/src/communication/gossip.rs
index eccafa23df6d..ba9c75218f28 100644
--- a/substrate/client/consensus/beefy/src/communication/gossip.rs
+++ b/substrate/client/consensus/beefy/src/communication/gossip.rs
@@ -285,36 +285,22 @@ where
 
 	fn validate_vote(
 		&self,
-		vote: VoteMessage<NumberFor<B>, AuthorityId, Signature>,
+		vote: &VoteMessage<NumberFor<B>, AuthorityId, Signature>,
 		sender: &PeerId,
 	) -> Action<B::Hash> {
 		let round = vote.commitment.block_number;
 		let set_id = vote.commitment.validator_set_id;
 		self.known_peers.lock().note_vote_for(*sender, round);
 
-		// Verify general utility of the message.
-		// Have fisherman check the vote for equivocations against finalized
-		// state if the vote is rejected since it could either equivocate against finalized state or
-		// be for a future non-active round that should not be voted for yet (from the client's
-		// perspective). The check is best-effort and ignores errors such as state pruned. We do not
-		// check for equivocations on accepted votes here since this check is performed later in
-		// `sc_consensus_beefy::worker::BeefyWorker::handle_vote`, and in case the vote does not
-		// equivocate, we'd otherwise only produce a false-positive report here. We are going to
-		// discard old votes right away (without verification).
+		// Verify general utility of the message. A vote is only useful if for an active round.
+		// We are going to discard old votes right away (without verification).
 		{
 			let filter = self.gossip_filter.read();
 
 			match filter.consider_vote(round, set_id) {
-				Consider::RejectFuture => {
-					let _ = self.fisherman.check_vote(vote);
-					return Action::Discard(cost::FUTURE_MESSAGE)
-				},
-				Consider::RejectOutOfScope => {
-					let _ = self.fisherman.check_vote(vote);
-					return Action::Discard(cost::OUT_OF_SCOPE_MESSAGE)
-				},
+				Consider::RejectFuture => return Action::Discard(cost::FUTURE_MESSAGE),
+				Consider::RejectOutOfScope => return Action::Discard(cost::OUT_OF_SCOPE_MESSAGE),
 				Consider::RejectPast => {
-					let _ = self.fisherman.check_vote(vote);
 					// TODO: maybe raise cost reputation when seeing votes that are intentional
 					// spam: votes that trigger fisherman reports, but don't go through either
 					// because signer is/was not authority or similar reasons.
@@ -348,10 +334,10 @@ where
 
 	fn validate_finality_proof(
 		&self,
-		proof: BeefyVersionedFinalityProof<B>,
+		proof: &BeefyVersionedFinalityProof<B>,
 		sender: &PeerId,
 	) -> Action<B::Hash> {
-		let (round, set_id) = proof_block_num_and_set_id::<B>(&proof);
+		let (round, set_id) = proof_block_num_and_set_id::<B>(proof);
 		self.known_peers.lock().note_vote_for(*sender, round);
 
 		let action = {
@@ -360,26 +346,17 @@ where
 			// Verify general utility of the justification.
 			// The justification is only useful if it is for an active round: voters should
 			// broadcast finality proofs once they have seen sufficient affirming votes to build a
-			// valid one. If the proof is not for an active round, the fisherman will check the
-			// proof for equivocations against its state (and report if applicable). The check is
-			// best-effort and ignores errors such as state pruned.
+			// valid one.
 			match guard.consider_finality_proof(round, set_id) {
 				Consider::RejectPast => {
-					let _ = self.fisherman.check_proof(proof);
 					// TODO: maybe raise cost reputation when seeing votes that are intentional
 					// spam: votes that trigger fisherman reports, but don't go through either
 					// because signer is/was not authority or similar reasons.
 					// The idea is to more quickly disconnect neighbors which are attempting DoS.
 					return Action::Discard(cost::OUTDATED_MESSAGE)
 				},
-				Consider::RejectFuture => {
-					let _ = self.fisherman.check_proof(proof);
-					return Action::Discard(cost::FUTURE_MESSAGE)
-				},
-				Consider::RejectOutOfScope => {
-					let _ = self.fisherman.check_proof(proof);
-					return Action::Discard(cost::OUT_OF_SCOPE_MESSAGE)
-				},
+				Consider::RejectFuture => return Action::Discard(cost::FUTURE_MESSAGE),
+				Consider::RejectOutOfScope => return Action::Discard(cost::OUT_OF_SCOPE_MESSAGE),
 				Consider::Accept => {},
 			}
 
@@ -392,7 +369,7 @@ where
 				.validator_set()
 				.map(|validator_set| {
 					if let Err((_, signatures_checked)) =
-						verify_with_validator_set::<B>(round, validator_set, &proof)
+						verify_with_validator_set::<B>(round, validator_set, proof)
 					{
 						debug!(
 							target: LOG_TARGET,
@@ -434,9 +411,28 @@ where
 		mut data: &[u8],
 	) -> ValidationResult<B::Hash> {
 		let raw = data;
+		// Have fisherman check the vote or proof for fork equivocations regardless of
+		// the action on it:
+		// 1. We check votes/proofs on past rounds and active rounds since they might equivocate
+		//    against grandpa-finalized state.
+		// 2. We check votes/proofs on future non-active rounds since these should not have been
+		//    voted on yet (from the client's perspective). In case the block is not even in the
+		//    client's history, but is in fact already finalized, the resulting equivocation report
+		//    will be ineffectual.
+		// The check is best-effort and ignores errors such as state pruned. Accepted votes are also
+		// checked against vote equivocations in
+		// `sc_consensus_beefy::worker::BeefyWorker::handle_vote`
 		let action = match GossipMessage::<B>::decode_all(&mut data) {
-			Ok(GossipMessage::Vote(msg)) => self.validate_vote(msg, sender),
-			Ok(GossipMessage::FinalityProof(proof)) => self.validate_finality_proof(proof, sender),
+			Ok(GossipMessage::Vote(msg)) => {
+				let action = self.validate_vote(&msg, sender);
+				let _ = self.fisherman.check_vote(msg);
+				action
+			},
+			Ok(GossipMessage::FinalityProof(proof)) => {
+				let action = self.validate_finality_proof(&proof, sender);
+				let _ = self.fisherman.check_proof(proof);
+				action
+			},
 			Err(e) => {
 				debug!(target: LOG_TARGET, "Error decoding message: {}", e);
 				let bytes = raw.len().min(i32::MAX as usize) as i32;

From 6be328f021305dbd0b218e6cf916e6c946408a28 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 25 Jan 2024 08:41:36 +0100
Subject: [PATCH 140/188] add CanonicalHashHeaderPayload struct

---
 .../beefy/src/communication/fisherman.rs      | 32 +++++++++++--------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index c3c494352075..41b326bcaefc 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -50,6 +50,12 @@ pub(crate) struct Fisherman<B: Block, BE, P, R> {
 	pub _phantom: PhantomData<B>,
 }
 
+struct CanonicalHashHeaderPayload<B: Block> {
+	hash: B::Hash,
+	header: B::Header,
+	payload: Payload,
+}
+
 impl<B, BE, R, P> Fisherman<B, BE, P, R>
 where
 	B: Block,
@@ -58,10 +64,10 @@ where
 	R: ProvideRuntimeApi<B> + Send + Sync,
 	R::Api: BeefyApi<B, AuthorityId, MmrRootHash> + MmrApi<B, MmrRootHash, NumberFor<B>>,
 {
-	fn canonical_hash_header_payload_tuple(
+	fn canonical_hash_header_payload(
 		&self,
 		number: NumberFor<B>,
-	) -> Result<(B::Hash, B::Header, Payload), Error> {
+	) -> Result<CanonicalHashHeaderPayload<B>, Error> {
 		// This should be un-ambiguous since `number` is finalized.
 		let hash = self
 			.backend
@@ -75,7 +81,7 @@ where
 			.map_err(|e| Error::Backend(e.to_string()))?;
 		self.payload_provider
 			.payload(&header)
-			.map(|payload| (hash, header, payload))
+			.map(|payload| CanonicalHashHeaderPayload { hash, header, payload })
 			.ok_or_else(|| Error::Backend("BEEFY Payload not found".into()))
 	}
 
@@ -233,13 +239,12 @@ where
 			};
 			self.report_fork_equivocation(proof)?;
 		} else {
-			let (correct_hash, canonical_header, canonical_payload) =
-				self.canonical_hash_header_payload_tuple(number)?;
-			if vote.commitment.payload != canonical_payload {
+			let canonical_hhp = self.canonical_hash_header_payload(number)?;
+			if vote.commitment.payload != canonical_hhp.payload {
 				let ancestry_proof: Option<_> = match self
 					.runtime
 					.runtime_api()
-					.generate_ancestry_proof(correct_hash, number, None)
+					.generate_ancestry_proof(canonical_hhp.hash, number, None)
 				{
 					Ok(Ok(ancestry_proof)) => Some(ancestry_proof),
 					Ok(Err(e)) => {
@@ -254,7 +259,7 @@ where
 				let proof = ForkEquivocationProof {
 					commitment: vote.commitment,
 					signatories: vec![(vote.id, vote.signature)],
-					canonical_header: Some(canonical_header),
+					canonical_header: Some(canonical_hhp.header),
 					ancestry_proof,
 				};
 				self.report_fork_equivocation(proof)?;
@@ -309,11 +314,10 @@ where
 				self.report_fork_equivocation(proof)?;
 			}
 		} else {
-			let (correct_hash, canonical_header, canonical_payload) =
-				self.canonical_hash_header_payload_tuple(number)?;
-			if commitment.payload != canonical_payload {
+			let canonical_hhp = self.canonical_hash_header_payload(number)?;
+			if commitment.payload != canonical_hhp.payload {
 				let ancestry_proof = match self.runtime.runtime_api().generate_ancestry_proof(
-					correct_hash,
+					canonical_hhp.hash,
 					number,
 					None,
 				) {
@@ -327,7 +331,7 @@ where
 						None
 					},
 				};
-				let validator_set = self.active_validator_set_at(correct_hash)?;
+				let validator_set = self.active_validator_set_at(canonical_hhp.hash)?;
 				if signatures.len() != validator_set.validators().len() {
 					// invalid proof
 					return Ok(())
@@ -345,7 +349,7 @@ where
 					let proof = ForkEquivocationProof {
 						commitment,
 						signatories,
-						canonical_header: Some(canonical_header),
+						canonical_header: Some(canonical_hhp.header),
 						ancestry_proof,
 					};
 					self.report_fork_equivocation(proof)?;

From e401534071242a65e7c069488955e0cb98ccf515 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 25 Jan 2024 08:45:23 +0100
Subject: [PATCH 141/188] update TODO

---
 substrate/client/consensus/beefy/src/worker.rs | 4 ++--
 substrate/frame/beefy/src/default_weights.rs   | 1 +
 substrate/frame/beefy/src/lib.rs               | 1 -
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index 46a7f64a9a87..f5a9c8b181a2 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -77,7 +77,7 @@ pub(crate) enum RoundAction {
 pub(crate) struct VoterOracle<B: Block> {
 	/// Queue of known sessions. Keeps track of voting rounds (block numbers) within each session.
 	///
-	/// There are three voter states coresponding to three queue states:
+	/// There are three voter states corresponding to three queue states:
 	/// 1. voter uninitialized: queue empty,
 	/// 2. up-to-date - all mandatory blocks leading up to current GRANDPA finalized: queue has ONE
 	///    element, the 'current session' where `mandatory_done == true`,
@@ -575,7 +575,7 @@ where
 					target: LOG_TARGET,
 					"🥩 Round #{} concluded, finality_proof: {:?}.", block_number, finality_proof
 				);
-				// We created the `finality_proof` and know to be valid.
+				// We created the `finality_proof` and know it to be valid.
 				// New state is persisted after finalization.
 				self.finalize(finality_proof.clone())?;
 				metric_inc!(self, beefy_good_votes_processed);
diff --git a/substrate/frame/beefy/src/default_weights.rs b/substrate/frame/beefy/src/default_weights.rs
index de63001aa3db..ad61c3ade5c8 100644
--- a/substrate/frame/beefy/src/default_weights.rs
+++ b/substrate/frame/beefy/src/default_weights.rs
@@ -50,6 +50,7 @@ impl crate::WeightInfo for () {
 			.saturating_add(DbWeight::get().reads(2))
 	}
 
+	// TODO: update weight calculation
 	fn report_fork_equivocation(
 		validator_count: u32,
 		max_nominators_per_validator: u32,
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index b83a8273080c..12e985fb69f3 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -341,7 +341,6 @@ pub mod pallet {
 		/// block authors will call it (validated in `ValidateUnsigned`), as such
 		/// if the block author is defined it will be defined as the equivocation
 		/// reporter.
-		// TODO: fix key_owner_proofs[0].validator_count()
 		#[pallet::call_index(4)]
 		#[pallet::weight(<T as pallet::Config>::WeightInfo::report_fork_equivocation(key_owner_proofs[0].validator_count(), T::MaxNominators::get(), key_owner_proofs.len()))]
 		pub fn report_fork_equivocation_unsigned(

From 4dbfcf3c9d9b3a4a823038c7229a9472461c6979 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 25 Jan 2024 09:24:44 +0100
Subject: [PATCH 142/188] filter signatories without key ownership proofs

if we couldn't generate a key ownership proof for a signatory, it is
either not part of the authority set for that session, or the generation failed for
another reason. In either case, the runtime would reject the proof for
the signatory, so we filter it.

Suggested-by: Serban Iorga <serban300@gmail.com>
---
 .../beefy/src/communication/fisherman.rs         | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 41b326bcaefc..0b52a2e70fc7 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -171,6 +171,7 @@ where
 
 		let runtime_api = self.runtime.runtime_api();
 
+		let mut filtered_signatories = Vec::new();
 		// generate key ownership proof at that block
 		let key_owner_proofs: Vec<_> = offender_ids
 			.iter()
@@ -187,11 +188,16 @@ where
 							target: LOG_TARGET,
 							"🥩 Invalid fork vote offender not part of the authority set."
 						);
+						// if signatory is not part of the authority set, we ignore the signatory
+						filtered_signatories.push(id);
 						None
 					},
 					Err(e) => {
 						debug!(target: LOG_TARGET,
 							   "🥩 Failed to generate key ownership proof for {:?}: {:?}", id, e);
+						// if a key ownership proof couldn't be generated for signatory, we ignore
+						// the signatory
+						filtered_signatories.push(id);
 						None
 					},
 				}
@@ -199,6 +205,16 @@ where
 			.collect::<Result<_, _>>()?;
 
 		if key_owner_proofs.len() > 0 {
+			// filter out the signatories that a key ownership proof could not be generated for
+			let proof = ForkEquivocationProof {
+				signatories: proof
+					.signatories
+					.clone()
+					.into_iter()
+					.filter(|(id, _)| !filtered_signatories.contains(&id))
+					.collect(),
+				..proof
+			};
 			// submit invalid fork vote report at **best** block
 			runtime_api
 				.submit_report_fork_equivocation_unsigned_extrinsic(

From e7718fc8e75ed7133b3c06061374f32beefa7b8f Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 25 Jan 2024 10:06:14 +0100
Subject: [PATCH 143/188] debug->warn

Co-authored-by: Adrian Catangiu <adrian@parity.io>
---
 .../client/consensus/beefy/src/communication/fisherman.rs    | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 0b52a2e70fc7..063094e5f1a2 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -23,7 +23,7 @@ use crate::{
 	keystore::{BeefyKeystore, BeefySignatureHasher},
 	LOG_TARGET,
 };
-use log::debug;
+use log::{debug, warn};
 use sc_client_api::Backend;
 use sp_api::ProvideRuntimeApi;
 use sp_blockchain::HeaderBackend;
@@ -163,8 +163,7 @@ where
 		let offender_ids = proof.offender_ids();
 		if let Some(local_id) = self.key_store.authority_id(validator_set.validators()) {
 			if offender_ids.contains(&&local_id) {
-				debug!(target: LOG_TARGET, "🥩 Skip equivocation report for own equivocation");
-				// TODO: maybe error here instead?
+				warn!(target: LOG_TARGET, "🥩 Skip equivocation report for own equivocation");
 				return Ok(false)
 			}
 		}

From fb183c9ef8af1422ba60741aaeb718d105b36d1a Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 13 Feb 2024 00:25:22 +0100
Subject: [PATCH 144/188] check session index first as cheap failfast

Suggested-by: Serban Iorga <serban300@gmail.com>
---
 substrate/frame/beefy/src/equivocation.rs | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 210eab1fed94..79a8a44c0649 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -269,6 +269,14 @@ where
 		let session_index = key_owner_proofs[0].session();
 		let validator_set_count = key_owner_proofs[0].validator_count();
 
+		// Check that the session id for the membership proof is within the
+		// bounds of the set id reported in the equivocation.
+		let set_id_session_index = crate::SetIdSession::<T>::get(set_id)
+			.ok_or(Error::<T>::InvalidEquivocationProofSession)?;
+		if session_index != set_id_session_index {
+			return Err(Error::<T>::InvalidEquivocationProofSession.into())
+		}
+
 		// Validate the key ownership proof extracting the ids of the offenders.
 		let offenders = offenders
 			.into_iter()
@@ -332,14 +340,6 @@ where
 			},
 		}
 
-		// Check that the session id for the membership proof is within the
-		// bounds of the set id reported in the equivocation.
-		let set_id_session_index = crate::SetIdSession::<T>::get(set_id)
-			.ok_or(Error::<T>::InvalidEquivocationProofSession)?;
-		if session_index != set_id_session_index {
-			return Err(Error::<T>::InvalidEquivocationProofSession.into())
-		}
-
 		let offence = EquivocationOffence {
 			time_slot: TimeSlot { set_id, round },
 			session_index,

From 20b828b69efb22cd1bf471c2bb8179a81ef445e3 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 13 Feb 2024 01:08:04 +0100
Subject: [PATCH 145/188] deduplicate ancestry proof generation

Suggested-by: Serban Iorga <serban300@gmail.com>
---
 .../beefy/src/communication/fisherman.rs      | 57 +++++++++----------
 1 file changed, 26 insertions(+), 31 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 063094e5f1a2..2583956bcb69 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -33,7 +33,7 @@ use sp_consensus_beefy::{
 	BeefyApi, ForkEquivocationProof, MmrHashing, MmrRootHash, Payload, PayloadProvider,
 	SignedCommitment, ValidatorSet, VoteMessage,
 };
-use sp_mmr_primitives::MmrApi;
+use sp_mmr_primitives::{AncestryProof, MmrApi};
 use sp_runtime::{
 	generic::BlockId,
 	traits::{Block, Header, NumberFor},
@@ -228,6 +228,29 @@ where
 		}
 	}
 
+	/// Generates an ancestry proof for the given ancestoring block's mmr root.
+	fn generate_ancestry_proof_opt(
+		&self,
+		best_block_hash: B::Hash,
+		prev_block_num: NumberFor<B>,
+	) -> Option<AncestryProof<sp_consensus_beefy::MmrRootHash>> {
+		match self.runtime.runtime_api().generate_ancestry_proof(
+			best_block_hash,
+			prev_block_num,
+			None,
+		) {
+			Ok(Ok(ancestry_proof)) => Some(ancestry_proof),
+			Ok(Err(e)) => {
+				debug!(target: LOG_TARGET, "🥩 Failed to generate ancestry proof: {:?}", e);
+				None
+			},
+			Err(e) => {
+				debug!(target: LOG_TARGET, "🥩 Failed to generate ancestry proof: {:?}", e);
+				None
+			},
+		}
+	}
+
 	/// Check `vote` for contained block against canonical payload. If an equivocation is detected,
 	/// this also reports it.
 	pub(crate) fn check_vote(
@@ -256,21 +279,7 @@ where
 		} else {
 			let canonical_hhp = self.canonical_hash_header_payload(number)?;
 			if vote.commitment.payload != canonical_hhp.payload {
-				let ancestry_proof: Option<_> = match self
-					.runtime
-					.runtime_api()
-					.generate_ancestry_proof(canonical_hhp.hash, number, None)
-				{
-					Ok(Ok(ancestry_proof)) => Some(ancestry_proof),
-					Ok(Err(e)) => {
-						debug!(target: LOG_TARGET, "🥩 Failed to generate ancestry proof: {:?}", e);
-						None
-					},
-					Err(e) => {
-						debug!(target: LOG_TARGET, "🥩 Failed to generate ancestry proof: {:?}", e);
-						None
-					},
-				};
+				let ancestry_proof = self.generate_ancestry_proof_opt(canonical_hhp.hash, number);
 				let proof = ForkEquivocationProof {
 					commitment: vote.commitment,
 					signatories: vec![(vote.id, vote.signature)],
@@ -331,21 +340,7 @@ where
 		} else {
 			let canonical_hhp = self.canonical_hash_header_payload(number)?;
 			if commitment.payload != canonical_hhp.payload {
-				let ancestry_proof = match self.runtime.runtime_api().generate_ancestry_proof(
-					canonical_hhp.hash,
-					number,
-					None,
-				) {
-					Ok(Ok(ancestry_proof)) => Some(ancestry_proof),
-					Ok(Err(e)) => {
-						debug!(target: LOG_TARGET, "🥩 Failed to generate ancestry proof: {:?}", e);
-						None
-					},
-					Err(e) => {
-						debug!(target: LOG_TARGET, "🥩 Failed to generate ancestry proof: {:?}", e);
-						None
-					},
-				};
+				let ancestry_proof = self.generate_ancestry_proof_opt(canonical_hhp.hash, number);
 				let validator_set = self.active_validator_set_at(canonical_hhp.hash)?;
 				if signatures.len() != validator_set.validators().len() {
 					// invalid proof

From 09f95563fabe664a2fd2e474c951cff9972c8b82 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 13 Feb 2024 01:14:04 +0100
Subject: [PATCH 146/188] {canonical->finalized} hash for anc. proof gen

---
 .../consensus/beefy/src/communication/fisherman.rs     | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 2583956bcb69..cb9e203cc12b 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -279,7 +279,10 @@ where
 		} else {
 			let canonical_hhp = self.canonical_hash_header_payload(number)?;
 			if vote.commitment.payload != canonical_hhp.payload {
-				let ancestry_proof = self.generate_ancestry_proof_opt(canonical_hhp.hash, number);
+				let ancestry_proof = self.generate_ancestry_proof_opt(
+					self.backend.blockchain().info().finalized_hash,
+					number,
+				);
 				let proof = ForkEquivocationProof {
 					commitment: vote.commitment,
 					signatories: vec![(vote.id, vote.signature)],
@@ -340,7 +343,10 @@ where
 		} else {
 			let canonical_hhp = self.canonical_hash_header_payload(number)?;
 			if commitment.payload != canonical_hhp.payload {
-				let ancestry_proof = self.generate_ancestry_proof_opt(canonical_hhp.hash, number);
+				let ancestry_proof = self.generate_ancestry_proof_opt(
+					self.backend.blockchain().info().finalized_hash,
+					number,
+				);
 				let validator_set = self.active_validator_set_at(canonical_hhp.hash)?;
 				if signatures.len() != validator_set.validators().len() {
 					// invalid proof

From 7c529a67db245ae50cb869ad85b28901181541e8 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 22 Feb 2024 11:26:24 +0100
Subject: [PATCH 147/188] fixup! filter invalidly signed commitments at call
 site

---
 .../beefy/src/communication/fisherman.rs         | 16 +++++++++++++++-
 substrate/client/consensus/beefy/src/tests.rs    |  4 ++--
 substrate/frame/beefy-mmr/src/lib.rs             |  2 +-
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index cb9e203cc12b..f26f915c03a5 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -358,7 +358,21 @@ where
 					.iter()
 					.cloned()
 					.zip(signatures.into_iter())
-					.filter_map(|(id, signature)| signature.map(|sig| (id, sig)))
+					.filter_map(|(id, signature)| match signature {
+						Some(sig) => {
+							if sp_consensus_beefy::check_commitment_signature::<
+								_,
+								_,
+								BeefySignatureHasher,
+							>(&commitment, &id, &sig)
+							{
+								Some((id, sig))
+							} else {
+								None
+							}
+						},
+						None => None,
+					})
 					.collect();
 
 				if signatories.len() > 0 {
diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index 4ce3f4da8b97..62711f0c2f70 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -166,7 +166,7 @@ impl BeefyTestNet {
 		// of hashes, otherwise indexing would be broken
 		assert!(self.peer(0).client().as_backend().blockchain().hash(1).unwrap().is_none());
 
-		// push genesis to make indexing human readable (index equals to block number)
+		// push genesis to make indexing human-readable (index equals to block number)
 		all_hashes.push(self.peer(0).client().info().genesis_hash);
 
 		let mut block_num: NumberFor<Block> = self.peer(0).client().info().best_number;
@@ -1451,7 +1451,7 @@ async fn beefy_reports_vote_equivocations() {
 	assert_eq!(equivocation_proof.first.id, BeefyKeyring::Bob.public());
 	assert_eq!(equivocation_proof.first.commitment.block_number, 1);
 
-	// Verify neither Bob or Bob_Prime report themselves as equivocating.
+	// Verify neither Bob nor Bob_Prime report themselves as equivocating.
 	assert!(api_bob.reported_vote_equivocations.as_ref().unwrap().lock().is_empty());
 	assert!(api_bob_prime.reported_vote_equivocations.as_ref().unwrap().lock().is_empty());
 
diff --git a/substrate/frame/beefy-mmr/src/lib.rs b/substrate/frame/beefy-mmr/src/lib.rs
index fa3caba7977d..7af4a7a14706 100644
--- a/substrate/frame/beefy-mmr/src/lib.rs
+++ b/substrate/frame/beefy-mmr/src/lib.rs
@@ -105,7 +105,7 @@ pub mod pallet {
 	pub trait Config: pallet_mmr::Config + pallet_beefy::Config {
 		/// Current leaf version.
 		///
-		/// Specifies the version number added to every leaf that get's appended to the MMR.
+		/// Specifies the version number added to every leaf that gets appended to the MMR.
 		/// Read more in [`MmrLeafVersion`] docs about versioning leaves.
 		type LeafVersion: Get<MmrLeafVersion>;
 

From b21c6e1958f6a87c579bf07a24177f222d88c24c Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 27 Feb 2024 09:48:45 +0100
Subject: [PATCH 148/188] merge fixes

---
 .../beefy/src/communication/fisherman.rs      | 13 +++-----
 .../beefy/src/communication/gossip.rs         |  8 ++---
 substrate/client/consensus/beefy/src/lib.rs   |  2 +-
 substrate/client/consensus/beefy/src/tests.rs | 33 ++++++++++---------
 .../client/consensus/beefy/src/worker.rs      | 22 +++++++------
 substrate/frame/beefy/src/tests.rs            | 22 +++++++------
 .../primitives/consensus/beefy/src/lib.rs     |  1 -
 .../consensus/beefy/src/test_utils.rs         | 12 ++++---
 8 files changed, 59 insertions(+), 54 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index f26f915c03a5..56121f5a634a 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -17,11 +17,8 @@
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 use crate::{
-	error::Error,
-	expect_validator_set_nonblocking,
-	justification::BeefyVersionedFinalityProof,
-	keystore::{BeefyKeystore, BeefySignatureHasher},
-	LOG_TARGET,
+	error::Error, expect_validator_set_nonblocking, justification::BeefyVersionedFinalityProof,
+	keystore::BeefyKeystore, LOG_TARGET,
 };
 use log::{debug, warn};
 use sc_client_api::Backend;
@@ -30,8 +27,8 @@ use sp_blockchain::HeaderBackend;
 use sp_consensus_beefy::{
 	check_fork_equivocation_proof,
 	ecdsa_crypto::{AuthorityId, Signature},
-	BeefyApi, ForkEquivocationProof, MmrHashing, MmrRootHash, Payload, PayloadProvider,
-	SignedCommitment, ValidatorSet, VoteMessage,
+	BeefyApi, BeefySignatureHasher, ForkEquivocationProof, MmrHashing, MmrRootHash, Payload,
+	PayloadProvider, SignedCommitment, ValidatorSet, VoteMessage,
 };
 use sp_mmr_primitives::{AncestryProof, MmrApi};
 use sp_runtime::{
@@ -45,7 +42,7 @@ use std::{marker::PhantomData, sync::Arc};
 pub(crate) struct Fisherman<B: Block, BE, P, R> {
 	pub backend: Arc<BE>,
 	pub runtime: Arc<R>,
-	pub key_store: Arc<BeefyKeystore>,
+	pub key_store: Arc<BeefyKeystore<AuthorityId>>,
 	pub payload_provider: P,
 	pub _phantom: PhantomData<B>,
 }
diff --git a/substrate/client/consensus/beefy/src/communication/gossip.rs b/substrate/client/consensus/beefy/src/communication/gossip.rs
index ce0604daf87e..934e20e96da4 100644
--- a/substrate/client/consensus/beefy/src/communication/gossip.rs
+++ b/substrate/client/consensus/beefy/src/communication/gossip.rs
@@ -247,7 +247,7 @@ where
 	B: Block,
 	BE: Backend<B> + Send + Sync,
 	P: PayloadProvider<B> + Send + Sync,
-	R: ProvideRuntimeApi<B> + Send + Sync,
+	R: ProvideRuntimeApi<B>,
 	R::Api: BeefyApi<B, AuthorityId, MmrRootHash> + MmrApi<B, MmrRootHash, NumberFor<B>>,
 {
 	pub(crate) fn new(
@@ -612,7 +612,7 @@ pub(crate) mod tests {
 		let api = TestApi::new(0, &validator_set, MmrRootHash::repeat_byte(0xbf));
 		let mut net = BeefyTestNet::new(1);
 		let backend = net.peer(0).client().as_backend();
-		let fisherman = create_fisherman(keyring, Arc::new(api.clone()), backend.clone());
+		let fisherman = create_fisherman(&keyring, Arc::new(api.clone()), backend.clone());
 
 		let (gv, mut report_stream) =
 			GossipValidator::new(Arc::new(Mutex::new(KnownPeers::new())), fisherman);
@@ -733,7 +733,7 @@ pub(crate) mod tests {
 		let api = TestApi::new(0, &validator_set, MmrRootHash::repeat_byte(0xbf));
 		let mut net = BeefyTestNet::new(1);
 		let backend = net.peer(0).client().as_backend();
-		let fisherman = create_fisherman(keyring, Arc::new(api.clone()), backend.clone());
+		let fisherman = create_fisherman(&keyring, Arc::new(api.clone()), backend.clone());
 
 		let (gv, _) = GossipValidator::new(Arc::new(Mutex::new(KnownPeers::new())), fisherman);
 		gv.update_filter(GossipFilterCfg { start: 0, end: 10, validator_set: &validator_set });
@@ -817,7 +817,7 @@ pub(crate) mod tests {
 		let api = TestApi::new(0, &validator_set, MmrRootHash::repeat_byte(0xbf));
 		let mut net = BeefyTestNet::new(1);
 		let backend = net.peer(0).client().as_backend();
-		let fisherman = create_fisherman(keyring, Arc::new(api.clone()), backend.clone());
+		let fisherman = create_fisherman(&keyring, Arc::new(api.clone()), backend.clone());
 
 		let (gv, _) = GossipValidator::new(Arc::new(Mutex::new(KnownPeers::new())), fisherman);
 		gv.update_filter(GossipFilterCfg { start: 0, end: 10, validator_set: &validator_set });
diff --git a/substrate/client/consensus/beefy/src/lib.rs b/substrate/client/consensus/beefy/src/lib.rs
index cfefd5a7ff75..679b3e340797 100644
--- a/substrate/client/consensus/beefy/src/lib.rs
+++ b/substrate/client/consensus/beefy/src/lib.rs
@@ -240,7 +240,7 @@ pub async fn start_beefy_gadget<B, BE, C, N, P, R, S>(
 		mut on_demand_justifications_handler,
 	} = beefy_params;
 
-	let key_store: Arc<BeefyKeystore> = Arc::new(key_store.into());
+	let key_store: Arc<BeefyKeystore<AuthorityId>> = Arc::new(key_store.into());
 
 	let BeefyNetworkParams {
 		network,
diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index 6f5f398a04da..282b74235abe 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -410,11 +410,13 @@ pub(crate) fn create_beefy_keystore(authority: &BeefyKeyring<AuthorityId>) -> Ke
 }
 
 pub(crate) fn create_fisherman<BE>(
-	beefy_keyring: BeefyKeyring, // Assuming BeefyKeyring contains necessary keys for the Fisherman
+	beefy_keyring: &BeefyKeyring<AuthorityId>, /* Assuming BeefyKeyring contains
+	                                            * necessary keys for the Fisherman */
 	api: Arc<TestApi>,
 	backend: Arc<BE>,
 ) -> Fisherman<Block, BE, MmrRootProvider<Block, TestApi>, TestApi> {
-	let key_store: Arc<BeefyKeystore> = Arc::new(Some(create_beefy_keystore(beefy_keyring)).into());
+	let key_store: Arc<BeefyKeystore<AuthorityId>> =
+		Arc::new(Some(create_beefy_keystore(beefy_keyring)).into());
 	let payload_provider = MmrRootProvider::new(api.clone());
 
 	Fisherman {
@@ -427,7 +429,7 @@ pub(crate) fn create_fisherman<BE>(
 }
 
 async fn voter_init_setup(
-	keyring: BeefyKeyring,
+	keyring: &BeefyKeyring<AuthorityId>,
 	net: &mut BeefyTestNet,
 	finality: &mut futures::stream::Fuse<FinalityNotifications<Block>>,
 	api: &TestApi,
@@ -450,7 +452,7 @@ async fn voter_init_setup(
 	let mut worker_base = BeefyWorkerBase {
 		backend,
 		runtime: Arc::new(api.clone()),
-		key_store: None.into(),
+		key_store: Arc::new(None.into()),
 		metrics: None,
 		_phantom: Default::default(),
 	};
@@ -1090,7 +1092,7 @@ async fn should_initialize_voter_at_genesis() {
 
 	let api = TestApi::with_validator_set(&validator_set);
 	// load persistent state - nothing in DB, should init at genesis
-	let persisted_state = voter_init_setup(keys[0], &mut net, &mut finality, &api).await.unwrap();
+	let persisted_state = voter_init_setup(&keys[0], &mut net, &mut finality, &api).await.unwrap();
 
 	// Test initialization at session boundary.
 	// verify voter initialized with two sessions starting at blocks 1 and 10
@@ -1136,7 +1138,7 @@ async fn should_initialize_voter_at_custom_genesis() {
 	// NOTE: code from `voter_init_setup()` is moved here because the new network event system
 	// doesn't allow creating a new `GossipEngine` as the notification handle is consumed by the
 	// first `GossipEngine`
-	let fisherman = create_fisherman(BeefyKeyring::Alice, Arc::new(api.clone()), backend.clone());
+	let fisherman = create_fisherman(&BeefyKeyring::Alice, Arc::new(api.clone()), backend.clone());
 	let known_peers = Arc::new(Mutex::new(KnownPeers::new()));
 	let (gossip_validator, _) = GossipValidator::new(known_peers, fisherman);
 	let gossip_validator = Arc::new(gossip_validator);
@@ -1153,7 +1155,7 @@ async fn should_initialize_voter_at_custom_genesis() {
 	let mut worker_base = BeefyWorkerBase {
 		backend: backend.clone(),
 		runtime: Arc::new(api),
-		key_store: None.into(),
+		key_store: Arc::new(None.into()),
 		metrics: None,
 		_phantom: Default::default(),
 	};
@@ -1194,7 +1196,7 @@ async fn should_initialize_voter_at_custom_genesis() {
 	let mut worker_base = BeefyWorkerBase {
 		backend: backend.clone(),
 		runtime: Arc::new(api),
-		key_store: None.into(),
+		key_store: Arc::new(None.into()),
 		metrics: None,
 		_phantom: Default::default(),
 	};
@@ -1254,7 +1256,7 @@ async fn should_initialize_voter_when_last_final_is_session_boundary() {
 
 	let api = TestApi::with_validator_set(&validator_set);
 	// load persistent state - nothing in DB, should init at session boundary
-	let persisted_state = voter_init_setup(keys[0], &mut net, &mut finality, &api).await.unwrap();
+	let persisted_state = voter_init_setup(&keys[0], &mut net, &mut finality, &api).await.unwrap();
 
 	// verify voter initialized with single session starting at block 10
 	assert_eq!(persisted_state.voting_oracle().sessions().len(), 1);
@@ -1307,7 +1309,7 @@ async fn should_initialize_voter_at_latest_finalized() {
 
 	let api = TestApi::with_validator_set(&validator_set);
 	// load persistent state - nothing in DB, should init at last BEEFY finalized
-	let persisted_state = voter_init_setup(keys[0], &mut net, &mut finality, &api).await.unwrap();
+	let persisted_state = voter_init_setup(&keys[0], &mut net, &mut finality, &api).await.unwrap();
 
 	// verify voter initialized with single session starting at block 12
 	assert_eq!(persisted_state.voting_oracle().sessions().len(), 1);
@@ -1345,7 +1347,7 @@ async fn should_initialize_voter_at_custom_genesis_when_state_unavailable() {
 	net.peer(0).client().as_client().finalize_block(hashes[30], None).unwrap();
 
 	// load persistent state - nothing in DB, should init at genesis
-	let persisted_state = voter_init_setup(keys[0], &mut net, &mut finality, &api).await.unwrap();
+	let persisted_state = voter_init_setup(&keys[0], &mut net, &mut finality, &api).await.unwrap();
 
 	// Test initialization at session boundary.
 	// verify voter initialized with all sessions pending, first one starting at block 5 (start of
@@ -1395,7 +1397,8 @@ async fn should_catch_up_when_loading_saved_voter_state() {
 	// doesn't allow creating a new `GossipEngine` as the notification handle is consumed by the
 	// first `GossipEngine`
 	let known_peers = Arc::new(Mutex::new(KnownPeers::new()));
-	let (gossip_validator, _) = GossipValidator::new(known_peers);
+	let fisherman = create_fisherman(&keys[0], Arc::new(api.clone()), backend.clone());
+	let (gossip_validator, _) = GossipValidator::new(known_peers, fisherman);
 	let gossip_validator = Arc::new(gossip_validator);
 	let mut gossip_engine = sc_network_gossip::GossipEngine::new(
 		net.peer(0).network_service().clone(),
@@ -1410,7 +1413,7 @@ async fn should_catch_up_when_loading_saved_voter_state() {
 	let mut worker_base = BeefyWorkerBase {
 		backend: backend.clone(),
 		runtime: Arc::new(api.clone()),
-		key_store: None.into(),
+		key_store: Arc::new(None.into()),
 		metrics: None,
 		_phantom: Default::default(),
 	};
@@ -1448,7 +1451,7 @@ async fn should_catch_up_when_loading_saved_voter_state() {
 	let mut worker_base = BeefyWorkerBase {
 		backend: backend.clone(),
 		runtime: Arc::new(api),
-		key_store: None.into(),
+		key_store: Arc::new(None.into()),
 		metrics: None,
 		_phantom: Default::default(),
 	};
@@ -1596,7 +1599,7 @@ async fn gossiped_finality_proofs() {
 	let beefy_peers = peers.iter().enumerate().map(|(id, key)| (id, key, api.clone())).collect();
 
 	let charlie = &mut net.peers[2];
-	let fisherman = create_fisherman(BeefyKeyring::Alice, api, backend.clone());
+	let fisherman = create_fisherman(&BeefyKeyring::Alice, api, backend.clone());
 	let known_peers = Arc::new(Mutex::new(KnownPeers::<Block>::new()));
 	// Charlie will run just the gossip engine and not the full voter.
 	let (gossip_validator, _) = GossipValidator::new(known_peers, fisherman);
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index d6f8cd0ce204..2d958b26fae4 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -582,7 +582,7 @@ pub(crate) struct BeefyWorker<B: Block, BE, P, RuntimeApi, S> {
 	pub sync: Arc<S>,
 
 	// communication (created once, but returned and reused if worker is restarted/reinitialized)
-	pub comms: BeefyComms<B>,
+	pub comms: BeefyComms<B, BE, P, RuntimeApi>,
 
 	// channels
 	/// Links between the block importer, the background voter and the RPC layer.
@@ -602,7 +602,7 @@ where
 	P: PayloadProvider<B>,
 	S: SyncOracle,
 	R: ProvideRuntimeApi<B>,
-	R::Api: BeefyApi<B, AuthorityId>,
+	R::Api: BeefyApi<B, AuthorityId, MmrRootHash> + MmrApi<B, MmrRootHash, NumberFor<B>>,
 {
 	fn best_grandpa_block(&self) -> NumberFor<B> {
 		*self.persisted_state.voting_oracle.best_grandpa_block_header.number()
@@ -1292,9 +1292,9 @@ pub(crate) mod tests {
 		mmr::MmrRootProvider,
 		test_utils::{
 			generate_fork_equivocation_proof_sc, generate_fork_equivocation_proof_vote,
-			generate_vote_equivocation_proof,
+			generate_vote_equivocation_proof, Keyring,
 		},
-		ForkEquivocationProof, Keyring, Payload, SignedCommitment,
+		ForkEquivocationProof, Payload, SignedCommitment,
 	};
 	use sp_runtime::traits::{Header as HeaderT, One};
 	use std::marker::PhantomData;
@@ -1336,7 +1336,8 @@ pub(crate) mod tests {
 		TestApi,
 		Arc<SyncingService<Block>>,
 	> {
-		let key_store: Arc<BeefyKeystore> = Arc::new(Some(create_beefy_keystore(*key)).into());
+		let key_store: Arc<BeefyKeystore<AuthorityId>> =
+			Arc::new(Some(create_beefy_keystore(key)).into());
 
 		let (to_rpc_justif_sender, from_voter_justif_stream) =
 			BeefyVersionedFinalityProofStream::<Block>::channel();
@@ -1418,7 +1419,7 @@ pub(crate) mod tests {
 			base: BeefyWorkerBase {
 				backend,
 				runtime: api,
-				key_store: Some(keystore).into(),
+				key_store,
 				metrics,
 				_phantom: Default::default(),
 			},
@@ -1955,6 +1956,7 @@ pub(crate) mod tests {
 
 		// verify: Alice reports Bob
 		let ancestry_proof = alice_worker
+			.base
 			.runtime
 			.runtime_api()
 			.generate_ancestry_proof(*hashes.last().unwrap(), block_number, None)
@@ -1978,7 +1980,7 @@ pub(crate) mod tests {
 			);
 			// verify Alice reports Bob's equivocation to runtime
 			let reported =
-				alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
+				alice_worker.base.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
 			assert_eq!(reported.len(), 1);
 			assert_eq!(*reported.get(0).unwrap(), proof);
 		}
@@ -2001,7 +2003,7 @@ pub(crate) mod tests {
 			);
 			// verify Alice does *not* report her own equivocation to runtime
 			let reported =
-				alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
+				alice_worker.base.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
 			assert_eq!(reported.len(), 1);
 			assert!(*reported.get(0).unwrap() != proof);
 		}
@@ -2042,7 +2044,7 @@ pub(crate) mod tests {
 				Ok(true)
 			);
 			let mut reported =
-				alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
+				alice_worker.base.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
 			// verify Alice report Bob's and Charlie's equivocation to runtime
 			assert_eq!(reported.len(), 2);
 			assert_eq!(reported.pop(), Some(proof));
@@ -2087,7 +2089,7 @@ pub(crate) mod tests {
 			Ok(true)
 		);
 		let mut reported =
-			alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
+			alice_worker.base.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
 		assert_eq!(reported.len(), 2);
 		assert_eq!(reported.pop(), Some(future_proof));
 	}
diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index 5b2c73711919..0e0b9c4136f1 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -24,14 +24,16 @@ use frame_support::{
 	traits::{Currency, KeyOwnerProofSystem, OnInitialize},
 };
 use sp_consensus_beefy::{
+	check_vote_equivocation_proof,
 	known_payloads::MMR_ROOT_ID,
 	test_utils::{
-		check_vote_equivocation_proof, generate_fork_equivocation_proof_sc,
-		generate_fork_equivocation_proof_vote, generate_vote_equivocation_proof,
+		generate_fork_equivocation_proof_sc, generate_fork_equivocation_proof_vote,
+		generate_vote_equivocation_proof, Keyring as BeefyKeyring,
 	},
-	Commitment, Keyring as BeefyKeyring, Payload, ValidatorSet, KEY_TYPE as BEEFY_KEY_TYPE,
+	Commitment, Payload, ValidatorSet, KEY_TYPE as BEEFY_KEY_TYPE,
 };
 use sp_core::offchain::{testing::TestOffchainExt, OffchainDbExt, OffchainWorkerExt};
+use sp_runtime::DigestItem;
 
 use crate::{mock::*, Call, Config, Error, Weight, WeightInfo};
 
@@ -1587,7 +1589,7 @@ fn report_fork_equivocation_sc_current_set_works() {
 			.collect::<Vec<_>>();
 		let equivocation_keyrings = equivocation_keys
 			.iter()
-			.map(|k| BeefyKeyring::from_public(k).unwrap())
+			.map(|k| BeefyKeyring::from_public(k.clone()).unwrap())
 			.collect();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
@@ -1824,7 +1826,7 @@ fn report_fork_equivocation_sc_future_block_works() {
 			.collect::<Vec<_>>();
 		let equivocation_keyrings = equivocation_keys
 			.iter()
-			.map(|k| BeefyKeyring::from_public(k).unwrap())
+			.map(|k| BeefyKeyring::from_public(k.clone()).unwrap())
 			.collect();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
@@ -1916,7 +1918,7 @@ fn report_fork_equivocation_sc_invalid_set_id() {
 			.collect::<Vec<_>>();
 		let equivocation_keyrings = equivocation_keys
 			.iter()
-			.map(|k| BeefyKeyring::from_public(k).unwrap())
+			.map(|k| BeefyKeyring::from_public(k.clone()).unwrap())
 			.collect();
 
 		let key_owner_proofs = equivocation_keys
@@ -2060,7 +2062,7 @@ fn report_fork_equivocation_sc_invalid_key_owner_proof() {
 			.collect::<Vec<_>>();
 		let equivocation_keyrings = equivocation_keys
 			.iter()
-			.map(|k| BeefyKeyring::from_public(k).unwrap())
+			.map(|k| BeefyKeyring::from_public(k.clone()).unwrap())
 			.collect();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
@@ -2125,7 +2127,7 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 			.collect::<Vec<_>>();
 		let equivocation_keyrings: Vec<_> = equivocation_keys
 			.iter()
-			.map(|k| BeefyKeyring::from_public(k).unwrap())
+			.map(|k| BeefyKeyring::from_public(k.clone()).unwrap())
 			.collect();
 
 		// generate a key ownership proof at set id in era 1
@@ -2441,7 +2443,7 @@ fn report_fork_equivocation_sc_stacked_reports_stack_correctly() {
 			.collect::<Vec<_>>();
 		let equivocation_keyrings: Vec<_> = equivocation_keys
 			.iter()
-			.map(|k| BeefyKeyring::from_public(k).unwrap())
+			.map(|k| BeefyKeyring::from_public(k.clone()).unwrap())
 			.collect();
 
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
@@ -2453,7 +2455,7 @@ fn report_fork_equivocation_sc_stacked_reports_stack_correctly() {
 		// 2. the second equivocation proof is for all equivocators
 		let equivocation_proof_singleton = generate_fork_equivocation_proof_sc(
 			commitment.clone(),
-			vec![equivocation_keyrings[0]],
+			vec![equivocation_keyrings[0].clone()],
 			None,
 			Some(ancestry_proof.clone()),
 		);
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 3241f823fc5d..63ab1cfe698a 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -36,7 +36,6 @@ mod payload;
 
 pub mod mmr;
 pub mod witness;
-use core::fmt::Debug;
 
 /// Test utilities
 #[cfg(feature = "std")]
diff --git a/substrate/primitives/consensus/beefy/src/test_utils.rs b/substrate/primitives/consensus/beefy/src/test_utils.rs
index 0657963c2101..b1c797a382d3 100644
--- a/substrate/primitives/consensus/beefy/src/test_utils.rs
+++ b/substrate/primitives/consensus/beefy/src/test_utils.rs
@@ -18,8 +18,8 @@
 #[cfg(feature = "bls-experimental")]
 use crate::ecdsa_bls_crypto;
 use crate::{
-	ecdsa_crypto, AuthorityIdBound, BeefySignatureHasher, Commitment, EquivocationProof, Payload,
-	ValidatorSetId, VoteMessage,
+	ecdsa_crypto, AuthorityIdBound, BeefySignatureHasher, Commitment, ForkEquivocationProof,
+	Payload, ValidatorSetId, VoteEquivocationProof, VoteMessage,
 };
 use sp_application_crypto::{AppCrypto, AppPair, RuntimeAppPublic, Wraps};
 use sp_core::{ecdsa, Pair};
@@ -27,7 +27,7 @@ use sp_runtime::traits::Hash;
 
 use codec::Encode;
 use sp_mmr_primitives::AncestryProof;
-use std::collections::{marker::PhantomData, HashMap};
+use std::{collections::HashMap, marker::PhantomData};
 use strum::IntoEnumIterator;
 
 /// Set of test accounts using [`crate::ecdsa_crypto`] types.
@@ -45,6 +45,8 @@ pub enum Keyring<AuthorityId> {
 	_Marker(PhantomData<AuthorityId>),
 }
 
+// impl AuthorityIdBound for Keyring<ecdsa_crypto::AuthorityId> {}
+
 /// Trait representing BEEFY specific generation and signing behavior of authority id
 ///
 /// Accepts custom hashing fn for the message and custom convertor fn for the signer.
@@ -161,7 +163,7 @@ pub fn generate_vote_equivocation_proof(
 
 /// Create a new `ForkEquivocationProof` based on vote & canonical header.
 pub fn generate_fork_equivocation_proof_vote<Header, Hash>(
-	vote: (u64, Payload, ValidatorSetId, &Keyring),
+	vote: (u64, Payload, ValidatorSetId, &Keyring<ecdsa_crypto::AuthorityId>),
 	canonical_header: Option<Header>,
 	ancestry_proof: Option<AncestryProof<Hash>>,
 ) -> ForkEquivocationProof<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature, Header, Hash> {
@@ -178,7 +180,7 @@ pub fn generate_fork_equivocation_proof_vote<Header, Hash>(
 /// Create a new `ForkEquivocationProof` based on signed commitment & canonical header.
 pub fn generate_fork_equivocation_proof_sc<Header, Hash>(
 	commitment: Commitment<u64>,
-	keyrings: Vec<Keyring>,
+	keyrings: Vec<Keyring<ecdsa_crypto::AuthorityId>>,
 	canonical_header: Option<Header>,
 	ancestry_proof: Option<AncestryProof<Hash>>,
 ) -> ForkEquivocationProof<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature, Header, Hash> {

From be3cbd8eac047916461f0bbf64655bc63cccbada Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 27 Feb 2024 11:39:10 +0100
Subject: [PATCH 149/188] merge fixes

---
 .../beefy/src/communication/gossip.rs         |  2 +-
 substrate/client/consensus/beefy/src/lib.rs   | 24 ++++++++++---------
 .../client/consensus/beefy/src/worker.rs      | 19 +++++++--------
 .../frame/merkle-mountain-range/src/lib.rs    |  2 +-
 4 files changed, 24 insertions(+), 23 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/gossip.rs b/substrate/client/consensus/beefy/src/communication/gossip.rs
index 3960d3bade91..939a906847dd 100644
--- a/substrate/client/consensus/beefy/src/communication/gossip.rs
+++ b/substrate/client/consensus/beefy/src/communication/gossip.rs
@@ -251,7 +251,7 @@ where
 	pub(crate) fn new(
 		known_peers: Arc<Mutex<KnownPeers<B>>>,
 		fisherman: Fisherman<B, BE, P, R>,
-	) -> (GossipValidator<B>, TracingUnboundedReceiver<PeerReport>) {
+	) -> (GossipValidator<B, BE, P, R>, TracingUnboundedReceiver<PeerReport>) {
 		let (tx, rx) = tracing_unbounded("mpsc_beefy_gossip_validator", 100_000);
 		let val = GossipValidator {
 			votes_topic: votes_topic::<B>(),
diff --git a/substrate/client/consensus/beefy/src/lib.rs b/substrate/client/consensus/beefy/src/lib.rs
index bc838db832b6..0dc5d9a9e5de 100644
--- a/substrate/client/consensus/beefy/src/lib.rs
+++ b/substrate/client/consensus/beefy/src/lib.rs
@@ -42,7 +42,9 @@ use sc_consensus::BlockImport;
 use sc_network::{NetworkRequest, NotificationService, ProtocolName};
 use sc_network_gossip::{GossipEngine, Network as GossipNetwork, Syncing as GossipSyncing};
 use sp_api::ProvideRuntimeApi;
-use sp_blockchain::{Backend as BlockchainBackend, HeaderBackend};
+use sp_blockchain::{
+	Backend as BlockchainBackend, Error as ClientError, HeaderBackend, Result as ClientResult,
+};
 use sp_consensus::{Error as ConsensusError, SyncOracle};
 use sp_consensus_beefy::{
 	ecdsa_crypto::AuthorityId, BeefyApi, ConsensusLog, MmrRootHash, PayloadProvider, ValidatorSet,
@@ -227,9 +229,9 @@ pub struct BeefyParams<B: Block, BE, C, N, P, R, S> {
 /// Helper object holding BEEFY worker communication/gossip components.
 ///
 /// These are created once, but will be reused if worker is restarted/reinitialized.
-pub(crate) struct BeefyComms<B: Block> {
+pub(crate) struct BeefyComms<B: Block, BE, P, R> {
 	pub gossip_engine: GossipEngine<B>,
-	pub gossip_validator: Arc<GossipValidator<B>>,
+	pub gossip_validator: Arc<GossipValidator<B, BE, P, R>>,
 	pub gossip_report_stream: TracingUnboundedReceiver<PeerReport>,
 	pub on_demand_justifications: OnDemandJustificationsEngine<B>,
 }
@@ -244,7 +246,7 @@ pub(crate) struct BeefyWorkerBuilder<B: Block, BE, RuntimeApi> {
 	// utilities
 	backend: Arc<BE>,
 	runtime: Arc<RuntimeApi>,
-	key_store: BeefyKeystore<AuthorityId>,
+	key_store: Arc<BeefyKeystore<AuthorityId>>,
 	// voter metrics
 	metrics: Option<VoterMetrics>,
 	persisted_state: PersistedState<B>,
@@ -255,7 +257,7 @@ where
 	B: Block + codec::Codec,
 	BE: Backend<B>,
 	R: ProvideRuntimeApi<B>,
-	R::Api: BeefyApi<B, AuthorityId>,
+	R::Api: BeefyApi<B, AuthorityId, MmrRootHash> + MmrApi<B, MmrRootHash, NumberFor<B>>,
 {
 	/// This will wait for the chain to enable BEEFY (if not yet enabled) and also wait for the
 	/// backend to sync all headers required by the voter to build a contiguous chain of mandatory
@@ -263,13 +265,13 @@ where
 	/// persisted state in AUX DB and latest chain information/progress.
 	///
 	/// Returns a sane `BeefyWorkerBuilder` that can build the `BeefyWorker`.
-	pub async fn async_initialize(
+	pub async fn async_initialize<P: PayloadProvider<B> + Send + Sync>(
 		backend: Arc<BE>,
 		runtime: Arc<R>,
-		key_store: BeefyKeystore<AuthorityId>,
+		key_store: Arc<BeefyKeystore<AuthorityId>>,
 		metrics: Option<VoterMetrics>,
 		min_block_delta: u32,
-		gossip_validator: Arc<GossipValidator<B>>,
+		gossip_validator: Arc<GossipValidator<B, BE, P, R>>,
 		finality_notifications: &mut Fuse<FinalityNotifications<B>>,
 	) -> Result<Self, Error> {
 		// Wait for BEEFY pallet to be active before starting voter.
@@ -299,7 +301,7 @@ where
 		self,
 		payload_provider: P,
 		sync: Arc<S>,
-		comms: BeefyComms<B>,
+		comms: BeefyComms<B, BE, P, R>,
 		links: BeefyVoterLinks<B>,
 		pending_justifications: BTreeMap<NumberFor<B>, BeefyVersionedFinalityProof<B>>,
 	) -> BeefyWorker<B, BE, P, R, S> {
@@ -558,7 +560,7 @@ pub async fn start_beefy_gadget<B, BE, C, N, P, R, S>(
 				builder_init_result = BeefyWorkerBuilder::async_initialize(
 					backend.clone(),
 					runtime.clone(),
-					key_store.clone().into(),
+					key_store.clone(),
 					metrics.clone(),
 					min_block_delta,
 					beefy_comms.gossip_validator.clone(),
@@ -757,7 +759,7 @@ where
 		if let Ok(Some(active)) = runtime.runtime_api().validator_set(header.hash()) {
 			return Ok(active)
 		} else {
-			match worker::find_authorities_change::<B>(&header) {
+			match crate::find_authorities_change::<B>(&header) {
 				Some(active) => return Ok(active),
 				// Move up the chain. Ultimately we'll get it from chain genesis state, or error out
 				// there.
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index 1abb141c85f3..1641393d4590 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -373,7 +373,7 @@ pub(crate) struct BeefyWorker<B: Block, BE, P, RuntimeApi, S> {
 	// utilities
 	pub backend: Arc<BE>,
 	pub runtime: Arc<RuntimeApi>,
-	pub key_store: BeefyKeystore<AuthorityId>,
+	pub key_store: Arc<BeefyKeystore<AuthorityId>>,
 	pub payload_provider: P,
 	pub sync: Arc<S>,
 
@@ -585,7 +585,7 @@ where
 			},
 			VoteImportResult::Equivocation(proof) => {
 				metric_inc!(self.metrics, beefy_equivocation_votes);
-				self.report_equivocation(proof)?;
+				self.report_vote_equivocation(proof)?;
 			},
 			VoteImportResult::Invalid => metric_inc!(self.metrics, beefy_invalid_votes),
 			VoteImportResult::Stale => metric_inc!(self.metrics, beefy_stale_votes),
@@ -1223,7 +1223,7 @@ pub(crate) mod tests {
 		BeefyWorker {
 			backend,
 			runtime: api,
-			key_store: Some(keystore).into(),
+			key_store: key_store.into(),
 			metrics,
 			payload_provider,
 			sync: Arc::new(sync),
@@ -1512,7 +1512,7 @@ pub(crate) mod tests {
 		assert_eq!(verify_validator_set::<Block>(&1, &validator_set, &worker.key_store), expected);
 
 		// worker has no keystore
-		worker.key_store = None.into();
+		worker.key_store = Arc::new(None.into());
 		let expected_err = Err(Error::Keystore("no Keystore".into()));
 		assert_eq!(
 			verify_validator_set::<Block>(&1, &validator_set, &worker.key_store),
@@ -1668,7 +1668,7 @@ pub(crate) mod tests {
 		let api_alice = Arc::new(api_alice);
 
 		let mut net = BeefyTestNet::new(1);
-		let mut worker = create_beefy_worker(net.peer(0), &keys[0], 1, validator_set.clone());
+		let mut worker = create_beefy_worker(net.peer(0), &keys[0], 1, validator_set.clone(), None);
 		worker.runtime = api_alice.clone();
 
 		// let there be a block with num = 1:
@@ -1756,7 +1756,6 @@ pub(crate) mod tests {
 
 		// verify: Alice reports Bob
 		let ancestry_proof = alice_worker
-			.base
 			.runtime
 			.runtime_api()
 			.generate_ancestry_proof(*hashes.last().unwrap(), block_number, None)
@@ -1780,7 +1779,7 @@ pub(crate) mod tests {
 			);
 			// verify Alice reports Bob's equivocation to runtime
 			let reported =
-				alice_worker.base.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
+				alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
 			assert_eq!(reported.len(), 1);
 			assert_eq!(*reported.get(0).unwrap(), proof);
 		}
@@ -1803,7 +1802,7 @@ pub(crate) mod tests {
 			);
 			// verify Alice does *not* report her own equivocation to runtime
 			let reported =
-				alice_worker.base.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
+				alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
 			assert_eq!(reported.len(), 1);
 			assert!(*reported.get(0).unwrap() != proof);
 		}
@@ -1844,7 +1843,7 @@ pub(crate) mod tests {
 				Ok(true)
 			);
 			let mut reported =
-				alice_worker.base.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
+				alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
 			// verify Alice report Bob's and Charlie's equivocation to runtime
 			assert_eq!(reported.len(), 2);
 			assert_eq!(reported.pop(), Some(proof));
@@ -1889,7 +1888,7 @@ pub(crate) mod tests {
 			Ok(true)
 		);
 		let mut reported =
-			alice_worker.base.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
+			alice_worker.runtime.reported_fork_equivocations.as_ref().unwrap().lock();
 		assert_eq!(reported.len(), 2);
 		assert_eq!(reported.pop(), Some(future_proof));
 	}
diff --git a/substrate/frame/merkle-mountain-range/src/lib.rs b/substrate/frame/merkle-mountain-range/src/lib.rs
index 04b0199aff94..737e987eda22 100644
--- a/substrate/frame/merkle-mountain-range/src/lib.rs
+++ b/substrate/frame/merkle-mountain-range/src/lib.rs
@@ -27,7 +27,7 @@
 //! - on-chain storage - hashes only; not full leaf content;
 //! - off-chain storage - via Indexing API we push full leaf content (and all internal nodes as
 //! well) to the Off-chain DB, so that the data is available for Off-chain workers.
-//! Hashing used for MMR is configurable independently from the rest of the runtime (i.e. not using
+//! Hashing used for MMR is configurable independently of the rest of the runtime (i.e. not using
 //! `frame_system::Hashing`) so something compatible with external chains can be used (like
 //! Keccak256 for Ethereum compatibility).
 //!

From 11095c73ab2cab37cf6b568ea46bc065e94615a0 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 27 Feb 2024 13:22:21 +0100
Subject: [PATCH 150/188] clippy

---
 substrate/primitives/merkle-mountain-range/src/utils.rs | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/substrate/primitives/merkle-mountain-range/src/utils.rs b/substrate/primitives/merkle-mountain-range/src/utils.rs
index 9a960533c802..f168238570bf 100644
--- a/substrate/primitives/merkle-mountain-range/src/utils.rs
+++ b/substrate/primitives/merkle-mountain-range/src/utils.rs
@@ -53,12 +53,7 @@ where
 {
 	let p: mmr_lib::NodeMerkleProof<H, _> = mmr_lib::NodeMerkleProof::<H, M>::new(
 		mmr_size,
-		ancestry_proof
-			.proof
-			.items
-			.into_iter()
-			.map(|(index, hash)| (index, hash))
-			.collect(),
+		ancestry_proof.proof.items.into_iter().collect(),
 	);
 
 	let ancestry_proof = mmr_lib::AncestryProof::<H, _> {

From e2080144b0390fc411040bbc17583dc379ddbf16 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 27 Feb 2024 18:39:55 +0100
Subject: [PATCH 151/188] align set_id in beefy key ownership tests

---
 substrate/frame/beefy/src/tests.rs | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index 0e0b9c4136f1..1b730475b828 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -534,8 +534,8 @@ fn report_vote_equivocation_invalid_key_owner_proof() {
 		let payload2 = Payload::from_single_entry(MMR_ROOT_ID, vec![128]);
 		// generate an equivocation proof for the authority at index 0
 		let equivocation_proof = generate_vote_equivocation_proof(
-			(block_num, payload1, set_id + 1, &equivocation_keyring),
-			(block_num, payload2, set_id + 1, &equivocation_keyring),
+			(block_num, payload1, set_id, &equivocation_keyring),
+			(block_num, payload2, set_id, &equivocation_keyring),
 		);
 
 		// we need to start a new era otherwise the key ownership proof won't be
@@ -1250,7 +1250,7 @@ fn report_fork_equivocation_vote_invalid_key_owner_proof() {
 		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		// generate an equivocation for a future set
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
-			(block_num, payload, set_id + 1, &equivocation_keyring),
+			(block_num, payload, set_id, &equivocation_keyring),
 			None,
 			Some(ancestry_proof),
 		);
@@ -2068,8 +2068,7 @@ fn report_fork_equivocation_sc_invalid_key_owner_proof() {
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 		// generate an equivocation proof for the authorities at indices [0, 2]
-		let commitment =
-			Commitment { validator_set_id: set_id + 1, block_number: block_num, payload };
+		let commitment = Commitment { validator_set_id: set_id, block_number: block_num, payload };
 		let equivocation_proof = generate_fork_equivocation_proof_sc(
 			commitment,
 			equivocation_keyrings,

From ab3f4171a8eae71cb40181edc4e817983c9e97fc Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Fri, 1 Mar 2024 07:39:06 +0100
Subject: [PATCH 152/188] sp beefy: feature gate Block trait requirement

---
 substrate/primitives/consensus/beefy/src/mmr.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/substrate/primitives/consensus/beefy/src/mmr.rs b/substrate/primitives/consensus/beefy/src/mmr.rs
index f8fbaf7f40b7..cebc6d1489e8 100644
--- a/substrate/primitives/consensus/beefy/src/mmr.rs
+++ b/substrate/primitives/consensus/beefy/src/mmr.rs
@@ -29,10 +29,10 @@
 use crate::{ecdsa_crypto::AuthorityId, ConsensusLog, MmrRootHash, Vec, BEEFY_ENGINE_ID};
 use codec::{Decode, Encode, MaxEncodedLen};
 use scale_info::TypeInfo;
-use sp_runtime::{
-	generic::OpaqueDigestItemId,
-	traits::{Block, Header},
-};
+use sp_runtime::{generic::OpaqueDigestItemId, traits::Header};
+
+#[cfg(feature = "std")]
+use sp_runtime::traits::Block;
 
 /// A provider for extra data that gets added to the Mmr leaf
 pub trait BeefyDataProvider<ExtraData> {

From 914caf9ed5fe90b8cce90a6c2e70920a2a61776b Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Sun, 3 Mar 2024 23:10:38 +0100
Subject: [PATCH 153/188] constrain NodeHash as HashOutput

---
 substrate/frame/beefy/src/equivocation.rs       | 5 +++--
 substrate/primitives/consensus/beefy/src/lib.rs | 6 +++---
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 79a8a44c0649..277f824a4f5c 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -42,6 +42,7 @@ use sp_consensus_beefy::{
 	ForkEquivocationProof, ValidatorSetId, VoteEquivocationProof, KEY_TYPE as BEEFY_KEY_TYPE,
 };
 use sp_runtime::{
+	traits::Hash,
 	transaction_validity::{
 		InvalidTransaction, TransactionPriority, TransactionSource, TransactionValidity,
 		TransactionValidityError, ValidTransaction,
@@ -139,7 +140,7 @@ pub enum EquivocationEvidenceFor<T: Config + pallet_mmr::Config> {
 			<T as Config>::BeefyId,
 			<<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
 			HeaderFor<T>,
-			<<T as pallet_mmr::Config>::Hashing as sp_runtime::traits::Hash>::Output,
+			<<T as pallet_mmr::Config>::Hashing as Hash>::Output,
 		>,
 		Vec<<T as Config>::KeyOwnerProof>,
 	),
@@ -325,7 +326,7 @@ where
 					_,
 					_,
 					_,
-					<<T as pallet_mmr::Config>::Hashing as sp_runtime::traits::Hash>::Output,
+					<<T as pallet_mmr::Config>::Hashing as Hash>::Output,
 					sp_mmr_primitives::utils::AncestryHasher<<T as pallet_mmr::Config>::Hashing>,
 				>(
 					equivocation_proof,
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 63ab1cfe698a..cfa6cb1f8938 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -50,7 +50,7 @@ use core::fmt::{Debug, Display};
 use scale_info::TypeInfo;
 use sp_application_crypto::{AppCrypto, AppPublic, ByteArray, RuntimeAppPublic};
 use sp_core::H256;
-use sp_runtime::traits::{Hash, Header as HeaderT, Keccak256, NumberFor};
+use sp_runtime::traits::{Hash, HashOutput, Header as HeaderT, Keccak256, NumberFor};
 use sp_std::prelude::*;
 
 /// Key type for BEEFY module.
@@ -449,7 +449,7 @@ fn check_ancestry_proof<Header, NodeHash, Hasher>(
 ) -> bool
 where
 	Header: HeaderT,
-	NodeHash: Clone + Debug + PartialEq + Encode + Decode,
+	NodeHash: HashOutput,
 	Hasher: mmr_lib::Merge<Item = NodeHash>,
 {
 	if let Some(ancestry_proof) = ancestry_proof {
@@ -538,7 +538,7 @@ where
 	Id: BeefyAuthorityId<MsgHash> + PartialEq,
 	MsgHash: Hash,
 	Header: HeaderT,
-	NodeHash: Clone + Debug + PartialEq + Encode + Decode,
+	NodeHash: sp_runtime::traits::HashOutput,
 	Hasher: mmr_lib::Merge<Item = NodeHash>,
 {
 	let ForkEquivocationProof { commitment, signatories, canonical_header, ancestry_proof } = proof;

From 730df52f95a21ce19e605445df5305bd165243f2 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Sun, 3 Mar 2024 23:41:27 +0100
Subject: [PATCH 154/188] straight plumbing impl using wrapper for mmr_root

---
 Cargo.lock                                    |  2 +
 substrate/frame/beefy-mmr/Cargo.toml          |  3 ++
 substrate/frame/beefy-mmr/src/lib.rs          | 46 +++++++++++++++--
 substrate/frame/beefy-mmr/src/mock.rs         |  1 +
 substrate/frame/beefy/src/equivocation.rs     | 20 ++++----
 substrate/frame/beefy/src/lib.rs              |  3 ++
 substrate/frame/beefy/src/mock.rs             |  1 +
 .../primitives/consensus/beefy/src/lib.rs     | 50 +++++++++++++++++++
 8 files changed, 114 insertions(+), 12 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index e7747139b9e1..ebf064ee66fd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9410,9 +9410,11 @@ dependencies = [
  "scale-info",
  "serde",
  "sp-api",
+ "sp-application-crypto",
  "sp-consensus-beefy",
  "sp-core",
  "sp-io",
+ "sp-mmr-primitives",
  "sp-runtime",
  "sp-staking",
  "sp-state-machine",
diff --git a/substrate/frame/beefy-mmr/Cargo.toml b/substrate/frame/beefy-mmr/Cargo.toml
index 177077317731..11074cf073dd 100644
--- a/substrate/frame/beefy-mmr/Cargo.toml
+++ b/substrate/frame/beefy-mmr/Cargo.toml
@@ -1,4 +1,5 @@
 [package]
+
 name = "pallet-beefy-mmr"
 version = "28.0.0"
 authors.workspace = true
@@ -24,6 +25,8 @@ pallet-beefy = { path = "../beefy", default-features = false }
 pallet-mmr = { path = "../merkle-mountain-range", default-features = false }
 pallet-session = { path = "../session", default-features = false }
 sp-consensus-beefy = { path = "../../primitives/consensus/beefy", default-features = false }
+sp-mmr-primitives = { path = "../../primitives/merkle-mountain-range", default-features = false }
+sp-application-crypto = { path = "../../primitives/application-crypto", default-features = false, features = ["serde"] }
 sp-core = { path = "../../primitives/core", default-features = false }
 sp-io = { path = "../../primitives/io", default-features = false }
 sp-runtime = { path = "../../primitives/runtime", default-features = false }
diff --git a/substrate/frame/beefy-mmr/src/lib.rs b/substrate/frame/beefy-mmr/src/lib.rs
index 7af4a7a14706..1417ea68647a 100644
--- a/substrate/frame/beefy-mmr/src/lib.rs
+++ b/substrate/frame/beefy-mmr/src/lib.rs
@@ -33,14 +33,14 @@
 //!
 //! and thanks to versioning can be easily updated in the future.
 
-use sp_runtime::traits::{Convert, Member};
+use sp_runtime::traits::{Convert, Hash, Member};
 use sp_std::prelude::*;
 
 use codec::Decode;
-use pallet_mmr::{LeafDataProvider, ParentNumberAndHash};
+use pallet_mmr::{primitives::utils::AncestryHasher, LeafDataProvider, ParentNumberAndHash};
 use sp_consensus_beefy::{
 	mmr::{BeefyAuthoritySet, BeefyDataProvider, BeefyNextAuthoritySet, MmrLeaf, MmrLeafVersion},
-	ValidatorSet as BeefyValidatorSet,
+	CheckForkEquivocationProof, ForkEquivocationProof, ValidatorSet as BeefyValidatorSet,
 };
 
 use frame_support::{crypto::ecdsa::ECDSAExt, traits::Get};
@@ -174,6 +174,46 @@ where
 	}
 }
 
+impl<T: pallet_mmr::Config> CheckForkEquivocationProof for Pallet<T> {
+	fn check_fork_equivocation_proof<Id, MsgHash, Header, NodeHash, Hasher>(
+		proof: &ForkEquivocationProof<
+			Header::Number,
+			Id,
+			<Id as sp_application_crypto::RuntimeAppPublic>::Signature,
+			Header,
+			NodeHash,
+		>,
+		canonical_root: NodeHash,
+		mmr_size: u64,
+		canonical_header_hash: &Header::Hash,
+		first_mmr_block_num: Header::Number,
+		best_block_num: Header::Number,
+	) -> bool
+	where
+		Id: sp_consensus_beefy::BeefyAuthorityId<MsgHash> + PartialEq,
+		MsgHash: sp_runtime::traits::Hash,
+		Header: sp_runtime::traits::Header,
+		NodeHash: sp_runtime::traits::HashOutput,
+		Hasher: sp_mmr_primitives::mmr_lib::Merge<Item = NodeHash>,
+	{
+		sp_consensus_beefy::check_fork_equivocation_proof::<
+			_,
+			_,
+			_,
+			_,
+			// AncestryHasher<<T as pallet_mmr::Config>::Hashing>,
+			Hasher,
+		>(
+			proof,
+			canonical_root,
+			mmr_size,
+			canonical_header_hash,
+			first_mmr_block_num,
+			best_block_num,
+		)
+	}
+}
+
 impl<T: Config> Pallet<T> {
 	/// Return the currently active BEEFY authority set proof.
 	pub fn authority_set_proof() -> BeefyAuthoritySet<MerkleRootOf<T>> {
diff --git a/substrate/frame/beefy-mmr/src/mock.rs b/substrate/frame/beefy-mmr/src/mock.rs
index 94149ac67765..a1957d739155 100644
--- a/substrate/frame/beefy-mmr/src/mock.rs
+++ b/substrate/frame/beefy-mmr/src/mock.rs
@@ -99,6 +99,7 @@ impl pallet_beefy::Config for Test {
 	type MaxNominators = ConstU32<1000>;
 	type MaxSetIdSessionEntries = ConstU64<100>;
 	type OnNewValidatorSet = BeefyMmr;
+	type CheckForkEquivocationProof = BeefyMmr;
 	type WeightInfo = ();
 	type KeyOwnerProof = sp_core::Void;
 	type EquivocationReportSystem = ();
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 277f824a4f5c..745df275207a 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -39,7 +39,8 @@ use frame_support::traits::{Get, KeyOwnerProofSystem};
 use frame_system::pallet_prelude::{BlockNumberFor, HeaderFor};
 use log::{error, info};
 use sp_consensus_beefy::{
-	ForkEquivocationProof, ValidatorSetId, VoteEquivocationProof, KEY_TYPE as BEEFY_KEY_TYPE,
+	CheckForkEquivocationProof, ForkEquivocationProof, ValidatorSetId, VoteEquivocationProof,
+	KEY_TYPE as BEEFY_KEY_TYPE,
 };
 use sp_runtime::{
 	traits::Hash,
@@ -301,7 +302,9 @@ where
 				let mmr_size =
 					sp_mmr_primitives::utils::NodesUtils::new(<pallet_mmr::Pallet<T>>::mmr_size())
 						.size();
-				let expected_mmr_root = <pallet_mmr::Pallet<T>>::mmr_root();
+				// let expected_mmr_root = <pallet_mmr::Pallet<T>>::mmr_root();
+				let expected_mmr_root = mmr_root_hash_wrapper::<T>();
+				// let expected_mmr_root = sp_consensus_beefy::MmrRootHash::default();
 				let best_block_num = <frame_system::Pallet<T>>::block_number();
 				// if first_mmr_block_num is invalid, then presumably beefy is not active.
 				// TODO: should we slash in this case?
@@ -322,13 +325,7 @@ where
 				// beefy light client at least once every 4096 blocks. See
 				// https://github.com/paritytech/polkadot-sdk/issues/1441 for
 				// replacement solution.
-				if !sp_consensus_beefy::check_fork_equivocation_proof::<
-					_,
-					_,
-					_,
-					<<T as pallet_mmr::Config>::Hashing as Hash>::Output,
-					sp_mmr_primitives::utils::AncestryHasher<<T as pallet_mmr::Config>::Hashing>,
-				>(
+				if !<T::CheckForkEquivocationProof as CheckForkEquivocationProof>::check_fork_equivocation_proof::<_, _, _, _, sp_mmr_primitives::utils::AncestryHasher<<T as pallet_mmr::Config>::Hashing>>(
 					equivocation_proof,
 					expected_mmr_root,
 					mmr_size,
@@ -355,6 +352,11 @@ where
 	}
 }
 
+fn mmr_root_hash_wrapper<T: pallet_mmr::Config>(
+) -> <<T as pallet_mmr::Config>::Hashing as sp_runtime::traits::Hash>::Output {
+	<pallet_mmr::Pallet<T>>::mmr_root()
+}
+
 /// Methods for the `ValidateUnsigned` implementation:
 /// It restricts calls to `report_vote_equivocation_unsigned` to local calls (i.e. extrinsics
 /// generated on this node) or that already in a block. This guarantees that only block authors can
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 12e985fb69f3..66c86e41cdfc 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -99,6 +99,9 @@ pub mod pallet {
 		/// weight MMR root over validators and make it available for Light Clients.
 		type OnNewValidatorSet: OnNewValidatorSet<<Self as Config>::BeefyId>;
 
+		/// Hook for checking fork equivocation proofs
+		type CheckForkEquivocationProof: sp_consensus_beefy::CheckForkEquivocationProof;
+
 		/// Weights for this pallet.
 		type WeightInfo: WeightInfo;
 
diff --git a/substrate/frame/beefy/src/mock.rs b/substrate/frame/beefy/src/mock.rs
index 94a8f7fcf824..2bbefedf8900 100644
--- a/substrate/frame/beefy/src/mock.rs
+++ b/substrate/frame/beefy/src/mock.rs
@@ -94,6 +94,7 @@ impl pallet_beefy::Config for Test {
 	type MaxNominators = ConstU32<1000>;
 	type MaxSetIdSessionEntries = MaxSetIdSessionEntries;
 	type OnNewValidatorSet = ();
+	type CheckForkEquivocationProof = ();
 	type WeightInfo = ();
 	type KeyOwnerProof = <Historical as KeyOwnerProofSystem<(KeyTypeId, BeefyId)>>::Proof;
 	type EquivocationReportSystem =
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index cfa6cb1f8938..e2f93b23a63e 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -587,6 +587,56 @@ impl<AuthorityId> OnNewValidatorSet<AuthorityId> for () {
 	fn on_new_validator_set(_: &ValidatorSet<AuthorityId>, _: &ValidatorSet<AuthorityId>) {}
 }
 
+/// Hook for checking fork equivocation proof.
+pub trait CheckForkEquivocationProof {
+	fn check_fork_equivocation_proof<Id, MsgHash, Header, NodeHash, Hasher>(
+		proof: &ForkEquivocationProof<
+			Header::Number,
+			Id,
+			<Id as RuntimeAppPublic>::Signature,
+			Header,
+			NodeHash,
+		>,
+		canonical_root: Hasher::Item,
+		mmr_size: u64,
+		canonical_header_hash: &Header::Hash,
+		first_mmr_block_num: Header::Number,
+		best_block_num: Header::Number,
+	) -> bool
+	where
+		Id: BeefyAuthorityId<MsgHash> + PartialEq,
+		MsgHash: Hash,
+		Header: HeaderT,
+		NodeHash: sp_runtime::traits::HashOutput,
+		Hasher: mmr_lib::Merge<Item = NodeHash>;
+}
+
+impl CheckForkEquivocationProof for () {
+	fn check_fork_equivocation_proof<Id, MsgHash, Header, NodeHash, Hasher>(
+		_proof: &ForkEquivocationProof<
+			Header::Number,
+			Id,
+			<Id as RuntimeAppPublic>::Signature,
+			Header,
+			NodeHash,
+		>,
+		_canonical_root: Hasher::Item,
+		_mmr_size: u64,
+		_canonical_header_hash: &Header::Hash,
+		_first_mmr_block_num: Header::Number,
+		_best_block_num: Header::Number,
+	) -> bool
+	where
+		Id: BeefyAuthorityId<MsgHash> + PartialEq,
+		MsgHash: Hash,
+		Header: HeaderT,
+		NodeHash: sp_runtime::traits::HashOutput,
+		Hasher: mmr_lib::Merge<Item = NodeHash>,
+	{
+		true
+	}
+}
+
 /// An opaque type used to represent the key ownership proof at the runtime API
 /// boundary. The inner value is an encoded representation of the actual key
 /// ownership proof which will be parameterized when defining the runtime. At

From 42b6e4fd083db6e660a4e0dd273d46f8a24d456c Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 4 Mar 2024 00:38:22 +0100
Subject: [PATCH 155/188] use Hashing trait generic

---
 substrate/frame/beefy-mmr/src/lib.rs          | 22 ++++++++++++++-----
 substrate/frame/beefy/src/equivocation.rs     |  2 +-
 substrate/frame/beefy/src/lib.rs              |  2 +-
 .../primitives/consensus/beefy/src/lib.rs     | 22 +++++++++----------
 4 files changed, 29 insertions(+), 19 deletions(-)

diff --git a/substrate/frame/beefy-mmr/src/lib.rs b/substrate/frame/beefy-mmr/src/lib.rs
index 1417ea68647a..49cac71c9b43 100644
--- a/substrate/frame/beefy-mmr/src/lib.rs
+++ b/substrate/frame/beefy-mmr/src/lib.rs
@@ -174,16 +174,23 @@ where
 	}
 }
 
-impl<T: pallet_mmr::Config> CheckForkEquivocationProof for Pallet<T> {
-	fn check_fork_equivocation_proof<Id, MsgHash, Header, NodeHash, Hasher>(
+fn mmr_root_hash_wrapper<T: pallet_mmr::Config>(
+) -> <<T as pallet_mmr::Config>::Hashing as sp_runtime::traits::Hash>::Output {
+	<pallet_mmr::Pallet<T>>::mmr_root()
+}
+
+impl<T: pallet_mmr::Config> CheckForkEquivocationProof<<T as pallet_mmr::Config>::Hashing>
+	for Pallet<T>
+{
+	fn check_fork_equivocation_proof<Id, MsgHash, Header, Hasher>(
 		proof: &ForkEquivocationProof<
 			Header::Number,
 			Id,
 			<Id as sp_application_crypto::RuntimeAppPublic>::Signature,
 			Header,
-			NodeHash,
+			<<T as pallet_mmr::Config>::Hashing as Hash>::Output,
 		>,
-		canonical_root: NodeHash,
+		canonical_root: <<T as pallet_mmr::Config>::Hashing as Hash>::Output,
 		mmr_size: u64,
 		canonical_header_hash: &Header::Hash,
 		first_mmr_block_num: Header::Number,
@@ -193,9 +200,12 @@ impl<T: pallet_mmr::Config> CheckForkEquivocationProof for Pallet<T> {
 		Id: sp_consensus_beefy::BeefyAuthorityId<MsgHash> + PartialEq,
 		MsgHash: sp_runtime::traits::Hash,
 		Header: sp_runtime::traits::Header,
-		NodeHash: sp_runtime::traits::HashOutput,
-		Hasher: sp_mmr_primitives::mmr_lib::Merge<Item = NodeHash>,
+		Hasher: sp_mmr_primitives::mmr_lib::Merge<
+			Item = <<T as pallet_mmr::Config>::Hashing as Hash>::Output,
+		>,
 	{
+		let canonical_root = mmr_root_hash_wrapper::<T>();
+		// let canonical_root = <pallet_mmr::Pallet<T>>::mmr_root();
 		sp_consensus_beefy::check_fork_equivocation_proof::<
 			_,
 			_,
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 745df275207a..f252cc2a5e92 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -325,7 +325,7 @@ where
 				// beefy light client at least once every 4096 blocks. See
 				// https://github.com/paritytech/polkadot-sdk/issues/1441 for
 				// replacement solution.
-				if !<T::CheckForkEquivocationProof as CheckForkEquivocationProof>::check_fork_equivocation_proof::<_, _, _, _, sp_mmr_primitives::utils::AncestryHasher<<T as pallet_mmr::Config>::Hashing>>(
+				if !<T::CheckForkEquivocationProof<<T as pallet_mmr::Config>::Hashing> as CheckForkEquivocationProof<<T as pallet_mmr::Config>::Hashing>>::check_fork_equivocation_proof::<_, _,  _, sp_mmr_primitives::utils::AncestryHasher<<T as pallet_mmr::Config>::Hashing>>(
 					equivocation_proof,
 					expected_mmr_root,
 					mmr_size,
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 66c86e41cdfc..de4b22be422b 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -100,7 +100,7 @@ pub mod pallet {
 		type OnNewValidatorSet: OnNewValidatorSet<<Self as Config>::BeefyId>;
 
 		/// Hook for checking fork equivocation proofs
-		type CheckForkEquivocationProof: sp_consensus_beefy::CheckForkEquivocationProof;
+		type CheckForkEquivocationProof<H: sp_runtime::traits::Hash>: sp_consensus_beefy::CheckForkEquivocationProof<H>;
 
 		/// Weights for this pallet.
 		type WeightInfo: WeightInfo;
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index e2f93b23a63e..cf7d4bd476b9 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -588,16 +588,17 @@ impl<AuthorityId> OnNewValidatorSet<AuthorityId> for () {
 }
 
 /// Hook for checking fork equivocation proof.
-pub trait CheckForkEquivocationProof {
-	fn check_fork_equivocation_proof<Id, MsgHash, Header, NodeHash, Hasher>(
+pub trait CheckForkEquivocationProof<H: Hash> {
+	// type NodeHash: HashOutput;
+	fn check_fork_equivocation_proof<Id, MsgHash, Header, Hasher>(
 		proof: &ForkEquivocationProof<
 			Header::Number,
 			Id,
 			<Id as RuntimeAppPublic>::Signature,
 			Header,
-			NodeHash,
+			H::Output,
 		>,
-		canonical_root: Hasher::Item,
+		canonical_root: H::Output,
 		mmr_size: u64,
 		canonical_header_hash: &Header::Hash,
 		first_mmr_block_num: Header::Number,
@@ -607,18 +608,18 @@ pub trait CheckForkEquivocationProof {
 		Id: BeefyAuthorityId<MsgHash> + PartialEq,
 		MsgHash: Hash,
 		Header: HeaderT,
-		NodeHash: sp_runtime::traits::HashOutput,
-		Hasher: mmr_lib::Merge<Item = NodeHash>;
+		Hasher: mmr_lib::Merge<Item = H::Output>;
 }
 
-impl CheckForkEquivocationProof for () {
-	fn check_fork_equivocation_proof<Id, MsgHash, Header, NodeHash, Hasher>(
+impl<H: Hash> CheckForkEquivocationProof<H> for () {
+	// type NodeHash = H256;
+	fn check_fork_equivocation_proof<Id, MsgHash, Header, Hasher>(
 		_proof: &ForkEquivocationProof<
 			Header::Number,
 			Id,
 			<Id as RuntimeAppPublic>::Signature,
 			Header,
-			NodeHash,
+			H::Output,
 		>,
 		_canonical_root: Hasher::Item,
 		_mmr_size: u64,
@@ -630,8 +631,7 @@ impl CheckForkEquivocationProof for () {
 		Id: BeefyAuthorityId<MsgHash> + PartialEq,
 		MsgHash: Hash,
 		Header: HeaderT,
-		NodeHash: sp_runtime::traits::HashOutput,
-		Hasher: mmr_lib::Merge<Item = NodeHash>,
+		Hasher: mmr_lib::Merge<Item = H::Output>,
 	{
 		true
 	}

From 6199919b87caaec6a99aa62937c3c9c792afaad3 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 4 Mar 2024 00:41:23 +0100
Subject: [PATCH 156/188] remove canonical root from outer wrapper

---
 substrate/frame/beefy-mmr/src/lib.rs            | 1 -
 substrate/frame/beefy/src/equivocation.rs       | 1 -
 substrate/primitives/consensus/beefy/src/lib.rs | 2 --
 3 files changed, 4 deletions(-)

diff --git a/substrate/frame/beefy-mmr/src/lib.rs b/substrate/frame/beefy-mmr/src/lib.rs
index 49cac71c9b43..34795b16851e 100644
--- a/substrate/frame/beefy-mmr/src/lib.rs
+++ b/substrate/frame/beefy-mmr/src/lib.rs
@@ -190,7 +190,6 @@ impl<T: pallet_mmr::Config> CheckForkEquivocationProof<<T as pallet_mmr::Config>
 			Header,
 			<<T as pallet_mmr::Config>::Hashing as Hash>::Output,
 		>,
-		canonical_root: <<T as pallet_mmr::Config>::Hashing as Hash>::Output,
 		mmr_size: u64,
 		canonical_header_hash: &Header::Hash,
 		first_mmr_block_num: Header::Number,
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index f252cc2a5e92..135ac4e37a90 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -327,7 +327,6 @@ where
 				// replacement solution.
 				if !<T::CheckForkEquivocationProof<<T as pallet_mmr::Config>::Hashing> as CheckForkEquivocationProof<<T as pallet_mmr::Config>::Hashing>>::check_fork_equivocation_proof::<_, _,  _, sp_mmr_primitives::utils::AncestryHasher<<T as pallet_mmr::Config>::Hashing>>(
 					equivocation_proof,
-					expected_mmr_root,
 					mmr_size,
 					&expected_block_hash,
 					first_mmr_block_num,
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index cf7d4bd476b9..0d49947bfbce 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -598,7 +598,6 @@ pub trait CheckForkEquivocationProof<H: Hash> {
 			Header,
 			H::Output,
 		>,
-		canonical_root: H::Output,
 		mmr_size: u64,
 		canonical_header_hash: &Header::Hash,
 		first_mmr_block_num: Header::Number,
@@ -621,7 +620,6 @@ impl<H: Hash> CheckForkEquivocationProof<H> for () {
 			Header,
 			H::Output,
 		>,
-		_canonical_root: Hasher::Item,
 		_mmr_size: u64,
 		_canonical_header_hash: &Header::Hash,
 		_first_mmr_block_num: Header::Number,

From 4a6633f385b230abd959e79d40fb2337cf4eae5f Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 4 Mar 2024 01:03:55 +0100
Subject: [PATCH 157/188] remove generic from beefy pallet CheckFEP type

---
 substrate/frame/beefy/src/equivocation.rs | 9 ++++++++-
 substrate/frame/beefy/src/lib.rs          | 4 +++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 135ac4e37a90..6309df5dd6cc 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -325,7 +325,14 @@ where
 				// beefy light client at least once every 4096 blocks. See
 				// https://github.com/paritytech/polkadot-sdk/issues/1441 for
 				// replacement solution.
-				if !<T::CheckForkEquivocationProof<<T as pallet_mmr::Config>::Hashing> as CheckForkEquivocationProof<<T as pallet_mmr::Config>::Hashing>>::check_fork_equivocation_proof::<_, _,  _, sp_mmr_primitives::utils::AncestryHasher<<T as pallet_mmr::Config>::Hashing>>(
+				if !<T::CheckForkEquivocationProof as CheckForkEquivocationProof<
+					<T as pallet_mmr::Config>::Hashing,
+				>>::check_fork_equivocation_proof::<
+					_,
+					_,
+					_,
+					sp_mmr_primitives::utils::AncestryHasher<<T as pallet_mmr::Config>::Hashing>,
+				>(
 					equivocation_proof,
 					mmr_size,
 					&expected_block_hash,
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index de4b22be422b..764d32b070f1 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -100,7 +100,9 @@ pub mod pallet {
 		type OnNewValidatorSet: OnNewValidatorSet<<Self as Config>::BeefyId>;
 
 		/// Hook for checking fork equivocation proofs
-		type CheckForkEquivocationProof<H: sp_runtime::traits::Hash>: sp_consensus_beefy::CheckForkEquivocationProof<H>;
+		type CheckForkEquivocationProof: sp_consensus_beefy::CheckForkEquivocationProof<
+			<Self as pallet_mmr::Config>::Hashing,
+		>;
 
 		/// Weights for this pallet.
 		type WeightInfo: WeightInfo;

From b0c3ce3e6e8b35c183ebba3a4b4c01871e11495b Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 4 Mar 2024 08:34:02 +0100
Subject: [PATCH 158/188] add CheckForkEquivocationProof type to runtimes

---
 polkadot/runtime/rococo/src/lib.rs    | 1 +
 polkadot/runtime/westend/src/lib.rs   | 1 +
 substrate/bin/node/runtime/src/lib.rs | 1 +
 3 files changed, 3 insertions(+)

diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs
index f4054c6240e3..ed33a91e772c 100644
--- a/polkadot/runtime/rococo/src/lib.rs
+++ b/polkadot/runtime/rococo/src/lib.rs
@@ -1211,6 +1211,7 @@ impl pallet_beefy::Config for Runtime {
 	type MaxNominators = ConstU32<0>;
 	type MaxSetIdSessionEntries = BeefySetIdSessionEntries;
 	type OnNewValidatorSet = MmrLeaf;
+	type CheckForkEquivocationProof = MmrLeaf;
 	type WeightInfo = ();
 	type KeyOwnerProof = <Historical as KeyOwnerProofSystem<(KeyTypeId, BeefyId)>>::Proof;
 	type EquivocationReportSystem =
diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs
index 75dc2f90665c..c1e90a23ffe3 100644
--- a/polkadot/runtime/westend/src/lib.rs
+++ b/polkadot/runtime/westend/src/lib.rs
@@ -319,6 +319,7 @@ impl pallet_beefy::Config for Runtime {
 	type MaxNominators = MaxNominators;
 	type MaxSetIdSessionEntries = BeefySetIdSessionEntries;
 	type OnNewValidatorSet = BeefyMmrLeaf;
+	type CheckForkEquivocationProof = BeefyMmrLeaf;
 	type WeightInfo = ();
 	type KeyOwnerProof = sp_session::MembershipProof;
 	type EquivocationReportSystem =
diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs
index aed0ab166d22..77131ee8430e 100644
--- a/substrate/bin/node/runtime/src/lib.rs
+++ b/substrate/bin/node/runtime/src/lib.rs
@@ -2323,6 +2323,7 @@ impl pallet_beefy::Config for Runtime {
 	type MaxNominators = ConstU32<0>;
 	type MaxSetIdSessionEntries = BeefySetIdSessionEntries;
 	type OnNewValidatorSet = MmrLeaf;
+	type CheckForkEquivocationProof = MmrLeaf;
 	type WeightInfo = ();
 	type KeyOwnerProof = <Historical as KeyOwnerProofSystem<(KeyTypeId, BeefyId)>>::Proof;
 	type EquivocationReportSystem =

From b4aa1587b6fd24fc45971d60f0204bd89c8d31d5 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 4 Mar 2024 08:38:44 +0100
Subject: [PATCH 159/188] remove Hasher generic from outer call

---
 substrate/frame/beefy-mmr/src/lib.rs            | 9 +++------
 substrate/frame/beefy/src/equivocation.rs       | 9 ++-------
 substrate/primitives/consensus/beefy/src/lib.rs | 8 +++-----
 3 files changed, 8 insertions(+), 18 deletions(-)

diff --git a/substrate/frame/beefy-mmr/src/lib.rs b/substrate/frame/beefy-mmr/src/lib.rs
index 34795b16851e..d13bc25dda12 100644
--- a/substrate/frame/beefy-mmr/src/lib.rs
+++ b/substrate/frame/beefy-mmr/src/lib.rs
@@ -182,7 +182,7 @@ fn mmr_root_hash_wrapper<T: pallet_mmr::Config>(
 impl<T: pallet_mmr::Config> CheckForkEquivocationProof<<T as pallet_mmr::Config>::Hashing>
 	for Pallet<T>
 {
-	fn check_fork_equivocation_proof<Id, MsgHash, Header, Hasher>(
+	fn check_fork_equivocation_proof<Id, MsgHash, Header>(
 		proof: &ForkEquivocationProof<
 			Header::Number,
 			Id,
@@ -199,9 +199,6 @@ impl<T: pallet_mmr::Config> CheckForkEquivocationProof<<T as pallet_mmr::Config>
 		Id: sp_consensus_beefy::BeefyAuthorityId<MsgHash> + PartialEq,
 		MsgHash: sp_runtime::traits::Hash,
 		Header: sp_runtime::traits::Header,
-		Hasher: sp_mmr_primitives::mmr_lib::Merge<
-			Item = <<T as pallet_mmr::Config>::Hashing as Hash>::Output,
-		>,
 	{
 		let canonical_root = mmr_root_hash_wrapper::<T>();
 		// let canonical_root = <pallet_mmr::Pallet<T>>::mmr_root();
@@ -210,8 +207,8 @@ impl<T: pallet_mmr::Config> CheckForkEquivocationProof<<T as pallet_mmr::Config>
 			_,
 			_,
 			_,
-			// AncestryHasher<<T as pallet_mmr::Config>::Hashing>,
-			Hasher,
+			AncestryHasher<<T as pallet_mmr::Config>::Hashing>,
+			// Hasher,
 		>(
 			proof,
 			canonical_root,
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 6309df5dd6cc..dbeff7b3803c 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -303,7 +303,7 @@ where
 					sp_mmr_primitives::utils::NodesUtils::new(<pallet_mmr::Pallet<T>>::mmr_size())
 						.size();
 				// let expected_mmr_root = <pallet_mmr::Pallet<T>>::mmr_root();
-				let expected_mmr_root = mmr_root_hash_wrapper::<T>();
+				// let expected_mmr_root = mmr_root_hash_wrapper::<T>();
 				// let expected_mmr_root = sp_consensus_beefy::MmrRootHash::default();
 				let best_block_num = <frame_system::Pallet<T>>::block_number();
 				// if first_mmr_block_num is invalid, then presumably beefy is not active.
@@ -327,12 +327,7 @@ where
 				// replacement solution.
 				if !<T::CheckForkEquivocationProof as CheckForkEquivocationProof<
 					<T as pallet_mmr::Config>::Hashing,
-				>>::check_fork_equivocation_proof::<
-					_,
-					_,
-					_,
-					sp_mmr_primitives::utils::AncestryHasher<<T as pallet_mmr::Config>::Hashing>,
-				>(
+				>>::check_fork_equivocation_proof::<_, _, _>(
 					equivocation_proof,
 					mmr_size,
 					&expected_block_hash,
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 0d49947bfbce..e7493042963a 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -590,7 +590,7 @@ impl<AuthorityId> OnNewValidatorSet<AuthorityId> for () {
 /// Hook for checking fork equivocation proof.
 pub trait CheckForkEquivocationProof<H: Hash> {
 	// type NodeHash: HashOutput;
-	fn check_fork_equivocation_proof<Id, MsgHash, Header, Hasher>(
+	fn check_fork_equivocation_proof<Id, MsgHash, Header>(
 		proof: &ForkEquivocationProof<
 			Header::Number,
 			Id,
@@ -606,13 +606,12 @@ pub trait CheckForkEquivocationProof<H: Hash> {
 	where
 		Id: BeefyAuthorityId<MsgHash> + PartialEq,
 		MsgHash: Hash,
-		Header: HeaderT,
-		Hasher: mmr_lib::Merge<Item = H::Output>;
+		Header: HeaderT;
 }
 
 impl<H: Hash> CheckForkEquivocationProof<H> for () {
 	// type NodeHash = H256;
-	fn check_fork_equivocation_proof<Id, MsgHash, Header, Hasher>(
+	fn check_fork_equivocation_proof<Id, MsgHash, Header>(
 		_proof: &ForkEquivocationProof<
 			Header::Number,
 			Id,
@@ -629,7 +628,6 @@ impl<H: Hash> CheckForkEquivocationProof<H> for () {
 		Id: BeefyAuthorityId<MsgHash> + PartialEq,
 		MsgHash: Hash,
 		Header: HeaderT,
-		Hasher: mmr_lib::Merge<Item = H::Output>,
 	{
 		true
 	}

From e4caca43379f772541c89d66f3aaaa8ee7d554ec Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 4 Mar 2024 10:15:11 +0100
Subject: [PATCH 160/188] replace trait generic & pallet_mmr dependency

---
 substrate/frame/beefy-mmr/src/lib.rs          | 24 ++++---------------
 substrate/frame/beefy/src/equivocation.rs     |  6 ++---
 substrate/frame/beefy/src/lib.rs              | 14 +++++------
 .../primitives/consensus/beefy/src/lib.rs     | 10 ++++----
 4 files changed, 19 insertions(+), 35 deletions(-)

diff --git a/substrate/frame/beefy-mmr/src/lib.rs b/substrate/frame/beefy-mmr/src/lib.rs
index d13bc25dda12..e828e29410dc 100644
--- a/substrate/frame/beefy-mmr/src/lib.rs
+++ b/substrate/frame/beefy-mmr/src/lib.rs
@@ -174,21 +174,15 @@ where
 	}
 }
 
-fn mmr_root_hash_wrapper<T: pallet_mmr::Config>(
-) -> <<T as pallet_mmr::Config>::Hashing as sp_runtime::traits::Hash>::Output {
-	<pallet_mmr::Pallet<T>>::mmr_root()
-}
-
-impl<T: pallet_mmr::Config> CheckForkEquivocationProof<<T as pallet_mmr::Config>::Hashing>
-	for Pallet<T>
-{
+impl<T: pallet_mmr::Config> CheckForkEquivocationProof for Pallet<T> {
+	type HashT = <T as pallet_mmr::Config>::Hashing;
 	fn check_fork_equivocation_proof<Id, MsgHash, Header>(
 		proof: &ForkEquivocationProof<
 			Header::Number,
 			Id,
 			<Id as sp_application_crypto::RuntimeAppPublic>::Signature,
 			Header,
-			<<T as pallet_mmr::Config>::Hashing as Hash>::Output,
+			<Self::HashT as Hash>::Output,
 		>,
 		mmr_size: u64,
 		canonical_header_hash: &Header::Hash,
@@ -200,16 +194,8 @@ impl<T: pallet_mmr::Config> CheckForkEquivocationProof<<T as pallet_mmr::Config>
 		MsgHash: sp_runtime::traits::Hash,
 		Header: sp_runtime::traits::Header,
 	{
-		let canonical_root = mmr_root_hash_wrapper::<T>();
-		// let canonical_root = <pallet_mmr::Pallet<T>>::mmr_root();
-		sp_consensus_beefy::check_fork_equivocation_proof::<
-			_,
-			_,
-			_,
-			_,
-			AncestryHasher<<T as pallet_mmr::Config>::Hashing>,
-			// Hasher,
-		>(
+		let canonical_root = <pallet_mmr::Pallet<T>>::mmr_root();
+		sp_consensus_beefy::check_fork_equivocation_proof::<_, _, _, _, AncestryHasher<Self::HashT>>(
 			proof,
 			canonical_root,
 			mmr_size,
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index dbeff7b3803c..a5d24022aa85 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -141,7 +141,7 @@ pub enum EquivocationEvidenceFor<T: Config + pallet_mmr::Config> {
 			<T as Config>::BeefyId,
 			<<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
 			HeaderFor<T>,
-			<<T as pallet_mmr::Config>::Hashing as Hash>::Output,
+			<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof>::HashT as Hash>::Output,
 		>,
 		Vec<<T as Config>::KeyOwnerProof>,
 	),
@@ -325,9 +325,7 @@ where
 				// beefy light client at least once every 4096 blocks. See
 				// https://github.com/paritytech/polkadot-sdk/issues/1441 for
 				// replacement solution.
-				if !<T::CheckForkEquivocationProof as CheckForkEquivocationProof<
-					<T as pallet_mmr::Config>::Hashing,
-				>>::check_fork_equivocation_proof::<_, _, _>(
+				if !<T::CheckForkEquivocationProof as CheckForkEquivocationProof>::check_fork_equivocation_proof(
 					equivocation_proof,
 					mmr_size,
 					&expected_block_hash,
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 764d32b070f1..72cefb965701 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -41,8 +41,8 @@ use sp_staking::{offence::OffenceReportSystem, SessionIndex};
 use sp_std::prelude::*;
 
 use sp_consensus_beefy::{
-	AuthorityIndex, BeefyAuthorityId, ConsensusLog, OnNewValidatorSet, ValidatorSet,
-	VoteEquivocationProof, BEEFY_ENGINE_ID, GENESIS_AUTHORITY_SET_ID,
+	AuthorityIndex, BeefyAuthorityId, CheckForkEquivocationProof, ConsensusLog, OnNewValidatorSet,
+	ValidatorSet, VoteEquivocationProof, BEEFY_ENGINE_ID, GENESIS_AUTHORITY_SET_ID,
 };
 
 mod default_weights;
@@ -100,9 +100,7 @@ pub mod pallet {
 		type OnNewValidatorSet: OnNewValidatorSet<<Self as Config>::BeefyId>;
 
 		/// Hook for checking fork equivocation proofs
-		type CheckForkEquivocationProof: sp_consensus_beefy::CheckForkEquivocationProof<
-			<Self as pallet_mmr::Config>::Hashing,
-		>;
+		type CheckForkEquivocationProof: sp_consensus_beefy::CheckForkEquivocationProof;
 
 		/// Weights for this pallet.
 		type WeightInfo: WeightInfo;
@@ -319,7 +317,7 @@ pub mod pallet {
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
 					HeaderFor<T>,
-					<<T as pallet_mmr::Config>::Hashing as sp_runtime::traits::Hash>::Output,
+					<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof>::HashT as sp_runtime::traits::Hash>::Output,
 				>,
 			>,
 			key_owner_proofs: Vec<T::KeyOwnerProof>,
@@ -356,7 +354,7 @@ pub mod pallet {
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
 					HeaderFor<T>,
-					<<T as pallet_mmr::Config>::Hashing as sp_runtime::traits::Hash>::Output,
+					<<<T as Config>::CheckForkEquivocationProof as sp_consensus_beefy::CheckForkEquivocationProof>::HashT as sp_runtime::traits::Hash>::Output,
 				>,
 			>,
 			key_owner_proofs: Vec<T::KeyOwnerProof>,
@@ -427,7 +425,7 @@ impl<T: Config> Pallet<T> {
 			T::BeefyId,
 			<T::BeefyId as RuntimeAppPublic>::Signature,
 			HeaderFor<T>,
-			<<T as pallet_mmr::Config>::Hashing as sp_runtime::traits::Hash>::Output,
+			<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof>::HashT as sp_runtime::traits::Hash>::Output,
 		>,
 		key_owner_proofs: Vec<T::KeyOwnerProof>,
 	) -> Option<()> {
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index e7493042963a..3e328107ec03 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -588,7 +588,8 @@ impl<AuthorityId> OnNewValidatorSet<AuthorityId> for () {
 }
 
 /// Hook for checking fork equivocation proof.
-pub trait CheckForkEquivocationProof<H: Hash> {
+pub trait CheckForkEquivocationProof {
+	type HashT: Hash;
 	// type NodeHash: HashOutput;
 	fn check_fork_equivocation_proof<Id, MsgHash, Header>(
 		proof: &ForkEquivocationProof<
@@ -596,7 +597,7 @@ pub trait CheckForkEquivocationProof<H: Hash> {
 			Id,
 			<Id as RuntimeAppPublic>::Signature,
 			Header,
-			H::Output,
+			<Self::HashT as Hash>::Output,
 		>,
 		mmr_size: u64,
 		canonical_header_hash: &Header::Hash,
@@ -609,15 +610,16 @@ pub trait CheckForkEquivocationProof<H: Hash> {
 		Header: HeaderT;
 }
 
-impl<H: Hash> CheckForkEquivocationProof<H> for () {
+impl CheckForkEquivocationProof for () {
 	// type NodeHash = H256;
+	type HashT = Keccak256;
 	fn check_fork_equivocation_proof<Id, MsgHash, Header>(
 		_proof: &ForkEquivocationProof<
 			Header::Number,
 			Id,
 			<Id as RuntimeAppPublic>::Signature,
 			Header,
-			H::Output,
+			<Self::HashT as Hash>::Output,
 		>,
 		_mmr_size: u64,
 		_canonical_header_hash: &Header::Hash,

From c42d22a901284c70fa31165d051be9287f787f27 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 4 Mar 2024 12:11:04 +0100
Subject: [PATCH 161/188] remove all pallet_mmr::Config deps of pallet_beefy

Reported-by: Adrian Catangiu <adrian@parity.io>
---
 substrate/frame/beefy-mmr/src/lib.rs          | 28 ++++++++---
 substrate/frame/beefy/src/equivocation.rs     | 47 +++++--------------
 substrate/frame/beefy/src/lib.rs              | 18 ++++---
 .../primitives/consensus/beefy/src/lib.rs     | 14 ++----
 4 files changed, 52 insertions(+), 55 deletions(-)

diff --git a/substrate/frame/beefy-mmr/src/lib.rs b/substrate/frame/beefy-mmr/src/lib.rs
index e828e29410dc..db18c0220756 100644
--- a/substrate/frame/beefy-mmr/src/lib.rs
+++ b/substrate/frame/beefy-mmr/src/lib.rs
@@ -174,7 +174,7 @@ where
 	}
 }
 
-impl<T: pallet_mmr::Config> CheckForkEquivocationProof for Pallet<T> {
+impl<T: pallet_mmr::Config> CheckForkEquivocationProof<pallet_beefy::Error<T>> for Pallet<T> {
 	type HashT = <T as pallet_mmr::Config>::Hashing;
 	fn check_fork_equivocation_proof<Id, MsgHash, Header>(
 		proof: &ForkEquivocationProof<
@@ -184,25 +184,41 @@ impl<T: pallet_mmr::Config> CheckForkEquivocationProof for Pallet<T> {
 			Header,
 			<Self::HashT as Hash>::Output,
 		>,
-		mmr_size: u64,
 		canonical_header_hash: &Header::Hash,
-		first_mmr_block_num: Header::Number,
 		best_block_num: Header::Number,
-	) -> bool
+	) -> Result<bool, pallet_beefy::Error<T>>
 	where
 		Id: sp_consensus_beefy::BeefyAuthorityId<MsgHash> + PartialEq,
 		MsgHash: sp_runtime::traits::Hash,
 		Header: sp_runtime::traits::Header,
 	{
 		let canonical_root = <pallet_mmr::Pallet<T>>::mmr_root();
-		sp_consensus_beefy::check_fork_equivocation_proof::<_, _, _, _, AncestryHasher<Self::HashT>>(
+		let mmr_size =
+			sp_mmr_primitives::utils::NodesUtils::new(<pallet_mmr::Pallet<T>>::mmr_size()).size();
+		// if first_mmr_block_num is invalid, then presumably beefy is not active.
+		// TODO: should we slash in this case?
+		let first_mmr_block_num = {
+			let mmr_leaf_count = <pallet_mmr::Pallet<T>>::mmr_size();
+			sp_mmr_primitives::utils::first_mmr_block_num::<Header>(best_block_num, mmr_leaf_count)
+				.map_err(|_| pallet_beefy::Error::<T>::InvalidForkEquivocationProof)?
+		};
+		if !sp_consensus_beefy::check_fork_equivocation_proof::<
+			_,
+			_,
+			_,
+			_,
+			AncestryHasher<Self::HashT>,
+		>(
 			proof,
 			canonical_root,
 			mmr_size,
 			canonical_header_hash,
 			first_mmr_block_num,
 			best_block_num,
-		)
+		) {
+			return Ok(false)
+		}
+		Ok(true)
 	}
 }
 
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index a5d24022aa85..16f5370408da 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -126,7 +126,7 @@ where
 pub struct EquivocationReportSystem<T, R, P, L>(sp_std::marker::PhantomData<(T, R, P, L)>);
 
 /// Equivocation evidence convenience alias.
-pub enum EquivocationEvidenceFor<T: Config + pallet_mmr::Config> {
+pub enum EquivocationEvidenceFor<T: Config> {
 	VoteEquivocationProof(
 		VoteEquivocationProof<
 			BlockNumberFor<T>,
@@ -137,12 +137,14 @@ pub enum EquivocationEvidenceFor<T: Config + pallet_mmr::Config> {
 	),
 	ForkEquivocationProof(
 		ForkEquivocationProof<
-			BlockNumberFor<T>,
-			<T as Config>::BeefyId,
-			<<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
-			HeaderFor<T>,
-			<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof>::HashT as Hash>::Output,
-		>,
+				BlockNumberFor<T>,
+				<T as Config>::BeefyId,
+				<<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
+				HeaderFor<T>,
+				<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof<
+					Error<T>,
+				>>::HashT as Hash>::Output,
+			>,
 		Vec<<T as Config>::KeyOwnerProof>,
 	),
 }
@@ -150,10 +152,7 @@ pub enum EquivocationEvidenceFor<T: Config + pallet_mmr::Config> {
 impl<T, R, P, L> OffenceReportSystem<Option<T::AccountId>, EquivocationEvidenceFor<T>>
 	for EquivocationReportSystem<T, R, P, L>
 where
-	T: Config
-		+ pallet_authorship::Config
-		+ pallet_mmr::Config
-		+ frame_system::offchain::SendTransactionTypes<Call<T>>,
+	T: Config + pallet_authorship::Config + frame_system::offchain::SendTransactionTypes<Call<T>>,
 	R: ReportOffence<
 		T::AccountId,
 		P::IdentificationTuple,
@@ -299,23 +298,9 @@ where
 			EquivocationEvidenceFor::ForkEquivocationProof(equivocation_proof, _) => {
 				let block_number = equivocation_proof.commitment.block_number;
 				let expected_block_hash = <frame_system::Pallet<T>>::block_hash(block_number);
-				let mmr_size =
-					sp_mmr_primitives::utils::NodesUtils::new(<pallet_mmr::Pallet<T>>::mmr_size())
-						.size();
-				// let expected_mmr_root = <pallet_mmr::Pallet<T>>::mmr_root();
 				// let expected_mmr_root = mmr_root_hash_wrapper::<T>();
 				// let expected_mmr_root = sp_consensus_beefy::MmrRootHash::default();
 				let best_block_num = <frame_system::Pallet<T>>::block_number();
-				// if first_mmr_block_num is invalid, then presumably beefy is not active.
-				// TODO: should we slash in this case?
-				let first_mmr_block_num = {
-					let mmr_leaf_count = <pallet_mmr::Pallet<T>>::mmr_size();
-					sp_mmr_primitives::utils::first_mmr_block_num::<HeaderFor<T>>(
-						best_block_num,
-						mmr_leaf_count,
-					)
-					.map_err(|_| Error::<T>::InvalidForkEquivocationProof)?
-				};
 
 				// Validate equivocation proof (check commitment is to unexpected payload and
 				// signatures are valid).
@@ -325,14 +310,13 @@ where
 				// beefy light client at least once every 4096 blocks. See
 				// https://github.com/paritytech/polkadot-sdk/issues/1441 for
 				// replacement solution.
-				if !<T::CheckForkEquivocationProof as CheckForkEquivocationProof>::check_fork_equivocation_proof(
+				match <T::CheckForkEquivocationProof as CheckForkEquivocationProof<Error<T>>>::check_fork_equivocation_proof(
 					equivocation_proof,
-					mmr_size,
 					&expected_block_hash,
-					first_mmr_block_num,
 					best_block_num,
 				) {
-					return Err(Error::<T>::InvalidForkEquivocationProof.into())
+					Ok(true) => {},
+					_ => return Err(Error::<T>::InvalidForkEquivocationProof.into())
 				}
 			},
 		}
@@ -351,11 +335,6 @@ where
 	}
 }
 
-fn mmr_root_hash_wrapper<T: pallet_mmr::Config>(
-) -> <<T as pallet_mmr::Config>::Hashing as sp_runtime::traits::Hash>::Output {
-	<pallet_mmr::Pallet<T>>::mmr_root()
-}
-
 /// Methods for the `ValidateUnsigned` implementation:
 /// It restricts calls to `report_vote_equivocation_unsigned` to local calls (i.e. extrinsics
 /// generated on this node) or that already in a block. This guarantees that only block authors can
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 72cefb965701..d6fcadd1a859 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -33,7 +33,7 @@ use frame_system::{
 use log;
 use sp_runtime::{
 	generic::DigestItem,
-	traits::{IsMember, Member, One},
+	traits::{Hash, IsMember, Member, One},
 	RuntimeAppPublic,
 };
 use sp_session::{GetSessionNumber, GetValidatorCount};
@@ -66,7 +66,7 @@ pub mod pallet {
 	use sp_consensus_beefy::ForkEquivocationProof;
 
 	#[pallet::config]
-	pub trait Config: frame_system::Config + pallet_mmr::Config {
+	pub trait Config: frame_system::Config {
 		/// Authority identifier type
 		type BeefyId: Member
 			+ Parameter
@@ -100,7 +100,9 @@ pub mod pallet {
 		type OnNewValidatorSet: OnNewValidatorSet<<Self as Config>::BeefyId>;
 
 		/// Hook for checking fork equivocation proofs
-		type CheckForkEquivocationProof: sp_consensus_beefy::CheckForkEquivocationProof;
+		type CheckForkEquivocationProof: sp_consensus_beefy::CheckForkEquivocationProof<
+			pallet::Error<Self>,
+		>;
 
 		/// Weights for this pallet.
 		type WeightInfo: WeightInfo;
@@ -317,7 +319,9 @@ pub mod pallet {
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
 					HeaderFor<T>,
-					<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof>::HashT as sp_runtime::traits::Hash>::Output,
+					<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof<
+						Error<T>,
+					>>::HashT as Hash>::Output,
 				>,
 			>,
 			key_owner_proofs: Vec<T::KeyOwnerProof>,
@@ -354,7 +358,7 @@ pub mod pallet {
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
 					HeaderFor<T>,
-					<<<T as Config>::CheckForkEquivocationProof as sp_consensus_beefy::CheckForkEquivocationProof>::HashT as sp_runtime::traits::Hash>::Output,
+					<<<T as Config>::CheckForkEquivocationProof as sp_consensus_beefy::CheckForkEquivocationProof<Error<T>>>::HashT as sp_runtime::traits::Hash>::Output,
 				>,
 			>,
 			key_owner_proofs: Vec<T::KeyOwnerProof>,
@@ -425,7 +429,9 @@ impl<T: Config> Pallet<T> {
 			T::BeefyId,
 			<T::BeefyId as RuntimeAppPublic>::Signature,
 			HeaderFor<T>,
-			<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof>::HashT as sp_runtime::traits::Hash>::Output,
+			<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof<
+				pallet::Error<T>,
+			>>::HashT as Hash>::Output,
 		>,
 		key_owner_proofs: Vec<T::KeyOwnerProof>,
 	) -> Option<()> {
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 3e328107ec03..2f9081d77817 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -588,7 +588,7 @@ impl<AuthorityId> OnNewValidatorSet<AuthorityId> for () {
 }
 
 /// Hook for checking fork equivocation proof.
-pub trait CheckForkEquivocationProof {
+pub trait CheckForkEquivocationProof<Err> {
 	type HashT: Hash;
 	// type NodeHash: HashOutput;
 	fn check_fork_equivocation_proof<Id, MsgHash, Header>(
@@ -599,18 +599,16 @@ pub trait CheckForkEquivocationProof {
 			Header,
 			<Self::HashT as Hash>::Output,
 		>,
-		mmr_size: u64,
 		canonical_header_hash: &Header::Hash,
-		first_mmr_block_num: Header::Number,
 		best_block_num: Header::Number,
-	) -> bool
+	) -> Result<bool, Err>
 	where
 		Id: BeefyAuthorityId<MsgHash> + PartialEq,
 		MsgHash: Hash,
 		Header: HeaderT;
 }
 
-impl CheckForkEquivocationProof for () {
+impl<Err> CheckForkEquivocationProof<Err> for () {
 	// type NodeHash = H256;
 	type HashT = Keccak256;
 	fn check_fork_equivocation_proof<Id, MsgHash, Header>(
@@ -621,17 +619,15 @@ impl CheckForkEquivocationProof for () {
 			Header,
 			<Self::HashT as Hash>::Output,
 		>,
-		_mmr_size: u64,
 		_canonical_header_hash: &Header::Hash,
-		_first_mmr_block_num: Header::Number,
 		_best_block_num: Header::Number,
-	) -> bool
+	) -> Result<bool, Err>
 	where
 		Id: BeefyAuthorityId<MsgHash> + PartialEq,
 		MsgHash: Hash,
 		Header: HeaderT,
 	{
-		true
+		Ok(true)
 	}
 }
 

From 95ab3fe8dd0c5e48a3ee23781c37671df639436a Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 4 Mar 2024 13:23:22 +0100
Subject: [PATCH 162/188] move all fork equiv. checking to assoc fn of trait

improved compartmentalization
---
 substrate/frame/beefy-mmr/src/lib.rs          | 31 ++++++++------
 substrate/frame/beefy/src/equivocation.rs     | 40 +++++++------------
 substrate/frame/beefy/src/lib.rs              |  5 ++-
 .../primitives/consensus/beefy/src/lib.rs     | 33 ++++++++-------
 4 files changed, 55 insertions(+), 54 deletions(-)

diff --git a/substrate/frame/beefy-mmr/src/lib.rs b/substrate/frame/beefy-mmr/src/lib.rs
index db18c0220756..c84aae5151fd 100644
--- a/substrate/frame/beefy-mmr/src/lib.rs
+++ b/substrate/frame/beefy-mmr/src/lib.rs
@@ -33,7 +33,7 @@
 //!
 //! and thanks to versioning can be easily updated in the future.
 
-use sp_runtime::traits::{Convert, Hash, Member};
+use sp_runtime::traits::{Convert, Hash, Header as HeaderT, Member};
 use sp_std::prelude::*;
 
 use codec::Decode;
@@ -44,7 +44,7 @@ use sp_consensus_beefy::{
 };
 
 use frame_support::{crypto::ecdsa::ECDSAExt, traits::Get};
-use frame_system::pallet_prelude::BlockNumberFor;
+use frame_system::pallet_prelude::{BlockNumberFor, HeaderFor};
 
 pub use pallet::*;
 
@@ -174,45 +174,50 @@ where
 	}
 }
 
-impl<T: pallet_mmr::Config> CheckForkEquivocationProof<pallet_beefy::Error<T>> for Pallet<T> {
+impl<T: pallet_mmr::Config> CheckForkEquivocationProof<pallet_beefy::Error<T>, HeaderFor<T>>
+	for Pallet<T>
+{
 	type HashT = <T as pallet_mmr::Config>::Hashing;
-	fn check_fork_equivocation_proof<Id, MsgHash, Header>(
+	fn check_fork_equivocation_proof<Id, MsgHash>(
 		proof: &ForkEquivocationProof<
-			Header::Number,
+			<HeaderFor<T> as HeaderT>::Number,
 			Id,
 			<Id as sp_application_crypto::RuntimeAppPublic>::Signature,
-			Header,
+			HeaderFor<T>,
 			<Self::HashT as Hash>::Output,
 		>,
-		canonical_header_hash: &Header::Hash,
-		best_block_num: Header::Number,
 	) -> Result<bool, pallet_beefy::Error<T>>
 	where
 		Id: sp_consensus_beefy::BeefyAuthorityId<MsgHash> + PartialEq,
 		MsgHash: sp_runtime::traits::Hash,
-		Header: sp_runtime::traits::Header,
 	{
 		let canonical_root = <pallet_mmr::Pallet<T>>::mmr_root();
 		let mmr_size =
 			sp_mmr_primitives::utils::NodesUtils::new(<pallet_mmr::Pallet<T>>::mmr_size()).size();
 		// if first_mmr_block_num is invalid, then presumably beefy is not active.
 		// TODO: should we slash in this case?
+		let block_number = proof.commitment.block_number;
+		let canonical_header_hash = <frame_system::Pallet<T>>::block_hash(block_number);
+		let best_block_num = <frame_system::Pallet<T>>::block_number();
 		let first_mmr_block_num = {
 			let mmr_leaf_count = <pallet_mmr::Pallet<T>>::mmr_size();
-			sp_mmr_primitives::utils::first_mmr_block_num::<Header>(best_block_num, mmr_leaf_count)
-				.map_err(|_| pallet_beefy::Error::<T>::InvalidForkEquivocationProof)?
+			sp_mmr_primitives::utils::first_mmr_block_num::<HeaderFor<T>>(
+				best_block_num,
+				mmr_leaf_count,
+			)
+			.map_err(|_| pallet_beefy::Error::<T>::InvalidForkEquivocationProof)?
 		};
 		if !sp_consensus_beefy::check_fork_equivocation_proof::<
 			_,
 			_,
-			_,
+			HeaderFor<T>,
 			_,
 			AncestryHasher<Self::HashT>,
 		>(
 			proof,
 			canonical_root,
 			mmr_size,
-			canonical_header_hash,
+			&canonical_header_hash,
 			first_mmr_block_num,
 			best_block_num,
 		) {
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 16f5370408da..19f79796c5c3 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -137,14 +137,16 @@ pub enum EquivocationEvidenceFor<T: Config> {
 	),
 	ForkEquivocationProof(
 		ForkEquivocationProof<
-				BlockNumberFor<T>,
-				<T as Config>::BeefyId,
-				<<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
+			BlockNumberFor<T>,
+			<T as Config>::BeefyId,
+			<<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
+			// <<T as frame_system::Config>::Block as Block>::Header,
+			HeaderFor<T>,
+			<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof<
+				Error<T>,
 				HeaderFor<T>,
-				<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof<
-					Error<T>,
-				>>::HashT as Hash>::Output,
-			>,
+			>>::HashT as Hash>::Output,
+		>,
 		Vec<<T as Config>::KeyOwnerProof>,
 	),
 }
@@ -296,27 +298,15 @@ where
 				}
 			},
 			EquivocationEvidenceFor::ForkEquivocationProof(equivocation_proof, _) => {
-				let block_number = equivocation_proof.commitment.block_number;
-				let expected_block_hash = <frame_system::Pallet<T>>::block_hash(block_number);
-				// let expected_mmr_root = mmr_root_hash_wrapper::<T>();
-				// let expected_mmr_root = sp_consensus_beefy::MmrRootHash::default();
-				let best_block_num = <frame_system::Pallet<T>>::block_number();
-
 				// Validate equivocation proof (check commitment is to unexpected payload and
 				// signatures are valid).
-				// NOTE: Fork equivocation proof currently only prevents attacks
-				// assuming 2/3rds of validators honestly participate in BEEFY
-				// finalization and at least one honest relayer can update the
-				// beefy light client at least once every 4096 blocks. See
-				// https://github.com/paritytech/polkadot-sdk/issues/1441 for
-				// replacement solution.
-				match <T::CheckForkEquivocationProof as CheckForkEquivocationProof<Error<T>>>::check_fork_equivocation_proof(
-					equivocation_proof,
-					&expected_block_hash,
-					best_block_num,
-				) {
+				match <T::CheckForkEquivocationProof as CheckForkEquivocationProof<
+					Error<T>,
+					HeaderFor<T>,
+				>>::check_fork_equivocation_proof(equivocation_proof)
+				{
 					Ok(true) => {},
-					_ => return Err(Error::<T>::InvalidForkEquivocationProof.into())
+					_ => return Err(Error::<T>::InvalidForkEquivocationProof.into()),
 				}
 			},
 		}
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index d6fcadd1a859..53b71f5b41b4 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -102,6 +102,7 @@ pub mod pallet {
 		/// Hook for checking fork equivocation proofs
 		type CheckForkEquivocationProof: sp_consensus_beefy::CheckForkEquivocationProof<
 			pallet::Error<Self>,
+			HeaderFor<Self>,
 		>;
 
 		/// Weights for this pallet.
@@ -321,6 +322,7 @@ pub mod pallet {
 					HeaderFor<T>,
 					<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof<
 						Error<T>,
+						HeaderFor<T>,
 					>>::HashT as Hash>::Output,
 				>,
 			>,
@@ -358,7 +360,7 @@ pub mod pallet {
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
 					HeaderFor<T>,
-					<<<T as Config>::CheckForkEquivocationProof as sp_consensus_beefy::CheckForkEquivocationProof<Error<T>>>::HashT as sp_runtime::traits::Hash>::Output,
+					<<<T as Config>::CheckForkEquivocationProof as sp_consensus_beefy::CheckForkEquivocationProof<Error<T>, HeaderFor<T>>>::HashT as sp_runtime::traits::Hash>::Output,
 				>,
 			>,
 			key_owner_proofs: Vec<T::KeyOwnerProof>,
@@ -431,6 +433,7 @@ impl<T: Config> Pallet<T> {
 			HeaderFor<T>,
 			<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof<
 				pallet::Error<T>,
+				HeaderFor<T>,
 			>>::HashT as Hash>::Output,
 		>,
 		key_owner_proofs: Vec<T::KeyOwnerProof>,
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 2f9081d77817..9516aa737553 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -588,44 +588,47 @@ impl<AuthorityId> OnNewValidatorSet<AuthorityId> for () {
 }
 
 /// Hook for checking fork equivocation proof.
-pub trait CheckForkEquivocationProof<Err> {
+pub trait CheckForkEquivocationProof<Err, Header: HeaderT> {
+	/// Associated hash type for hashing ancestry proof.
 	type HashT: Hash;
-	// type NodeHash: HashOutput;
-	fn check_fork_equivocation_proof<Id, MsgHash, Header>(
+	/// Validate equivocation proof (check commitment is to unexpected payload and
+	/// signatures are valid).
+	/// NOTE: Fork equivocation proof currently only prevents attacks
+	/// assuming 2/3rds of validators honestly participate in BEEFY
+	/// finalization and at least one honest relayer can update the
+	/// beefy light client at least once every 4096 blocks. See
+	/// https://github.com/paritytech/polkadot-sdk/issues/1441 for
+	/// replacement solution.
+	fn check_fork_equivocation_proof<Id, MsgHash>(
 		proof: &ForkEquivocationProof<
-			Header::Number,
+			<Header as HeaderT>::Number,
 			Id,
 			<Id as RuntimeAppPublic>::Signature,
 			Header,
 			<Self::HashT as Hash>::Output,
 		>,
-		canonical_header_hash: &Header::Hash,
-		best_block_num: Header::Number,
 	) -> Result<bool, Err>
 	where
 		Id: BeefyAuthorityId<MsgHash> + PartialEq,
-		MsgHash: Hash,
-		Header: HeaderT;
+		MsgHash: Hash;
 }
 
-impl<Err> CheckForkEquivocationProof<Err> for () {
-	// type NodeHash = H256;
+impl<Err, Header: HeaderT> CheckForkEquivocationProof<Err, Header> for () {
 	type HashT = Keccak256;
-	fn check_fork_equivocation_proof<Id, MsgHash, Header>(
+	fn check_fork_equivocation_proof<Id, MsgHash>(
 		_proof: &ForkEquivocationProof<
-			Header::Number,
+			<Header as HeaderT>::Number,
 			Id,
 			<Id as RuntimeAppPublic>::Signature,
 			Header,
 			<Self::HashT as Hash>::Output,
 		>,
-		_canonical_header_hash: &Header::Hash,
-		_best_block_num: Header::Number,
+		// _canonical_header_hash: &<Self::Header as HeaderT>::Hash,
+		// _best_block_num: <Self::Header as HeaderT>::Number,
 	) -> Result<bool, Err>
 	where
 		Id: BeefyAuthorityId<MsgHash> + PartialEq,
 		MsgHash: Hash,
-		Header: HeaderT,
 	{
 		Ok(true)
 	}

From a543c2811129b44f48154e43cb83d25e75108193 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 4 Mar 2024 13:52:49 +0100
Subject: [PATCH 163/188] remove contrived degree of freedom: BlockNumber

The block number in ForkEquivocationProof was in fact not independent,
but should be strictly coupled to the header's associated type
---
 polkadot/node/service/src/fake_runtime_api.rs |  2 +-
 polkadot/runtime/rococo/src/lib.rs            |  2 +-
 polkadot/runtime/test-runtime/src/lib.rs      |  2 +-
 polkadot/runtime/westend/src/lib.rs           |  2 +-
 substrate/bin/node/runtime/src/lib.rs         |  2 +-
 .../beefy/src/communication/fisherman.rs      |  2 +-
 substrate/client/consensus/beefy/src/tests.rs | 19 +++-------------
 substrate/frame/beefy-mmr/src/lib.rs          |  1 -
 substrate/frame/beefy/src/equivocation.rs     |  2 --
 substrate/frame/beefy/src/lib.rs              |  3 ---
 .../primitives/consensus/beefy/src/lib.rs     | 22 +++++--------------
 .../consensus/beefy/src/test_utils.rs         | 22 +++++++++----------
 12 files changed, 26 insertions(+), 55 deletions(-)

diff --git a/polkadot/node/service/src/fake_runtime_api.rs b/polkadot/node/service/src/fake_runtime_api.rs
index 2d3d3cc89537..7547e9c996de 100644
--- a/polkadot/node/service/src/fake_runtime_api.rs
+++ b/polkadot/node/service/src/fake_runtime_api.rs
@@ -252,7 +252,7 @@ sp_api::impl_runtime_apis! {
 		}
 
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
-			_: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, <Block as BlockT>::Header, Hash>,
+			_: beefy_primitives::ForkEquivocationProof<BeefyId, BeefySignature, <Block as BlockT>::Header, Hash>,
 			_: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
 			unimplemented!()
diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs
index ed33a91e772c..75b96c369632 100644
--- a/polkadot/runtime/rococo/src/lib.rs
+++ b/polkadot/runtime/rococo/src/lib.rs
@@ -2015,7 +2015,7 @@ sp_api::impl_runtime_apis! {
 		}
 
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
-			fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, Header, Hash>,
+			fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BeefyId, BeefySignature, Header, Hash>,
 			key_owner_proofs: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
 			let key_owner_proofs = key_owner_proofs.iter().cloned().map(|p| p.decode()).collect::<Option<Vec<_>>>()?;
diff --git a/polkadot/runtime/test-runtime/src/lib.rs b/polkadot/runtime/test-runtime/src/lib.rs
index 74f04a4845db..8532b4e255c2 100644
--- a/polkadot/runtime/test-runtime/src/lib.rs
+++ b/polkadot/runtime/test-runtime/src/lib.rs
@@ -981,7 +981,7 @@ sp_api::impl_runtime_apis! {
 		}
 
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
-			_fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, Header, Hash>,
+			_fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BeefyId, BeefySignature, Header, Hash>,
 			_key_owner_proofs: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
 			None
diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs
index c1e90a23ffe3..e1c4dd407733 100644
--- a/polkadot/runtime/westend/src/lib.rs
+++ b/polkadot/runtime/westend/src/lib.rs
@@ -2039,7 +2039,7 @@ sp_api::impl_runtime_apis! {
 		}
 
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
-			fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, Header, Hash>,
+			fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BeefyId, BeefySignature, Header, Hash>,
 			key_owner_proofs: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
 			let key_owner_proofs = key_owner_proofs.iter().cloned().map(|p| p.decode()).collect::<Option<Vec<_>>>()?;
diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs
index 77131ee8430e..9cda6dc085c7 100644
--- a/substrate/bin/node/runtime/src/lib.rs
+++ b/substrate/bin/node/runtime/src/lib.rs
@@ -2812,7 +2812,7 @@ impl_runtime_apis! {
 		}
 
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
-			fork_equivocation_proof: sp_consensus_beefy::ForkEquivocationProof<BlockNumber, BeefyId, BeefySignature, Header, Hash>,
+			fork_equivocation_proof: sp_consensus_beefy::ForkEquivocationProof<BeefyId, BeefySignature, Header, Hash>,
 			key_owner_proofs: Vec<sp_consensus_beefy::OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
 			let key_owner_proofs = key_owner_proofs.iter().cloned().map(|p| p.decode()).collect::<Option<Vec<_>>>()?;
diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 56121f5a634a..99fe2c78ac8f 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -97,7 +97,7 @@ where
 
 	pub(crate) fn report_fork_equivocation(
 		&self,
-		proof: ForkEquivocationProof<NumberFor<B>, AuthorityId, Signature, B::Header, MmrRootHash>,
+		proof: ForkEquivocationProof<AuthorityId, Signature, B::Header, MmrRootHash>,
 	) -> Result<bool, Error> {
 		let best_block_number = self.backend.blockchain().info().best_number;
 		let best_block_hash = self.backend.blockchain().info().best_hash;
diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index 19658b374cf2..fdad9b58a47a 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -257,21 +257,8 @@ pub(crate) struct TestApi {
 	pub mmr_root_hash: MmrRootHash,
 	pub reported_vote_equivocations:
 		Option<Arc<Mutex<Vec<VoteEquivocationProof<NumberFor<Block>, AuthorityId, Signature>>>>>,
-	pub reported_fork_equivocations: Option<
-		Arc<
-			Mutex<
-				Vec<
-					ForkEquivocationProof<
-						NumberFor<Block>,
-						AuthorityId,
-						Signature,
-						Header,
-						MmrRootHash,
-					>,
-				>,
-			>,
-		>,
-	>,
+	pub reported_fork_equivocations:
+		Option<Arc<Mutex<Vec<ForkEquivocationProof<AuthorityId, Signature, Header, MmrRootHash>>>>>,
 }
 
 impl TestApi {
@@ -340,7 +327,7 @@ sp_api::mock_impl_runtime_apis! {
 		}
 
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
-			proof: ForkEquivocationProof<NumberFor<Block>, AuthorityId, Signature, Header, MmrRootHash>,
+			proof: ForkEquivocationProof<AuthorityId, Signature, Header, MmrRootHash>,
 			_dummy: Vec<OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
 			if let Some(equivocations_buf) = self.inner.reported_fork_equivocations.as_ref() {
diff --git a/substrate/frame/beefy-mmr/src/lib.rs b/substrate/frame/beefy-mmr/src/lib.rs
index c84aae5151fd..284df6e4718a 100644
--- a/substrate/frame/beefy-mmr/src/lib.rs
+++ b/substrate/frame/beefy-mmr/src/lib.rs
@@ -180,7 +180,6 @@ impl<T: pallet_mmr::Config> CheckForkEquivocationProof<pallet_beefy::Error<T>, H
 	type HashT = <T as pallet_mmr::Config>::Hashing;
 	fn check_fork_equivocation_proof<Id, MsgHash>(
 		proof: &ForkEquivocationProof<
-			<HeaderFor<T> as HeaderT>::Number,
 			Id,
 			<Id as sp_application_crypto::RuntimeAppPublic>::Signature,
 			HeaderFor<T>,
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 19f79796c5c3..f6d11f77f9fb 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -137,10 +137,8 @@ pub enum EquivocationEvidenceFor<T: Config> {
 	),
 	ForkEquivocationProof(
 		ForkEquivocationProof<
-			BlockNumberFor<T>,
 			<T as Config>::BeefyId,
 			<<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
-			// <<T as frame_system::Config>::Block as Block>::Header,
 			HeaderFor<T>,
 			<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof<
 				Error<T>,
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 53b71f5b41b4..c8aa4aedc6fe 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -316,7 +316,6 @@ pub mod pallet {
 			origin: OriginFor<T>,
 			equivocation_proof: Box<
 				ForkEquivocationProof<
-					BlockNumberFor<T>,
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
 					HeaderFor<T>,
@@ -356,7 +355,6 @@ pub mod pallet {
 			origin: OriginFor<T>,
 			equivocation_proof: Box<
 				ForkEquivocationProof<
-					BlockNumberFor<T>,
 					T::BeefyId,
 					<T::BeefyId as RuntimeAppPublic>::Signature,
 					HeaderFor<T>,
@@ -427,7 +425,6 @@ impl<T: Config> Pallet<T> {
 	/// to the pool. Only useful in an offchain context.
 	pub fn submit_unsigned_fork_equivocation_report(
 		fork_equivocation_proof: sp_consensus_beefy::ForkEquivocationProof<
-			BlockNumberFor<T>,
 			T::BeefyId,
 			<T::BeefyId as RuntimeAppPublic>::Signature,
 			HeaderFor<T>,
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 9516aa737553..a62ee900942e 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -331,10 +331,10 @@ impl<Number, Id, Signature> VoteEquivocationProof<Number, Id, Signature> {
 /// This proof shows commitment signed on a different fork.
 /// See [check_fork_equivocation_proof] for proof validity conditions.
 #[derive(Clone, Debug, Decode, Encode, PartialEq, TypeInfo)]
-pub struct ForkEquivocationProof<Number, Id, Signature, Header, Hash> {
+pub struct ForkEquivocationProof<Id, Signature, Header: HeaderT, Hash> {
 	/// Commitment for a block on a different fork than one at the same height in
 	/// the chain where this proof is submitted.
-	pub commitment: Commitment<Number>,
+	pub commitment: Commitment<Header::Number>,
 	/// Signatures on this block
 	pub signatories: Vec<(Id, Signature)>,
 	/// Canonical header at the same height as `commitment.block_number`.
@@ -343,15 +343,13 @@ pub struct ForkEquivocationProof<Number, Id, Signature, Header, Hash> {
 	pub ancestry_proof: Option<AncestryProof<Hash>>,
 }
 
-impl<Number, Id, Signature, H: HeaderT, Hash>
-	ForkEquivocationProof<Number, Id, Signature, H, Hash>
-{
+impl<Id, Signature, H: HeaderT, Hash> ForkEquivocationProof<Id, Signature, H, Hash> {
 	/// Returns the authority ids of the misbehaving voters.
 	pub fn offender_ids(&self) -> Vec<&Id> {
 		self.signatories.iter().map(|(id, _)| id).collect()
 	}
 	/// Returns the round number at which the infringement occurred.
-	pub fn round_number(&self) -> &Number {
+	pub fn round_number(&self) -> &H::Number {
 		&self.commitment.block_number
 	}
 	/// Returns the set id at which the infringement occurred.
@@ -521,13 +519,7 @@ where
 /// incorrect block implies validators will only sign blocks they *know* will be finalized by
 /// GRANDPA.
 pub fn check_fork_equivocation_proof<Id, MsgHash, Header, NodeHash, Hasher>(
-	proof: &ForkEquivocationProof<
-		Header::Number,
-		Id,
-		<Id as RuntimeAppPublic>::Signature,
-		Header,
-		NodeHash,
-	>,
+	proof: &ForkEquivocationProof<Id, <Id as RuntimeAppPublic>::Signature, Header, NodeHash>,
 	canonical_root: Hasher::Item,
 	mmr_size: u64,
 	canonical_header_hash: &Header::Hash,
@@ -601,7 +593,6 @@ pub trait CheckForkEquivocationProof<Err, Header: HeaderT> {
 	/// replacement solution.
 	fn check_fork_equivocation_proof<Id, MsgHash>(
 		proof: &ForkEquivocationProof<
-			<Header as HeaderT>::Number,
 			Id,
 			<Id as RuntimeAppPublic>::Signature,
 			Header,
@@ -617,7 +608,6 @@ impl<Err, Header: HeaderT> CheckForkEquivocationProof<Err, Header> for () {
 	type HashT = Keccak256;
 	fn check_fork_equivocation_proof<Id, MsgHash>(
 		_proof: &ForkEquivocationProof<
-			<Header as HeaderT>::Number,
 			Id,
 			<Id as RuntimeAppPublic>::Signature,
 			Header,
@@ -694,7 +684,7 @@ sp_api::decl_runtime_apis! {
 		/// hardcoded to return `None`). Only useful in an offchain context.
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
 			fork_equivocation_proof:
-				ForkEquivocationProof<NumberFor<Block>, AuthorityId, <AuthorityId as RuntimeAppPublic>::Signature, Block::Header, Hash>,
+				ForkEquivocationProof<AuthorityId, <AuthorityId as RuntimeAppPublic>::Signature, Block::Header, Hash>,
 			key_owner_proofs: Vec<OpaqueKeyOwnershipProof>,
 		) -> Option<()>;
 
diff --git a/substrate/primitives/consensus/beefy/src/test_utils.rs b/substrate/primitives/consensus/beefy/src/test_utils.rs
index b1c797a382d3..72be3731e671 100644
--- a/substrate/primitives/consensus/beefy/src/test_utils.rs
+++ b/substrate/primitives/consensus/beefy/src/test_utils.rs
@@ -23,7 +23,7 @@ use crate::{
 };
 use sp_application_crypto::{AppCrypto, AppPair, RuntimeAppPublic, Wraps};
 use sp_core::{ecdsa, Pair};
-use sp_runtime::traits::Hash;
+use sp_runtime::traits::{BlockNumber, Hash, Header as HeaderT};
 
 use codec::Encode;
 use sp_mmr_primitives::AncestryProof;
@@ -140,12 +140,12 @@ impl From<Keyring<ecdsa_crypto::AuthorityId>> for ecdsa_crypto::Public {
 }
 
 /// Create a new `VoteMessage` from commitment primitives and keyring
-fn signed_vote(
-	block_number: u64,
+fn signed_vote<Number: BlockNumber>(
+	block_number: Number,
 	payload: Payload,
 	validator_set_id: ValidatorSetId,
 	keyring: &Keyring<ecdsa_crypto::AuthorityId>,
-) -> VoteMessage<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature> {
+) -> VoteMessage<Number, ecdsa_crypto::Public, ecdsa_crypto::Signature> {
 	let commitment = Commitment { validator_set_id, block_number, payload };
 	let signature = keyring.sign(&commitment.encode());
 	VoteMessage { commitment, id: keyring.public(), signature }
@@ -162,12 +162,12 @@ pub fn generate_vote_equivocation_proof(
 }
 
 /// Create a new `ForkEquivocationProof` based on vote & canonical header.
-pub fn generate_fork_equivocation_proof_vote<Header, Hash>(
-	vote: (u64, Payload, ValidatorSetId, &Keyring<ecdsa_crypto::AuthorityId>),
+pub fn generate_fork_equivocation_proof_vote<Header: HeaderT, Hash>(
+	vote: (Header::Number, Payload, ValidatorSetId, &Keyring<ecdsa_crypto::AuthorityId>),
 	canonical_header: Option<Header>,
 	ancestry_proof: Option<AncestryProof<Hash>>,
-) -> ForkEquivocationProof<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature, Header, Hash> {
-	let signed_vote = signed_vote(vote.0, vote.1, vote.2, vote.3);
+) -> ForkEquivocationProof<ecdsa_crypto::Public, ecdsa_crypto::Signature, Header, Hash> {
+	let signed_vote = signed_vote::<Header::Number>(vote.0, vote.1, vote.2, vote.3);
 	let signatories = vec![(signed_vote.id, signed_vote.signature)];
 	ForkEquivocationProof {
 		commitment: signed_vote.commitment,
@@ -178,12 +178,12 @@ pub fn generate_fork_equivocation_proof_vote<Header, Hash>(
 }
 
 /// Create a new `ForkEquivocationProof` based on signed commitment & canonical header.
-pub fn generate_fork_equivocation_proof_sc<Header, Hash>(
-	commitment: Commitment<u64>,
+pub fn generate_fork_equivocation_proof_sc<Header: HeaderT, Hash>(
+	commitment: Commitment<Header::Number>,
 	keyrings: Vec<Keyring<ecdsa_crypto::AuthorityId>>,
 	canonical_header: Option<Header>,
 	ancestry_proof: Option<AncestryProof<Hash>>,
-) -> ForkEquivocationProof<u64, ecdsa_crypto::Public, ecdsa_crypto::Signature, Header, Hash> {
+) -> ForkEquivocationProof<ecdsa_crypto::Public, ecdsa_crypto::Signature, Header, Hash> {
 	let signatories = keyrings
 		.into_iter()
 		.map(|k| (k.public(), k.sign(&commitment.encode())))

From e17b7ca2b739c8778509d07fbf263a3820cf94e3 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 4 Mar 2024 14:05:37 +0100
Subject: [PATCH 164/188] remove contrived degree of freedom: Signature

The signature in ForkEquivocationProof was in fact not independent,
but is de facto coupled to RuntimeAppPublic's associated signature type.
---
 polkadot/node/service/src/fake_runtime_api.rs |  2 +-
 polkadot/runtime/rococo/src/lib.rs            |  2 +-
 polkadot/runtime/test-runtime/src/lib.rs      |  2 +-
 polkadot/runtime/westend/src/lib.rs           |  2 +-
 substrate/bin/node/runtime/src/lib.rs         |  2 +-
 .../beefy/src/communication/fisherman.rs      |  2 +-
 substrate/client/consensus/beefy/src/tests.rs |  4 ++--
 substrate/frame/beefy-mmr/src/lib.rs          |  7 +-----
 substrate/frame/beefy/src/equivocation.rs     |  1 -
 substrate/frame/beefy/src/lib.rs              |  3 ---
 .../primitives/consensus/beefy/src/lib.rs     | 24 ++++++-------------
 .../consensus/beefy/src/test_utils.rs         |  4 ++--
 12 files changed, 18 insertions(+), 37 deletions(-)

diff --git a/polkadot/node/service/src/fake_runtime_api.rs b/polkadot/node/service/src/fake_runtime_api.rs
index 7547e9c996de..d7edb12cba10 100644
--- a/polkadot/node/service/src/fake_runtime_api.rs
+++ b/polkadot/node/service/src/fake_runtime_api.rs
@@ -252,7 +252,7 @@ sp_api::impl_runtime_apis! {
 		}
 
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
-			_: beefy_primitives::ForkEquivocationProof<BeefyId, BeefySignature, <Block as BlockT>::Header, Hash>,
+			_: beefy_primitives::ForkEquivocationProof<BeefyId, <Block as BlockT>::Header, Hash>,
 			_: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
 			unimplemented!()
diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs
index 75b96c369632..66402a772bbb 100644
--- a/polkadot/runtime/rococo/src/lib.rs
+++ b/polkadot/runtime/rococo/src/lib.rs
@@ -2015,7 +2015,7 @@ sp_api::impl_runtime_apis! {
 		}
 
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
-			fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BeefyId, BeefySignature, Header, Hash>,
+			fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BeefyId, Header, Hash>,
 			key_owner_proofs: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
 			let key_owner_proofs = key_owner_proofs.iter().cloned().map(|p| p.decode()).collect::<Option<Vec<_>>>()?;
diff --git a/polkadot/runtime/test-runtime/src/lib.rs b/polkadot/runtime/test-runtime/src/lib.rs
index 8532b4e255c2..10817455fb3d 100644
--- a/polkadot/runtime/test-runtime/src/lib.rs
+++ b/polkadot/runtime/test-runtime/src/lib.rs
@@ -981,7 +981,7 @@ sp_api::impl_runtime_apis! {
 		}
 
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
-			_fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BeefyId, BeefySignature, Header, Hash>,
+			_fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BeefyId, Header, Hash>,
 			_key_owner_proofs: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
 			None
diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs
index e1c4dd407733..7843791d76e5 100644
--- a/polkadot/runtime/westend/src/lib.rs
+++ b/polkadot/runtime/westend/src/lib.rs
@@ -2039,7 +2039,7 @@ sp_api::impl_runtime_apis! {
 		}
 
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
-			fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BeefyId, BeefySignature, Header, Hash>,
+			fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BeefyId, Header, Hash>,
 			key_owner_proofs: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
 			let key_owner_proofs = key_owner_proofs.iter().cloned().map(|p| p.decode()).collect::<Option<Vec<_>>>()?;
diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs
index 9cda6dc085c7..b178c67ff562 100644
--- a/substrate/bin/node/runtime/src/lib.rs
+++ b/substrate/bin/node/runtime/src/lib.rs
@@ -2812,7 +2812,7 @@ impl_runtime_apis! {
 		}
 
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
-			fork_equivocation_proof: sp_consensus_beefy::ForkEquivocationProof<BeefyId, BeefySignature, Header, Hash>,
+			fork_equivocation_proof: sp_consensus_beefy::ForkEquivocationProof<BeefyId, Header, Hash>,
 			key_owner_proofs: Vec<sp_consensus_beefy::OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
 			let key_owner_proofs = key_owner_proofs.iter().cloned().map(|p| p.decode()).collect::<Option<Vec<_>>>()?;
diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 99fe2c78ac8f..36104b26d71b 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -97,7 +97,7 @@ where
 
 	pub(crate) fn report_fork_equivocation(
 		&self,
-		proof: ForkEquivocationProof<AuthorityId, Signature, B::Header, MmrRootHash>,
+		proof: ForkEquivocationProof<AuthorityId, B::Header, MmrRootHash>,
 	) -> Result<bool, Error> {
 		let best_block_number = self.backend.blockchain().info().best_number;
 		let best_block_hash = self.backend.blockchain().info().best_hash;
diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index fdad9b58a47a..3c88599aded8 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -258,7 +258,7 @@ pub(crate) struct TestApi {
 	pub reported_vote_equivocations:
 		Option<Arc<Mutex<Vec<VoteEquivocationProof<NumberFor<Block>, AuthorityId, Signature>>>>>,
 	pub reported_fork_equivocations:
-		Option<Arc<Mutex<Vec<ForkEquivocationProof<AuthorityId, Signature, Header, MmrRootHash>>>>>,
+		Option<Arc<Mutex<Vec<ForkEquivocationProof<AuthorityId, Header, MmrRootHash>>>>>,
 }
 
 impl TestApi {
@@ -327,7 +327,7 @@ sp_api::mock_impl_runtime_apis! {
 		}
 
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
-			proof: ForkEquivocationProof<AuthorityId, Signature, Header, MmrRootHash>,
+			proof: ForkEquivocationProof<AuthorityId, Header, MmrRootHash>,
 			_dummy: Vec<OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
 			if let Some(equivocations_buf) = self.inner.reported_fork_equivocations.as_ref() {
diff --git a/substrate/frame/beefy-mmr/src/lib.rs b/substrate/frame/beefy-mmr/src/lib.rs
index 284df6e4718a..b79d1e283c81 100644
--- a/substrate/frame/beefy-mmr/src/lib.rs
+++ b/substrate/frame/beefy-mmr/src/lib.rs
@@ -179,12 +179,7 @@ impl<T: pallet_mmr::Config> CheckForkEquivocationProof<pallet_beefy::Error<T>, H
 {
 	type HashT = <T as pallet_mmr::Config>::Hashing;
 	fn check_fork_equivocation_proof<Id, MsgHash>(
-		proof: &ForkEquivocationProof<
-			Id,
-			<Id as sp_application_crypto::RuntimeAppPublic>::Signature,
-			HeaderFor<T>,
-			<Self::HashT as Hash>::Output,
-		>,
+		proof: &ForkEquivocationProof<Id, HeaderFor<T>, <Self::HashT as Hash>::Output>,
 	) -> Result<bool, pallet_beefy::Error<T>>
 	where
 		Id: sp_consensus_beefy::BeefyAuthorityId<MsgHash> + PartialEq,
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index f6d11f77f9fb..d6334cbbfd17 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -138,7 +138,6 @@ pub enum EquivocationEvidenceFor<T: Config> {
 	ForkEquivocationProof(
 		ForkEquivocationProof<
 			<T as Config>::BeefyId,
-			<<T as Config>::BeefyId as RuntimeAppPublic>::Signature,
 			HeaderFor<T>,
 			<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof<
 				Error<T>,
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index c8aa4aedc6fe..674605527187 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -317,7 +317,6 @@ pub mod pallet {
 			equivocation_proof: Box<
 				ForkEquivocationProof<
 					T::BeefyId,
-					<T::BeefyId as RuntimeAppPublic>::Signature,
 					HeaderFor<T>,
 					<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof<
 						Error<T>,
@@ -356,7 +355,6 @@ pub mod pallet {
 			equivocation_proof: Box<
 				ForkEquivocationProof<
 					T::BeefyId,
-					<T::BeefyId as RuntimeAppPublic>::Signature,
 					HeaderFor<T>,
 					<<<T as Config>::CheckForkEquivocationProof as sp_consensus_beefy::CheckForkEquivocationProof<Error<T>, HeaderFor<T>>>::HashT as sp_runtime::traits::Hash>::Output,
 				>,
@@ -426,7 +424,6 @@ impl<T: Config> Pallet<T> {
 	pub fn submit_unsigned_fork_equivocation_report(
 		fork_equivocation_proof: sp_consensus_beefy::ForkEquivocationProof<
 			T::BeefyId,
-			<T::BeefyId as RuntimeAppPublic>::Signature,
 			HeaderFor<T>,
 			<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof<
 				pallet::Error<T>,
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index a62ee900942e..f251de5fb415 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -331,19 +331,19 @@ impl<Number, Id, Signature> VoteEquivocationProof<Number, Id, Signature> {
 /// This proof shows commitment signed on a different fork.
 /// See [check_fork_equivocation_proof] for proof validity conditions.
 #[derive(Clone, Debug, Decode, Encode, PartialEq, TypeInfo)]
-pub struct ForkEquivocationProof<Id, Signature, Header: HeaderT, Hash> {
+pub struct ForkEquivocationProof<Id: RuntimeAppPublic, Header: HeaderT, Hash> {
 	/// Commitment for a block on a different fork than one at the same height in
 	/// the chain where this proof is submitted.
 	pub commitment: Commitment<Header::Number>,
 	/// Signatures on this block
-	pub signatories: Vec<(Id, Signature)>,
+	pub signatories: Vec<(Id, Id::Signature)>,
 	/// Canonical header at the same height as `commitment.block_number`.
 	pub canonical_header: Option<Header>,
 	/// Ancestry proof showing mmr root
 	pub ancestry_proof: Option<AncestryProof<Hash>>,
 }
 
-impl<Id, Signature, H: HeaderT, Hash> ForkEquivocationProof<Id, Signature, H, Hash> {
+impl<Id: RuntimeAppPublic, H: HeaderT, Hash> ForkEquivocationProof<Id, H, Hash> {
 	/// Returns the authority ids of the misbehaving voters.
 	pub fn offender_ids(&self) -> Vec<&Id> {
 		self.signatories.iter().map(|(id, _)| id).collect()
@@ -519,7 +519,7 @@ where
 /// incorrect block implies validators will only sign blocks they *know* will be finalized by
 /// GRANDPA.
 pub fn check_fork_equivocation_proof<Id, MsgHash, Header, NodeHash, Hasher>(
-	proof: &ForkEquivocationProof<Id, <Id as RuntimeAppPublic>::Signature, Header, NodeHash>,
+	proof: &ForkEquivocationProof<Id, Header, NodeHash>,
 	canonical_root: Hasher::Item,
 	mmr_size: u64,
 	canonical_header_hash: &Header::Hash,
@@ -592,12 +592,7 @@ pub trait CheckForkEquivocationProof<Err, Header: HeaderT> {
 	/// https://github.com/paritytech/polkadot-sdk/issues/1441 for
 	/// replacement solution.
 	fn check_fork_equivocation_proof<Id, MsgHash>(
-		proof: &ForkEquivocationProof<
-			Id,
-			<Id as RuntimeAppPublic>::Signature,
-			Header,
-			<Self::HashT as Hash>::Output,
-		>,
+		proof: &ForkEquivocationProof<Id, Header, <Self::HashT as Hash>::Output>,
 	) -> Result<bool, Err>
 	where
 		Id: BeefyAuthorityId<MsgHash> + PartialEq,
@@ -607,12 +602,7 @@ pub trait CheckForkEquivocationProof<Err, Header: HeaderT> {
 impl<Err, Header: HeaderT> CheckForkEquivocationProof<Err, Header> for () {
 	type HashT = Keccak256;
 	fn check_fork_equivocation_proof<Id, MsgHash>(
-		_proof: &ForkEquivocationProof<
-			Id,
-			<Id as RuntimeAppPublic>::Signature,
-			Header,
-			<Self::HashT as Hash>::Output,
-		>,
+		_proof: &ForkEquivocationProof<Id, Header, <Self::HashT as Hash>::Output>,
 		// _canonical_header_hash: &<Self::Header as HeaderT>::Hash,
 		// _best_block_num: <Self::Header as HeaderT>::Number,
 	) -> Result<bool, Err>
@@ -684,7 +674,7 @@ sp_api::decl_runtime_apis! {
 		/// hardcoded to return `None`). Only useful in an offchain context.
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
 			fork_equivocation_proof:
-				ForkEquivocationProof<AuthorityId, <AuthorityId as RuntimeAppPublic>::Signature, Block::Header, Hash>,
+				ForkEquivocationProof<AuthorityId, Block::Header, Hash>,
 			key_owner_proofs: Vec<OpaqueKeyOwnershipProof>,
 		) -> Option<()>;
 
diff --git a/substrate/primitives/consensus/beefy/src/test_utils.rs b/substrate/primitives/consensus/beefy/src/test_utils.rs
index 72be3731e671..48995ace4763 100644
--- a/substrate/primitives/consensus/beefy/src/test_utils.rs
+++ b/substrate/primitives/consensus/beefy/src/test_utils.rs
@@ -166,7 +166,7 @@ pub fn generate_fork_equivocation_proof_vote<Header: HeaderT, Hash>(
 	vote: (Header::Number, Payload, ValidatorSetId, &Keyring<ecdsa_crypto::AuthorityId>),
 	canonical_header: Option<Header>,
 	ancestry_proof: Option<AncestryProof<Hash>>,
-) -> ForkEquivocationProof<ecdsa_crypto::Public, ecdsa_crypto::Signature, Header, Hash> {
+) -> ForkEquivocationProof<ecdsa_crypto::Public, Header, Hash> {
 	let signed_vote = signed_vote::<Header::Number>(vote.0, vote.1, vote.2, vote.3);
 	let signatories = vec![(signed_vote.id, signed_vote.signature)];
 	ForkEquivocationProof {
@@ -183,7 +183,7 @@ pub fn generate_fork_equivocation_proof_sc<Header: HeaderT, Hash>(
 	keyrings: Vec<Keyring<ecdsa_crypto::AuthorityId>>,
 	canonical_header: Option<Header>,
 	ancestry_proof: Option<AncestryProof<Hash>>,
-) -> ForkEquivocationProof<ecdsa_crypto::Public, ecdsa_crypto::Signature, Header, Hash> {
+) -> ForkEquivocationProof<ecdsa_crypto::Public, Header, Hash> {
 	let signatories = keyrings
 		.into_iter()
 		.map(|k| (k.public(), k.sign(&commitment.encode())))

From 6081f25a101cbf0421ffdc2a506a28e720873b5d Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 4 Mar 2024 16:13:03 +0100
Subject: [PATCH 165/188] cleanup return type

---
 substrate/frame/beefy-mmr/src/lib.rs            |  8 ++++----
 substrate/frame/beefy/src/equivocation.rs       |  8 ++------
 substrate/primitives/consensus/beefy/src/lib.rs | 10 ++++------
 3 files changed, 10 insertions(+), 16 deletions(-)

diff --git a/substrate/frame/beefy-mmr/src/lib.rs b/substrate/frame/beefy-mmr/src/lib.rs
index b79d1e283c81..224f358ec56b 100644
--- a/substrate/frame/beefy-mmr/src/lib.rs
+++ b/substrate/frame/beefy-mmr/src/lib.rs
@@ -33,7 +33,7 @@
 //!
 //! and thanks to versioning can be easily updated in the future.
 
-use sp_runtime::traits::{Convert, Hash, Header as HeaderT, Member};
+use sp_runtime::traits::{Convert, Hash, Member};
 use sp_std::prelude::*;
 
 use codec::Decode;
@@ -180,7 +180,7 @@ impl<T: pallet_mmr::Config> CheckForkEquivocationProof<pallet_beefy::Error<T>, H
 	type HashT = <T as pallet_mmr::Config>::Hashing;
 	fn check_fork_equivocation_proof<Id, MsgHash>(
 		proof: &ForkEquivocationProof<Id, HeaderFor<T>, <Self::HashT as Hash>::Output>,
-	) -> Result<bool, pallet_beefy::Error<T>>
+	) -> Result<(), pallet_beefy::Error<T>>
 	where
 		Id: sp_consensus_beefy::BeefyAuthorityId<MsgHash> + PartialEq,
 		MsgHash: sp_runtime::traits::Hash,
@@ -215,9 +215,9 @@ impl<T: pallet_mmr::Config> CheckForkEquivocationProof<pallet_beefy::Error<T>, H
 			first_mmr_block_num,
 			best_block_num,
 		) {
-			return Ok(false)
+			return Err(pallet_beefy::Error::<T>::InvalidForkEquivocationProof)
 		}
-		Ok(true)
+		Ok(())
 	}
 }
 
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index d6334cbbfd17..f58160e03321 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -297,14 +297,10 @@ where
 			EquivocationEvidenceFor::ForkEquivocationProof(equivocation_proof, _) => {
 				// Validate equivocation proof (check commitment is to unexpected payload and
 				// signatures are valid).
-				match <T::CheckForkEquivocationProof as CheckForkEquivocationProof<
+				<T::CheckForkEquivocationProof as CheckForkEquivocationProof<
 					Error<T>,
 					HeaderFor<T>,
-				>>::check_fork_equivocation_proof(equivocation_proof)
-				{
-					Ok(true) => {},
-					_ => return Err(Error::<T>::InvalidForkEquivocationProof.into()),
-				}
+				>>::check_fork_equivocation_proof(equivocation_proof)?
 			},
 		}
 
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index f251de5fb415..f0235a7798f4 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -579,7 +579,7 @@ impl<AuthorityId> OnNewValidatorSet<AuthorityId> for () {
 	fn on_new_validator_set(_: &ValidatorSet<AuthorityId>, _: &ValidatorSet<AuthorityId>) {}
 }
 
-/// Hook for checking fork equivocation proof.
+/// Hook for checking fork equivocation proof for validity.
 pub trait CheckForkEquivocationProof<Err, Header: HeaderT> {
 	/// Associated hash type for hashing ancestry proof.
 	type HashT: Hash;
@@ -593,7 +593,7 @@ pub trait CheckForkEquivocationProof<Err, Header: HeaderT> {
 	/// replacement solution.
 	fn check_fork_equivocation_proof<Id, MsgHash>(
 		proof: &ForkEquivocationProof<Id, Header, <Self::HashT as Hash>::Output>,
-	) -> Result<bool, Err>
+	) -> Result<(), Err>
 	where
 		Id: BeefyAuthorityId<MsgHash> + PartialEq,
 		MsgHash: Hash;
@@ -603,14 +603,12 @@ impl<Err, Header: HeaderT> CheckForkEquivocationProof<Err, Header> for () {
 	type HashT = Keccak256;
 	fn check_fork_equivocation_proof<Id, MsgHash>(
 		_proof: &ForkEquivocationProof<Id, Header, <Self::HashT as Hash>::Output>,
-		// _canonical_header_hash: &<Self::Header as HeaderT>::Hash,
-		// _best_block_num: <Self::Header as HeaderT>::Number,
-	) -> Result<bool, Err>
+	) -> Result<(), Err>
 	where
 		Id: BeefyAuthorityId<MsgHash> + PartialEq,
 		MsgHash: Hash,
 	{
-		Ok(true)
+		Ok(())
 	}
 }
 

From a814740e39dbe4f6e231a023a82827fe4baf4f19 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 4 Mar 2024 16:30:07 +0100
Subject: [PATCH 166/188] remove contrived degree of freedom: Hasher

The `Hasher` and `HashOutput` types in `check_fork_equivocation` were in
fact not independent, but are both dependent on `Hash`, so shouldn't
entertain a contrived independence of them.
---
 .../beefy/src/communication/fisherman.rs      |  3 +-
 substrate/frame/beefy-mmr/src/lib.rs          | 10 +---
 .../primitives/consensus/beefy/src/lib.rs     | 47 ++++++++++---------
 3 files changed, 28 insertions(+), 32 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 36104b26d71b..288de9afa5fa 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -143,8 +143,7 @@ where
 				AuthorityId,
 				BeefySignatureHasher,
 				B::Header,
-				MmrRootHash,
-				sp_mmr_primitives::utils::AncestryHasher<MmrHashing>,
+				MmrHashing,
 			>(
 				&proof,
 				best_mmr_root,
diff --git a/substrate/frame/beefy-mmr/src/lib.rs b/substrate/frame/beefy-mmr/src/lib.rs
index 224f358ec56b..9ad8ffdb03af 100644
--- a/substrate/frame/beefy-mmr/src/lib.rs
+++ b/substrate/frame/beefy-mmr/src/lib.rs
@@ -37,7 +37,7 @@ use sp_runtime::traits::{Convert, Hash, Member};
 use sp_std::prelude::*;
 
 use codec::Decode;
-use pallet_mmr::{primitives::utils::AncestryHasher, LeafDataProvider, ParentNumberAndHash};
+use pallet_mmr::{LeafDataProvider, ParentNumberAndHash};
 use sp_consensus_beefy::{
 	mmr::{BeefyAuthoritySet, BeefyDataProvider, BeefyNextAuthoritySet, MmrLeaf, MmrLeafVersion},
 	CheckForkEquivocationProof, ForkEquivocationProof, ValidatorSet as BeefyValidatorSet,
@@ -201,13 +201,7 @@ impl<T: pallet_mmr::Config> CheckForkEquivocationProof<pallet_beefy::Error<T>, H
 			)
 			.map_err(|_| pallet_beefy::Error::<T>::InvalidForkEquivocationProof)?
 		};
-		if !sp_consensus_beefy::check_fork_equivocation_proof::<
-			_,
-			_,
-			HeaderFor<T>,
-			_,
-			AncestryHasher<Self::HashT>,
-		>(
+		if !sp_consensus_beefy::check_fork_equivocation_proof::<_, _, HeaderFor<T>, Self::HashT>(
 			proof,
 			canonical_root,
 			mmr_size,
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index f0235a7798f4..741eabdb985f 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -43,14 +43,18 @@ pub mod test_utils;
 
 pub use commitment::{Commitment, SignedCommitment, VersionedFinalityProof};
 pub use payload::{known_payloads, BeefyPayloadId, Payload, PayloadProvider};
-use sp_mmr_primitives::{mmr_lib, AncestryProof};
+use sp_mmr_primitives::{
+	mmr_lib,
+	utils::{self, AncestryHasher},
+	AncestryProof,
+};
 
 use codec::{Codec, Decode, Encode};
 use core::fmt::{Debug, Display};
 use scale_info::TypeInfo;
 use sp_application_crypto::{AppCrypto, AppPublic, ByteArray, RuntimeAppPublic};
 use sp_core::H256;
-use sp_runtime::traits::{Hash, HashOutput, Header as HeaderT, Keccak256, NumberFor};
+use sp_runtime::traits::{Hash, Header as HeaderT, Keccak256, NumberFor};
 use sp_std::prelude::*;
 
 /// Key type for BEEFY module.
@@ -438,17 +442,16 @@ where
 
 /// Checks whether an ancestry proof has the correct size and its calculated root differs from the
 /// commitment's payload's.
-fn check_ancestry_proof<Header, NodeHash, Hasher>(
+fn check_ancestry_proof<Header, NodeHash>(
 	commitment: &Commitment<Header::Number>,
-	ancestry_proof: &Option<AncestryProof<Hasher::Item>>,
+	ancestry_proof: &Option<AncestryProof<NodeHash::Output>>,
 	first_mmr_block_num: Header::Number,
-	canonical_root: Hasher::Item,
+	canonical_root: NodeHash::Output,
 	mmr_size: u64,
 ) -> bool
 where
 	Header: HeaderT,
-	NodeHash: HashOutput,
-	Hasher: mmr_lib::Merge<Item = NodeHash>,
+	NodeHash: Hash,
 {
 	if let Some(ancestry_proof) = ancestry_proof {
 		let expected_leaf_count = sp_mmr_primitives::utils::block_num_to_leaf_index::<Header>(
@@ -471,11 +474,11 @@ where
 			if expected_mmr_size != ancestry_proof.prev_size {
 				return false
 			}
-			if sp_mmr_primitives::utils::verify_ancestry_proof::<NodeHash, Hasher>(
-				canonical_root,
-				mmr_size,
-				ancestry_proof.clone(),
-			) != Ok(true)
+			if sp_mmr_primitives::utils::verify_ancestry_proof::<
+				NodeHash::Output,
+				utils::AncestryHasher<NodeHash>,
+			>(canonical_root, mmr_size, ancestry_proof.clone()) !=
+				Ok(true)
 			{
 				return false
 			}
@@ -489,13 +492,14 @@ where
 
 		// once the ancestry proof is verified, calculate the prev_root to compare it
 		// with the commitment's prev_root
-		let ancestry_prev_root = mmr_lib::ancestry_proof::bagging_peaks_hashes::<NodeHash, Hasher>(
-			ancestry_proof.prev_peaks.clone(),
-		);
+		let ancestry_prev_root = mmr_lib::ancestry_proof::bagging_peaks_hashes::<
+			NodeHash::Output,
+			AncestryHasher<NodeHash>,
+		>(ancestry_proof.prev_peaks.clone());
 		// if the commitment payload does not commit to an MMR root, then this
 		// commitment may have another purpose and should not be slashed
 		let commitment_prev_root =
-			commitment.payload.get_decoded::<NodeHash>(&known_payloads::MMR_ROOT_ID);
+			commitment.payload.get_decoded::<NodeHash::Output>(&known_payloads::MMR_ROOT_ID);
 		return commitment_prev_root != ancestry_prev_root.ok()
 	}
 	// if no ancestry proof provided, the proof is also not correct
@@ -518,9 +522,9 @@ where
 /// being finalized by GRANDPA. This is fine too, since the slashing risk of committing to an
 /// incorrect block implies validators will only sign blocks they *know* will be finalized by
 /// GRANDPA.
-pub fn check_fork_equivocation_proof<Id, MsgHash, Header, NodeHash, Hasher>(
-	proof: &ForkEquivocationProof<Id, Header, NodeHash>,
-	canonical_root: Hasher::Item,
+pub fn check_fork_equivocation_proof<Id, MsgHash, Header, NodeHash>(
+	proof: &ForkEquivocationProof<Id, Header, NodeHash::Output>,
+	canonical_root: NodeHash::Output,
 	mmr_size: u64,
 	canonical_header_hash: &Header::Hash,
 	first_mmr_block_num: Header::Number,
@@ -530,8 +534,7 @@ where
 	Id: BeefyAuthorityId<MsgHash> + PartialEq,
 	MsgHash: Hash,
 	Header: HeaderT,
-	NodeHash: sp_runtime::traits::HashOutput,
-	Hasher: mmr_lib::Merge<Item = NodeHash>,
+	NodeHash: Hash,
 {
 	let ForkEquivocationProof { commitment, signatories, canonical_header, ancestry_proof } = proof;
 
@@ -546,7 +549,7 @@ where
 		// if neither the ancestry proof nor the header proof is correct, the proof is invalid
 		// avoid verifying the ancestry proof if a valid header proof has been provided
 		if !check_header_proof(commitment, canonical_header, canonical_header_hash) &&
-			!check_ancestry_proof::<Header, NodeHash, Hasher>(
+			!check_ancestry_proof::<Header, NodeHash>(
 				commitment,
 				ancestry_proof,
 				first_mmr_block_num,

From ee4c6c55a9008e1fc8cdef017f8df100d7b0e30a Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 4 Mar 2024 16:40:10 +0100
Subject: [PATCH 167/188] consistent Hash/HashT usage

convention: xT := x/Trait/
---
 substrate/frame/beefy-mmr/src/lib.rs          |  6 +--
 substrate/frame/beefy/src/equivocation.rs     |  4 +-
 substrate/frame/beefy/src/lib.rs              |  8 ++--
 .../primitives/consensus/beefy/src/lib.rs     | 44 +++++++++----------
 4 files changed, 31 insertions(+), 31 deletions(-)

diff --git a/substrate/frame/beefy-mmr/src/lib.rs b/substrate/frame/beefy-mmr/src/lib.rs
index 9ad8ffdb03af..4b8bfeb5ac9b 100644
--- a/substrate/frame/beefy-mmr/src/lib.rs
+++ b/substrate/frame/beefy-mmr/src/lib.rs
@@ -177,9 +177,9 @@ where
 impl<T: pallet_mmr::Config> CheckForkEquivocationProof<pallet_beefy::Error<T>, HeaderFor<T>>
 	for Pallet<T>
 {
-	type HashT = <T as pallet_mmr::Config>::Hashing;
+	type Hash = <T as pallet_mmr::Config>::Hashing;
 	fn check_fork_equivocation_proof<Id, MsgHash>(
-		proof: &ForkEquivocationProof<Id, HeaderFor<T>, <Self::HashT as Hash>::Output>,
+		proof: &ForkEquivocationProof<Id, HeaderFor<T>, <Self::Hash as Hash>::Output>,
 	) -> Result<(), pallet_beefy::Error<T>>
 	where
 		Id: sp_consensus_beefy::BeefyAuthorityId<MsgHash> + PartialEq,
@@ -201,7 +201,7 @@ impl<T: pallet_mmr::Config> CheckForkEquivocationProof<pallet_beefy::Error<T>, H
 			)
 			.map_err(|_| pallet_beefy::Error::<T>::InvalidForkEquivocationProof)?
 		};
-		if !sp_consensus_beefy::check_fork_equivocation_proof::<_, _, HeaderFor<T>, Self::HashT>(
+		if !sp_consensus_beefy::check_fork_equivocation_proof::<_, _, HeaderFor<T>, Self::Hash>(
 			proof,
 			canonical_root,
 			mmr_size,
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index f58160e03321..da0007c7b433 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -43,7 +43,7 @@ use sp_consensus_beefy::{
 	KEY_TYPE as BEEFY_KEY_TYPE,
 };
 use sp_runtime::{
-	traits::Hash,
+	traits::Hash as HashT,
 	transaction_validity::{
 		InvalidTransaction, TransactionPriority, TransactionSource, TransactionValidity,
 		TransactionValidityError, ValidTransaction,
@@ -142,7 +142,7 @@ pub enum EquivocationEvidenceFor<T: Config> {
 			<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof<
 				Error<T>,
 				HeaderFor<T>,
-			>>::HashT as Hash>::Output,
+			>>::Hash as HashT>::Output,
 		>,
 		Vec<<T as Config>::KeyOwnerProof>,
 	),
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 674605527187..4da0993fb31d 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -33,7 +33,7 @@ use frame_system::{
 use log;
 use sp_runtime::{
 	generic::DigestItem,
-	traits::{Hash, IsMember, Member, One},
+	traits::{Hash as HashT, IsMember, Member, One},
 	RuntimeAppPublic,
 };
 use sp_session::{GetSessionNumber, GetValidatorCount};
@@ -321,7 +321,7 @@ pub mod pallet {
 					<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof<
 						Error<T>,
 						HeaderFor<T>,
-					>>::HashT as Hash>::Output,
+					>>::Hash as HashT>::Output,
 				>,
 			>,
 			key_owner_proofs: Vec<T::KeyOwnerProof>,
@@ -356,7 +356,7 @@ pub mod pallet {
 				ForkEquivocationProof<
 					T::BeefyId,
 					HeaderFor<T>,
-					<<<T as Config>::CheckForkEquivocationProof as sp_consensus_beefy::CheckForkEquivocationProof<Error<T>, HeaderFor<T>>>::HashT as sp_runtime::traits::Hash>::Output,
+					<<<T as Config>::CheckForkEquivocationProof as sp_consensus_beefy::CheckForkEquivocationProof<Error<T>, HeaderFor<T>>>::Hash as HashT>::Output,
 				>,
 			>,
 			key_owner_proofs: Vec<T::KeyOwnerProof>,
@@ -428,7 +428,7 @@ impl<T: Config> Pallet<T> {
 			<<<T as Config>::CheckForkEquivocationProof as CheckForkEquivocationProof<
 				pallet::Error<T>,
 				HeaderFor<T>,
-			>>::HashT as Hash>::Output,
+			>>::Hash as HashT>::Output,
 		>,
 		key_owner_proofs: Vec<T::KeyOwnerProof>,
 	) -> Option<()> {
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 741eabdb985f..9ee44038d5f7 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -54,7 +54,7 @@ use core::fmt::{Debug, Display};
 use scale_info::TypeInfo;
 use sp_application_crypto::{AppCrypto, AppPublic, ByteArray, RuntimeAppPublic};
 use sp_core::H256;
-use sp_runtime::traits::{Hash, Header as HeaderT, Keccak256, NumberFor};
+use sp_runtime::traits::{Hash as HashT, Header as HeaderT, Keccak256, NumberFor};
 use sp_std::prelude::*;
 
 /// Key type for BEEFY module.
@@ -63,7 +63,7 @@ pub const KEY_TYPE: sp_core::crypto::KeyTypeId = sp_application_crypto::key_type
 /// Trait representing BEEFY authority id, including custom signature verification.
 ///
 /// Accepts custom hashing fn for the message and custom convertor fn for the signer.
-pub trait BeefyAuthorityId<MsgHash: Hash>: RuntimeAppPublic {
+pub trait BeefyAuthorityId<MsgHash: HashT>: RuntimeAppPublic {
 	/// Verify a signature.
 	///
 	/// Return `true` if signature over `msg` is valid for this id.
@@ -100,7 +100,7 @@ pub trait AuthorityIdBound:
 /// Your code should use the above types as concrete types for all crypto related
 /// functionality.
 pub mod ecdsa_crypto {
-	use super::{AuthorityIdBound, BeefyAuthorityId, Hash, RuntimeAppPublic, KEY_TYPE};
+	use super::{AuthorityIdBound, BeefyAuthorityId, HashT, RuntimeAppPublic, KEY_TYPE};
 	use sp_application_crypto::{app_crypto, ecdsa};
 	use sp_core::crypto::Wraps;
 
@@ -112,12 +112,12 @@ pub mod ecdsa_crypto {
 	/// Signature for a BEEFY authority using ECDSA as its crypto.
 	pub type AuthoritySignature = Signature;
 
-	impl<MsgHash: Hash> BeefyAuthorityId<MsgHash> for AuthorityId
+	impl<MsgHash: HashT> BeefyAuthorityId<MsgHash> for AuthorityId
 	where
-		<MsgHash as Hash>::Output: Into<[u8; 32]>,
+		<MsgHash as HashT>::Output: Into<[u8; 32]>,
 	{
 		fn verify(&self, signature: &<Self as RuntimeAppPublic>::Signature, msg: &[u8]) -> bool {
-			let msg_hash = <MsgHash as Hash>::hash(msg).into();
+			let msg_hash = <MsgHash as HashT>::hash(msg).into();
 			match sp_io::crypto::secp256k1_ecdsa_recover_compressed(
 				signature.as_inner_ref().as_ref(),
 				&msg_hash,
@@ -143,7 +143,7 @@ pub mod ecdsa_crypto {
 
 #[cfg(feature = "bls-experimental")]
 pub mod bls_crypto {
-	use super::{AuthorityIdBound, BeefyAuthorityId, Hash, RuntimeAppPublic, KEY_TYPE};
+	use super::{AuthorityIdBound, BeefyAuthorityId, HashT, RuntimeAppPublic, KEY_TYPE};
 	use sp_application_crypto::{app_crypto, bls377};
 	use sp_core::{bls377::Pair as BlsPair, crypto::Wraps, Pair as _};
 
@@ -155,9 +155,9 @@ pub mod bls_crypto {
 	/// Signature for a BEEFY authority using BLS as its crypto.
 	pub type AuthoritySignature = Signature;
 
-	impl<MsgHash: Hash> BeefyAuthorityId<MsgHash> for AuthorityId
+	impl<MsgHash: HashT> BeefyAuthorityId<MsgHash> for AuthorityId
 	where
-		<MsgHash as Hash>::Output: Into<[u8; 32]>,
+		<MsgHash as HashT>::Output: Into<[u8; 32]>,
 	{
 		fn verify(&self, signature: &<Self as RuntimeAppPublic>::Signature, msg: &[u8]) -> bool {
 			// `w3f-bls` library uses IETF hashing standard and as such does not expose
@@ -183,7 +183,7 @@ pub mod bls_crypto {
 /// functionality.
 #[cfg(feature = "bls-experimental")]
 pub mod ecdsa_bls_crypto {
-	use super::{AuthorityIdBound, BeefyAuthorityId, Hash, RuntimeAppPublic, KEY_TYPE};
+	use super::{AuthorityIdBound, BeefyAuthorityId, HashT, RuntimeAppPublic, KEY_TYPE};
 	use sp_application_crypto::{app_crypto, ecdsa_bls377};
 	use sp_core::{crypto::Wraps, ecdsa_bls377::Pair as EcdsaBlsPair};
 
@@ -197,7 +197,7 @@ pub mod ecdsa_bls_crypto {
 
 	impl<H> BeefyAuthorityId<H> for AuthorityId
 	where
-		H: Hash,
+		H: HashT,
 		H::Output: Into<[u8; 32]>,
 	{
 		fn verify(&self, signature: &<Self as RuntimeAppPublic>::Signature, msg: &[u8]) -> bool {
@@ -372,7 +372,7 @@ pub fn check_commitment_signature<Number, Id, MsgHash>(
 where
 	Id: BeefyAuthorityId<MsgHash>,
 	Number: Clone + Encode + PartialEq,
-	MsgHash: Hash,
+	MsgHash: HashT,
 {
 	let encoded_commitment = commitment.encode();
 	BeefyAuthorityId::<MsgHash>::verify(authority_id, signature, &encoded_commitment)
@@ -386,7 +386,7 @@ pub fn check_vote_equivocation_proof<Number, Id, MsgHash>(
 where
 	Id: BeefyAuthorityId<MsgHash> + PartialEq,
 	Number: Clone + Encode + PartialEq,
-	MsgHash: Hash,
+	MsgHash: HashT,
 {
 	let first = &report.first;
 	let second = &report.second;
@@ -451,7 +451,7 @@ fn check_ancestry_proof<Header, NodeHash>(
 ) -> bool
 where
 	Header: HeaderT,
-	NodeHash: Hash,
+	NodeHash: HashT,
 {
 	if let Some(ancestry_proof) = ancestry_proof {
 		let expected_leaf_count = sp_mmr_primitives::utils::block_num_to_leaf_index::<Header>(
@@ -532,9 +532,9 @@ pub fn check_fork_equivocation_proof<Id, MsgHash, Header, NodeHash>(
 ) -> bool
 where
 	Id: BeefyAuthorityId<MsgHash> + PartialEq,
-	MsgHash: Hash,
+	MsgHash: HashT,
 	Header: HeaderT,
-	NodeHash: Hash,
+	NodeHash: HashT,
 {
 	let ForkEquivocationProof { commitment, signatories, canonical_header, ancestry_proof } = proof;
 
@@ -585,7 +585,7 @@ impl<AuthorityId> OnNewValidatorSet<AuthorityId> for () {
 /// Hook for checking fork equivocation proof for validity.
 pub trait CheckForkEquivocationProof<Err, Header: HeaderT> {
 	/// Associated hash type for hashing ancestry proof.
-	type HashT: Hash;
+	type Hash: HashT;
 	/// Validate equivocation proof (check commitment is to unexpected payload and
 	/// signatures are valid).
 	/// NOTE: Fork equivocation proof currently only prevents attacks
@@ -595,21 +595,21 @@ pub trait CheckForkEquivocationProof<Err, Header: HeaderT> {
 	/// https://github.com/paritytech/polkadot-sdk/issues/1441 for
 	/// replacement solution.
 	fn check_fork_equivocation_proof<Id, MsgHash>(
-		proof: &ForkEquivocationProof<Id, Header, <Self::HashT as Hash>::Output>,
+		proof: &ForkEquivocationProof<Id, Header, <Self::Hash as HashT>::Output>,
 	) -> Result<(), Err>
 	where
 		Id: BeefyAuthorityId<MsgHash> + PartialEq,
-		MsgHash: Hash;
+		MsgHash: HashT;
 }
 
 impl<Err, Header: HeaderT> CheckForkEquivocationProof<Err, Header> for () {
-	type HashT = Keccak256;
+	type Hash = Keccak256;
 	fn check_fork_equivocation_proof<Id, MsgHash>(
-		_proof: &ForkEquivocationProof<Id, Header, <Self::HashT as Hash>::Output>,
+		_proof: &ForkEquivocationProof<Id, Header, <Self::Hash as HashT>::Output>,
 	) -> Result<(), Err>
 	where
 		Id: BeefyAuthorityId<MsgHash> + PartialEq,
-		MsgHash: Hash,
+		MsgHash: HashT,
 	{
 		Ok(())
 	}

From 5502a8ee32697104498e924de0b78686174b3e2f Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 4 Mar 2024 18:00:38 +0100
Subject: [PATCH 168/188] remove redundant mmr_size alias

Reported-by: Adrian Catangiu <adrian@parity.io>
---
 substrate/frame/beefy-mmr/src/lib.rs             | 4 ++--
 substrate/frame/merkle-mountain-range/src/lib.rs | 5 -----
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/substrate/frame/beefy-mmr/src/lib.rs b/substrate/frame/beefy-mmr/src/lib.rs
index 4b8bfeb5ac9b..1ee5f0382be2 100644
--- a/substrate/frame/beefy-mmr/src/lib.rs
+++ b/substrate/frame/beefy-mmr/src/lib.rs
@@ -187,14 +187,14 @@ impl<T: pallet_mmr::Config> CheckForkEquivocationProof<pallet_beefy::Error<T>, H
 	{
 		let canonical_root = <pallet_mmr::Pallet<T>>::mmr_root();
 		let mmr_size =
-			sp_mmr_primitives::utils::NodesUtils::new(<pallet_mmr::Pallet<T>>::mmr_size()).size();
+			sp_mmr_primitives::utils::NodesUtils::new(<pallet_mmr::Pallet<T>>::mmr_leaves()).size();
 		// if first_mmr_block_num is invalid, then presumably beefy is not active.
 		// TODO: should we slash in this case?
 		let block_number = proof.commitment.block_number;
 		let canonical_header_hash = <frame_system::Pallet<T>>::block_hash(block_number);
 		let best_block_num = <frame_system::Pallet<T>>::block_number();
 		let first_mmr_block_num = {
-			let mmr_leaf_count = <pallet_mmr::Pallet<T>>::mmr_size();
+			let mmr_leaf_count = <pallet_mmr::Pallet<T>>::mmr_leaves();
 			sp_mmr_primitives::utils::first_mmr_block_num::<HeaderFor<T>>(
 				best_block_num,
 				mmr_leaf_count,
diff --git a/substrate/frame/merkle-mountain-range/src/lib.rs b/substrate/frame/merkle-mountain-range/src/lib.rs
index 737e987eda22..c7754feccd44 100644
--- a/substrate/frame/merkle-mountain-range/src/lib.rs
+++ b/substrate/frame/merkle-mountain-range/src/lib.rs
@@ -360,11 +360,6 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 		Self::mmr_root_hash()
 	}
 
-	/// Return the on-chain MMR root hash.
-	pub fn mmr_size() -> NodeIndex {
-		Self::mmr_leaves()
-	}
-
 	/// Verify MMR proof for given `leaves`.
 	///
 	/// This method is safe to use within the runtime code.

From d27fc06a279f40f18c5255f48559c199c9b0a3a6 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Tue, 12 Mar 2024 10:26:13 +0100
Subject: [PATCH 169/188] reuse verify_with_validator_set

Suggested-by: Adrian Catangiu <adrian@parity.io>
---
 .../consensus/beefy/src/communication/fisherman.rs   | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 288de9afa5fa..b68c004e6e60 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -297,7 +297,7 @@ where
 		&self,
 		signed_commitment: SignedCommitment<NumberFor<B>, Signature>,
 	) -> Result<(), Error> {
-		let SignedCommitment { commitment, signatures } = signed_commitment;
+		let SignedCommitment { commitment, signatures } = signed_commitment.clone();
 		let number = commitment.block_number;
 		// if the vote is for a block number exceeding our best block number, there shouldn't even
 		// be a payload to sign yet, hence we assume it is an equivocation and report it
@@ -344,7 +344,13 @@ where
 					number,
 				);
 				let validator_set = self.active_validator_set_at(canonical_hhp.hash)?;
-				if signatures.len() != validator_set.validators().len() {
+				if crate::justification::verify_with_validator_set::<B>(
+					commitment.block_number,
+					&validator_set,
+					&BeefyVersionedFinalityProof::<B>::V1(signed_commitment),
+				)
+				.is_err()
+				{
 					// invalid proof
 					return Ok(())
 				}
@@ -362,7 +368,7 @@ where
 								BeefySignatureHasher,
 							>(&commitment, &id, &sig)
 							{
-								Some((id, sig))
+								Some((id, sig.clone()))
 							} else {
 								None
 							}

From 7dd7c3de8c8705410217587cee88047d26294527 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 14 Mar 2024 10:26:33 +0100
Subject: [PATCH 170/188] deduplicate OpaqueKeyOwnershipProof decoding

Suggested-by: Serban Iorga <serban300@gmail.com>
---
 polkadot/runtime/rococo/src/lib.rs             |  2 --
 polkadot/runtime/westend/src/lib.rs            |  2 --
 substrate/bin/node/runtime/src/lib.rs          |  2 --
 substrate/frame/beefy/src/lib.rs               |  5 ++++-
 .../primitives/consensus/beefy/src/lib.rs      | 18 ++++++++++--------
 5 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs
index b5730a919260..93deae9e8ab2 100644
--- a/polkadot/runtime/rococo/src/lib.rs
+++ b/polkadot/runtime/rococo/src/lib.rs
@@ -2019,8 +2019,6 @@ sp_api::impl_runtime_apis! {
 			fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BeefyId, Header, Hash>,
 			key_owner_proofs: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
-			let key_owner_proofs = key_owner_proofs.iter().cloned().map(|p| p.decode()).collect::<Option<Vec<_>>>()?;
-
 			Beefy::submit_unsigned_fork_equivocation_report(
 				fork_equivocation_proof,
 				key_owner_proofs,
diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs
index 306c6ec4f686..ba01fb62739c 100644
--- a/polkadot/runtime/westend/src/lib.rs
+++ b/polkadot/runtime/westend/src/lib.rs
@@ -2043,8 +2043,6 @@ sp_api::impl_runtime_apis! {
 			fork_equivocation_proof: beefy_primitives::ForkEquivocationProof<BeefyId, Header, Hash>,
 			key_owner_proofs: Vec<beefy_primitives::OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
-			let key_owner_proofs = key_owner_proofs.iter().cloned().map(|p| p.decode()).collect::<Option<Vec<_>>>()?;
-
 			Beefy::submit_unsigned_fork_equivocation_report(
 				fork_equivocation_proof,
 				key_owner_proofs,
diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs
index ffe72dc7c0ab..cafcebd3c22d 100644
--- a/substrate/bin/node/runtime/src/lib.rs
+++ b/substrate/bin/node/runtime/src/lib.rs
@@ -2837,8 +2837,6 @@ impl_runtime_apis! {
 			fork_equivocation_proof: sp_consensus_beefy::ForkEquivocationProof<BeefyId, Header, Hash>,
 			key_owner_proofs: Vec<sp_consensus_beefy::OpaqueKeyOwnershipProof>,
 		) -> Option<()> {
-			let key_owner_proofs = key_owner_proofs.iter().cloned().map(|p| p.decode()).collect::<Option<Vec<_>>>()?;
-
 			Beefy::submit_unsigned_fork_equivocation_report(
 				fork_equivocation_proof,
 				key_owner_proofs,
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 4da0993fb31d..f8cca2633e6a 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -430,8 +430,11 @@ impl<T: Config> Pallet<T> {
 				HeaderFor<T>,
 			>>::Hash as HashT>::Output,
 		>,
-		key_owner_proofs: Vec<T::KeyOwnerProof>,
+		key_owner_proofs: Vec<sp_consensus_beefy::OpaqueKeyOwnershipProof>,
 	) -> Option<()> {
+		let key_owner_proofs =
+			key_owner_proofs.into_iter().map(|p| p.decode()).collect::<Option<Vec<_>>>()?;
+
 		T::EquivocationReportSystem::publish_evidence(
 			EquivocationEvidenceFor::<T>::ForkEquivocationProof(
 				fork_equivocation_proof,
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 9ee44038d5f7..67df1eac4345 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -665,14 +665,16 @@ sp_api::decl_runtime_apis! {
 			key_owner_proof: OpaqueKeyOwnershipProof,
 		) -> Option<()>;
 
-		/// Submits an unsigned extrinsic to report commitments to an invalid fork.
-		/// The caller must provide the invalid commitments proof and key ownership proofs
-		/// (should be obtained using `generate_key_ownership_proof`) for the offenders. The
-		/// extrinsic will be unsigned and should only be accepted for local
-		/// authorship (not to be broadcast to the network). This method returns
-		/// `None` when creation of the extrinsic fails, e.g. if equivocation
-		/// reporting is disabled for the given runtime (i.e. this method is
-		/// hardcoded to return `None`). Only useful in an offchain context.
+		/// Submits an unsigned extrinsic to report commitments to an invalid
+		/// fork. The caller must provide the invalid commitments proof and key
+		/// ownership proofs (should be obtained using
+		/// `generate_key_ownership_proof`) for the offenders. The extrinsic
+		/// will be unsigned and should only be accepted for local authorship
+		/// (not to be broadcast to the network). This method returns `None`
+		/// when creation of the extrinsic fails, e.g. if the key owner proofs
+		/// are not validly encoded or if equivocation reporting is disabled for
+		/// the given runtime (i.e. this method is hardcoded to return `None`).
+		/// Only useful in an offchain context.
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
 			fork_equivocation_proof:
 				ForkEquivocationProof<AuthorityId, Block::Header, Hash>,

From bb6bd8d846f5db726d5d58d27e3144d76ca8679c Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 14 Mar 2024 10:57:24 +0100
Subject: [PATCH 171/188] specific mock impl for CheckForkEquivocationProof

avoids requiring a general () impl.

Suggested-by: Serban Iorga <serban300@gmail.com>
---
 substrate/frame/beefy/src/mock.rs             | 22 +++++++++++++++++--
 .../primitives/consensus/beefy/src/lib.rs     | 13 -----------
 2 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/substrate/frame/beefy/src/mock.rs b/substrate/frame/beefy/src/mock.rs
index 2bbefedf8900..70d0ad09243d 100644
--- a/substrate/frame/beefy/src/mock.rs
+++ b/substrate/frame/beefy/src/mock.rs
@@ -26,6 +26,7 @@ use frame_support::{
 	traits::{ConstU32, ConstU64, KeyOwnerProofSystem, OnFinalize, OnInitialize},
 };
 use pallet_session::historical as pallet_session_historical;
+use sp_consensus_beefy::{BeefyAuthorityId, CheckForkEquivocationProof, ForkEquivocationProof};
 use sp_core::{crypto::KeyTypeId, ConstU128};
 use sp_io::TestExternalities;
 use sp_runtime::{
@@ -33,7 +34,7 @@ use sp_runtime::{
 	curve::PiecewiseLinear,
 	impl_opaque_keys,
 	testing::TestXt,
-	traits::{Keccak256, OpaqueKeys},
+	traits::{Hash as HashT, Header as HeaderT, Keccak256, OpaqueKeys},
 	BuildStorage, Perbill,
 };
 use sp_staking::{EraIndex, SessionIndex};
@@ -88,13 +89,30 @@ parameter_types! {
 	pub const MaxSetIdSessionEntries: u32 = BondingDuration::get() * SessionsPerEra::get();
 }
 
+pub struct AlwaysValidForkEquivocationProof;
+
+impl<Err, Header: HeaderT> CheckForkEquivocationProof<Err, Header>
+	for AlwaysValidForkEquivocationProof
+{
+	type Hash = Keccak256;
+	fn check_fork_equivocation_proof<Id, MsgHash>(
+		_proof: &ForkEquivocationProof<Id, Header, <Self::Hash as HashT>::Output>,
+	) -> Result<(), Err>
+	where
+		Id: BeefyAuthorityId<MsgHash> + PartialEq,
+		MsgHash: HashT,
+	{
+		Ok(())
+	}
+}
+
 impl pallet_beefy::Config for Test {
 	type BeefyId = BeefyId;
 	type MaxAuthorities = ConstU32<100>;
 	type MaxNominators = ConstU32<1000>;
 	type MaxSetIdSessionEntries = MaxSetIdSessionEntries;
 	type OnNewValidatorSet = ();
-	type CheckForkEquivocationProof = ();
+	type CheckForkEquivocationProof = AlwaysValidForkEquivocationProof;
 	type WeightInfo = ();
 	type KeyOwnerProof = <Historical as KeyOwnerProofSystem<(KeyTypeId, BeefyId)>>::Proof;
 	type EquivocationReportSystem =
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 67df1eac4345..03ddd7287589 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -602,19 +602,6 @@ pub trait CheckForkEquivocationProof<Err, Header: HeaderT> {
 		MsgHash: HashT;
 }
 
-impl<Err, Header: HeaderT> CheckForkEquivocationProof<Err, Header> for () {
-	type Hash = Keccak256;
-	fn check_fork_equivocation_proof<Id, MsgHash>(
-		_proof: &ForkEquivocationProof<Id, Header, <Self::Hash as HashT>::Output>,
-	) -> Result<(), Err>
-	where
-		Id: BeefyAuthorityId<MsgHash> + PartialEq,
-		MsgHash: HashT,
-	{
-		Ok(())
-	}
-}
-
 /// An opaque type used to represent the key ownership proof at the runtime API
 /// boundary. The inner value is an encoded representation of the actual key
 /// ownership proof which will be parameterized when defining the runtime. At

From e291372ef81b3d57be29fd3a8f4bf82bd9f80b09 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 14 Mar 2024 11:12:27 +0100
Subject: [PATCH 172/188] add missing versioning on westend's BeefyApi

Reported-by: Adrian Catangiu <adrian@parity.io>
---
 polkadot/runtime/westend/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs
index ba01fb62739c..d041e6f737de 100644
--- a/polkadot/runtime/westend/src/lib.rs
+++ b/polkadot/runtime/westend/src/lib.rs
@@ -2014,6 +2014,7 @@ sp_api::impl_runtime_apis! {
 		}
 	}
 
+	#[api_version(4)]
 	impl beefy_primitives::BeefyApi<Block, BeefyId, Hash> for Runtime {
 		fn beefy_genesis() -> Option<BlockNumber> {
 			Beefy::genesis_block()

From d8625d8bdae36b71e78394613ec00264b1159f3f Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Sun, 17 Mar 2024 19:20:45 +0100
Subject: [PATCH 173/188] =?UTF-8?q?{canonical=E2=86=92best}=5Froot?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Conflated use of canonical:
Using canonical elsewhere to refer to the payload etc. that should live
at the same height as the equivocating payload. But here, the ancestry
proof is being checked for being able to reproduce the *best* root, not
against the root that *should* have been at
`commitment.block_number` (this root is derived from the ancestry proof
itself).
---
 substrate/frame/beefy-mmr/src/lib.rs                  |  4 ++--
 substrate/primitives/consensus/beefy/src/lib.rs       | 11 ++++++-----
 .../primitives/merkle-mountain-range/src/utils.rs     |  3 ++-
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/substrate/frame/beefy-mmr/src/lib.rs b/substrate/frame/beefy-mmr/src/lib.rs
index 1ee5f0382be2..c0e0c27a82dd 100644
--- a/substrate/frame/beefy-mmr/src/lib.rs
+++ b/substrate/frame/beefy-mmr/src/lib.rs
@@ -185,7 +185,7 @@ impl<T: pallet_mmr::Config> CheckForkEquivocationProof<pallet_beefy::Error<T>, H
 		Id: sp_consensus_beefy::BeefyAuthorityId<MsgHash> + PartialEq,
 		MsgHash: sp_runtime::traits::Hash,
 	{
-		let canonical_root = <pallet_mmr::Pallet<T>>::mmr_root();
+		let best_root = <pallet_mmr::Pallet<T>>::mmr_root();
 		let mmr_size =
 			sp_mmr_primitives::utils::NodesUtils::new(<pallet_mmr::Pallet<T>>::mmr_leaves()).size();
 		// if first_mmr_block_num is invalid, then presumably beefy is not active.
@@ -203,7 +203,7 @@ impl<T: pallet_mmr::Config> CheckForkEquivocationProof<pallet_beefy::Error<T>, H
 		};
 		if !sp_consensus_beefy::check_fork_equivocation_proof::<_, _, HeaderFor<T>, Self::Hash>(
 			proof,
-			canonical_root,
+			best_root,
 			mmr_size,
 			&canonical_header_hash,
 			first_mmr_block_num,
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 03ddd7287589..2dc64ab42fff 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -343,7 +343,8 @@ pub struct ForkEquivocationProof<Id: RuntimeAppPublic, Header: HeaderT, Hash> {
 	pub signatories: Vec<(Id, Id::Signature)>,
 	/// Canonical header at the same height as `commitment.block_number`.
 	pub canonical_header: Option<Header>,
-	/// Ancestry proof showing mmr root
+	/// Ancestry proof showing that the current best mmr root descends from another mmr root at
+	/// `commitment.block_number` than commitment.payload
 	pub ancestry_proof: Option<AncestryProof<Hash>>,
 }
 
@@ -446,7 +447,7 @@ fn check_ancestry_proof<Header, NodeHash>(
 	commitment: &Commitment<Header::Number>,
 	ancestry_proof: &Option<AncestryProof<NodeHash::Output>>,
 	first_mmr_block_num: Header::Number,
-	canonical_root: NodeHash::Output,
+	best_root: NodeHash::Output,
 	mmr_size: u64,
 ) -> bool
 where
@@ -477,7 +478,7 @@ where
 			if sp_mmr_primitives::utils::verify_ancestry_proof::<
 				NodeHash::Output,
 				utils::AncestryHasher<NodeHash>,
-			>(canonical_root, mmr_size, ancestry_proof.clone()) !=
+			>(best_root, mmr_size, ancestry_proof.clone()) !=
 				Ok(true)
 			{
 				return false
@@ -524,7 +525,7 @@ where
 /// GRANDPA.
 pub fn check_fork_equivocation_proof<Id, MsgHash, Header, NodeHash>(
 	proof: &ForkEquivocationProof<Id, Header, NodeHash::Output>,
-	canonical_root: NodeHash::Output,
+	best_root: NodeHash::Output,
 	mmr_size: u64,
 	canonical_header_hash: &Header::Hash,
 	first_mmr_block_num: Header::Number,
@@ -553,7 +554,7 @@ where
 				commitment,
 				ancestry_proof,
 				first_mmr_block_num,
-				canonical_root,
+				best_root,
 				mmr_size,
 			) {
 			return false;
diff --git a/substrate/primitives/merkle-mountain-range/src/utils.rs b/substrate/primitives/merkle-mountain-range/src/utils.rs
index f168238570bf..37e2f65db236 100644
--- a/substrate/primitives/merkle-mountain-range/src/utils.rs
+++ b/substrate/primitives/merkle-mountain-range/src/utils.rs
@@ -41,7 +41,8 @@ impl<H: sp_runtime::traits::Hash> mmr_lib::Merge for AncestryHasher<H> {
 	}
 }
 
-/// Stateless verification of the ancestry proof of a previous root.
+/// Stateless verification of the ancestry proof of `root` against a prev_root implicit in the
+/// ancestry proof.
 pub fn verify_ancestry_proof<H, M>(
 	root: H,
 	mmr_size: NodeIndex,

From 8d7e6dfe1460535f9deb5340f753b17f3b4fdfae Mon Sep 17 00:00:00 2001
From: Serban Iorga <serban@parity.io>
Date: Tue, 16 Apr 2024 17:15:21 +0300
Subject: [PATCH 174/188] Fix correct_beefy_payload() test

---
 substrate/client/consensus/beefy/src/tests.rs | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index 6da9e68ce6b2..48023355692e 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -325,10 +325,9 @@ sp_api::mock_impl_runtime_apis! {
 		) -> Option<()> {
 			if let Some(equivocations_buf) = self.inner.reported_vote_equivocations.as_ref() {
 				equivocations_buf.lock().push(proof);
-				None
-			} else {
-				panic!("Equivocations not expected, but following proof was reported: {:?}", proof);
 			}
+
+			None
 		}
 
 		fn submit_report_fork_equivocation_unsigned_extrinsic(
@@ -337,10 +336,9 @@ sp_api::mock_impl_runtime_apis! {
 		) -> Option<()> {
 			if let Some(equivocations_buf) = self.inner.reported_fork_equivocations.as_ref() {
 				equivocations_buf.lock().push(proof);
-				None
-			} else {
-				panic!("Equivocations not expected, but following proof was reported: {:?}", proof);
 			}
+
+			None
 		}
 
 		fn generate_key_ownership_proof(

From c23f4d2afea5525e0fdf4099d36182c15debba0a Mon Sep 17 00:00:00 2001
From: Serban Iorga <serban@parity.io>
Date: Wed, 17 Apr 2024 09:41:26 +0300
Subject: [PATCH 175/188] Address review comments

---
 .../beefy/src/communication/fisherman.rs      |  16 +-
 .../client/consensus/beefy/src/worker.rs      |   5 +-
 substrate/frame/beefy-mmr/src/lib.rs          |   3 +-
 substrate/frame/beefy/src/equivocation.rs     | 286 ++++++----------
 substrate/frame/beefy/src/lib.rs              |  54 +++-
 .../merkle-mountain-range/src/mmr/mmr.rs      |   2 +-
 .../primitives/consensus/beefy/src/lib.rs     | 305 ++++++++++--------
 .../consensus/beefy/src/test_utils.rs         |   2 -
 8 files changed, 325 insertions(+), 348 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index b68c004e6e60..d0255886bc9e 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -25,10 +25,9 @@ use sc_client_api::Backend;
 use sp_api::ProvideRuntimeApi;
 use sp_blockchain::HeaderBackend;
 use sp_consensus_beefy::{
-	check_fork_equivocation_proof,
 	ecdsa_crypto::{AuthorityId, Signature},
-	BeefyApi, BeefySignatureHasher, ForkEquivocationProof, MmrHashing, MmrRootHash, Payload,
-	PayloadProvider, SignedCommitment, ValidatorSet, VoteMessage,
+	BeefyApi, BeefyEquivocationProof, BeefySignatureHasher, ForkEquivocationProof, MmrHashing,
+	MmrRootHash, Payload, PayloadProvider, SignedCommitment, ValidatorSet, VoteMessage,
 };
 use sp_mmr_primitives::{AncestryProof, MmrApi};
 use sp_runtime::{
@@ -65,7 +64,8 @@ where
 		&self,
 		number: NumberFor<B>,
 	) -> Result<CanonicalHashHeaderPayload<B>, Error> {
-		// This should be un-ambiguous since `number` is finalized.
+		// This should be un-ambiguous since `number` should be finalized for
+		// incoming votes and proofs.
 		let hash = self
 			.backend
 			.blockchain()
@@ -139,13 +139,7 @@ where
 		.map_err(|e| Error::Backend(e.to_string()))?;
 
 		if proof.commitment.validator_set_id != set_id ||
-			!check_fork_equivocation_proof::<
-				AuthorityId,
-				BeefySignatureHasher,
-				B::Header,
-				MmrHashing,
-			>(
-				&proof,
+			!proof.check::<BeefySignatureHasher, MmrHashing>(
 				best_mmr_root,
 				leaf_count,
 				&canonical_commitment_block_hash,
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index a3b05fc486aa..3633806ef6e4 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -41,8 +41,9 @@ use sp_consensus::SyncOracle;
 use sp_consensus_beefy::{
 	check_vote_equivocation_proof,
 	ecdsa_crypto::{AuthorityId, Signature},
-	BeefyApi, BeefySignatureHasher, Commitment, MmrRootHash, PayloadProvider, ValidatorSet,
-	VersionedFinalityProof, VoteEquivocationProof, VoteMessage, BEEFY_ENGINE_ID,
+	BeefyApi, BeefyEquivocationProof, BeefySignatureHasher, Commitment, MmrRootHash,
+	PayloadProvider, ValidatorSet, VersionedFinalityProof, VoteEquivocationProof, VoteMessage,
+	BEEFY_ENGINE_ID,
 };
 use sp_mmr_primitives::MmrApi;
 use sp_runtime::{
diff --git a/substrate/frame/beefy-mmr/src/lib.rs b/substrate/frame/beefy-mmr/src/lib.rs
index d81e750f52a3..b155ec5b4816 100644
--- a/substrate/frame/beefy-mmr/src/lib.rs
+++ b/substrate/frame/beefy-mmr/src/lib.rs
@@ -199,8 +199,7 @@ impl<T: pallet_mmr::Config> CheckForkEquivocationProof<pallet_beefy::Error<T>, H
 			)
 			.map_err(|_| pallet_beefy::Error::<T>::InvalidForkEquivocationProof)?
 		};
-		if !sp_consensus_beefy::check_fork_equivocation_proof::<_, _, HeaderFor<T>, Self::Hash>(
-			proof,
+		if !proof.check::<_, Self::Hash>(
 			best_root,
 			mmr_size,
 			&canonical_header_hash,
diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index da0007c7b433..96436234d331 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -39,8 +39,8 @@ use frame_support::traits::{Get, KeyOwnerProofSystem};
 use frame_system::pallet_prelude::{BlockNumberFor, HeaderFor};
 use log::{error, info};
 use sp_consensus_beefy::{
-	CheckForkEquivocationProof, ForkEquivocationProof, ValidatorSetId, VoteEquivocationProof,
-	KEY_TYPE as BEEFY_KEY_TYPE,
+	BeefyEquivocationProof, CheckForkEquivocationProof, ForkEquivocationProof, ValidatorSetId,
+	VoteEquivocationProof, KEY_TYPE as BEEFY_KEY_TYPE,
 };
 use sp_runtime::{
 	traits::Hash as HashT,
@@ -148,6 +148,60 @@ pub enum EquivocationEvidenceFor<T: Config> {
 	),
 }
 
+impl<T: Config> EquivocationEvidenceFor<T> {
+	fn proof(&self) -> Box<&dyn BeefyEquivocationProof<<T as Config>::BeefyId, BlockNumberFor<T>>> {
+		match self {
+			EquivocationEvidenceFor::VoteEquivocationProof(proof, _) => Box::new(proof),
+			EquivocationEvidenceFor::ForkEquivocationProof(proof, _) => Box::new(proof),
+		}
+	}
+
+	fn key_owner_proofs(&self) -> Vec<&<T as Config>::KeyOwnerProof> {
+		match self {
+			EquivocationEvidenceFor::VoteEquivocationProof(_, key_owner_proof) =>
+				vec![key_owner_proof],
+			EquivocationEvidenceFor::ForkEquivocationProof(_, key_owner_proofs) =>
+				key_owner_proofs.iter().collect(),
+		}
+	}
+
+	// Validate the key ownership proofs extracting the ids of the offenders.
+	fn checked_offenders<P>(&self) -> Option<Vec<P::IdentificationTuple>>
+	where
+		P: KeyOwnerProofSystem<(KeyTypeId, T::BeefyId), Proof = T::KeyOwnerProof>,
+	{
+		self.proof()
+			.offender_ids()
+			.into_iter()
+			.zip(self.key_owner_proofs().iter())
+			.map(|(key, &key_owner_proof)| {
+				P::check_proof((BEEFY_KEY_TYPE, key.clone()), key_owner_proof.clone())
+			})
+			.collect::<Option<Vec<_>>>()
+	}
+
+	fn check_equivocation_proof(&self) -> Result<(), Error<T>> {
+		match self {
+			EquivocationEvidenceFor::VoteEquivocationProof(equivocation_proof, _) => {
+				// Validate equivocation proof (check votes are different and signatures are valid).
+				if !sp_consensus_beefy::check_vote_equivocation_proof(&equivocation_proof) {
+					return Err(Error::<T>::InvalidVoteEquivocationProof.into());
+				}
+
+				return Ok(())
+			},
+			EquivocationEvidenceFor::ForkEquivocationProof(equivocation_proof, _) => {
+				// Validate equivocation proof (check commitment is to unexpected payload and
+				// signatures are valid).
+				<T::CheckForkEquivocationProof as CheckForkEquivocationProof<
+					Error<T>,
+					HeaderFor<T>,
+				>>::check_fork_equivocation_proof(equivocation_proof)
+			},
+		}
+	}
+}
+
 impl<T, R, P, L> OffenceReportSystem<Option<T::AccountId>, EquivocationEvidenceFor<T>>
 	for EquivocationReportSystem<T, R, P, L>
 where
@@ -166,21 +220,7 @@ where
 	fn publish_evidence(evidence: EquivocationEvidenceFor<T>) -> Result<(), ()> {
 		use frame_system::offchain::SubmitTransaction;
 
-		let call = match evidence {
-			EquivocationEvidenceFor::VoteEquivocationProof(equivocation_proof, key_owner_proof) =>
-				Call::report_vote_equivocation_unsigned {
-					equivocation_proof: Box::new(equivocation_proof),
-					key_owner_proof,
-				},
-			EquivocationEvidenceFor::ForkEquivocationProof(
-				equivocation_proof,
-				key_owner_proofs,
-			) => Call::report_fork_equivocation_unsigned {
-				equivocation_proof: Box::new(equivocation_proof),
-				key_owner_proofs,
-			},
-		};
-
+		let call: Call<T> = evidence.into();
 		let res = SubmitTransaction::<T, Call<T>>::submit_unsigned_transaction(call.into());
 		match res {
 			Ok(_) => info!(target: LOG_TARGET, "Submitted equivocation report."),
@@ -192,41 +232,10 @@ where
 	fn check_evidence(
 		evidence: EquivocationEvidenceFor<T>,
 	) -> Result<(), TransactionValidityError> {
-		let (offenders, key_owner_proofs, time_slot) = match &evidence {
-			EquivocationEvidenceFor::VoteEquivocationProof(equivocation_proof, key_owner_proof) => {
-				// Check if the offence has already been reported, and if so then we can discard the
-				// report.
-				let time_slot = TimeSlot {
-					set_id: equivocation_proof.set_id(),
-					round: *equivocation_proof.round_number(),
-				};
-				(vec![equivocation_proof.offender_id()], vec![key_owner_proof.clone()], time_slot)
-			},
-			EquivocationEvidenceFor::ForkEquivocationProof(
-				equivocation_proof,
-				key_owner_proofs,
-			) => {
-				// Check if the offence has already been reported, and if so then we can discard the
-				// report.
-				let time_slot = TimeSlot {
-					set_id: equivocation_proof.set_id(),
-					round: *equivocation_proof.round_number(),
-				};
-				let offenders = equivocation_proof.offender_ids(); // clone data here
-				(offenders, key_owner_proofs.to_owned(), time_slot)
-			},
-		};
-
-		// Validate the key ownership proof extracting the id of the offender.
-		let offenders = offenders
-			.into_iter()
-			.zip(key_owner_proofs.iter())
-			.map(|(key, key_owner_proof)| {
-				P::check_proof((BEEFY_KEY_TYPE, key.clone()), key_owner_proof.clone())
-			})
-			.collect::<Option<Vec<_>>>()
-			.ok_or(InvalidTransaction::BadProof)?;
+		let offenders = evidence.checked_offenders::<P>().ok_or(InvalidTransaction::BadProof)?;
 
+		let time_slot =
+			TimeSlot { set_id: evidence.proof().set_id(), round: *evidence.proof().round_number() };
 		if R::is_known_offence(&offenders, &time_slot) {
 			Err(InvalidTransaction::Stale.into())
 		} else {
@@ -240,81 +249,31 @@ where
 	) -> Result<(), DispatchError> {
 		let reporter = reporter.or_else(|| <pallet_authorship::Pallet<T>>::author());
 
-		let (offenders, key_owner_proofs, set_id, round) = match &evidence {
-			EquivocationEvidenceFor::VoteEquivocationProof(equivocation_proof, key_owner_proof) =>
-				(
-					vec![equivocation_proof.offender_id()],
-					vec![key_owner_proof.clone()],
-					equivocation_proof.set_id(),
-					*equivocation_proof.round_number(),
-				),
-			EquivocationEvidenceFor::ForkEquivocationProof(
-				equivocation_proof,
-				key_owner_proofs,
-			) => {
-				let offenders = equivocation_proof.offender_ids(); // clone data here
-				(
-					offenders,
-					key_owner_proofs.to_owned(),
-					equivocation_proof.set_id(),
-					*equivocation_proof.round_number(),
-				)
-			},
-		};
-
-		// We check the equivocation within the context of its set id (and
-		// associated session) and round. We also need to know the validator
-		// set count at the time of the offence since it is required to calculate
-		// the slash amount.
-		let session_index = key_owner_proofs[0].session();
-		let validator_set_count = key_owner_proofs[0].validator_count();
-
-		// Check that the session id for the membership proof is within the
-		// bounds of the set id reported in the equivocation.
+		// We check the equivocation within the context of its set id (and associated session).
+		let set_id = evidence.proof().set_id();
 		let set_id_session_index = crate::SetIdSession::<T>::get(set_id)
 			.ok_or(Error::<T>::InvalidEquivocationProofSession)?;
+
+		// Check that the session id for the membership proof is within the bounds
+		// of the set id reported in the equivocation.
+		let key_owner_proofs = evidence.key_owner_proofs();
+		let session_index = key_owner_proofs[0].session();
 		if session_index != set_id_session_index {
 			return Err(Error::<T>::InvalidEquivocationProofSession.into())
 		}
 
-		// Validate the key ownership proof extracting the ids of the offenders.
-		let offenders = offenders
-			.into_iter()
-			.zip(key_owner_proofs.iter())
-			.map(|(key, key_owner_proof)| {
-				P::check_proof((BEEFY_KEY_TYPE, key.clone()), key_owner_proof.clone())
-			})
-			.collect::<Option<Vec<_>>>()
-			.ok_or(Error::<T>::InvalidKeyOwnershipProof)?;
-
-		match &evidence {
-			EquivocationEvidenceFor::VoteEquivocationProof(equivocation_proof, _) => {
-				// Validate equivocation proof (check votes are different and signatures are valid).
-				if !sp_consensus_beefy::check_vote_equivocation_proof(&equivocation_proof) {
-					return Err(Error::<T>::InvalidVoteEquivocationProof.into())
-				}
-			},
-			EquivocationEvidenceFor::ForkEquivocationProof(equivocation_proof, _) => {
-				// Validate equivocation proof (check commitment is to unexpected payload and
-				// signatures are valid).
-				<T::CheckForkEquivocationProof as CheckForkEquivocationProof<
-					Error<T>,
-					HeaderFor<T>,
-				>>::check_fork_equivocation_proof(equivocation_proof)?
-			},
-		}
+		let offenders =
+			evidence.checked_offenders::<P>().ok_or(Error::<T>::InvalidKeyOwnershipProof)?;
+		evidence.check_equivocation_proof()?;
 
 		let offence = EquivocationOffence {
-			time_slot: TimeSlot { set_id, round },
+			time_slot: TimeSlot { set_id, round: *evidence.proof().round_number() },
 			session_index,
-			validator_set_count,
+			validator_set_count: key_owner_proofs[0].validator_count(),
 			offenders,
 		};
-
 		R::report_offence(reporter.into_iter().collect(), offence)
-			.map_err(|_| Error::<T>::DuplicateOffenceReport)?;
-
-		Ok(())
+			.map_err(|_| Error::<T>::DuplicateOffenceReport.into())
 	}
 }
 
@@ -335,85 +294,30 @@ impl<T: Config> Pallet<T> {
 				return InvalidTransaction::Call.into()
 			},
 		}
-		match call {
-			Call::report_vote_equivocation_unsigned { equivocation_proof, key_owner_proof } => {
-				let evidence = EquivocationEvidenceFor::<T>::VoteEquivocationProof(
-					*equivocation_proof.clone(),
-					key_owner_proof.clone(),
-				);
-				T::EquivocationReportSystem::check_evidence(evidence)?;
-
-				let longevity =
-					<T::EquivocationReportSystem as OffenceReportSystem<_, _>>::Longevity::get();
-
-				ValidTransaction::with_tag_prefix("BeefyEquivocation")
-					// We assign the maximum priority for any equivocation report.
-					.priority(TransactionPriority::MAX)
-					// Only one equivocation report for the same offender at the same slot.
-					.and_provides((
-						equivocation_proof.offender_id().clone(),
-						equivocation_proof.set_id(),
-						*equivocation_proof.round_number(),
-					))
-					.longevity(longevity)
-					// We don't propagate this. This can never be included on a remote node.
-					.propagate(false)
-					.build()
-			},
-			Call::report_fork_equivocation_unsigned { equivocation_proof, key_owner_proofs } => {
-				let evidence = EquivocationEvidenceFor::<T>::ForkEquivocationProof(
-					*equivocation_proof.clone(),
-					key_owner_proofs.clone(),
-				);
-				log::error!("checking evidence");
-				T::EquivocationReportSystem::check_evidence(evidence)?;
-
-				log::error!("passed evidence check");
-				let longevity =
-					<T::EquivocationReportSystem as OffenceReportSystem<_, _>>::Longevity::get();
-				log::error!("retrieved longevity");
-
-				ValidTransaction::with_tag_prefix("BeefyEquivocation")
-					// We assign the maximum priority for any equivocation report.
-					.priority(TransactionPriority::MAX)
-					// Only one equivocation report for the same offender at the same slot.
-					.and_provides((
-						equivocation_proof.offender_ids().clone(),
-						equivocation_proof.set_id(),
-						*equivocation_proof.round_number(),
-					))
-					.longevity(longevity)
-					// We don't propagate this. This can never be included on a remote node.
-					.propagate(false)
-					.build()
-			},
-			_ => {
-				log::error!(
-					target: LOG_TARGET,
-					"lolalol"
-				);
-				InvalidTransaction::Call.into()
-			},
-		}
+
+		let (evidence, equivocation_proof) =
+			call.to_equivocation_evidence_for().ok_or(InvalidTransaction::Call)?;
+		T::EquivocationReportSystem::check_evidence(evidence)?;
+
+		let longevity =
+			<T::EquivocationReportSystem as OffenceReportSystem<_, _>>::Longevity::get();
+		ValidTransaction::with_tag_prefix("BeefyEquivocation")
+			// We assign the maximum priority for any equivocation report.
+			.priority(TransactionPriority::MAX)
+			// Only one equivocation report for the same offender at the same slot.
+			.and_provides((
+				equivocation_proof.offender_ids(),
+				equivocation_proof.set_id(),
+				*equivocation_proof.round_number(),
+			))
+			.longevity(longevity)
+			// We don't propagate this. This can never be included on a remote node.
+			.propagate(false)
+			.build()
 	}
 
 	pub fn pre_dispatch(call: &Call<T>) -> Result<(), TransactionValidityError> {
-		match call {
-			Call::report_vote_equivocation_unsigned { equivocation_proof, key_owner_proof } => {
-				let evidence = EquivocationEvidenceFor::<T>::VoteEquivocationProof(
-					*equivocation_proof.clone(),
-					key_owner_proof.clone(),
-				);
-				T::EquivocationReportSystem::check_evidence(evidence)
-			},
-			Call::report_fork_equivocation_unsigned { equivocation_proof, key_owner_proofs } => {
-				let evidence = EquivocationEvidenceFor::<T>::ForkEquivocationProof(
-					*equivocation_proof.clone(),
-					key_owner_proofs.clone(),
-				);
-				T::EquivocationReportSystem::check_evidence(evidence)
-			},
-			_ => Err(InvalidTransaction::Call.into()),
-		}
+		let (evidence, _) = call.to_equivocation_evidence_for().ok_or(InvalidTransaction::Call)?;
+		T::EquivocationReportSystem::check_evidence(evidence)
 	}
 }
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index 984efa169833..f902f0d88b73 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -63,7 +63,7 @@ const LOG_TARGET: &str = "runtime::beefy";
 pub mod pallet {
 	use super::*;
 	use frame_system::{ensure_root, pallet_prelude::BlockNumberFor};
-	use sp_consensus_beefy::ForkEquivocationProof;
+	use sp_consensus_beefy::{BeefyEquivocationProof, ForkEquivocationProof};
 
 	#[pallet::config]
 	pub trait Config: frame_system::Config {
@@ -389,6 +389,58 @@ pub mod pallet {
 			Self::validate_unsigned(source, call)
 		}
 	}
+
+	impl<T: Config> Call<T> {
+		pub fn to_equivocation_evidence_for(
+			&self,
+		) -> Option<(
+			EquivocationEvidenceFor<T>,
+			Box<&dyn BeefyEquivocationProof<<T as Config>::BeefyId, BlockNumberFor<T>>>,
+		)> {
+			match self {
+				Call::report_vote_equivocation_unsigned { equivocation_proof, key_owner_proof } =>
+					Some((
+						EquivocationEvidenceFor::<T>::VoteEquivocationProof(
+							*equivocation_proof.clone(),
+							key_owner_proof.clone(),
+						),
+						Box::new(equivocation_proof.as_ref()),
+					)),
+				Call::report_fork_equivocation_unsigned {
+					equivocation_proof,
+					key_owner_proofs,
+				} => Some((
+					EquivocationEvidenceFor::<T>::ForkEquivocationProof(
+						*equivocation_proof.clone(),
+						key_owner_proofs.clone(),
+					),
+					Box::new(equivocation_proof.as_ref()),
+				)),
+				_ => None,
+			}
+		}
+	}
+
+	impl<T: Config> From<EquivocationEvidenceFor<T>> for Call<T> {
+		fn from(evidence: EquivocationEvidenceFor<T>) -> Self {
+			match evidence {
+				EquivocationEvidenceFor::VoteEquivocationProof(
+					equivocation_proof,
+					key_owner_proof,
+				) => Call::report_vote_equivocation_unsigned {
+					equivocation_proof: Box::new(equivocation_proof),
+					key_owner_proof,
+				},
+				EquivocationEvidenceFor::ForkEquivocationProof(
+					equivocation_proof,
+					key_owner_proofs,
+				) => Call::report_fork_equivocation_unsigned {
+					equivocation_proof: Box::new(equivocation_proof),
+					key_owner_proofs,
+				},
+			}
+		}
+	}
 }
 
 #[cfg(any(feature = "try-runtime", test))]
diff --git a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
index 99167683829e..c11a1639fb55 100644
--- a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
+++ b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
@@ -182,7 +182,7 @@ where
 
 	/// Commit the changes to underlying storage, return current number of leaves and
 	/// calculate the new MMR's root hash.
-	pub fn finalize(&mut self) -> Result<(NodeIndex, HashOf<T, I>), Error> {
+	pub fn finalize(mut self) -> Result<(NodeIndex, HashOf<T, I>), Error> {
 		let root = self.mmr.get_root().map_err(|e| Error::GetRoot.log_error(e))?;
 		self.mmr.commit().map_err(|e| Error::Commit.log_error(e))?;
 		Ok((self.leaves, root.hash()))
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 27514de8f6b1..d5806a83be0f 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -57,7 +57,7 @@ use core::fmt::{Debug, Display};
 use scale_info::TypeInfo;
 use sp_application_crypto::{AppCrypto, AppPublic, ByteArray, RuntimeAppPublic};
 use sp_core::H256;
-use sp_runtime::traits::{Hash as HashT, Header as HeaderT, Keccak256, NumberFor};
+use sp_runtime::traits::{Hash as HashT, HashOutput, Header as HeaderT, Keccak256, NumberFor};
 
 /// Key type for BEEFY module.
 pub const KEY_TYPE: sp_core::crypto::KeyTypeId = sp_application_crypto::key_types::BEEFY;
@@ -307,6 +307,18 @@ pub struct VoteMessage<Number, Id, Signature> {
 	pub signature: Signature,
 }
 
+/// Trait containing generic methods for BEEFY equivocation proofs.
+pub trait BeefyEquivocationProof<Id, Number> {
+	/// Returns the authority ids of the misbehaving voters.
+	fn offender_ids(&self) -> Vec<&Id>;
+
+	/// Returns the round number at which the infringement occurred.
+	fn round_number(&self) -> &Number;
+
+	/// Returns the set id at which the infringement occurred.
+	fn set_id(&self) -> ValidatorSetId;
+}
+
 /// Proof of voter misbehavior on a given set id. Misbehavior/equivocation in
 /// BEEFY happens when a voter votes on the same round/block for different payloads.
 /// Proving is achieved by collecting the signed commitments of conflicting votes.
@@ -323,12 +335,20 @@ impl<Number, Id, Signature> VoteEquivocationProof<Number, Id, Signature> {
 	pub fn offender_id(&self) -> &Id {
 		&self.first.id
 	}
-	/// Returns the round number at which the equivocation occurred.
-	pub fn round_number(&self) -> &Number {
+}
+
+impl<Number, Id, Signature> BeefyEquivocationProof<Id, Number>
+	for VoteEquivocationProof<Number, Id, Signature>
+{
+	fn offender_ids(&self) -> Vec<&Id> {
+		vec![self.offender_id()]
+	}
+
+	fn round_number(&self) -> &Number {
 		&self.first.commitment.block_number
 	}
-	/// Returns the set id at which the equivocation occurred.
-	pub fn set_id(&self) -> ValidatorSetId {
+
+	fn set_id(&self) -> ValidatorSetId {
 		self.first.commitment.validator_set_id
 	}
 }
@@ -350,17 +370,97 @@ pub struct ForkEquivocationProof<Id: RuntimeAppPublic, Header: HeaderT, Hash> {
 	pub ancestry_proof: Option<AncestryProof<Hash>>,
 }
 
-impl<Id: RuntimeAppPublic, H: HeaderT, Hash> ForkEquivocationProof<Id, H, Hash> {
-	/// Returns the authority ids of the misbehaving voters.
-	pub fn offender_ids(&self) -> Vec<&Id> {
+impl<Id: RuntimeAppPublic, Header: HeaderT, Hash: HashOutput>
+	ForkEquivocationProof<Id, Header, Hash>
+{
+	fn check_fork<NodeHash: HashT<Output = Hash>>(
+		&self,
+		best_root: Hash,
+		mmr_size: u64,
+		canonical_header_hash: &Header::Hash,
+		first_mmr_block_num: Header::Number,
+		best_block_num: Header::Number,
+	) -> bool {
+		if self.commitment.block_number <= best_block_num {
+			if let Some(canonical_header) = &self.canonical_header {
+				if check_header_proof(&self.commitment, canonical_header, canonical_header_hash) {
+					// avoid verifying the ancestry proof if a valid header proof has been provided
+					return true;
+				}
+			}
+
+			if let Some(ancestry_proof) = &self.ancestry_proof {
+				return check_ancestry_proof::<Header, NodeHash>(
+					&self.commitment,
+					ancestry_proof,
+					first_mmr_block_num,
+					best_root,
+					mmr_size,
+				);
+			}
+
+			return false;
+		}
+
+		true
+	}
+
+	/// Validates [ForkEquivocationProof] with the following checks:
+	/// - if the commitment is to a block in our history, then at least a header or an ancestry
+	///   proof is provided:
+	///   - the proof is correct if the provided `canonical_header` is at height
+	/// `commitment.block_number` and `commitment.payload` != `canonical_payload(canonical_header)`
+	///   - the proof is correct if the provided `ancestry_proof` proves
+	///   `mmr_root(commitment.block_number) != mmr_root(commitment.payload)`
+	/// - `commitment` is signed by all claimed signatories
+	///
+	/// NOTE: GRANDPA finalization proof is not checked, which leads to slashing on forks. This is
+	/// fine since honest validators will not be slashed on the chain finalized by GRANDPA, which is
+	/// the only chain that ultimately matters. The only material difference not checking GRANDPA
+	/// proofs makes is that validators are not slashed for signing BEEFY commitments prior to the
+	/// blocks committed to being finalized by GRANDPA. This is fine too, since the slashing risk of
+	/// committing to an incorrect block implies validators will only sign blocks they *know* will
+	/// be finalized by GRANDPA.
+	pub fn check<MsgHash: HashT, NodeHash: HashT<Output = Hash>>(
+		&self,
+		best_root: Hash,
+		mmr_size: u64,
+		canonical_header_hash: &Header::Hash,
+		first_mmr_block_num: Header::Number,
+		best_block_num: Header::Number,
+	) -> bool
+	where
+		Id: BeefyAuthorityId<MsgHash> + PartialEq,
+	{
+		if !self.check_fork::<NodeHash>(
+			best_root,
+			mmr_size,
+			canonical_header_hash,
+			first_mmr_block_num,
+			best_block_num,
+		) {
+			return false;
+		}
+
+		return self.signatories.iter().all(|(authority_id, signature)| {
+			// TODO: refactor check_commitment_signature to take a slice of signatories
+			check_commitment_signature(&self.commitment, authority_id, signature)
+		})
+	}
+}
+
+impl<Id: RuntimeAppPublic, Header: HeaderT, Hash: HashOutput>
+	BeefyEquivocationProof<Id, Header::Number> for ForkEquivocationProof<Id, Header, Hash>
+{
+	fn offender_ids(&self) -> Vec<&Id> {
 		self.signatories.iter().map(|(id, _)| id).collect()
 	}
-	/// Returns the round number at which the infringement occurred.
-	pub fn round_number(&self) -> &H::Number {
+
+	fn round_number(&self) -> &Header::Number {
 		&self.commitment.block_number
 	}
-	/// Returns the set id at which the infringement occurred.
-	pub fn set_id(&self) -> ValidatorSetId {
+
+	fn set_id(&self) -> ValidatorSetId {
 		self.commitment.validator_set_id
 	}
 }
@@ -419,35 +519,30 @@ where
 /// Checks whether the provided header's payload differs from the commitment's payload.
 fn check_header_proof<Header>(
 	commitment: &Commitment<Header::Number>,
-	canonical_header: &Option<Header>,
+	canonical_header: &Header,
 	canonical_header_hash: &Header::Hash,
 ) -> bool
 where
 	Header: HeaderT,
 {
-	if let Some(canonical_header) = canonical_header {
-		let canonical_mmr_root_digest = mmr::find_mmr_root_digest::<Header>(canonical_header);
-		let canonical_payload = canonical_mmr_root_digest.map(|mmr_root| {
-			Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode())
-		});
-		// Check header's hash and that the `payload` of the `commitment` differs from the
-		// `canonical_payload`. Note that if the signatories signed a payload when there should be
-		// none (for instance for a block prior to BEEFY activation), then canonical_payload = None,
-		// and they will likewise be slashed.
-		// Note that we can only check this if a valid header has been provided - we cannot
-		// slash for this with an ancestry proof - by necessity)
-		return canonical_header.hash() == *canonical_header_hash &&
-			Some(&commitment.payload) != canonical_payload.as_ref()
-	}
-	// if no header provided, the header proof is also not correct
-	false
+	let canonical_mmr_root_digest = mmr::find_mmr_root_digest::<Header>(canonical_header);
+	let canonical_payload = canonical_mmr_root_digest
+		.map(|mmr_root| Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode()));
+	// Check header's hash and that the `payload` of the `commitment` differs from the
+	// `canonical_payload`. Note that if the signatories signed a payload when there should be
+	// none (for instance for a block prior to BEEFY activation), then canonical_payload = None,
+	// and they will likewise be slashed.
+	// Note that we can only check this if a valid header has been provided - we cannot
+	// slash for this with an ancestry proof - by necessity)
+	return canonical_header.hash() == *canonical_header_hash &&
+		Some(&commitment.payload) != canonical_payload.as_ref()
 }
 
 /// Checks whether an ancestry proof has the correct size and its calculated root differs from the
 /// commitment's payload's.
 fn check_ancestry_proof<Header, NodeHash>(
 	commitment: &Commitment<Header::Number>,
-	ancestry_proof: &Option<AncestryProof<NodeHash::Output>>,
+	ancestry_proof: &AncestryProof<NodeHash::Output>,
 	first_mmr_block_num: Header::Number,
 	best_root: NodeHash::Output,
 	mmr_size: u64,
@@ -456,119 +551,53 @@ where
 	Header: HeaderT,
 	NodeHash: HashT,
 {
-	if let Some(ancestry_proof) = ancestry_proof {
-		let expected_leaf_count = sp_mmr_primitives::utils::block_num_to_leaf_index::<Header>(
-			commitment.block_number,
-			first_mmr_block_num,
-		)
-		.and_then(|leaf_index| {
-			leaf_index.checked_add(1).ok_or_else(|| {
-				sp_mmr_primitives::Error::InvalidNumericOp.log_debug("leaf_index + 1 overflowed")
-			})
-		});
-
-		if let Ok(expected_leaf_count) = expected_leaf_count {
-			let expected_mmr_size =
-				sp_mmr_primitives::utils::NodesUtils::new(expected_leaf_count).size();
-			// verify that the prev_root is at the correct block number
-			// this can be inferred from the leaf_count / mmr_size of the prev_root:
-			// we've converted the commitment.block_number to an mmr size and now
-			// compare with the value in the ancestry proof
-			if expected_mmr_size != ancestry_proof.prev_size {
-				return false
-			}
-			if sp_mmr_primitives::utils::verify_ancestry_proof::<
-				NodeHash::Output,
-				utils::AncestryHasher<NodeHash>,
-			>(best_root, mmr_size, ancestry_proof.clone()) !=
-				Ok(true)
-			{
-				return false
-			}
-		} else {
-			// if the block number either under- or overflowed, the
-			// commitment.block_number was not valid and the commitment should not have
-			// been signed, hence we can skip the ancestry proof and slash the
-			// signatories
-			return true
+	let expected_leaf_count = sp_mmr_primitives::utils::block_num_to_leaf_index::<Header>(
+		commitment.block_number,
+		first_mmr_block_num,
+	)
+	.and_then(|leaf_index| {
+		leaf_index.checked_add(1).ok_or_else(|| {
+			sp_mmr_primitives::Error::InvalidNumericOp.log_debug("leaf_index + 1 overflowed")
+		})
+	});
+
+	if let Ok(expected_leaf_count) = expected_leaf_count {
+		let expected_mmr_size =
+			sp_mmr_primitives::utils::NodesUtils::new(expected_leaf_count).size();
+		// verify that the prev_root is at the correct block number
+		// this can be inferred from the leaf_count / mmr_size of the prev_root:
+		// we've converted the commitment.block_number to an mmr size and now
+		// compare with the value in the ancestry proof
+		if expected_mmr_size != ancestry_proof.prev_size {
+			return false
 		}
-
-		// once the ancestry proof is verified, calculate the prev_root to compare it
-		// with the commitment's prev_root
-		let ancestry_prev_root = mmr_lib::ancestry_proof::bagging_peaks_hashes::<
+		if sp_mmr_primitives::utils::verify_ancestry_proof::<
 			NodeHash::Output,
-			AncestryHasher<NodeHash>,
-		>(ancestry_proof.prev_peaks.clone());
-		// if the commitment payload does not commit to an MMR root, then this
-		// commitment may have another purpose and should not be slashed
-		let commitment_prev_root =
-			commitment.payload.get_decoded::<NodeHash::Output>(&known_payloads::MMR_ROOT_ID);
-		return commitment_prev_root != ancestry_prev_root.ok()
-	}
-	// if no ancestry proof provided, the proof is also not correct
-	false
-}
-
-/// Validates [ForkEquivocationProof] with the following checks:
-/// - if the commitment is to a block in our history, then at least a header or an ancestry proof is
-///   provided:
-///   - a `canonical_header` is correct if it's at height `commitment.block_number` and
-///   commitment.payload` != `canonical_payload(canonical_header)`
-///   - an `ancestry_proof` is correct if it proves mmr_root(commitment.block_number) !=
-///   mmr_root(commitment.payload)`
-/// - `commitment` is signed by all claimed signatories
-///
-/// NOTE: GRANDPA finalization proof is not checked, which leads to slashing on forks. This is fine
-/// since honest validators will not be slashed on the chain finalized by GRANDPA, which is the only
-/// chain that ultimately matters. The only material difference not checking GRANDPA proofs makes is
-/// that validators are not slashed for signing BEEFY commitments prior to the blocks committed to
-/// being finalized by GRANDPA. This is fine too, since the slashing risk of committing to an
-/// incorrect block implies validators will only sign blocks they *know* will be finalized by
-/// GRANDPA.
-pub fn check_fork_equivocation_proof<Id, MsgHash, Header, NodeHash>(
-	proof: &ForkEquivocationProof<Id, Header, NodeHash::Output>,
-	best_root: NodeHash::Output,
-	mmr_size: u64,
-	canonical_header_hash: &Header::Hash,
-	first_mmr_block_num: Header::Number,
-	best_block_num: Header::Number,
-) -> bool
-where
-	Id: BeefyAuthorityId<MsgHash> + PartialEq,
-	MsgHash: HashT,
-	Header: HeaderT,
-	NodeHash: HashT,
-{
-	let ForkEquivocationProof { commitment, signatories, canonical_header, ancestry_proof } = proof;
-
-	// if commitment is to a block in the future, it's an equivocation as long as it's been signed
-	if commitment.block_number <= best_block_num {
-		if (canonical_header, ancestry_proof) == (&None, &None) {
-			// if commitment isn't to a block number in the future, at least a header or ancestry
-			// proof must be provided, otherwise the proof is entirely invalid
-			return false;
-		}
-
-		// if neither the ancestry proof nor the header proof is correct, the proof is invalid
-		// avoid verifying the ancestry proof if a valid header proof has been provided
-		if !check_header_proof(commitment, canonical_header, canonical_header_hash) &&
-			!check_ancestry_proof::<Header, NodeHash>(
-				commitment,
-				ancestry_proof,
-				first_mmr_block_num,
-				best_root,
-				mmr_size,
-			) {
-			return false;
+			utils::AncestryHasher<NodeHash>,
+		>(best_root, mmr_size, ancestry_proof.clone()) !=
+			Ok(true)
+		{
+			return false
 		}
+	} else {
+		// if the block number either under- or overflowed, the
+		// commitment.block_number was not valid and the commitment should not have
+		// been signed, hence we can skip the ancestry proof and slash the
+		// signatories
+		return true
 	}
 
-	// if commitment is to future block or either proof is valid, check the validator's signatures.
-	// The proof is verified if they are all correct.
-	return signatories.iter().all(|(authority_id, signature)| {
-		// TODO: refactor check_commitment_signature to take a slice of signatories
-		check_commitment_signature(&commitment, authority_id, signature)
-	})
+	// once the ancestry proof is verified, calculate the prev_root to compare it
+	// with the commitment's prev_root
+	let ancestry_prev_root = mmr_lib::ancestry_proof::bagging_peaks_hashes::<
+		NodeHash::Output,
+		AncestryHasher<NodeHash>,
+	>(ancestry_proof.prev_peaks.clone());
+	// if the commitment payload does not commit to an MMR root, then this
+	// commitment may have another purpose and should not be slashed
+	let commitment_prev_root =
+		commitment.payload.get_decoded::<NodeHash::Output>(&known_payloads::MMR_ROOT_ID);
+	return commitment_prev_root != ancestry_prev_root.ok()
 }
 
 /// New BEEFY validator set notification hook.
diff --git a/substrate/primitives/consensus/beefy/src/test_utils.rs b/substrate/primitives/consensus/beefy/src/test_utils.rs
index 48995ace4763..8e5bd83a002b 100644
--- a/substrate/primitives/consensus/beefy/src/test_utils.rs
+++ b/substrate/primitives/consensus/beefy/src/test_utils.rs
@@ -45,8 +45,6 @@ pub enum Keyring<AuthorityId> {
 	_Marker(PhantomData<AuthorityId>),
 }
 
-// impl AuthorityIdBound for Keyring<ecdsa_crypto::AuthorityId> {}
-
 /// Trait representing BEEFY specific generation and signing behavior of authority id
 ///
 /// Accepts custom hashing fn for the message and custom convertor fn for the signer.

From dd6ff5665fdfa69549cad2f0e5090bdbdb0fcf95 Mon Sep 17 00:00:00 2001
From: Serban Iorga <serban@parity.io>
Date: Wed, 17 Apr 2024 17:58:53 +0300
Subject: [PATCH 176/188] fix CI

---
 substrate/frame/beefy-mmr/Cargo.toml                  | 2 ++
 substrate/primitives/consensus/beefy/src/lib.rs       | 2 +-
 substrate/primitives/merkle-mountain-range/Cargo.toml | 3 ++-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/substrate/frame/beefy-mmr/Cargo.toml b/substrate/frame/beefy-mmr/Cargo.toml
index c89ae741ab6d..aef962a48604 100644
--- a/substrate/frame/beefy-mmr/Cargo.toml
+++ b/substrate/frame/beefy-mmr/Cargo.toml
@@ -53,9 +53,11 @@ std = [
 	"scale-info/std",
 	"serde",
 	"sp-api/std",
+	"sp-application-crypto/std",
 	"sp-consensus-beefy/std",
 	"sp-core/std",
 	"sp-io/std",
+	"sp-mmr-primitives/std",
 	"sp-runtime/std",
 	"sp-staking/std",
 	"sp-state-machine/std",
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index d5806a83be0f..89307ba60407 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -51,7 +51,7 @@ use sp_mmr_primitives::{
 	AncestryProof,
 };
 
-use alloc::vec::Vec;
+use alloc::{vec, vec::Vec};
 use codec::{Codec, Decode, Encode};
 use core::fmt::{Debug, Display};
 use scale_info::TypeInfo;
diff --git a/substrate/primitives/merkle-mountain-range/Cargo.toml b/substrate/primitives/merkle-mountain-range/Cargo.toml
index 3cded0cd3bc2..9b0b60cac567 100644
--- a/substrate/primitives/merkle-mountain-range/Cargo.toml
+++ b/substrate/primitives/merkle-mountain-range/Cargo.toml
@@ -21,7 +21,7 @@ log = { workspace = true }
 mmr-lib = { package = "ckb-merkle-mountain-range", git = "https://github.com/paritytech/merkle-mountain-range.git", branch = "leaf-node-proof-split", default-features = false }
 serde = { features = ["alloc", "derive"], optional = true, workspace = true }
 sp-api = { path = "../api", default-features = false }
-sp-std = { path = "../std" }
+sp-std = { path = "../std", default-features = false }
 sp-core = { path = "../core", default-features = false }
 sp-debug-derive = { path = "../debug-derive", default-features = false }
 sp-runtime = { path = "../runtime", default-features = false }
@@ -43,6 +43,7 @@ std = [
 	"sp-core/std",
 	"sp-debug-derive/std",
 	"sp-runtime/std",
+	"sp-std/std"
 ]
 
 # Serde support without relying on std features.

From cf5103703a0c34d1d56bc9d2a82b12ca70ebaeba Mon Sep 17 00:00:00 2001
From: Serban Iorga <serban@parity.io>
Date: Wed, 17 Apr 2024 18:00:51 +0300
Subject: [PATCH 177/188] taplo

---
 substrate/primitives/merkle-mountain-range/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/substrate/primitives/merkle-mountain-range/Cargo.toml b/substrate/primitives/merkle-mountain-range/Cargo.toml
index 9b0b60cac567..2ad636eca24e 100644
--- a/substrate/primitives/merkle-mountain-range/Cargo.toml
+++ b/substrate/primitives/merkle-mountain-range/Cargo.toml
@@ -43,7 +43,7 @@ std = [
 	"sp-core/std",
 	"sp-debug-derive/std",
 	"sp-runtime/std",
-	"sp-std/std"
+	"sp-std/std",
 ]
 
 # Serde support without relying on std features.

From 114a894bb45f20fa25c668085c55c3353e1b5a1f Mon Sep 17 00:00:00 2001
From: Serban Iorga <serban@parity.io>
Date: Thu, 18 Apr 2024 09:18:44 +0300
Subject: [PATCH 178/188] Fix tests

---
 substrate/frame/beefy/src/mock.rs  | 17 +++++++----
 substrate/frame/beefy/src/tests.rs | 49 ++++++++++++++++--------------
 2 files changed, 37 insertions(+), 29 deletions(-)

diff --git a/substrate/frame/beefy/src/mock.rs b/substrate/frame/beefy/src/mock.rs
index 9ba57a312ec6..73b4a0bcac9d 100644
--- a/substrate/frame/beefy/src/mock.rs
+++ b/substrate/frame/beefy/src/mock.rs
@@ -87,22 +87,27 @@ parameter_types! {
 	pub const ReportLongevity: u64 =
 		BondingDuration::get() as u64 * SessionsPerEra::get() as u64 * Period::get();
 	pub const MaxSetIdSessionEntries: u32 = BondingDuration::get() * SessionsPerEra::get();
+
+	pub storage IsValidForkEquivocationProof: bool = true;
 }
 
-pub struct AlwaysValidForkEquivocationProof;
+pub struct MockForkEquivocationProofChecker;
 
-impl<Err, Header: HeaderT> CheckForkEquivocationProof<Err, Header>
-	for AlwaysValidForkEquivocationProof
+impl<Header: HeaderT> CheckForkEquivocationProof<crate::pallet::Error<Test>, Header>
+	for MockForkEquivocationProofChecker
 {
 	type Hash = Keccak256;
 	fn check_fork_equivocation_proof<Id, MsgHash>(
 		_proof: &ForkEquivocationProof<Id, Header, <Self::Hash as HashT>::Output>,
-	) -> Result<(), Err>
+	) -> Result<(), crate::pallet::Error<Test>>
 	where
 		Id: BeefyAuthorityId<MsgHash> + PartialEq,
 		MsgHash: HashT,
 	{
-		Ok(())
+		match IsValidForkEquivocationProof::get() {
+			true => Ok(()),
+			false => Err(crate::pallet::Error::InvalidForkEquivocationProof),
+		}
 	}
 }
 
@@ -112,7 +117,7 @@ impl pallet_beefy::Config for Test {
 	type MaxNominators = ConstU32<1000>;
 	type MaxSetIdSessionEntries = MaxSetIdSessionEntries;
 	type OnNewValidatorSet = ();
-	type CheckForkEquivocationProof = AlwaysValidForkEquivocationProof;
+	type CheckForkEquivocationProof = MockForkEquivocationProofChecker;
 	type WeightInfo = ();
 	type KeyOwnerProof = <Historical as KeyOwnerProofSystem<(KeyTypeId, BeefyId)>>::Proof;
 	type EquivocationReportSystem =
diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs
index 33761be18004..a136371cdd73 100644
--- a/substrate/frame/beefy/src/tests.rs
+++ b/substrate/frame/beefy/src/tests.rs
@@ -94,7 +94,8 @@ fn session_change_updates_authorities() {
 			assert!(2 == beefy::ValidatorSetId::<Test>::get());
 
 			let want = beefy_log(ConsensusLog::AuthoritiesChange(
-				ValidatorSet::new(vec![mock_beefy_id(2), mock_beefy_id(4)], 2).unwrap(),
+				ValidatorSet::new(vec![mock_beefy_id(2), mock_beefy_id(3), mock_beefy_id(4)], 2)
+					.unwrap(),
 			));
 
 			let log = System::digest().logs[1].clone();
@@ -678,7 +679,7 @@ fn report_vote_equivocation_validate_unsigned_prevents_duplicates() {
 		);
 
 		// the transaction is valid when passed as local
-		let tx_tag = (equivocation_key, set_id, 3u64);
+		let tx_tag = (vec![equivocation_key], set_id, 3u64);
 
 		assert_eq!(
 			<Beefy as sp_runtime::traits::ValidateUnsigned>::validate_unsigned(
@@ -1317,9 +1318,9 @@ fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 
-		// vote targets different round than finalized payload, there is no equivocation.
+		// vote signed with a key that isn't part of the authority set
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
-			(block_num + 1, payload.clone(), set_id, &equivocation_keyring),
+			(block_num, payload.clone(), set_id, &BeefyKeyring::Dave),
 			None,
 			Some(ancestry_proof.clone()),
 		);
@@ -1329,12 +1330,12 @@ fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 				Box::new(equivocation_proof),
 				vec![key_owner_proof.clone()],
 			),
-			Error::<Test>::InvalidForkEquivocationProof,
+			Error::<Test>::InvalidKeyOwnershipProof,
 		);
 
-		// vote signed with a key that isn't part of the authority set
+		// vote targets future set id
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
-			(block_num, payload.clone(), set_id, &BeefyKeyring::Dave),
+			(block_num, payload.clone(), set_id + 1, &equivocation_keyring),
 			None,
 			Some(ancestry_proof.clone()),
 		);
@@ -1344,12 +1345,13 @@ fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 				Box::new(equivocation_proof),
 				vec![key_owner_proof.clone()],
 			),
-			Error::<Test>::InvalidKeyOwnershipProof,
+			Error::<Test>::InvalidEquivocationProofSession,
 		);
 
-		// vote targets future set id
+		// Simulate InvalidForkEquivocationProof error.
+		IsValidForkEquivocationProof::set(&false);
 		let equivocation_proof = generate_fork_equivocation_proof_vote(
-			(block_num, payload.clone(), set_id + 1, &equivocation_keyring),
+			(block_num + 1, payload.clone(), set_id, &equivocation_keyring),
 			None,
 			Some(ancestry_proof),
 		);
@@ -1359,7 +1361,7 @@ fn report_fork_equivocation_vote_invalid_equivocation_proof() {
 				Box::new(equivocation_proof),
 				vec![key_owner_proof.clone()],
 			),
-			Error::<Test>::InvalidEquivocationProofSession,
+			Error::<Test>::InvalidForkEquivocationProof,
 		);
 	});
 }
@@ -2149,14 +2151,14 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 		let payload = Payload::from_single_entry(MMR_ROOT_ID, vec![42]);
 		let ancestry_proof = Mmr::generate_ancestry_proof(block_num, None).unwrap();
 
-		// commitment targets different round than finalized payload, there is no equivocation.
+		// commitment signed with a key that isn't part of the authority set
 		let equivocation_proof = generate_fork_equivocation_proof_sc(
 			Commitment {
 				validator_set_id: set_id,
-				block_number: block_num + 1,
+				block_number: block_num,
 				payload: payload.clone(),
 			},
-			equivocation_keyrings.clone(),
+			vec![BeefyKeyring::Eve],
 			None,
 			Some(ancestry_proof.clone()),
 		);
@@ -2166,17 +2168,17 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 				Box::new(equivocation_proof),
 				key_owner_proofs.clone(),
 			),
-			Error::<Test>::InvalidForkEquivocationProof,
+			Error::<Test>::InvalidKeyOwnershipProof,
 		);
 
-		// commitment signed with a key that isn't part of the authority set
+		// commitment targets future set id
 		let equivocation_proof = generate_fork_equivocation_proof_sc(
 			Commitment {
-				validator_set_id: set_id,
+				validator_set_id: set_id + 1,
 				block_number: block_num,
 				payload: payload.clone(),
 			},
-			vec![BeefyKeyring::Eve],
+			equivocation_keyrings.clone(),
 			None,
 			Some(ancestry_proof.clone()),
 		);
@@ -2186,14 +2188,15 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 				Box::new(equivocation_proof),
 				key_owner_proofs.clone(),
 			),
-			Error::<Test>::InvalidKeyOwnershipProof,
+			Error::<Test>::InvalidEquivocationProofSession,
 		);
 
-		// commitment targets future set id
+		// Simulate InvalidForkEquivocationProof error.
+		IsValidForkEquivocationProof::set(&false);
 		let equivocation_proof = generate_fork_equivocation_proof_sc(
 			Commitment {
-				validator_set_id: set_id + 1,
-				block_number: block_num,
+				validator_set_id: set_id,
+				block_number: block_num + 1,
 				payload: payload.clone(),
 			},
 			equivocation_keyrings,
@@ -2206,7 +2209,7 @@ fn report_fork_equivocation_sc_invalid_equivocation_proof() {
 				Box::new(equivocation_proof),
 				key_owner_proofs,
 			),
-			Error::<Test>::InvalidEquivocationProofSession,
+			Error::<Test>::InvalidForkEquivocationProof,
 		);
 	});
 }

From c8e680268cb67c2ee0e4c9241331af4a1cb7b416 Mon Sep 17 00:00:00 2001
From: Serban Iorga <serban@parity.io>
Date: Thu, 18 Apr 2024 15:23:39 +0300
Subject: [PATCH 179/188] fix rustdoc

---
 substrate/primitives/consensus/beefy/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 89307ba60407..2e9a1d0fd215 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -355,7 +355,7 @@ impl<Number, Id, Signature> BeefyEquivocationProof<Id, Number>
 
 /// Proof of authority misbehavior on a given set id.
 /// This proof shows commitment signed on a different fork.
-/// See [check_fork_equivocation_proof] for proof validity conditions.
+/// See [`CheckForkEquivocationProof`] for proof validity conditions.
 #[derive(Clone, Debug, Decode, Encode, PartialEq, TypeInfo)]
 pub struct ForkEquivocationProof<Id: RuntimeAppPublic, Header: HeaderT, Hash> {
 	/// Commitment for a block on a different fork than one at the same height in
@@ -624,7 +624,7 @@ pub trait CheckForkEquivocationProof<Err, Header: HeaderT> {
 	/// assuming 2/3rds of validators honestly participate in BEEFY
 	/// finalization and at least one honest relayer can update the
 	/// beefy light client at least once every 4096 blocks. See
-	/// https://github.com/paritytech/polkadot-sdk/issues/1441 for
+	/// <https://github.com/paritytech/polkadot-sdk/issues/1441> for
 	/// replacement solution.
 	fn check_fork_equivocation_proof<Id, MsgHash>(
 		proof: &ForkEquivocationProof<Id, Header, <Self::Hash as HashT>::Output>,

From 6aacc7dfa549d14339286685eba0ebe66ec4bf50 Mon Sep 17 00:00:00 2001
From: Serban Iorga <serban@parity.io>
Date: Thu, 18 Apr 2024 16:50:31 +0300
Subject: [PATCH 180/188] Use master branch of
 https://github.com/paritytech/merkle-mountain-range.git

---
 Cargo.lock                                                | 2 +-
 .../client/consensus/beefy/src/communication/fisherman.rs | 8 ++++----
 substrate/primitives/consensus/beefy/src/commitment.rs    | 2 +-
 substrate/primitives/merkle-mountain-range/Cargo.toml     | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index d37dc59a0a6a..8d18c2158876 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2674,7 +2674,7 @@ dependencies = [
 [[package]]
 name = "ckb-merkle-mountain-range"
 version = "0.6.0"
-source = "git+https://github.com/paritytech/merkle-mountain-range.git?branch=leaf-node-proof-split#7749d1ff5263f9bf1f56ea9193c0a357881695e6"
+source = "git+https://github.com/paritytech/merkle-mountain-range.git?branch=master#537f0e3f67c5adf7afff0800bbb81f02f17570a1"
 dependencies = [
  "cfg-if",
  "itertools 0.10.5",
diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index d0255886bc9e..ec6dc9ec2652 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -219,11 +219,11 @@ where
 	}
 
 	/// Generates an ancestry proof for the given ancestoring block's mmr root.
-	fn generate_ancestry_proof_opt(
+	fn try_generate_ancestry_proof(
 		&self,
 		best_block_hash: B::Hash,
 		prev_block_num: NumberFor<B>,
-	) -> Option<AncestryProof<sp_consensus_beefy::MmrRootHash>> {
+	) -> Option<AncestryProof<MmrRootHash>> {
 		match self.runtime.runtime_api().generate_ancestry_proof(
 			best_block_hash,
 			prev_block_num,
@@ -269,7 +269,7 @@ where
 		} else {
 			let canonical_hhp = self.canonical_hash_header_payload(number)?;
 			if vote.commitment.payload != canonical_hhp.payload {
-				let ancestry_proof = self.generate_ancestry_proof_opt(
+				let ancestry_proof = self.try_generate_ancestry_proof(
 					self.backend.blockchain().info().finalized_hash,
 					number,
 				);
@@ -333,7 +333,7 @@ where
 		} else {
 			let canonical_hhp = self.canonical_hash_header_payload(number)?;
 			if commitment.payload != canonical_hhp.payload {
-				let ancestry_proof = self.generate_ancestry_proof_opt(
+				let ancestry_proof = self.try_generate_ancestry_proof(
 					self.backend.blockchain().info().finalized_hash,
 					number,
 				);
diff --git a/substrate/primitives/consensus/beefy/src/commitment.rs b/substrate/primitives/consensus/beefy/src/commitment.rs
index 8abf08add15b..1a95a015b27c 100644
--- a/substrate/primitives/consensus/beefy/src/commitment.rs
+++ b/substrate/primitives/consensus/beefy/src/commitment.rs
@@ -131,7 +131,7 @@ struct CompactSignedCommitment<TBlockNumber, TSignature> {
 	/// A bitfield representing presence of a signature coming from a validator at some index.
 	///
 	/// The bit at index `0` is set to `1` in case we have a signature coming from a validator at
-	/// index `0` in in the original validator set. In case the [`SignedCommitment`] does not
+	/// index `0` in the original validator set. In case the [`SignedCommitment`] does not
 	/// contain that signature the `bit` will be set to `0`. Bits are packed into `Vec<u8>`
 	signatures_from: BitField,
 	/// Number of validators in the Validator Set and hence number of significant bits in the
diff --git a/substrate/primitives/merkle-mountain-range/Cargo.toml b/substrate/primitives/merkle-mountain-range/Cargo.toml
index 2ad636eca24e..edd7d3606068 100644
--- a/substrate/primitives/merkle-mountain-range/Cargo.toml
+++ b/substrate/primitives/merkle-mountain-range/Cargo.toml
@@ -18,7 +18,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false }
 scale-info = { version = "2.11.1", default-features = false, features = ["derive"] }
 log = { workspace = true }
-mmr-lib = { package = "ckb-merkle-mountain-range", git = "https://github.com/paritytech/merkle-mountain-range.git", branch = "leaf-node-proof-split", default-features = false }
+mmr-lib = { package = "ckb-merkle-mountain-range", git = "https://github.com/paritytech/merkle-mountain-range.git", branch = "master", default-features = false }
 serde = { features = ["alloc", "derive"], optional = true, workspace = true }
 sp-api = { path = "../api", default-features = false }
 sp-std = { path = "../std", default-features = false }

From 1f751735e2c21a59d5c0a198623de505e26f4166 Mon Sep 17 00:00:00 2001
From: Serban Iorga <serban@parity.io>
Date: Sat, 20 Apr 2024 16:13:20 +0300
Subject: [PATCH 181/188] Cosmetics

---
 .../beefy/src/communication/fisherman.rs      | 149 ++++++------------
 .../consensus/beefy/src/justification.rs      |  86 +++++-----
 2 files changed, 101 insertions(+), 134 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index ec6dc9ec2652..b87dc25f1060 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -27,7 +27,7 @@ use sp_blockchain::HeaderBackend;
 use sp_consensus_beefy::{
 	ecdsa_crypto::{AuthorityId, Signature},
 	BeefyApi, BeefyEquivocationProof, BeefySignatureHasher, ForkEquivocationProof, MmrHashing,
-	MmrRootHash, Payload, PayloadProvider, SignedCommitment, ValidatorSet, VoteMessage,
+	MmrRootHash, Payload, PayloadProvider, ValidatorSet, VoteMessage,
 };
 use sp_mmr_primitives::{AncestryProof, MmrApi};
 use sp_runtime::{
@@ -285,112 +285,67 @@ where
 		Ok(())
 	}
 
-	/// Check `signed_commitment` for contained block against canonical payload. If an equivocation
-	/// is detected, this also reports it.
-	fn check_signed_commitment(
-		&self,
-		signed_commitment: SignedCommitment<NumberFor<B>, Signature>,
-	) -> Result<(), Error> {
-		let SignedCommitment { commitment, signatures } = signed_commitment.clone();
-		let number = commitment.block_number;
+	/// Check `proof` for contained block against canonical payload. If an equivocation is detected,
+	/// this also reports it.
+	pub fn check_proof(&self, proof: BeefyVersionedFinalityProof<B>) -> Result<(), Error> {
+		let signed_commitment = match proof {
+			BeefyVersionedFinalityProof::<B>::V1(signed_commitment) => signed_commitment,
+		};
+		let commitment = &signed_commitment.commitment;
+
 		// if the vote is for a block number exceeding our best block number, there shouldn't even
 		// be a payload to sign yet, hence we assume it is an equivocation and report it
-		if number > self.backend.blockchain().info().best_number {
-			// if block number is in the future, we use the latest validator set
-			// as the assumed signatories (note: this assumption is fragile and can possibly be
-			// improved upon)
-			let best_hash = self.backend.blockchain().info().best_hash;
-			let validator_set = self.active_validator_set_at(best_hash)?;
-			let signatories: Vec<_> = validator_set
-				.validators()
-				.iter()
-				.cloned()
-				.zip(signatures.into_iter())
-				.filter_map(|(id, signature)| match signature {
-					Some(sig) =>
-						if sp_consensus_beefy::check_commitment_signature::<
-							_,
-							_,
-							BeefySignatureHasher,
-						>(&commitment, &id, &sig)
-						{
-							Some((id, sig))
-						} else {
-							None
-						},
-					None => None,
-				})
-				.collect();
-			if signatories.len() > 0 {
-				let proof = ForkEquivocationProof {
-					commitment,
-					signatories,
-					canonical_header: None,
-					ancestry_proof: None,
-				};
-				self.report_fork_equivocation(proof)?;
-			}
-		} else {
-			let canonical_hhp = self.canonical_hash_header_payload(number)?;
-			if commitment.payload != canonical_hhp.payload {
+		let (validator_set, canonical_header, ancestry_proof) =
+			if commitment.block_number > self.backend.blockchain().info().best_number {
+				// if block number is in the future, we use the latest validator set
+				// as the assumed signatories (note: this assumption is fragile and can possibly be
+				// improved upon)
+				let best_hash = self.backend.blockchain().info().best_hash;
+				let validator_set = self.active_validator_set_at(best_hash)?;
+
+				(validator_set, None, None)
+			} else {
+				let canonical_hhp = self.canonical_hash_header_payload(commitment.block_number)?;
+				if commitment.payload == canonical_hhp.payload {
+					// The commitment is valid
+					return Ok(())
+				}
+
 				let ancestry_proof = self.try_generate_ancestry_proof(
 					self.backend.blockchain().info().finalized_hash,
-					number,
+					commitment.block_number,
 				);
 				let validator_set = self.active_validator_set_at(canonical_hhp.hash)?;
-				if crate::justification::verify_with_validator_set::<B>(
-					commitment.block_number,
-					&validator_set,
-					&BeefyVersionedFinalityProof::<B>::V1(signed_commitment),
-				)
-				.is_err()
-				{
+				(validator_set, Some(canonical_hhp.header), ancestry_proof)
+			};
+
+		// let finality_proof = BeefyVersionedFinalityProof::<B>::V1(signed_commitment);
+		let signatories: Vec<_> =
+			match crate::justification::verify_signed_commitment_with_validator_set::<B>(
+				commitment.block_number,
+				&validator_set,
+				&signed_commitment,
+			) {
+				Ok(signatories_refs) => signatories_refs
+					.into_iter()
+					.map(|(id, signature)| (id.clone(), signature.clone()))
+					.collect(),
+				Err(_) => {
 					// invalid proof
 					return Ok(())
-				}
-				// report every signer of the bad justification
-				let signatories: Vec<_> = validator_set
-					.validators()
-					.iter()
-					.cloned()
-					.zip(signatures.into_iter())
-					.filter_map(|(id, signature)| match signature {
-						Some(sig) => {
-							if sp_consensus_beefy::check_commitment_signature::<
-								_,
-								_,
-								BeefySignatureHasher,
-							>(&commitment, &id, &sig)
-							{
-								Some((id, sig.clone()))
-							} else {
-								None
-							}
-						},
-						None => None,
-					})
-					.collect();
+				},
+			};
 
-				if signatories.len() > 0 {
-					let proof = ForkEquivocationProof {
-						commitment,
-						signatories,
-						canonical_header: Some(canonical_hhp.header),
-						ancestry_proof,
-					};
-					self.report_fork_equivocation(proof)?;
-				}
-			}
+		if signatories.len() > 0 {
+			let proof = ForkEquivocationProof {
+				commitment: signed_commitment.commitment,
+				signatories,
+				canonical_header,
+				ancestry_proof,
+			};
+			self.report_fork_equivocation(proof)?;
 		}
-		Ok(())
-	}
 
-	/// Check `proof` for contained block against canonical payload. If an equivocation is detected,
-	/// this also reports it.
-	pub(crate) fn check_proof(&self, proof: BeefyVersionedFinalityProof<B>) -> Result<(), Error> {
-		match proof {
-			BeefyVersionedFinalityProof::<B>::V1(signed_commitment) =>
-				self.check_signed_commitment(signed_commitment),
-		}
+		Ok(())
 	}
 }
diff --git a/substrate/client/consensus/beefy/src/justification.rs b/substrate/client/consensus/beefy/src/justification.rs
index 45c1ffff5097..ea0712d63fee 100644
--- a/substrate/client/consensus/beefy/src/justification.rs
+++ b/substrate/client/consensus/beefy/src/justification.rs
@@ -21,7 +21,7 @@ use codec::{DecodeAll, Encode};
 use sp_consensus::Error as ConsensusError;
 use sp_consensus_beefy::{
 	ecdsa_crypto::{AuthorityId, Signature},
-	ValidatorSet, ValidatorSetId, VersionedFinalityProof,
+	SignedCommitment, ValidatorSet, ValidatorSetId, VersionedFinalityProof,
 };
 use sp_runtime::traits::{Block as BlockT, NumberFor};
 
@@ -45,48 +45,60 @@ pub(crate) fn decode_and_verify_finality_proof<Block: BlockT>(
 ) -> Result<BeefyVersionedFinalityProof<Block>, (ConsensusError, u32)> {
 	let proof = <BeefyVersionedFinalityProof<Block>>::decode_all(&mut &*encoded)
 		.map_err(|_| (ConsensusError::InvalidJustification, 0))?;
-	verify_with_validator_set::<Block>(target_number, validator_set, &proof).map(|_| proof)
+	verify_with_validator_set::<Block>(target_number, validator_set, &proof)?;
+	Ok(proof)
 }
 
-/// Verify the BEEFY finality proof against the validator set at the block it was generated.
-pub(crate) fn verify_with_validator_set<Block: BlockT>(
+/// Verify the BEEFY signed commitment against the validator set at the block it was generated.
+pub(crate) fn verify_signed_commitment_with_validator_set<'a, Block: BlockT>(
 	target_number: NumberFor<Block>,
-	validator_set: &ValidatorSet<AuthorityId>,
-	proof: &BeefyVersionedFinalityProof<Block>,
-) -> Result<(), (ConsensusError, u32)> {
+	validator_set: &'a ValidatorSet<AuthorityId>,
+	signed_commitment: &'a SignedCommitment<NumberFor<Block>, Signature>,
+) -> Result<Vec<(&'a AuthorityId, &'a Signature)>, (ConsensusError, u32)> {
+	if signed_commitment.signatures.len() != validator_set.len() ||
+		signed_commitment.commitment.validator_set_id != validator_set.id() ||
+		signed_commitment.commitment.block_number != target_number
+	{
+		return Err((ConsensusError::InvalidJustification, 0))
+	}
+
+	let message = signed_commitment.commitment.encode();
+	// Arrangement of signatures in the commitment should be in the same order
+	// as validators for that set.
 	let mut signatures_checked = 0u32;
-	match proof {
-		VersionedFinalityProof::V1(signed_commitment) => {
-			if signed_commitment.signatures.len() != validator_set.len() ||
-				signed_commitment.commitment.validator_set_id != validator_set.id() ||
-				signed_commitment.commitment.block_number != target_number
-			{
-				return Err((ConsensusError::InvalidJustification, 0))
+	let signatories: Vec<_> = validator_set
+		.validators()
+		.into_iter()
+		.zip(signed_commitment.signatures.iter())
+		.filter_map(|(id, maybe_signature)| {
+			let signature = maybe_signature.as_ref()?;
+			signatures_checked += 1;
+			match BeefyKeystore::verify(id, signature, &message[..]) {
+				true => Some((id, signature)),
+				false => None,
 			}
+		})
+		.collect();
+	if signatories.len() >= crate::round::threshold(validator_set.len()) {
+		Ok(signatories)
+	} else {
+		Err((ConsensusError::InvalidJustification, signatures_checked))
+	}
+}
 
-			// Arrangement of signatures in the commitment should be in the same order
-			// as validators for that set.
-			let message = signed_commitment.commitment.encode();
-			let valid_signatures = validator_set
-				.validators()
-				.into_iter()
-				.zip(signed_commitment.signatures.iter())
-				.filter(|(id, signature)| {
-					signature
-						.as_ref()
-						.map(|sig| {
-							signatures_checked += 1;
-							BeefyKeystore::verify(*id, sig, &message[..])
-						})
-						.unwrap_or(false)
-				})
-				.count();
-			if valid_signatures >= crate::round::threshold(validator_set.len()) {
-				Ok(())
-			} else {
-				Err((ConsensusError::InvalidJustification, signatures_checked))
-			}
-		},
+/// Verify the BEEFY finality proof against the validator set at the block it was generated.
+pub(crate) fn verify_with_validator_set<'a, Block: BlockT>(
+	target_number: NumberFor<Block>,
+	validator_set: &'a ValidatorSet<AuthorityId>,
+	proof: &'a BeefyVersionedFinalityProof<Block>,
+) -> Result<Vec<(&'a AuthorityId, &'a Signature)>, (ConsensusError, u32)> {
+	match proof {
+		VersionedFinalityProof::V1(signed_commitment) =>
+			verify_signed_commitment_with_validator_set::<Block>(
+				target_number,
+				validator_set,
+				signed_commitment,
+			),
 	}
 }
 

From 938a2118cc25279cd95cb946041dbac1d97575f7 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 25 Apr 2024 11:11:16 +0200
Subject: [PATCH 182/188] destructure `KnownSignature` for incumbent tuple
 interface

temporary measure before replacing tuple interface
---
 .../client/consensus/beefy/src/communication/fisherman.rs    | 5 ++++-
 substrate/primitives/consensus/beefy/src/lib.rs              | 3 ++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index b87dc25f1060..6791b11e3c35 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -25,6 +25,7 @@ use sc_client_api::Backend;
 use sp_api::ProvideRuntimeApi;
 use sp_blockchain::HeaderBackend;
 use sp_consensus_beefy::{
+	commitment::KnownSignature,
 	ecdsa_crypto::{AuthorityId, Signature},
 	BeefyApi, BeefyEquivocationProof, BeefySignatureHasher, ForkEquivocationProof, MmrHashing,
 	MmrRootHash, Payload, PayloadProvider, ValidatorSet, VoteMessage,
@@ -328,7 +329,9 @@ where
 			) {
 				Ok(signatories_refs) => signatories_refs
 					.into_iter()
-					.map(|(id, signature)| (id.clone(), signature.clone()))
+					.map(|KnownSignature { validator_id, signature }| {
+						(validator_id.clone(), signature.clone())
+					})
 					.collect(),
 				Err(_) => {
 					// invalid proof
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 686aad8267bc..728de932b7a2 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -33,7 +33,8 @@
 
 extern crate alloc;
 
-mod commitment;
+// TODO: unpub
+pub mod commitment;
 mod payload;
 
 pub mod mmr;

From d4b479da500e5a5289e54e08059e687c25469589 Mon Sep 17 00:00:00 2001
From: Serban Iorga <serban@parity.io>
Date: Fri, 26 Apr 2024 10:02:32 +0300
Subject: [PATCH 183/188] Use KnownSignature instead of tuple

---
 .../beefy/src/communication/fisherman.rs      | 41 ++++++++++---------
 .../consensus/beefy/src/justification.rs      |  6 +--
 .../client/consensus/beefy/src/worker.rs      | 13 +++---
 .../consensus/beefy/src/commitment.rs         |  6 +--
 .../primitives/consensus/beefy/src/lib.rs     | 15 ++++---
 .../consensus/beefy/src/test_utils.rs         | 16 +++++---
 6 files changed, 54 insertions(+), 43 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index 6791b11e3c35..e3bd5f73664a 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -25,10 +25,9 @@ use sc_client_api::Backend;
 use sp_api::ProvideRuntimeApi;
 use sp_blockchain::HeaderBackend;
 use sp_consensus_beefy::{
-	commitment::KnownSignature,
 	ecdsa_crypto::{AuthorityId, Signature},
-	BeefyApi, BeefyEquivocationProof, BeefySignatureHasher, ForkEquivocationProof, MmrHashing,
-	MmrRootHash, Payload, PayloadProvider, ValidatorSet, VoteMessage,
+	BeefyApi, BeefyEquivocationProof, BeefySignatureHasher, ForkEquivocationProof, KnownSignature,
+	MmrHashing, MmrRootHash, Payload, PayloadProvider, ValidatorSet, VoteMessage,
 };
 use sp_mmr_primitives::{AncestryProof, MmrApi};
 use sp_runtime::{
@@ -161,7 +160,7 @@ where
 
 		let runtime_api = self.runtime.runtime_api();
 
-		let mut filtered_signatories = Vec::new();
+		let mut filtered_signatures = Vec::new();
 		// generate key ownership proof at that block
 		let key_owner_proofs: Vec<_> = offender_ids
 			.iter()
@@ -179,7 +178,7 @@ where
 							"🥩 Invalid fork vote offender not part of the authority set."
 						);
 						// if signatory is not part of the authority set, we ignore the signatory
-						filtered_signatories.push(id);
+						filtered_signatures.push(id);
 						None
 					},
 					Err(e) => {
@@ -187,7 +186,7 @@ where
 							   "🥩 Failed to generate key ownership proof for {:?}: {:?}", id, e);
 						// if a key ownership proof couldn't be generated for signatory, we ignore
 						// the signatory
-						filtered_signatories.push(id);
+						filtered_signatures.push(id);
 						None
 					},
 				}
@@ -197,11 +196,11 @@ where
 		if key_owner_proofs.len() > 0 {
 			// filter out the signatories that a key ownership proof could not be generated for
 			let proof = ForkEquivocationProof {
-				signatories: proof
-					.signatories
+				signatures: proof
+					.signatures
 					.clone()
 					.into_iter()
-					.filter(|(id, _)| !filtered_signatories.contains(&id))
+					.filter(|signature| !filtered_signatures.contains(&&signature.validator_id))
 					.collect(),
 				..proof
 			};
@@ -262,7 +261,10 @@ where
 		if number > self.backend.blockchain().info().best_number {
 			let proof = ForkEquivocationProof {
 				commitment: vote.commitment,
-				signatories: vec![(vote.id, vote.signature)],
+				signatures: vec![KnownSignature {
+					validator_id: vote.id,
+					signature: vote.signature,
+				}],
 				canonical_header: None,
 				ancestry_proof: None,
 			};
@@ -276,7 +278,10 @@ where
 				);
 				let proof = ForkEquivocationProof {
 					commitment: vote.commitment,
-					signatories: vec![(vote.id, vote.signature)],
+					signatures: vec![KnownSignature {
+						validator_id: vote.id,
+						signature: vote.signature,
+					}],
 					canonical_header: Some(canonical_hhp.header),
 					ancestry_proof,
 				};
@@ -321,28 +326,24 @@ where
 			};
 
 		// let finality_proof = BeefyVersionedFinalityProof::<B>::V1(signed_commitment);
-		let signatories: Vec<_> =
+		let signatures: Vec<_> =
 			match crate::justification::verify_signed_commitment_with_validator_set::<B>(
 				commitment.block_number,
 				&validator_set,
 				&signed_commitment,
 			) {
-				Ok(signatories_refs) => signatories_refs
-					.into_iter()
-					.map(|KnownSignature { validator_id, signature }| {
-						(validator_id.clone(), signature.clone())
-					})
-					.collect(),
+				Ok(signatures_refs) =>
+					signatures_refs.into_iter().map(|signature| signature.to_owned()).collect(),
 				Err(_) => {
 					// invalid proof
 					return Ok(())
 				},
 			};
 
-		if signatories.len() > 0 {
+		if signatures.len() > 0 {
 			let proof = ForkEquivocationProof {
 				commitment: signed_commitment.commitment,
-				signatories,
+				signatures,
 				canonical_header,
 				ancestry_proof,
 			};
diff --git a/substrate/client/consensus/beefy/src/justification.rs b/substrate/client/consensus/beefy/src/justification.rs
index fdd6b982a03a..e791ed48ebb4 100644
--- a/substrate/client/consensus/beefy/src/justification.rs
+++ b/substrate/client/consensus/beefy/src/justification.rs
@@ -55,11 +55,11 @@ pub(crate) fn verify_signed_commitment_with_validator_set<'a, Block: BlockT>(
 	validator_set: &'a ValidatorSet<AuthorityId>,
 	signed_commitment: &'a SignedCommitment<NumberFor<Block>, Signature>,
 ) -> Result<Vec<KnownSignature<&'a AuthorityId, &'a Signature>>, (ConsensusError, u32)> {
-	let signatories = signed_commitment
+	let signatures = signed_commitment
 		.verify_signatures::<_, BeefySignatureHasher>(target_number, validator_set)
 		.map_err(|checked_signatures| (ConsensusError::InvalidJustification, checked_signatures))?;
-	if signatories.len() >= crate::round::threshold(validator_set.len()) {
-		Ok(signatories)
+	if signatures.len() >= crate::round::threshold(validator_set.len()) {
+		Ok(signatures)
 	} else {
 		Err((ConsensusError::InvalidJustification, signed_commitment.signature_count() as u32))
 	}
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index 3633806ef6e4..193329489f6e 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -1075,7 +1075,7 @@ pub(crate) mod tests {
 			generate_fork_equivocation_proof_sc, generate_fork_equivocation_proof_vote,
 			generate_vote_equivocation_proof, Keyring,
 		},
-		ConsensusLog, ForkEquivocationProof, Payload, SignedCommitment,
+		ConsensusLog, ForkEquivocationProof, KnownSignature, Payload, SignedCommitment,
 	};
 	use sp_runtime::traits::{Header as HeaderT, One};
 	use std::marker::PhantomData;
@@ -1759,9 +1759,12 @@ pub(crate) mod tests {
 		};
 
 		// only Bob and Charlie sign
-		let signatories = &[Keyring::Bob, Keyring::Charlie]
+		let signatures = &[Keyring::Bob, Keyring::Charlie]
 			.into_iter()
-			.map(|k| (k.public(), k.sign(&commitment.encode())))
+			.map(|k| KnownSignature {
+				validator_id: k.public(),
+				signature: k.sign(&commitment.encode()),
+			})
 			.collect::<Vec<_>>();
 
 		// test over all permutations of header and ancestry proof being submitted (proof should
@@ -1773,7 +1776,7 @@ pub(crate) mod tests {
 		] {
 			let proof = ForkEquivocationProof {
 				commitment: commitment.clone(),
-				signatories: signatories.clone(),
+				signatures: signatures.clone(),
 				canonical_header,
 				ancestry_proof,
 			};
@@ -1796,7 +1799,7 @@ pub(crate) mod tests {
 		// test that Alice does not submit invalid proof
 		let proofless_proof = ForkEquivocationProof {
 			commitment: commitment.clone(),
-			signatories: signatories.clone(),
+			signatures: signatures.clone(),
 			canonical_header: None,
 			ancestry_proof: None,
 		};
diff --git a/substrate/primitives/consensus/beefy/src/commitment.rs b/substrate/primitives/consensus/beefy/src/commitment.rs
index 13a74bbab9df..1cd06be633bb 100644
--- a/substrate/primitives/consensus/beefy/src/commitment.rs
+++ b/substrate/primitives/consensus/beefy/src/commitment.rs
@@ -25,7 +25,7 @@ use sp_runtime::traits::Hash;
 use crate::{BeefyAuthorityId, Payload, ValidatorSet, ValidatorSetId};
 
 /// A commitment signature, accompanied by the id of the validator that it belongs to.
-#[derive(Debug)]
+#[derive(Clone, Debug, Decode, Encode, PartialEq, TypeInfo)]
 pub struct KnownSignature<TAuthorityId, TSignature> {
 	/// The signing validator.
 	pub validator_id: TAuthorityId,
@@ -163,7 +163,7 @@ impl<TBlockNumber, TSignature> SignedCommitment<TBlockNumber, TSignature> {
 		// Arrangement of signatures in the commitment should be in the same order
 		// as validators for that set.
 		let encoded_commitment = self.commitment.encode();
-		let signatories: Vec<_> = validator_set
+		let signatures: Vec<_> = validator_set
 			.validators()
 			.into_iter()
 			.zip(self.signatures.iter())
@@ -176,7 +176,7 @@ impl<TBlockNumber, TSignature> SignedCommitment<TBlockNumber, TSignature> {
 			})
 			.collect();
 
-		Ok(signatories)
+		Ok(signatures)
 	}
 }
 
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 728de932b7a2..b1a410fda962 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -33,8 +33,7 @@
 
 extern crate alloc;
 
-// TODO: unpub
-pub mod commitment;
+mod commitment;
 mod payload;
 
 pub mod mmr;
@@ -363,7 +362,7 @@ pub struct ForkEquivocationProof<Id: RuntimeAppPublic, Header: HeaderT, Hash> {
 	/// the chain where this proof is submitted.
 	pub commitment: Commitment<Header::Number>,
 	/// Signatures on this block
-	pub signatories: Vec<(Id, Id::Signature)>,
+	pub signatures: Vec<KnownSignature<Id, Id::Signature>>,
 	/// Canonical header at the same height as `commitment.block_number`.
 	pub canonical_header: Option<Header>,
 	/// Ancestry proof showing that the current best mmr root descends from another mmr root at
@@ -443,9 +442,13 @@ impl<Id: RuntimeAppPublic, Header: HeaderT, Hash: HashOutput>
 			return false;
 		}
 
-		return self.signatories.iter().all(|(authority_id, signature)| {
+		return self.signatures.iter().all(|signature| {
 			// TODO: refactor check_commitment_signature to take a slice of signatories
-			check_commitment_signature(&self.commitment, authority_id, signature)
+			check_commitment_signature(
+				&self.commitment,
+				&signature.validator_id,
+				&signature.signature,
+			)
 		})
 	}
 }
@@ -454,7 +457,7 @@ impl<Id: RuntimeAppPublic, Header: HeaderT, Hash: HashOutput>
 	BeefyEquivocationProof<Id, Header::Number> for ForkEquivocationProof<Id, Header, Hash>
 {
 	fn offender_ids(&self) -> Vec<&Id> {
-		self.signatories.iter().map(|(id, _)| id).collect()
+		self.signatures.iter().map(|signature| &signature.validator_id).collect()
 	}
 
 	fn round_number(&self) -> &Header::Number {
diff --git a/substrate/primitives/consensus/beefy/src/test_utils.rs b/substrate/primitives/consensus/beefy/src/test_utils.rs
index 8e5bd83a002b..e8424088597d 100644
--- a/substrate/primitives/consensus/beefy/src/test_utils.rs
+++ b/substrate/primitives/consensus/beefy/src/test_utils.rs
@@ -19,7 +19,7 @@
 use crate::ecdsa_bls_crypto;
 use crate::{
 	ecdsa_crypto, AuthorityIdBound, BeefySignatureHasher, Commitment, ForkEquivocationProof,
-	Payload, ValidatorSetId, VoteEquivocationProof, VoteMessage,
+	KnownSignature, Payload, ValidatorSetId, VoteEquivocationProof, VoteMessage,
 };
 use sp_application_crypto::{AppCrypto, AppPair, RuntimeAppPublic, Wraps};
 use sp_core::{ecdsa, Pair};
@@ -166,10 +166,11 @@ pub fn generate_fork_equivocation_proof_vote<Header: HeaderT, Hash>(
 	ancestry_proof: Option<AncestryProof<Hash>>,
 ) -> ForkEquivocationProof<ecdsa_crypto::Public, Header, Hash> {
 	let signed_vote = signed_vote::<Header::Number>(vote.0, vote.1, vote.2, vote.3);
-	let signatories = vec![(signed_vote.id, signed_vote.signature)];
+	let signatures =
+		vec![KnownSignature { validator_id: signed_vote.id, signature: signed_vote.signature }];
 	ForkEquivocationProof {
 		commitment: signed_vote.commitment,
-		signatories,
+		signatures,
 		canonical_header,
 		ancestry_proof,
 	}
@@ -182,9 +183,12 @@ pub fn generate_fork_equivocation_proof_sc<Header: HeaderT, Hash>(
 	canonical_header: Option<Header>,
 	ancestry_proof: Option<AncestryProof<Hash>>,
 ) -> ForkEquivocationProof<ecdsa_crypto::Public, Header, Hash> {
-	let signatories = keyrings
+	let signatures = keyrings
 		.into_iter()
-		.map(|k| (k.public(), k.sign(&commitment.encode())))
+		.map(|k| KnownSignature {
+			validator_id: k.public(),
+			signature: k.sign(&commitment.encode()),
+		})
 		.collect::<Vec<_>>();
-	ForkEquivocationProof { commitment, signatories, canonical_header, ancestry_proof }
+	ForkEquivocationProof { commitment, signatures, canonical_header, ancestry_proof }
 }

From 06492da09d0121d701182715a57a4b64b34fd65f Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 25 Apr 2024 18:58:24 +0200
Subject: [PATCH 184/188] enforce unique equivocator identities in evidence

To avoid potential DoS vectors of artifically inflated evidence size, we
accept equivocation evidence only if the reported identities are all
unique.

Reported-by: Serban Iorga <serban300@gmail.com>
---
 substrate/frame/beefy/src/equivocation.rs | 10 ++++++++++
 substrate/frame/beefy/src/lib.rs          |  3 ++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/substrate/frame/beefy/src/equivocation.rs b/substrate/frame/beefy/src/equivocation.rs
index 96436234d331..bdf49ae8139e 100644
--- a/substrate/frame/beefy/src/equivocation.rs
+++ b/substrate/frame/beefy/src/equivocation.rs
@@ -170,6 +170,16 @@ impl<T: Config> EquivocationEvidenceFor<T> {
 	where
 		P: KeyOwnerProofSystem<(KeyTypeId, T::BeefyId), Proof = T::KeyOwnerProof>,
 	{
+		let offenders_unique: sp_std::collections::btree_set::BTreeSet<_> =
+			self.proof().offender_ids().iter().cloned().collect();
+		if offenders_unique.len() != self.proof().offender_ids().len() {
+			log::warn!(
+					target: LOG_TARGET,
+					"ignoring equivocation evidence since it contains duplicate offenders."
+			);
+			return None
+		}
+
 		self.proof()
 			.offender_ids()
 			.into_iter()
diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs
index f902f0d88b73..6340cd789a9e 100644
--- a/substrate/frame/beefy/src/lib.rs
+++ b/substrate/frame/beefy/src/lib.rs
@@ -73,7 +73,8 @@ pub mod pallet {
 			// todo: use custom signature hashing type instead of hardcoded `Keccak256`
 			+ BeefyAuthorityId<sp_runtime::traits::Keccak256>
 			+ MaybeSerializeDeserialize
-			+ MaxEncodedLen;
+			+ MaxEncodedLen
+			+ Ord;
 
 		/// The maximum number of authorities that can be added.
 		#[pallet::constant]

From c34617a2b6860d022cc407b8f2df5642d27bd12b Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Fri, 26 Apr 2024 19:10:38 +0200
Subject: [PATCH 185/188] bump key ownership proof gen. failures to warnings

---
 .../client/consensus/beefy/src/communication/fisherman.rs     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/fisherman.rs b/substrate/client/consensus/beefy/src/communication/fisherman.rs
index e3bd5f73664a..f37a77f9032b 100644
--- a/substrate/client/consensus/beefy/src/communication/fisherman.rs
+++ b/substrate/client/consensus/beefy/src/communication/fisherman.rs
@@ -173,7 +173,7 @@ where
 				) {
 					Ok(Some(proof)) => Some(Ok(proof)),
 					Ok(None) => {
-						debug!(
+						warn!(
 							target: LOG_TARGET,
 							"🥩 Invalid fork vote offender not part of the authority set."
 						);
@@ -182,7 +182,7 @@ where
 						None
 					},
 					Err(e) => {
-						debug!(target: LOG_TARGET,
+						warn!(target: LOG_TARGET,
 							   "🥩 Failed to generate key ownership proof for {:?}: {:?}", id, e);
 						// if a key ownership proof couldn't be generated for signatory, we ignore
 						// the signatory

From b65b7a5f20a666f60a86ce93faf29eec6600a75b Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 16 May 2024 11:26:37 +0200
Subject: [PATCH 186/188] clarify `ForkEquivocationProof::check`'s params

---
 substrate/primitives/consensus/beefy/src/lib.rs | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index 995dffb6b686..d5ce603f4d17 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -408,8 +408,9 @@ impl<Id: RuntimeAppPublic, Header: HeaderT, Hash: HashOutput>
 	/// Validates [ForkEquivocationProof] with the following checks:
 	/// - if the commitment is to a block in our history, then at least a header or an ancestry
 	///   proof is provided:
-	///   - the proof is correct if the provided `canonical_header` is at height
-	/// `commitment.block_number` and `commitment.payload` != `canonical_payload(canonical_header)`
+	///   - the proof is correct if `self.canonical_header` hashes to `canonical_header_hash`, is at
+	///     height `commitment.block_number`, and `commitment.payload` !=
+	///     `canonical_payload(canonical_header)`
 	///   - the proof is correct if the provided `ancestry_proof` proves
 	///   `mmr_root(commitment.block_number) != mmr_root(commitment.payload)`
 	/// - `commitment` is signed by all claimed signatories
@@ -423,10 +424,15 @@ impl<Id: RuntimeAppPublic, Header: HeaderT, Hash: HashOutput>
 	/// be finalized by GRANDPA.
 	pub fn check<MsgHash: HashT, NodeHash: HashT<Output = Hash>>(
 		&self,
+		// The MMR root of the best block of the chain where this proof is submitted.
 		best_root: Hash,
+		// The size of the MMR at the best block.
 		mmr_size: u64,
+		// The hash of the canonical header at the height of `commitment.block_number`.
 		canonical_header_hash: &Header::Hash,
+		// The block number at which the mmr pallet was added to the runtime.
 		first_mmr_block_num: Header::Number,
+		// The best block number of the chain where this proof is submitted.
 		best_block_num: Header::Number,
 	) -> bool
 	where

From 3913018118711b1a77536a3fe4b70b51e28a6f5e Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Mon, 20 May 2024 20:24:51 +0200
Subject: [PATCH 187/188] account for #4430

---
 substrate/client/consensus/beefy/src/tests.rs           | 7 ++++---
 substrate/frame/merkle-mountain-range/src/mmr/mmr.rs    | 2 +-
 substrate/primitives/consensus/beefy/src/lib.rs         | 2 +-
 substrate/primitives/merkle-mountain-range/src/utils.rs | 8 +++-----
 4 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs
index d2cf43b06c48..b96b390d1346 100644
--- a/substrate/client/consensus/beefy/src/tests.rs
+++ b/substrate/client/consensus/beefy/src/tests.rs
@@ -67,7 +67,7 @@ use sp_consensus_beefy::{
 };
 use sp_core::H256;
 use sp_keystore::{testing::MemoryKeystore, Keystore, KeystorePtr};
-use sp_mmr_primitives::{AncestryProof, Error as MmrError, LeafIndex, MmrApi, NodeProof};
+use sp_mmr_primitives::{AncestryProof, Error as MmrError, LeafIndex, MmrApi};
 use sp_runtime::{
 	codec::{Decode, Encode},
 	traits::{Header as HeaderT, NumberFor},
@@ -362,8 +362,9 @@ sp_api::mock_impl_runtime_apis! {
 		) -> Result<AncestryProof<MmrRootHash>, MmrError> {
 			Ok(AncestryProof {
 				prev_peaks: vec![],
-				prev_size: 0,
-				proof: NodeProof { leaf_indices: vec![], leaf_count: 0, items: vec![] },
+				prev_leaf_count: 0,
+				leaf_count: 0,
+				items: vec![],
 			})
 		}
 	}
diff --git a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
index 71ae80e70650..5efc172d1e93 100644
--- a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
+++ b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
@@ -20,7 +20,7 @@ use crate::{
 		storage::{OffchainStorage, RuntimeStorage, Storage},
 		Hasher, Node, NodeOf,
 	},
-	primitives::{self, Error, LeafIndex, NodeIndex},
+	primitives::{self, Error, NodeIndex},
 	Config, HashOf, HashingOf,
 };
 use sp_mmr_primitives::{mmr_lib, mmr_lib::MMRStoreReadOps, utils::NodesUtils, LeafIndex};
diff --git a/substrate/primitives/consensus/beefy/src/lib.rs b/substrate/primitives/consensus/beefy/src/lib.rs
index d5ce603f4d17..bc0e2048aef8 100644
--- a/substrate/primitives/consensus/beefy/src/lib.rs
+++ b/substrate/primitives/consensus/beefy/src/lib.rs
@@ -578,7 +578,7 @@ where
 		// this can be inferred from the leaf_count / mmr_size of the prev_root:
 		// we've converted the commitment.block_number to an mmr size and now
 		// compare with the value in the ancestry proof
-		if expected_mmr_size != ancestry_proof.prev_size {
+		if expected_mmr_size != ancestry_proof.prev_leaf_count {
 			return false
 		}
 		if sp_mmr_primitives::utils::verify_ancestry_proof::<
diff --git a/substrate/primitives/merkle-mountain-range/src/utils.rs b/substrate/primitives/merkle-mountain-range/src/utils.rs
index 604a7f43404a..f5dd6940a3f2 100644
--- a/substrate/primitives/merkle-mountain-range/src/utils.rs
+++ b/substrate/primitives/merkle-mountain-range/src/utils.rs
@@ -52,14 +52,12 @@ where
 	H: Clone + Debug + PartialEq + Encode,
 	M: mmr_lib::Merge<Item = H>,
 {
-	let p: mmr_lib::NodeMerkleProof<H, _> = mmr_lib::NodeMerkleProof::<H, M>::new(
-		mmr_size,
-		ancestry_proof.proof.items.into_iter().collect(),
-	);
+	let p: mmr_lib::NodeMerkleProof<H, _> =
+		mmr_lib::NodeMerkleProof::<H, M>::new(mmr_size, ancestry_proof.items.into_iter().collect());
 
 	let ancestry_proof = mmr_lib::AncestryProof::<H, _> {
 		prev_peaks: ancestry_proof.prev_peaks,
-		prev_size: ancestry_proof.prev_size,
+		prev_size: ancestry_proof.prev_leaf_count,
 		proof: p,
 	};
 

From ada8bdbb99ca1287951b68394ebc64c06f54bf57 Mon Sep 17 00:00:00 2001
From: Robert Hambrock <roberthambrock@gmail.com>
Date: Thu, 30 May 2024 11:16:06 +0200
Subject: [PATCH 188/188] add fisherman to `OnDemandJustificationsEngine`

set cost for an equivocation to the worst case `INVALID_PROOF` on
Kusama, i.e. baseline 5k + 25 per bad signature.
---
 .../consensus/beefy/src/communication/mod.rs  |  2 +
 .../outgoing_requests_engine.rs               | 38 +++++++++++++++++--
 substrate/client/consensus/beefy/src/lib.rs   |  3 +-
 .../client/consensus/beefy/src/metrics.rs     |  9 +++++
 .../client/consensus/beefy/src/worker.rs      |  1 +
 5 files changed, 49 insertions(+), 4 deletions(-)

diff --git a/substrate/client/consensus/beefy/src/communication/mod.rs b/substrate/client/consensus/beefy/src/communication/mod.rs
index 3c93368be363..70fca0cfa8ed 100644
--- a/substrate/client/consensus/beefy/src/communication/mod.rs
+++ b/substrate/client/consensus/beefy/src/communication/mod.rs
@@ -103,6 +103,8 @@ mod cost {
 	pub(super) const UNKNOWN_VOTER: Rep = Rep::new(-150, "BEEFY: Unknown voter");
 	// Message containing invalid proof.
 	pub(super) const INVALID_PROOF: Rep = Rep::new(-5000, "BEEFY: Invalid commit");
+	// Message containing equivocating payload.
+	pub(super) const EQUIVOCATION: Rep = Rep::new(-30000, "BEEFY: Equivocation detected");
 	// Reputation cost per signature checked for invalid proof.
 	pub(super) const PER_SIGNATURE_CHECKED: i32 = -25;
 	// Reputation cost per byte for un-decodable message.
diff --git a/substrate/client/consensus/beefy/src/communication/request_response/outgoing_requests_engine.rs b/substrate/client/consensus/beefy/src/communication/request_response/outgoing_requests_engine.rs
index 2ab072960900..0e7596905ca5 100644
--- a/substrate/client/consensus/beefy/src/communication/request_response/outgoing_requests_engine.rs
+++ b/substrate/client/consensus/beefy/src/communication/request_response/outgoing_requests_engine.rs
@@ -22,12 +22,16 @@ use codec::Encode;
 use futures::channel::{oneshot, oneshot::Canceled};
 use log::{debug, warn};
 use parking_lot::Mutex;
+use sc_client_api::Backend;
 use sc_network::{
 	request_responses::{IfDisconnected, RequestFailure},
 	NetworkRequest, ProtocolName,
 };
 use sc_network_types::PeerId;
-use sp_consensus_beefy::{ecdsa_crypto::AuthorityId, ValidatorSet};
+use sp_api::ProvideRuntimeApi;
+use sp_consensus_beefy::{
+	ecdsa_crypto::AuthorityId, BeefyApi, MmrRootHash, PayloadProvider, ValidatorSet,
+};
 use sp_runtime::traits::{Block, NumberFor};
 use std::{collections::VecDeque, result::Result, sync::Arc};
 
@@ -37,11 +41,13 @@ use crate::{
 		peers::PeerReport,
 		request_response::{Error, JustificationRequest, BEEFY_SYNC_LOG_TARGET},
 	},
+	fisherman::Fisherman,
 	justification::{decode_and_verify_finality_proof, BeefyVersionedFinalityProof},
 	metric_inc,
 	metrics::{register_metrics, OnDemandOutgoingRequestsMetrics},
 	KnownPeers,
 };
+use sp_mmr_primitives::MmrApi;
 
 /// Response type received from network.
 type Response = Result<(Vec<u8>, ProtocolName), RequestFailure>;
@@ -69,7 +75,7 @@ pub(crate) enum ResponseInfo<B: Block> {
 	PeerReport(PeerReport),
 }
 
-pub struct OnDemandJustificationsEngine<B: Block> {
+pub struct OnDemandJustificationsEngine<B: Block, BE, R, P> {
 	network: Arc<dyn NetworkRequest + Send + Sync>,
 	protocol_name: ProtocolName,
 
@@ -78,14 +84,24 @@ pub struct OnDemandJustificationsEngine<B: Block> {
 
 	state: State<B>,
 	metrics: Option<OnDemandOutgoingRequestsMetrics>,
+
+	fisherman: Arc<Fisherman<B, BE, R, P>>,
 }
 
-impl<B: Block> OnDemandJustificationsEngine<B> {
+impl<B, BE, R, P> OnDemandJustificationsEngine<B, BE, R, P>
+where
+	B: Block,
+	BE: Backend<B> + Send + Sync,
+	P: PayloadProvider<B> + Send + Sync,
+	R: ProvideRuntimeApi<B> + Send + Sync,
+	R::Api: BeefyApi<B, AuthorityId, MmrRootHash> + MmrApi<B, MmrRootHash, NumberFor<B>>,
+{
 	pub fn new(
 		network: Arc<dyn NetworkRequest + Send + Sync>,
 		protocol_name: ProtocolName,
 		live_peers: Arc<Mutex<KnownPeers<B>>>,
 		prometheus_registry: Option<prometheus::Registry>,
+		fisherman: Arc<Fisherman<B, BE, R, P>>,
 	) -> Self {
 		let metrics = register_metrics(prometheus_registry);
 		Self {
@@ -95,6 +111,7 @@ impl<B: Block> OnDemandJustificationsEngine<B> {
 			peers_cache: VecDeque::new(),
 			state: State::Idle,
 			metrics,
+			fisherman,
 		}
 	}
 
@@ -225,6 +242,21 @@ impl<B: Block> OnDemandJustificationsEngine<B> {
 					Error::InvalidResponse(PeerReport { who: *peer, cost_benefit: cost })
 				})
 			})
+			.and_then(|proof| match self.fisherman.check_proof(proof.clone()) {
+				Ok(()) => Ok(proof),
+				Err(err) => {
+					metric_inc!(self.metrics, beefy_on_demand_justification_equivocation);
+					warn!(
+						target: BEEFY_SYNC_LOG_TARGET,
+						"🥩 for on demand justification #{:?}, peer {:?} responded with equivocation: {:?}",
+						req_info.block, peer, err
+					);
+					Err(Error::InvalidResponse(PeerReport {
+						who: *peer,
+						cost_benefit: cost::EQUIVOCATION,
+					}))
+				},
+			})
 	}
 
 	pub(crate) async fn next(&mut self) -> ResponseInfo<B> {
diff --git a/substrate/client/consensus/beefy/src/lib.rs b/substrate/client/consensus/beefy/src/lib.rs
index e55d78eb5f33..a36521c9c13e 100644
--- a/substrate/client/consensus/beefy/src/lib.rs
+++ b/substrate/client/consensus/beefy/src/lib.rs
@@ -235,7 +235,7 @@ pub struct BeefyParams<B: Block, BE, C, N, P, R, S> {
 pub(crate) struct BeefyComms<B: Block, N, BE, P, R> {
 	pub gossip_engine: GossipEngine<B>,
 	pub gossip_validator: Arc<GossipValidator<B, N, BE, P, R>>,
-	pub on_demand_justifications: OnDemandJustificationsEngine<B>,
+	pub on_demand_justifications: OnDemandJustificationsEngine<B, BE, R, P>,
 }
 
 /// Helper builder object for building [worker::BeefyWorker].
@@ -556,6 +556,7 @@ pub async fn start_beefy_gadget<B, BE, C, N, P, R, S>(
 		justifications_protocol_name.clone(),
 		known_peers,
 		prometheus_registry.clone(),
+		fisherman.clone(),
 	);
 	let mut beefy_comms = BeefyComms { gossip_engine, gossip_validator, on_demand_justifications };
 
diff --git a/substrate/client/consensus/beefy/src/metrics.rs b/substrate/client/consensus/beefy/src/metrics.rs
index ef3928d79faa..8620a29c2027 100644
--- a/substrate/client/consensus/beefy/src/metrics.rs
+++ b/substrate/client/consensus/beefy/src/metrics.rs
@@ -236,6 +236,8 @@ pub struct OnDemandOutgoingRequestsMetrics {
 	pub beefy_on_demand_justification_invalid_proof: Counter<U64>,
 	/// Number of on-demand justification good proof
 	pub beefy_on_demand_justification_good_proof: Counter<U64>,
+	/// Number of on-demand justification equivocation
+	pub beefy_on_demand_justification_equivocation: Counter<U64>,
 }
 
 impl PrometheusRegister for OnDemandOutgoingRequestsMetrics {
@@ -277,6 +279,13 @@ impl PrometheusRegister for OnDemandOutgoingRequestsMetrics {
 				)?,
 				registry,
 			)?,
+			beefy_on_demand_justification_equivocation: register(
+				Counter::new(
+					"substrate_beefy_on_demand_justification_equivocation",
+					"Number of on-demand justification equivocations",
+				)?,
+				registry,
+			)?,
 		})
 	}
 }
diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs
index e53aafa65cfe..97f349c67ea8 100644
--- a/substrate/client/consensus/beefy/src/worker.rs
+++ b/substrate/client/consensus/beefy/src/worker.rs
@@ -1115,6 +1115,7 @@ pub(crate) mod tests {
 			"/beefy/justifs/1".into(),
 			known_peers,
 			None,
+			fisherman.clone(),
 		);
 		// If chain's still at genesis, push 1 block to start first session.
 		if backend.blockchain().info().best_hash == backend.blockchain().info().genesis_hash {