From 9f7e8fda45b7a4f7af2e6546a81f96ea987f185b Mon Sep 17 00:00:00 2001 From: Bharath Date: Thu, 29 Aug 2024 20:47:42 +0530 Subject: [PATCH 01/43] execution api updates for trusted builder --- charts/evm-rollup/values.yaml | 2 +- .../astria-bridge-contracts | 1 + .../astria-conductor/src/executor/client.rs | 13 +- crates/astria-conductor/src/executor/mod.rs | 17 +- crates/astria-core/src/composer/mod.rs | 1 + .../astria-core/src/composer/v1alpha1/mod.rs | 178 ++++++++++++++ .../astria-core/src/execution/v1alpha2/mod.rs | 203 ++++++++++++++++ .../src/generated/astria.composer.v1alpha1.rs | 32 +++ .../astria.composer.v1alpha1.serde.rs | 223 ++++++++++++++++++ .../generated/astria.execution.v1alpha2.rs | 36 ++- .../astria.execution.v1alpha2.serde.rs | 127 ++++++++++ crates/astria-core/src/generated/mod.rs | 12 +- crates/astria-core/src/lib.rs | 1 + .../composer/v1alpha1/trusted_builder.proto | 15 ++ .../astria/execution/v1alpha2/execution.proto | 13 +- 15 files changed, 855 insertions(+), 19 deletions(-) create mode 160000 crates/astria-bridge-withdrawer/astria-bridge-contracts create mode 100644 crates/astria-core/src/composer/mod.rs create mode 100644 crates/astria-core/src/composer/v1alpha1/mod.rs create mode 100644 proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto diff --git a/charts/evm-rollup/values.yaml b/charts/evm-rollup/values.yaml index 89659c997..a7654228d 100644 --- a/charts/evm-rollup/values.yaml +++ b/charts/evm-rollup/values.yaml @@ -4,7 +4,7 @@ global: # Whether to use tty readable logging for astria services, when false use json. # Best to be false in production environments, true for clean logs on local dev. useTTY: false - dev: false + dev: true images: geth: diff --git a/crates/astria-bridge-withdrawer/astria-bridge-contracts b/crates/astria-bridge-withdrawer/astria-bridge-contracts new file mode 160000 index 000000000..4580ffc07 --- /dev/null +++ b/crates/astria-bridge-withdrawer/astria-bridge-contracts @@ -0,0 +1 @@ +Subproject commit 4580ffc0747f463e304214bb29848e21e4e93e32 diff --git a/crates/astria-conductor/src/executor/client.rs b/crates/astria-conductor/src/executor/client.rs index 6f9f9fcdb..fc815ae20 100644 --- a/crates/astria-conductor/src/executor/client.rs +++ b/crates/astria-conductor/src/executor/client.rs @@ -4,6 +4,7 @@ use astria_core::{ execution::v1alpha2::{ Block, CommitmentState, + ExecuteBlockResponse, GenesisInfo, }, generated::{ @@ -119,7 +120,7 @@ impl Client { prev_block_hash: Bytes, transactions: Vec, timestamp: Timestamp, - ) -> eyre::Result { + ) -> eyre::Result { use prost::Message; let transactions = transactions @@ -132,6 +133,7 @@ impl Client { prev_block_hash, transactions, timestamp: Some(timestamp), + simulate_only: false, }; let response = tryhard::retry_fn(|| { let mut client = self.inner.clone(); @@ -146,9 +148,12 @@ impl Client { code or because number of retries were exhausted", )? .into_inner(); - let block = Block::try_from_raw(response) - .wrap_err("failed converting raw response to validated block")?; - Ok(block) + let execute_block_response = ExecuteBlockResponse::try_from_raw(response) + .wrap_err("failed converting raw response to validated execute block response")?; + + // let block = Block::try_from_raw(response) + // .wrap_err("failed converting raw response to validated block")?; + Ok(execute_block_response) } /// Calls remote procedure `astria.execution.v1alpha2.GetCommitmentState` diff --git a/crates/astria-conductor/src/executor/mod.rs b/crates/astria-conductor/src/executor/mod.rs index 916a951e2..c5f9e59d2 100644 --- a/crates/astria-conductor/src/executor/mod.rs +++ b/crates/astria-conductor/src/executor/mod.rs @@ -4,6 +4,7 @@ use astria_core::{ execution::v1alpha2::{ Block, CommitmentState, + ExecuteBlockResponse, }, primitive::v1::RollupId, sequencerblock::v1alpha1::block::{ @@ -437,15 +438,15 @@ impl Executor { .await .wrap_err("failed to execute block")?; - self.does_block_response_fulfill_contract(ExecutionKind::Soft, &executed_block) + self.does_block_response_fulfill_contract(ExecutionKind::Soft, &executed_block.block()) .wrap_err("execution API server violated contract")?; - self.update_commitment_state(Update::OnlySoft(executed_block.clone())) + self.update_commitment_state(Update::OnlySoft(executed_block.block().clone())) .await .wrap_err("failed to update soft commitment state")?; self.blocks_pending_finalization - .insert(block_number, executed_block); + .insert(block_number, executed_block.block().clone()); // XXX: We set an absolute number value here to avoid any potential issues of the remote // rollup state and the local state falling out of lock-step. @@ -488,9 +489,9 @@ impl Executor { .execute_block(parent_hash, executable_block) .await .wrap_err("failed to execute block")?; - self.does_block_response_fulfill_contract(ExecutionKind::Firm, &executed_block) + self.does_block_response_fulfill_contract(ExecutionKind::Firm, &executed_block.block()) .wrap_err("execution API server violated contract")?; - Update::ToSame(executed_block, celestia_height) + Update::ToSame(executed_block.block().clone(), celestia_height) } else if let Some(block) = self.blocks_pending_finalization.remove(&block_number) { debug!( block_number, @@ -546,7 +547,7 @@ impl Executor { &mut self, parent_hash: Bytes, block: ExecutableBlock, - ) -> eyre::Result { + ) -> eyre::Result { let ExecutableBlock { transactions, timestamp, @@ -565,8 +566,8 @@ impl Executor { .record_transactions_per_executed_block(n_transactions); info!( - executed_block.hash = %telemetry::display::base64(&executed_block.hash()), - executed_block.number = executed_block.number(), + executed_block.hash = %telemetry::display::base64(&executed_block.block().hash()), + executed_block.number = executed_block.block().number(), "executed block", ); diff --git a/crates/astria-core/src/composer/mod.rs b/crates/astria-core/src/composer/mod.rs new file mode 100644 index 000000000..b9390b87c --- /dev/null +++ b/crates/astria-core/src/composer/mod.rs @@ -0,0 +1 @@ +pub mod v1alpha1; \ No newline at end of file diff --git a/crates/astria-core/src/composer/v1alpha1/mod.rs b/crates/astria-core/src/composer/v1alpha1/mod.rs new file mode 100644 index 000000000..0db2af8d1 --- /dev/null +++ b/crates/astria-core/src/composer/v1alpha1/mod.rs @@ -0,0 +1,178 @@ +use bytes::Bytes; + +use crate::{ + execution::v1alpha2::{ + RollupData, + RollupDataError, + }, + Protobuf, +}; + +#[derive(Debug, thiserror::Error)] +#[error(transparent)] +pub struct BuilderBundleError(BuilderBundleErrorKind); + +impl BuilderBundleError { + fn field_not_set(field: &'static str) -> Self { + Self(BuilderBundleErrorKind::FieldNotSet(field)) + } + + fn invalid_rollup_data(error: RollupDataError) -> Self { + Self(BuilderBundleErrorKind::InvalidRollupData(error)) + } +} + +#[derive(Debug, thiserror::Error)] +enum BuilderBundleErrorKind { + #[error("{0} field not set")] + FieldNotSet(&'static str), + #[error("{0} invalid rollup data")] + InvalidRollupData(#[source] RollupDataError), +} + +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize))] +#[cfg_attr( + feature = "serde", + serde(into = "crate::generated::composer::v1alpha1::BuilderBundle") +)] +pub struct BuilderBundle { + transactions: Vec, + parent_hash: Bytes, +} + +impl BuilderBundle { + pub fn transactions(&self) -> &[RollupData] { + self.transactions.as_slice() + } + + pub fn parent_hash(&self) -> Bytes { + self.parent_hash.clone() + } +} + +impl From for crate::generated::composer::v1alpha1::BuilderBundle { + fn from(value: BuilderBundle) -> Self { + value.to_raw() + } +} + +impl Protobuf for BuilderBundle { + type Error = BuilderBundleError; + type Raw = crate::generated::composer::v1alpha1::BuilderBundle; + + fn try_from_raw_ref(raw: &Self::Raw) -> Result { + let crate::generated::composer::v1alpha1::BuilderBundle { + transactions, + parent_hash, + } = raw; + + let mut rollup_data_transactions = vec![]; + for transaction in transactions { + let rollup_data = RollupData::try_from_raw_ref(&transaction) + .map_err(BuilderBundleError::invalid_rollup_data)?; + rollup_data_transactions.push(rollup_data); + } + + Ok(BuilderBundle { + transactions: rollup_data_transactions, + parent_hash: Bytes::from(parent_hash.clone()), + }) + } + + fn to_raw(&self) -> Self::Raw { + crate::generated::composer::v1alpha1::BuilderBundle { + transactions: self + .transactions + .iter() + .map(|transaction| transaction.to_raw()) + .collect(), + parent_hash: self.parent_hash.clone().to_vec(), + } + } +} + +#[derive(Debug, thiserror::Error)] +#[error(transparent)] +pub struct BuilderBundlePacketError(BuilderBundlePacketErrorKind); + +#[derive(Debug, thiserror::Error)] +enum BuilderBundlePacketErrorKind { + #[error("{0} field not set")] + FieldNotSet(&'static str), + #[error("{0} invalid bundle")] + InvalidBundle(#[source] BuilderBundleError), +} + +impl BuilderBundlePacketError { + fn field_not_set(field: &'static str) -> Self { + Self(BuilderBundlePacketErrorKind::FieldNotSet(field)) + } + + fn invalid_bundle(error: BuilderBundleError) -> Self { + Self(BuilderBundlePacketErrorKind::InvalidBundle(error)) + } + +} + +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize))] +#[cfg_attr( + feature = "serde", + serde(into = "crate::generated::composer::v1alpha1::BuilderBundlePacket") +)] +pub struct BuilderBundlePacket { + bundle: BuilderBundle, + signature: Bytes, +} + +impl BuilderBundlePacket { + pub fn bundle(&self) -> &BuilderBundle { + &self.bundle + } + + pub fn signature(&self) -> Bytes { + self.signature.clone() + } +} + +impl From for crate::generated::composer::v1alpha1::BuilderBundlePacket { + fn from(value: BuilderBundlePacket) -> Self { + value.to_raw() + } +} + +impl Protobuf for BuilderBundlePacket { + type Error = BuilderBundlePacketError; + type Raw = crate::generated::composer::v1alpha1::BuilderBundlePacket; + + fn try_from_raw_ref(raw: &Self::Raw) -> Result { + let crate::generated::composer::v1alpha1::BuilderBundlePacket { + bundle, + signature, + } = raw; + + let bundle = { + let Some(bundle) = bundle else { + return Err(BuilderBundlePacketError::field_not_set("bundle")); + }; + + BuilderBundle::try_from_raw_ref(bundle) + .map_err(BuilderBundlePacketError::invalid_bundle)? + }; + + + Ok(BuilderBundlePacket { + bundle, + signature: Bytes::from(signature.clone()), + }) + } + + fn to_raw(&self) -> Self::Raw { + crate::generated::composer::v1alpha1::BuilderBundlePacket { + bundle: Some(self.bundle.to_raw()), + signature: self.signature.clone().to_vec(), + } + } + +} \ No newline at end of file diff --git a/crates/astria-core/src/execution/v1alpha2/mod.rs b/crates/astria-core/src/execution/v1alpha2/mod.rs index 7ceb0c17e..1ca4e04bf 100644 --- a/crates/astria-core/src/execution/v1alpha2/mod.rs +++ b/crates/astria-core/src/execution/v1alpha2/mod.rs @@ -7,6 +7,10 @@ use crate::{ IncorrectRollupIdLength, RollupId, }, + sequencerblock::v1alpha1::block::{ + Deposit, + DepositError, + }, Protobuf, }; @@ -233,6 +237,205 @@ impl Protobuf for Block { } } +// TODO - move RollupData protobuf impl to its own file +#[derive(Debug, thiserror::Error)] +#[error(transparent)] +pub struct RollupDataError(RollupDataErrorKind); + +impl RollupDataError { + fn field_not_set(field: &'static str) -> Self { + Self(RollupDataErrorKind::FieldNotSet(field)) + } + + fn invalid_deposit(source: DepositError) -> Self { + Self(RollupDataErrorKind::InvalidDeposit(source)) + } +} + +#[derive(Debug, thiserror::Error)] +enum RollupDataErrorKind { + #[error("{0} field not set")] + FieldNotSet(&'static str), + #[error("{0} invalid deposit")] + InvalidDeposit(#[source] DepositError), +} + +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize))] +#[cfg_attr( + feature = "serde", + serde(into = "crate::generated::sequencerblock::v1alpha1::RollupData") +)] +pub struct RollupData { + rollup_data_value: RollupDataValue, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum RollupDataValue { + SequenceData(Vec), + Deposit(Deposit), +} + +impl RollupData { + pub fn rollup_data_value(&self) -> &RollupDataValue { + &self.rollup_data_value + } +} + +impl From for crate::generated::sequencerblock::v1alpha1::RollupData { + fn from(value: RollupData) -> Self { + value.to_raw() + } +} + +impl Protobuf for RollupData { + type Error = RollupDataError; + type Raw = crate::generated::sequencerblock::v1alpha1::RollupData; + + fn try_from_raw_ref(raw: &Self::Raw) -> Result { + let crate::generated::sequencerblock::v1alpha1::RollupData { + value, + } = raw; + let value = value.clone().ok_or(Self::Error::field_not_set(".value"))?; + + let rollup_data_value = match value { + crate::generated::sequencerblock::v1alpha1::rollup_data::Value::SequencedData( + sequence_data, + ) => RollupDataValue::SequenceData(sequence_data), + crate::generated::sequencerblock::v1alpha1::rollup_data::Value::Deposit(deposit) => { + RollupDataValue::Deposit( + Deposit::try_from_raw(deposit).map_err(RollupDataError::invalid_deposit)?, + ) + } + }; + + Ok(RollupData { + rollup_data_value, + }) + } + + fn to_raw(&self) -> Self::Raw { + let value = match &self.rollup_data_value { + RollupDataValue::SequenceData(sequence_data) => { + crate::generated::sequencerblock::v1alpha1::rollup_data::Value::SequencedData( + sequence_data.clone(), + ) + } + RollupDataValue::Deposit(deposit) => { + crate::generated::sequencerblock::v1alpha1::rollup_data::Value::Deposit( + deposit.clone().into_raw(), + ) + } + }; + + crate::generated::sequencerblock::v1alpha1::RollupData { + value: Some(value), + } + } +} + +#[derive(Debug, thiserror::Error)] +#[error(transparent)] +pub struct ExecuteBlockResponseError(ExecuteBlockResponseErrorKind); + +impl ExecuteBlockResponseError { + fn field_not_set(field: &'static str) -> Self { + Self(ExecuteBlockResponseErrorKind::FieldNotSet(field)) + } + + fn invalid_rollup_data(source: RollupDataError) -> Self { + Self(ExecuteBlockResponseErrorKind::InvalidRollupData(source)) + } +} + +#[derive(Debug, thiserror::Error)] +enum ExecuteBlockResponseErrorKind { + #[error("{0} field not set")] + FieldNotSet(&'static str), + #[error("{0} invalid rollup data")] + InvalidRollupData(#[source] RollupDataError), +} + +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize))] +#[cfg_attr( + feature = "serde", + serde(into = "crate::generated::execution::v1alpha2::ExecuteBlockResponse") +)] +pub struct ExecuteBlockResponse { + block: Block, + included_transactions: Vec, +} + +impl ExecuteBlockResponse { + #[must_use] + pub fn block(&self) -> &Block { + &self.block + } + + #[must_use] + pub fn included_transactions(&self) -> &[RollupData] { + &self.included_transactions + } +} + +impl From for raw::ExecuteBlockResponse { + fn from(value: ExecuteBlockResponse) -> Self { + value.to_raw() + } +} + +impl Protobuf for ExecuteBlockResponse { + type Error = ExecuteBlockResponseError; + type Raw = raw::ExecuteBlockResponse; + + fn try_from_raw_ref(raw: &Self::Raw) -> Result { + let raw::ExecuteBlockResponse { + block, + included_transactions, + } = raw; + let block = { + let Some(block) = block else { + return Err(Self::Error::field_not_set(".block")); + }; + if let Ok(parsed_block) = Block::try_from_raw_ref(block) { + Ok(parsed_block) + } else { + return Err(Self::Error::field_not_set(".block")); + } + }?; + + let included_transactions = included_transactions + .iter() + .map(|rollup_data| RollupData::try_from_raw_ref(rollup_data)) + .collect::, _>>() + .map_err(Self::Error::invalid_rollup_data)?; + + Ok(Self { + block, + included_transactions, + }) + } + + fn to_raw(&self) -> Self::Raw { + let Self { + block, + included_transactions, + } = self; + let block = block.to_raw(); + + let included_transactions = included_transactions + .iter() + .map(|rollup_data| rollup_data.to_raw()) + .collect(); + + Self::Raw { + block: Some(block), + included_transactions, + } + } +} + #[derive(Debug, thiserror::Error)] #[error(transparent)] pub struct CommitmentStateError(CommitmentStateErrorKind); diff --git a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs index 3a50686cc..49ae11e7e 100644 --- a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs +++ b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs @@ -345,3 +345,35 @@ pub mod grpc_collector_service_server { const NAME: &'static str = "astria.composer.v1alpha1.GrpcCollectorService"; } } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BuilderBundle { + #[prost(message, repeated, tag = "1")] + pub transactions: ::prost::alloc::vec::Vec< + super::super::sequencerblock::v1alpha1::RollupData, + >, + #[prost(bytes = "vec", tag = "2")] + pub parent_hash: ::prost::alloc::vec::Vec, +} +impl ::prost::Name for BuilderBundle { + const NAME: &'static str = "BuilderBundle"; + const PACKAGE: &'static str = "astria.composer.v1alpha1"; + fn full_name() -> ::prost::alloc::string::String { + ::prost::alloc::format!("astria.composer.v1alpha1.{}", Self::NAME) + } +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BuilderBundlePacket { + #[prost(message, optional, tag = "1")] + pub bundle: ::core::option::Option, + #[prost(bytes = "vec", tag = "2")] + pub signature: ::prost::alloc::vec::Vec, +} +impl ::prost::Name for BuilderBundlePacket { + const NAME: &'static str = "BuilderBundlePacket"; + const PACKAGE: &'static str = "astria.composer.v1alpha1"; + fn full_name() -> ::prost::alloc::string::String { + ::prost::alloc::format!("astria.composer.v1alpha1.{}", Self::NAME) + } +} diff --git a/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs b/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs index 91e8502f9..571e5a588 100644 --- a/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs +++ b/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs @@ -1,3 +1,226 @@ +impl serde::Serialize for BuilderBundle { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if !self.transactions.is_empty() { + len += 1; + } + if !self.parent_hash.is_empty() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("astria.composer.v1alpha1.BuilderBundle", len)?; + if !self.transactions.is_empty() { + struct_ser.serialize_field("transactions", &self.transactions)?; + } + if !self.parent_hash.is_empty() { + #[allow(clippy::needless_borrow)] + struct_ser.serialize_field("parentHash", pbjson::private::base64::encode(&self.parent_hash).as_str())?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for BuilderBundle { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "transactions", + "parent_hash", + "parentHash", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Transactions, + ParentHash, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "transactions" => Ok(GeneratedField::Transactions), + "parentHash" | "parent_hash" => Ok(GeneratedField::ParentHash), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = BuilderBundle; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct astria.composer.v1alpha1.BuilderBundle") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut transactions__ = None; + let mut parent_hash__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Transactions => { + if transactions__.is_some() { + return Err(serde::de::Error::duplicate_field("transactions")); + } + transactions__ = Some(map_.next_value()?); + } + GeneratedField::ParentHash => { + if parent_hash__.is_some() { + return Err(serde::de::Error::duplicate_field("parentHash")); + } + parent_hash__ = + Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) + ; + } + } + } + Ok(BuilderBundle { + transactions: transactions__.unwrap_or_default(), + parent_hash: parent_hash__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("astria.composer.v1alpha1.BuilderBundle", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for BuilderBundlePacket { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.bundle.is_some() { + len += 1; + } + if !self.signature.is_empty() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("astria.composer.v1alpha1.BuilderBundlePacket", len)?; + if let Some(v) = self.bundle.as_ref() { + struct_ser.serialize_field("bundle", v)?; + } + if !self.signature.is_empty() { + #[allow(clippy::needless_borrow)] + struct_ser.serialize_field("signature", pbjson::private::base64::encode(&self.signature).as_str())?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for BuilderBundlePacket { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "bundle", + "signature", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Bundle, + Signature, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "bundle" => Ok(GeneratedField::Bundle), + "signature" => Ok(GeneratedField::Signature), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = BuilderBundlePacket; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct astria.composer.v1alpha1.BuilderBundlePacket") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut bundle__ = None; + let mut signature__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Bundle => { + if bundle__.is_some() { + return Err(serde::de::Error::duplicate_field("bundle")); + } + bundle__ = map_.next_value()?; + } + GeneratedField::Signature => { + if signature__.is_some() { + return Err(serde::de::Error::duplicate_field("signature")); + } + signature__ = + Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) + ; + } + } + } + Ok(BuilderBundlePacket { + bundle: bundle__, + signature: signature__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("astria.composer.v1alpha1.BuilderBundlePacket", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for SubmitRollupTransactionRequest { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result diff --git a/crates/astria-core/src/generated/astria.execution.v1alpha2.rs b/crates/astria-core/src/generated/astria.execution.v1alpha2.rs index f050cdcd2..4b61c29ea 100644 --- a/crates/astria-core/src/generated/astria.execution.v1alpha2.rs +++ b/crates/astria-core/src/generated/astria.execution.v1alpha2.rs @@ -144,6 +144,9 @@ pub struct ExecuteBlockRequest { /// Timestamp to be used for new block. #[prost(message, optional, tag = "3")] pub timestamp: ::core::option::Option<::pbjson_types::Timestamp>, + /// If true, the block will be created but not persisted. + #[prost(bool, tag = "4")] + pub simulate_only: bool, } impl ::prost::Name for ExecuteBlockRequest { const NAME: &'static str = "ExecuteBlockRequest"; @@ -152,6 +155,27 @@ impl ::prost::Name for ExecuteBlockRequest { ::prost::alloc::format!("astria.execution.v1alpha2.{}", Self::NAME) } } +/// ExecuteBlockResponse contains the new block and the transactions that were +/// included in the block. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ExecuteBlockResponse { + /// The new block that was created. + #[prost(message, optional, tag = "1")] + pub block: ::core::option::Option, + /// The transactions that were included in the block. + #[prost(message, repeated, tag = "2")] + pub included_transactions: ::prost::alloc::vec::Vec< + super::super::sequencerblock::v1alpha1::RollupData, + >, +} +impl ::prost::Name for ExecuteBlockResponse { + const NAME: &'static str = "ExecuteBlockResponse"; + const PACKAGE: &'static str = "astria.execution.v1alpha2"; + fn full_name() -> ::prost::alloc::string::String { + ::prost::alloc::format!("astria.execution.v1alpha2.{}", Self::NAME) + } +} /// The CommitmentState holds the block at each stage of sequencer commitment /// level /// @@ -389,7 +413,10 @@ pub mod execution_service_client { pub async fn execute_block( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await @@ -510,7 +537,10 @@ pub mod execution_service_server { async fn execute_block( self: std::sync::Arc, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; /// GetCommitmentState fetches the current CommitmentState of the chain. async fn get_commitment_state( self: std::sync::Arc, @@ -754,7 +784,7 @@ pub mod execution_service_server { T: ExecutionService, > tonic::server::UnaryService for ExecuteBlockSvc { - type Response = super::Block; + type Response = super::ExecuteBlockResponse; type Future = BoxFuture< tonic::Response, tonic::Status, diff --git a/crates/astria-core/src/generated/astria.execution.v1alpha2.serde.rs b/crates/astria-core/src/generated/astria.execution.v1alpha2.serde.rs index 2fbadc3e3..c3b2eac76 100644 --- a/crates/astria-core/src/generated/astria.execution.v1alpha2.serde.rs +++ b/crates/astria-core/src/generated/astria.execution.v1alpha2.serde.rs @@ -587,6 +587,9 @@ impl serde::Serialize for ExecuteBlockRequest { if self.timestamp.is_some() { len += 1; } + if self.simulate_only { + len += 1; + } let mut struct_ser = serializer.serialize_struct("astria.execution.v1alpha2.ExecuteBlockRequest", len)?; if !self.prev_block_hash.is_empty() { #[allow(clippy::needless_borrow)] @@ -598,6 +601,9 @@ impl serde::Serialize for ExecuteBlockRequest { if let Some(v) = self.timestamp.as_ref() { struct_ser.serialize_field("timestamp", v)?; } + if self.simulate_only { + struct_ser.serialize_field("simulateOnly", &self.simulate_only)?; + } struct_ser.end() } } @@ -612,6 +618,8 @@ impl<'de> serde::Deserialize<'de> for ExecuteBlockRequest { "prevBlockHash", "transactions", "timestamp", + "simulate_only", + "simulateOnly", ]; #[allow(clippy::enum_variant_names)] @@ -619,6 +627,7 @@ impl<'de> serde::Deserialize<'de> for ExecuteBlockRequest { PrevBlockHash, Transactions, Timestamp, + SimulateOnly, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -643,6 +652,7 @@ impl<'de> serde::Deserialize<'de> for ExecuteBlockRequest { "prevBlockHash" | "prev_block_hash" => Ok(GeneratedField::PrevBlockHash), "transactions" => Ok(GeneratedField::Transactions), "timestamp" => Ok(GeneratedField::Timestamp), + "simulateOnly" | "simulate_only" => Ok(GeneratedField::SimulateOnly), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -665,6 +675,7 @@ impl<'de> serde::Deserialize<'de> for ExecuteBlockRequest { let mut prev_block_hash__ = None; let mut transactions__ = None; let mut timestamp__ = None; + let mut simulate_only__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::PrevBlockHash => { @@ -687,18 +698,134 @@ impl<'de> serde::Deserialize<'de> for ExecuteBlockRequest { } timestamp__ = map_.next_value()?; } + GeneratedField::SimulateOnly => { + if simulate_only__.is_some() { + return Err(serde::de::Error::duplicate_field("simulateOnly")); + } + simulate_only__ = Some(map_.next_value()?); + } } } Ok(ExecuteBlockRequest { prev_block_hash: prev_block_hash__.unwrap_or_default(), transactions: transactions__.unwrap_or_default(), timestamp: timestamp__, + simulate_only: simulate_only__.unwrap_or_default(), }) } } deserializer.deserialize_struct("astria.execution.v1alpha2.ExecuteBlockRequest", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for ExecuteBlockResponse { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.block.is_some() { + len += 1; + } + if !self.included_transactions.is_empty() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("astria.execution.v1alpha2.ExecuteBlockResponse", len)?; + if let Some(v) = self.block.as_ref() { + struct_ser.serialize_field("block", v)?; + } + if !self.included_transactions.is_empty() { + struct_ser.serialize_field("includedTransactions", &self.included_transactions)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for ExecuteBlockResponse { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "block", + "included_transactions", + "includedTransactions", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Block, + IncludedTransactions, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "block" => Ok(GeneratedField::Block), + "includedTransactions" | "included_transactions" => Ok(GeneratedField::IncludedTransactions), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = ExecuteBlockResponse; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct astria.execution.v1alpha2.ExecuteBlockResponse") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut block__ = None; + let mut included_transactions__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Block => { + if block__.is_some() { + return Err(serde::de::Error::duplicate_field("block")); + } + block__ = map_.next_value()?; + } + GeneratedField::IncludedTransactions => { + if included_transactions__.is_some() { + return Err(serde::de::Error::duplicate_field("includedTransactions")); + } + included_transactions__ = Some(map_.next_value()?); + } + } + } + Ok(ExecuteBlockResponse { + block: block__, + included_transactions: included_transactions__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("astria.execution.v1alpha2.ExecuteBlockResponse", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for GenesisInfo { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result diff --git a/crates/astria-core/src/generated/mod.rs b/crates/astria-core/src/generated/mod.rs index 4b78c07fe..cbadc195c 100644 --- a/crates/astria-core/src/generated/mod.rs +++ b/crates/astria-core/src/generated/mod.rs @@ -136,8 +136,16 @@ pub mod sequencerblock { #[path = ""] pub mod composer { - #[path = "astria.composer.v1alpha1.rs"] - pub mod v1alpha1; + + pub mod v1alpha1 { + include!("astria.composer.v1alpha1.rs"); + + #[cfg(feature = "serde")] + mod _serde_impl { + use super::*; + include!("astria.composer.v1alpha1.serde.rs"); + } + } } #[path = ""] diff --git a/crates/astria-core/src/lib.rs b/crates/astria-core/src/lib.rs index 4fecf8ddd..3d4189c4b 100644 --- a/crates/astria-core/src/lib.rs +++ b/crates/astria-core/src/lib.rs @@ -18,6 +18,7 @@ pub mod sequencerblock; pub mod brotli; #[cfg(feature = "celestia")] pub mod celestia; +mod composer; #[cfg(feature = "serde")] pub(crate) mod serde; diff --git a/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto b/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto new file mode 100644 index 000000000..a8cc53734 --- /dev/null +++ b/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto @@ -0,0 +1,15 @@ +syntax = 'proto3'; + +package astria.composer.v1alpha1; + +import "astria/sequencerblock/v1alpha1/block.proto"; + +message BuilderBundle { + repeated astria.sequencerblock.v1alpha1.RollupData transactions = 1; + bytes parent_hash = 2; +} + +message BuilderBundlePacket { + BuilderBundle bundle = 1; + bytes signature = 2; +} diff --git a/proto/executionapis/astria/execution/v1alpha2/execution.proto b/proto/executionapis/astria/execution/v1alpha2/execution.proto index 687b32d01..d11bce2f8 100644 --- a/proto/executionapis/astria/execution/v1alpha2/execution.proto +++ b/proto/executionapis/astria/execution/v1alpha2/execution.proto @@ -70,6 +70,17 @@ message ExecuteBlockRequest { repeated astria.sequencerblock.v1alpha1.RollupData transactions = 2; // Timestamp to be used for new block. google.protobuf.Timestamp timestamp = 3; + // If true, the block will be created but not persisted. + bool simulate_only = 4; +} + +// ExecuteBlockResponse contains the new block and the transactions that were +// included in the block. +message ExecuteBlockResponse { + // The new block that was created. + Block block = 1; + // The transactions that were included in the block. + repeated astria.sequencerblock.v1alpha1.RollupData included_transactions = 2; } // The CommitmentState holds the block at each stage of sequencer commitment @@ -115,7 +126,7 @@ service ExecutionService { // ExecuteBlock is called to deterministically derive a rollup block from // filtered sequencer block information. - rpc ExecuteBlock(ExecuteBlockRequest) returns (Block); + rpc ExecuteBlock(ExecuteBlockRequest) returns (ExecuteBlockResponse); // GetCommitmentState fetches the current CommitmentState of the chain. rpc GetCommitmentState(GetCommitmentStateRequest) returns (CommitmentState); From 6728aa8d38f01d155db56740f8ebf826a2846a7b Mon Sep 17 00:00:00 2001 From: Bharath Date: Fri, 2 Aug 2024 11:40:09 +0530 Subject: [PATCH 02/43] run fmt and lint --- crates/astria-conductor/src/executor/mod.rs | 4 ++-- crates/astria-core/src/composer/mod.rs | 2 +- crates/astria-core/src/composer/v1alpha1/mod.rs | 15 +++------------ 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/crates/astria-conductor/src/executor/mod.rs b/crates/astria-conductor/src/executor/mod.rs index c5f9e59d2..cb5e61043 100644 --- a/crates/astria-conductor/src/executor/mod.rs +++ b/crates/astria-conductor/src/executor/mod.rs @@ -438,7 +438,7 @@ impl Executor { .await .wrap_err("failed to execute block")?; - self.does_block_response_fulfill_contract(ExecutionKind::Soft, &executed_block.block()) + self.does_block_response_fulfill_contract(ExecutionKind::Soft, executed_block.block()) .wrap_err("execution API server violated contract")?; self.update_commitment_state(Update::OnlySoft(executed_block.block().clone())) @@ -489,7 +489,7 @@ impl Executor { .execute_block(parent_hash, executable_block) .await .wrap_err("failed to execute block")?; - self.does_block_response_fulfill_contract(ExecutionKind::Firm, &executed_block.block()) + self.does_block_response_fulfill_contract(ExecutionKind::Firm, executed_block.block()) .wrap_err("execution API server violated contract")?; Update::ToSame(executed_block.block().clone(), celestia_height) } else if let Some(block) = self.blocks_pending_finalization.remove(&block_number) { diff --git a/crates/astria-core/src/composer/mod.rs b/crates/astria-core/src/composer/mod.rs index b9390b87c..32a5a9d4f 100644 --- a/crates/astria-core/src/composer/mod.rs +++ b/crates/astria-core/src/composer/mod.rs @@ -1 +1 @@ -pub mod v1alpha1; \ No newline at end of file +pub mod v1alpha1; diff --git a/crates/astria-core/src/composer/v1alpha1/mod.rs b/crates/astria-core/src/composer/v1alpha1/mod.rs index 0db2af8d1..a7014571a 100644 --- a/crates/astria-core/src/composer/v1alpha1/mod.rs +++ b/crates/astria-core/src/composer/v1alpha1/mod.rs @@ -13,10 +13,6 @@ use crate::{ pub struct BuilderBundleError(BuilderBundleErrorKind); impl BuilderBundleError { - fn field_not_set(field: &'static str) -> Self { - Self(BuilderBundleErrorKind::FieldNotSet(field)) - } - fn invalid_rollup_data(error: RollupDataError) -> Self { Self(BuilderBundleErrorKind::InvalidRollupData(error)) } @@ -24,8 +20,6 @@ impl BuilderBundleError { #[derive(Debug, thiserror::Error)] enum BuilderBundleErrorKind { - #[error("{0} field not set")] - FieldNotSet(&'static str), #[error("{0} invalid rollup data")] InvalidRollupData(#[source] RollupDataError), } @@ -69,7 +63,7 @@ impl Protobuf for BuilderBundle { let mut rollup_data_transactions = vec![]; for transaction in transactions { - let rollup_data = RollupData::try_from_raw_ref(&transaction) + let rollup_data = RollupData::try_from_raw_ref(transaction) .map_err(BuilderBundleError::invalid_rollup_data)?; rollup_data_transactions.push(rollup_data); } @@ -85,7 +79,7 @@ impl Protobuf for BuilderBundle { transactions: self .transactions .iter() - .map(|transaction| transaction.to_raw()) + .map(Protobuf::to_raw) .collect(), parent_hash: self.parent_hash.clone().to_vec(), } @@ -112,7 +106,6 @@ impl BuilderBundlePacketError { fn invalid_bundle(error: BuilderBundleError) -> Self { Self(BuilderBundlePacketErrorKind::InvalidBundle(error)) } - } #[derive(Clone, Debug)] @@ -161,7 +154,6 @@ impl Protobuf for BuilderBundlePacket { .map_err(BuilderBundlePacketError::invalid_bundle)? }; - Ok(BuilderBundlePacket { bundle, signature: Bytes::from(signature.clone()), @@ -174,5 +166,4 @@ impl Protobuf for BuilderBundlePacket { signature: self.signature.clone().to_vec(), } } - -} \ No newline at end of file +} From 46038040b3a0babae1031321d7204adb7ea9c1bd Mon Sep 17 00:00:00 2001 From: Bharath Date: Fri, 6 Sep 2024 14:10:14 +0530 Subject: [PATCH 03/43] cover deposits case --- Cargo.lock | 2 + crates/astria-composer/Cargo.toml | 10 +- crates/astria-composer/src/composer.rs | 3 + crates/astria-composer/src/config.rs | 3 + .../astria-composer/src/executor/builder.rs | 29 +- .../src/executor/bundle_factory/mod.rs | 8 +- crates/astria-composer/src/executor/client.rs | 342 ++++++++++++++++++ crates/astria-composer/src/executor/mod.rs | 87 ++++- .../astria-composer/src/executor/simulator.rs | 99 +++++ .../astria-core/src/composer/v1alpha1/mod.rs | 6 +- crates/astria-core/src/lib.rs | 2 +- 11 files changed, 575 insertions(+), 16 deletions(-) create mode 100644 crates/astria-composer/src/executor/client.rs create mode 100644 crates/astria-composer/src/executor/simulator.rs diff --git a/Cargo.lock b/Cargo.lock index edcac97fb..a16282f9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -599,6 +599,7 @@ dependencies = [ "astria-test-utils", "async-trait", "axum", + "bytes", "ethers", "futures", "hex", @@ -607,6 +608,7 @@ dependencies = [ "insta", "itertools 0.12.1", "once_cell", + "pbjson-types", "pin-project-lite", "prost", "rand 0.8.5", diff --git a/crates/astria-composer/Cargo.toml b/crates/astria-composer/Cargo.toml index 9dad92be7..7f10a9ec4 100644 --- a/crates/astria-composer/Cargo.toml +++ b/crates/astria-composer/Cargo.toml @@ -13,7 +13,11 @@ name = "astria-composer" [dependencies] astria-build-info = { path = "../astria-build-info", features = ["runtime"] } -astria-core = { path = "../astria-core", features = ["serde", "server"] } +astria-core = { path = "../astria-core", features = [ + "client", + "serde", + "server", +] } astria-eyre = { path = "../astria-eyre" } config = { package = "astria-config", path = "../astria-config" } telemetry = { package = "astria-telemetry", path = "../astria-telemetry", features = [ @@ -53,6 +57,9 @@ tracing = { workspace = true, features = ["attributes"] } tryhard = { workspace = true } tonic = { workspace = true } tokio-stream = { workspace = true, features = ["net"] } +metrics = { workspace = true } +pbjson-types = { workspace = true } +bytes = { workspace = true } [dependencies.sequencer-client] package = "astria-sequencer-client" @@ -69,7 +76,6 @@ test_utils = { package = "astria-test-utils", path = "../astria-test-utils", fea insta = { workspace = true, features = ["json"] } tempfile = { workspace = true } tokio-test = { workspace = true } -astria-core = { path = "../astria-core", features = ["client"] } tendermint-rpc = { workspace = true } wiremock = { workspace = true } diff --git a/crates/astria-composer/src/composer.rs b/crates/astria-composer/src/composer.rs index a4b82f27b..ab8eb82d2 100644 --- a/crates/astria-composer/src/composer.rs +++ b/crates/astria-composer/src/composer.rs @@ -131,6 +131,9 @@ impl Composer { block_time_ms: cfg.block_time_ms, max_bytes_per_bundle: cfg.max_bytes_per_bundle, bundle_queue_capacity: cfg.bundle_queue_capacity, + execution_api_url: cfg.execution_api_url.clone(), + fee_asset: cfg.fee_asset.clone(), + chain_name: rollup_name.clone(), shutdown_token: shutdown_token.clone(), metrics, } diff --git a/crates/astria-composer/src/config.rs b/crates/astria-composer/src/config.rs index c8e125d96..58265be88 100644 --- a/crates/astria-composer/src/config.rs +++ b/crates/astria-composer/src/config.rs @@ -70,6 +70,9 @@ pub struct Config { /// The IBC asset to pay for transactions submiited to the sequencer. pub fee_asset: astria_core::primitive::v1::asset::Denom, + + /// The URL of the execution API server + pub execution_api_url: String, } impl Config { diff --git a/crates/astria-composer/src/executor/builder.rs b/crates/astria-composer/src/executor/builder.rs index 871a4f1b3..059294e36 100644 --- a/crates/astria-composer/src/executor/builder.rs +++ b/crates/astria-composer/src/executor/builder.rs @@ -6,7 +6,11 @@ use std::{ use astria_core::{ crypto::SigningKey, - primitive::v1::Address, + primitive::v1::{ + asset, + Address, + RollupId, + }, protocol::transaction::v1alpha1::action::SequenceAction, }; use astria_eyre::eyre::{ @@ -16,10 +20,14 @@ use astria_eyre::eyre::{ }; use tokio::sync::watch; use tokio_util::sync::CancellationToken; +use tracing::info; use crate::{ executor, - executor::Status, + executor::{ + simulator::BundleSimulator, + Status, + }, metrics::Metrics, }; @@ -32,6 +40,9 @@ pub(crate) struct Builder { pub(crate) max_bytes_per_bundle: usize, pub(crate) bundle_queue_capacity: usize, pub(crate) shutdown_token: CancellationToken, + pub(crate) execution_api_url: String, + pub(crate) chain_name: String, + pub(crate) fee_asset: asset::Denom, pub(crate) metrics: &'static Metrics, } @@ -46,6 +57,9 @@ impl Builder { max_bytes_per_bundle, bundle_queue_capacity, shutdown_token, + execution_api_url, + chain_name, + fee_asset, metrics, } = self; let sequencer_client = sequencer_client::HttpClient::new(sequencer_url.as_str()) @@ -65,6 +79,13 @@ impl Builder { let (serialized_rollup_transaction_tx, serialized_rollup_transaction_rx) = tokio::sync::mpsc::channel::(256); + let rollup_id = RollupId::from_unhashed_bytes(&chain_name); + info!( + rollup_name = %chain_name, + rollup_id = %rollup_id, + "created new geth collector for rollup", + ); + Ok(( super::Executor { status, @@ -76,7 +97,11 @@ impl Builder { block_time: Duration::from_millis(block_time_ms), max_bytes_per_bundle, bundle_queue_capacity, + bundle_simulator: BundleSimulator::new(execution_api_url.as_str()) + .wrap_err("failed constructing bundle simulator")?, shutdown_token, + rollup_id, + fee_asset, metrics, }, executor::Handle::new(serialized_rollup_transaction_tx), diff --git a/crates/astria-composer/src/executor/bundle_factory/mod.rs b/crates/astria-composer/src/executor/bundle_factory/mod.rs index 58a3e5a9b..d70f9829d 100644 --- a/crates/astria-composer/src/executor/bundle_factory/mod.rs +++ b/crates/astria-composer/src/executor/bundle_factory/mod.rs @@ -26,7 +26,7 @@ use tracing::trace; mod tests; #[derive(Debug, thiserror::Error)] -enum SizedBundleError { +pub(super) enum SizedBundleError { #[error("bundle does not have enough space left for the given sequence action")] NotEnoughSpace(SequenceAction), #[error("sequence action is larger than the max bundle size")] @@ -65,7 +65,7 @@ pub(super) struct SizedBundle { impl SizedBundle { /// Create a new empty bundle with the given max size. - fn new(max_size: usize) -> Self { + pub(super) fn new(max_size: usize) -> Self { Self { buffer: vec![], curr_size: 0, @@ -78,7 +78,7 @@ impl SizedBundle { /// # Errors /// - `seq_action` is beyond the max size allowed for the entire bundle /// - `seq_action` does not fit in the remaining space in the bundle - fn try_push(&mut self, seq_action: SequenceAction) -> Result<(), SizedBundleError> { + pub(super) fn try_push(&mut self, seq_action: SequenceAction) -> Result<(), SizedBundleError> { let seq_action_size = encoded_len(&seq_action); if seq_action_size > self.max_size { @@ -102,7 +102,7 @@ impl SizedBundle { } /// Replace self with a new empty bundle, returning the old bundle. - fn flush(&mut self) -> SizedBundle { + pub(super) fn flush(&mut self) -> SizedBundle { mem::replace(self, Self::new(self.max_size)) } diff --git a/crates/astria-composer/src/executor/client.rs b/crates/astria-composer/src/executor/client.rs new file mode 100644 index 000000000..c1c92b881 --- /dev/null +++ b/crates/astria-composer/src/executor/client.rs @@ -0,0 +1,342 @@ +use std::time::Duration; + +use astria_core::{ + execution::v1alpha2::{ + Block, + CommitmentState, + ExecuteBlockResponse, + }, + generated::{ + execution::{ + v1alpha2 as raw, + v1alpha2::execution_service_client::ExecutionServiceClient, + }, + sequencerblock::v1alpha1::RollupData, + }, + Protobuf as _, +}; +use astria_eyre::eyre::{ + self, + ensure, + WrapErr as _, +}; +use bytes::Bytes; +use pbjson_types::Timestamp; +use tonic::transport::{ + Channel, + Endpoint, + Uri, +}; +use tracing::{ + instrument, + warn, + Instrument, + Span, +}; +use tryhard::{ + backoff_strategies::BackoffStrategy, + RetryPolicy, +}; + +/// A newtype wrapper around [`ExecutionServiceClient`] to work with +/// idiomatic types. +#[derive(Clone)] +pub(crate) struct Client { + uri: Uri, + inner: ExecutionServiceClient, +} + +impl Client { + pub(crate) fn connect_lazy(uri: &str) -> eyre::Result { + let uri: Uri = uri + .parse() + .wrap_err("failed to parse provided string as uri")?; + let endpoint = Endpoint::from(uri.clone()).connect_lazy(); + let inner = ExecutionServiceClient::new(endpoint); + Ok(Self { + uri, + inner, + }) + } + + pub(crate) fn uri(&self) -> String { + self.uri.to_string() + } + + /// Calls RPC astria.execution.v1alpha2.GetBlock + #[instrument(skip_all, fields(block_number, uri = %self.uri), err)] + pub(crate) async fn get_block_with_retry(&mut self, block_number: u32) -> eyre::Result { + let raw_block = tryhard::retry_fn(|| { + let mut client = self.inner.clone(); + let request = raw::GetBlockRequest { + identifier: Some(block_identifier(block_number)), + }; + async move { client.get_block(request).await } + }) + .with_config(retry_config()) + .in_current_span() + .await + .wrap_err( + "failed to execute astria.execution.v1alpha2.GetBlocks RPC because of gRPC status \ + code or because number of retries were exhausted", + )? + .into_inner(); + ensure!( + block_number == raw_block.number, + "requested block at number `{block_number}`, but received block contained `{}`", + raw_block.number + ); + Block::try_from_raw(raw_block).wrap_err("failed validating received block") + } + + /// Calls remote procedure `astria.execution.v1alpha2.ExecuteBlock` + /// + /// # Arguments + /// + /// * `prev_block_hash` - Block hash of the parent block + /// * `transactions` - List of transactions extracted from the sequencer block + /// * `timestamp` - Optional timestamp of the sequencer block + #[instrument(skip_all, fields(uri = %self.uri), err)] + pub(super) async fn execute_block_with_retry( + &self, + prev_block_hash: Bytes, + transactions: Vec>, + timestamp: Timestamp, + simulate_only: bool, + ) -> eyre::Result { + use prost::Message; + + let transactions = transactions + .into_iter() + .map(|tx| RollupData::decode(tx.as_slice())) + .collect::>() + .wrap_err("failed to decode tx bytes as RollupData")?; + + let request = raw::ExecuteBlockRequest { + prev_block_hash, + transactions, + timestamp: Some(timestamp), + simulate_only, + }; + let response = tryhard::retry_fn(|| { + let mut client = self.inner.clone(); + let request = request.clone(); + async move { client.execute_block(request).await } + }) + .with_config(retry_config()) + .in_current_span() + .await + .wrap_err( + "failed to execute astria.execution.v1alpha2.ExecuteBlock RPC because of gRPC status \ + code or because number of retries were exhausted", + )? + .into_inner(); + let execute_block_response = ExecuteBlockResponse::try_from_raw(response) + .wrap_err("failed converting raw response to validated execute block response")?; + + Ok(execute_block_response) + } + + /// Calls remote procedure `astria.execution.v1alpha2.GetCommitmentState` + #[instrument(skip_all, fields(uri = %self.uri), err)] + pub(crate) async fn get_commitment_state_with_retry(&self) -> eyre::Result { + let response = tryhard::retry_fn(|| { + let mut client = self.inner.clone(); + async move { + let request = raw::GetCommitmentStateRequest {}; + client.get_commitment_state(request).await + } + }) + .with_config(retry_config()) + .in_current_span() + .await + .wrap_err( + "failed to execute astria.execution.v1alpha2.GetCommitmentState RPC because of gRPC \ + status code or because number of retries were exhausted", + )? + .into_inner(); + let commitment_state = CommitmentState::try_from_raw(response) + .wrap_err("failed converting raw response to validated commitment state")?; + Ok(commitment_state) + } +} + +/// Utility function to construct a `astria.execution.v1alpha2.BlockIdentifier` from `number` +/// to use in RPC requests. +fn block_identifier(number: u32) -> raw::BlockIdentifier { + raw::BlockIdentifier { + identifier: Some(raw::block_identifier::Identifier::BlockNumber(number)), + } +} + +struct OnRetry { + parent: Span, +} + +impl tryhard::OnRetry for OnRetry { + type Future = futures::future::Ready<()>; + + fn on_retry( + &mut self, + attempt: u32, + next_delay: Option, + previous_error: &tonic::Status, + ) -> Self::Future { + let wait_duration = next_delay + .map(humantime::format_duration) + .map(tracing::field::display); + warn!( + parent: self.parent.id(), + attempt, + wait_duration, + error = previous_error as &dyn std::error::Error, + "failed executing RPC; retrying after after backoff" + ); + futures::future::ready(()) + } +} + +fn retry_config() -> tryhard::RetryFutureConfig { + tryhard::RetryFutureConfig::new(u32::MAX) + .custom_backoff(ExecutionApiRetryStrategy { + delay: Duration::from_millis(100), + }) + // XXX: This should probably be configurable. + .max_delay(Duration::from_secs(10)) + .on_retry(OnRetry { + parent: Span::current(), + }) +} + +/// An exponential retry strategy branching on [`tonic::Status::code`]. +/// +/// This retry strategy behaves exactly like +/// [`tryhard::backoff_strategies::ExponentialBackoff`] but is specialized to +/// work with [`tonic::Status`]. +/// +/// Execution will be retried under the following conditions: +/// +/// ```text +/// Code::Cancelled +/// Code::Unknown +/// Code::DeadlineExceeded +/// Code::NotFound +/// Code::ResourceExhausted +/// Code::Aborted +/// Code::Unavailable +/// ``` +struct ExecutionApiRetryStrategy { + delay: Duration, +} + +impl<'a> BackoffStrategy<'a, tonic::Status> for ExecutionApiRetryStrategy { + type Output = RetryPolicy; + + fn delay(&mut self, _attempt: u32, error: &'a tonic::Status) -> Self::Output { + if should_retry(error) { + let prev_delay = self.delay; + self.delay = self.delay.saturating_mul(2); + RetryPolicy::Delay(prev_delay) + } else { + RetryPolicy::Break + } + } +} + +fn should_retry(status: &tonic::Status) -> bool { + use tonic::Code; + // gRPC return codes and if they should be retried. Also refer to + // [1] https://github.com/grpc/grpc/blob/1309eb283c3e11c471191f286ceab01b75477ffc/doc/statuscodes.md + // + // Code::Ok => no, success + // Code::Cancelled => yes, but should be revisited if "we" would cancel + // Code::Unknown => yes, could this be returned if the endpoint is unavailable? + // Code::InvalidArgument => no, no point retrying + // Code::DeadlineExceeded => yes, server might be slow + // Code::NotFound => yes, resource might not yet be available + // Code::AlreadyExists => no, no point retrying + // Code::PermissionDenied => no, execution API uses permission-denied restart-trigger + // Code::ResourceExhausted => yes, retry after a while + // Code::FailedPrecondition => no, failed precondition should not be retried unless the + // precondition is fixed, see [1] + // Code::Aborted => yes, although this applies to a read-modify-write sequence. We should + // implement this not per-request but per-request-sequence (for example, + // execute + update-commitment-state) + // Code::OutOfRange => no, we don't expect to send any out-of-range requests. + // Code::Unimplemented => no, no point retrying + // Code::Internal => no, this is a serious error on the backend; don't retry + // Code::Unavailable => yes, retry after backoff is desired + // Code::DataLoss => no, unclear how this would happen, but sounds very terminal + // Code::Unauthenticated => no, this status code will likely not change after retrying + matches!( + status.code(), + Code::Cancelled + | Code::Unknown + | Code::DeadlineExceeded + | Code::NotFound + | Code::ResourceExhausted + | Code::Aborted + | Code::Unavailable + ) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use tonic::{ + Code, + Status, + }; + + use super::{ + BackoffStrategy as _, + ExecutionApiRetryStrategy, + RetryPolicy, + }; + + #[track_caller] + fn assert_retry_policy(code: Code) { + let mut strat = ExecutionApiRetryStrategy { + delay: Duration::from_secs(1), + }; + let status = Status::new(code, ""); + let actual = strat.delay(1, &status); + if SHOULD_RETRY { + let expected = RetryPolicy::Delay(Duration::from_secs(1)); + assert_eq!( + expected, actual, + "gRPC code `{code}` should lead to retry, but instead gave break" + ); + } else { + let expected = RetryPolicy::Break; + assert_eq!( + expected, actual, + "gRPC code `{code}` should lead to break, but instead gave delay" + ); + } + } + + #[test] + fn status_codes_lead_to_expected_retry_policy() { + const SHOULD_RETRY: bool = true; + const SHOULD_BREAK: bool = false; + assert_retry_policy::(Code::Ok); + assert_retry_policy::(Code::Cancelled); + assert_retry_policy::(Code::Unknown); + assert_retry_policy::(Code::InvalidArgument); + assert_retry_policy::(Code::DeadlineExceeded); + assert_retry_policy::(Code::NotFound); + assert_retry_policy::(Code::AlreadyExists); + assert_retry_policy::(Code::PermissionDenied); + assert_retry_policy::(Code::ResourceExhausted); + assert_retry_policy::(Code::FailedPrecondition); + assert_retry_policy::(Code::Aborted); + assert_retry_policy::(Code::OutOfRange); + assert_retry_policy::(Code::Unimplemented); + assert_retry_policy::(Code::Internal); + assert_retry_policy::(Code::Unavailable); + assert_retry_policy::(Code::DataLoss); + assert_retry_policy::(Code::Unauthenticated); + } +} diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index a873119d7..a35009ebc 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -12,6 +12,20 @@ use std::{ use astria_core::{ crypto::SigningKey, + generated::{ + composer::v1alpha1::{ + BuilderBundle, + BuilderBundlePacket, + }, + sequencerblock::v1alpha1::{ + rollup_data, + RollupData, + }, + }, + primitive::v1::{ + asset, + RollupId, + }, protocol::{ abci::AbciErrorCode, transaction::v1alpha1::{ @@ -21,6 +35,7 @@ use astria_core::{ UnsignedTransaction, }, }, + Protobuf, }; use astria_eyre::eyre::{ self, @@ -91,11 +106,15 @@ use crate::{ mod bundle_factory; pub(crate) mod builder; +mod client; +mod simulator; #[cfg(test)] mod tests; pub(crate) use builder::Builder; +use crate::executor::simulator::BundleSimulator; + // Duration to wait for the executor to drain all the remaining bundles before shutting down. // This is 16s because the timeout for the higher level executor task is 17s to shut down. // The extra second is to prevent the higher level executor task from timing out before the @@ -140,6 +159,9 @@ pub(super) struct Executor { bundle_queue_capacity: usize, // Token to signal the executor to stop upon shutdown. shutdown_token: CancellationToken, + bundle_simulator: BundleSimulator, + rollup_id: RollupId, + fee_asset: asset::Denom, metrics: &'static Metrics, } @@ -190,6 +212,64 @@ impl Executor { self.status.subscribe() } + // TODO - maybe we should break this up into a separate simulate and submit step? + async fn simulate_and_submit_bundle( + &self, + nonce: u32, + bundle: SizedBundle, + metrics: &'static Metrics, + ) -> eyre::Result>> { + let bundle_simulator = self.bundle_simulator.clone(); + + // simulate the bundle + let bundle_simulation_result = bundle_simulator + .simulate_bundle(bundle.clone()) + .await + .wrap_err("failed to simulate bundle")?; + let included_actions = bundle_simulation_result.included_actions(); + + let rollup_data_items: Vec = included_actions + .iter() + .map(|action| action.to_raw()) + .collect(); + + // create a top of block bundle + // TODO - we need to sign the builder bundle packet + let builder_bundle_packet = BuilderBundlePacket { + bundle: Some(BuilderBundle { + transactions: rollup_data_items, + parent_hash: bundle_simulation_result.parent_hash().encode_to_vec(), + }), + signature: vec![], + }; + let encoded_builder_bundle_packet = builder_bundle_packet.encode_to_vec(); + + // TODO - we had to make sized bundle struct public, can we avoid that? + let mut final_bundle = SizedBundle::new(200000); + if let Err(e) = final_bundle.try_push(SequenceAction { + rollup_id: self.rollup_id, + data: encoded_builder_bundle_packet, + fee_asset: self.fee_asset.clone(), + }) { + // TODO - we had to make the SizedBundle error public across the crate, we should + // revisit that + return Err(eyre!(e.to_string())); + } + + Ok(SubmitFut { + client: self.sequencer_client.clone(), + address: self.address, + nonce, + chain_id: self.sequencer_chain_id.clone(), + signing_key: self.sequencer_key.clone(), + state: SubmitState::NotStarted, + bundle: final_bundle, + metrics, + } + .in_current_span() + .fuse()) + } + /// Create a future to submit a bundle to the sequencer. #[instrument(skip_all, fields(nonce.initial = %nonce))] fn submit_bundle( @@ -262,7 +342,7 @@ impl Executor { Some(next_bundle) = future::ready(bundle_factory.next_finished()), if submission_fut.is_terminated() => { let bundle = next_bundle.pop(); if !bundle.is_empty() { - submission_fut = self.submit_bundle(nonce, bundle, self.metrics); + submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; } } @@ -277,7 +357,10 @@ impl Executor { if bundle.is_empty() { block_timer.as_mut().reset(reset_time()); } else { - submission_fut = self.submit_bundle(nonce, bundle, self.metrics); + debug!( + "forcing bundle submission to sequencer due to block timer" + ); + submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; } } } diff --git a/crates/astria-composer/src/executor/simulator.rs b/crates/astria-composer/src/executor/simulator.rs new file mode 100644 index 000000000..c1e2e5226 --- /dev/null +++ b/crates/astria-composer/src/executor/simulator.rs @@ -0,0 +1,99 @@ +use astria_core::execution::v1alpha2::{ + RollupData, + RollupDataValue, +}; +use astria_eyre::{ + eyre, + eyre::WrapErr as _, +}; +use bytes::Bytes; +use pbjson_types::Timestamp; +use tracing::instrument; + +use crate::executor::{ + bundle_factory::SizedBundle, + client::Client, +}; + +#[derive(Clone)] +pub(crate) struct BundleSimulator { + execution_service_client: Client, +} + +pub(crate) struct BundleSimulationResult { + included_actions: Vec, + parent_hash: Bytes, +} + +impl BundleSimulationResult { + pub(crate) fn new(included_sequence_actions: Vec, parent_hash: Bytes) -> Self { + Self { + included_actions: included_sequence_actions, + parent_hash, + } + } + + pub(crate) fn included_actions(&self) -> &[RollupData] { + self.included_actions.as_slice() + } + + pub(crate) fn parent_hash(&self) -> Bytes { + self.parent_hash.clone() + } +} + +impl BundleSimulator { + pub(crate) fn new(execution_api_uri: &str) -> eyre::Result { + Ok(Self { + execution_service_client: Client::connect_lazy(execution_api_uri) + .wrap_err("failed to connect to execution service")?, + }) + } + + #[instrument(skip_all, fields(uri=self.execution_service_client.uri()))] + pub(crate) async fn simulate_bundle( + self, + bundle: SizedBundle, + ) -> eyre::Result { + // call GetCommitmentState to get the soft block + let commitment_state = self + .execution_service_client + .get_commitment_state_with_retry() + .await + .wrap_err("failed to get commitment state")?; + + let soft_block = commitment_state.soft(); + + // convert the sized bundle actions to a list of list of u8s + // TODO - bharath - revisit this and make the code better. The else stmt is a bit weird + let actions = bundle + .into_actions() + .iter() + .map(|action| { + return if let Some(seq_action) = action.as_sequence() { + seq_action.clone().data + } else { + vec![] + }; + }) + .filter(|data| !data.is_empty()) + .collect::>>(); + + // call execute block with the bundle to get back the included transactions + let execute_block_response = self + .execution_service_client + .execute_block_with_retry( + soft_block.hash().clone(), + actions, + Timestamp::from(soft_block.timestamp()), + true, + ) + .await + .wrap_err("failed to execute block")?; + + Ok(BundleSimulationResult::new( + execute_block_response.included_transactions().to_vec(), + soft_block.hash().clone(), + )) + } +} diff --git a/crates/astria-core/src/composer/v1alpha1/mod.rs b/crates/astria-core/src/composer/v1alpha1/mod.rs index a7014571a..496de140d 100644 --- a/crates/astria-core/src/composer/v1alpha1/mod.rs +++ b/crates/astria-core/src/composer/v1alpha1/mod.rs @@ -76,11 +76,7 @@ impl Protobuf for BuilderBundle { fn to_raw(&self) -> Self::Raw { crate::generated::composer::v1alpha1::BuilderBundle { - transactions: self - .transactions - .iter() - .map(Protobuf::to_raw) - .collect(), + transactions: self.transactions.iter().map(Protobuf::to_raw).collect(), parent_hash: self.parent_hash.clone().to_vec(), } } diff --git a/crates/astria-core/src/lib.rs b/crates/astria-core/src/lib.rs index 3d4189c4b..d411d2b00 100644 --- a/crates/astria-core/src/lib.rs +++ b/crates/astria-core/src/lib.rs @@ -18,7 +18,7 @@ pub mod sequencerblock; pub mod brotli; #[cfg(feature = "celestia")] pub mod celestia; -mod composer; +pub mod composer; #[cfg(feature = "serde")] pub(crate) mod serde; From 17806ece8fe9c82763ed96eedaab7563d3511027 Mon Sep 17 00:00:00 2001 From: Bharath Date: Fri, 6 Sep 2024 14:12:47 +0530 Subject: [PATCH 04/43] initial version of trusted builder mvp --- Cargo.lock | 2 + crates/astria-composer/Cargo.toml | 2 + crates/astria-composer/justfile | 1 + crates/astria-composer/local.env.example | 9 +- crates/astria-composer/src/composer.rs | 9 +- crates/astria-composer/src/config.rs | 24 +- crates/astria-composer/src/executor/client.rs | 42 +- .../astria-composer/src/executor/mock_grpc.rs | 331 ++++++++++++++++ crates/astria-composer/src/executor/mod.rs | 17 +- .../astria-composer/src/executor/simulator.rs | 24 +- crates/astria-composer/src/executor/tests.rs | 366 ++++++++++++++---- crates/astria-composer/src/metrics.rs | 81 ++++ crates/astria-composer/tests/blackbox/api.rs | 12 +- .../tests/blackbox/geth_collector.rs | 135 ++++++- .../tests/blackbox/grpc_collector.rs | 103 ++++- .../tests/blackbox/helper/mock_grpc.rs | 330 ++++++++++++++++ .../tests/blackbox/helper/mod.rs | 55 ++- .../tests/blackbox/helpers/macros.rs | 16 +- .../tests/blackbox/helpers/mock_grpc.rs | 3 +- .../tests/blackbox/helpers/mod.rs | 4 +- .../astria-core/src/composer/v1alpha1/mod.rs | 2 +- .../astria-core/src/execution/v1alpha2/mod.rs | 106 +---- crates/astria-core/src/protocol/mod.rs | 5 +- crates/astria-core/src/protocol/test_utils.rs | 3 +- .../src/sequencerblock/v1alpha1/block.rs | 48 ++- .../src/proposal/commitment.rs | 3 +- 26 files changed, 1416 insertions(+), 317 deletions(-) create mode 100644 crates/astria-composer/src/executor/mock_grpc.rs create mode 100644 crates/astria-composer/tests/blackbox/helper/mock_grpc.rs diff --git a/Cargo.lock b/Cargo.lock index a16282f9c..2ec8bb3c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -594,11 +594,13 @@ dependencies = [ "astria-config", "astria-core", "astria-eyre", + "astria-grpc-mock", "astria-sequencer-client", "astria-telemetry", "astria-test-utils", "async-trait", "axum", + "base64 0.21.7", "bytes", "ethers", "futures", diff --git a/crates/astria-composer/Cargo.toml b/crates/astria-composer/Cargo.toml index 7f10a9ec4..64fb91428 100644 --- a/crates/astria-composer/Cargo.toml +++ b/crates/astria-composer/Cargo.toml @@ -60,6 +60,7 @@ tokio-stream = { workspace = true, features = ["net"] } metrics = { workspace = true } pbjson-types = { workspace = true } bytes = { workspace = true } +astria-grpc-mock = { path = "../astria-grpc-mock" } [dependencies.sequencer-client] package = "astria-sequencer-client" @@ -78,6 +79,7 @@ tempfile = { workspace = true } tokio-test = { workspace = true } tendermint-rpc = { workspace = true } wiremock = { workspace = true } +base64 = { workspace = true } [build-dependencies] astria-build-info = { path = "../astria-build-info", features = ["build"] } diff --git a/crates/astria-composer/justfile b/crates/astria-composer/justfile index e5c9ef965..fbb5671b6 100644 --- a/crates/astria-composer/justfile +++ b/crates/astria-composer/justfile @@ -1,3 +1,4 @@ + default: @just --list diff --git a/crates/astria-composer/local.env.example b/crates/astria-composer/local.env.example index 8936089de..68065cf9e 100644 --- a/crates/astria-composer/local.env.example +++ b/crates/astria-composer/local.env.example @@ -33,7 +33,11 @@ ASTRIA_COMPOSER_SEQUENCER_CHAIN_ID="astria-dev-1" # A list of execution `::,::`. # Rollup names are not case sensitive. If a name is repeated, the last list item is used. # names are sha256 hashed and used as the `rollup_id` in `SequenceAction`s -ASTRIA_COMPOSER_ROLLUPS="astriachain::ws://127.0.0.1:8545" +ASTRIA_COMPOSER_ROLLUP="" + +# The websocket url of the Rollup from which we can stream transactions using the +# getPendingTransactions API. +ASTRIA_COMPOSER_ROLLUP_WEBSOCKET_URL="" # The path to the file storing the private key for the sequencer account used for signing # transactions. The file should contain a hex-encoded Ed25519 secret key. @@ -72,6 +76,9 @@ ASTRIA_COMPOSER_GRPC_ADDR="0.0.0.0:0" # The asset to use for paying for transactions submitted to sequencer. ASTRIA_COMPOSER_FEE_ASSET="nria" +# The url of the execution api server +ASTRIA_COMPOSER_EXECUTION_API_URL="0.0.0.0:5031" + # The OTEL specific config options follow the OpenTelemetry Protocol Exporter v1 # specification as defined here: # https://github.com/open-telemetry/opentelemetry-specification/blob/e94af89e3d0c01de30127a0f423e912f6cda7bed/specification/protocol/exporter.md diff --git a/crates/astria-composer/src/composer.rs b/crates/astria-composer/src/composer.rs index ab8eb82d2..c16b06383 100644 --- a/crates/astria-composer/src/composer.rs +++ b/crates/astria-composer/src/composer.rs @@ -133,7 +133,7 @@ impl Composer { bundle_queue_capacity: cfg.bundle_queue_capacity, execution_api_url: cfg.execution_api_url.clone(), fee_asset: cfg.fee_asset.clone(), - chain_name: rollup_name.clone(), + chain_name: cfg.rollup.clone(), shutdown_token: shutdown_token.clone(), metrics, } @@ -163,7 +163,11 @@ impl Composer { "API server listening" ); - let rollups = cfg.parse_rollups()?; + // TODO - we don't need a map here, we can just use a single collector. This is done to + // get things working for now. we need to clean up later + let mut rollups = HashMap::new(); + rollups.insert(cfg.rollup.clone(), cfg.rollup_websocket_url.clone()); + let geth_collectors = rollups .iter() .map(|(rollup_name, url)| { @@ -179,6 +183,7 @@ impl Composer { (rollup_name.clone(), collector) }) .collect::>(); + let geth_collector_statuses: HashMap> = geth_collectors .iter() diff --git a/crates/astria-composer/src/config.rs b/crates/astria-composer/src/config.rs index 58265be88..1e66a33de 100644 --- a/crates/astria-composer/src/config.rs +++ b/crates/astria-composer/src/config.rs @@ -1,5 +1,4 @@ use std::{ - collections::HashMap, net::SocketAddr, }; @@ -8,11 +7,14 @@ use serde::{ Serialize, }; +<<<<<<< HEAD use crate::rollup::{ ParseError, Rollup, }; +======= +>>>>>>> f151354e (initial version of trusted builder mvp) // this is a config, may have many boolean values #[allow(clippy::struct_excessive_bools)] #[derive(Debug, Deserialize, Serialize)] @@ -30,8 +32,11 @@ pub struct Config { /// The chain ID of the sequencer chain pub sequencer_chain_id: String, - /// A list of `::` pairs - pub rollups: String, + /// The rollup name + pub rollup: String, + + /// The URL of the websocket server for the rollup chain + pub rollup_websocket_url: String, /// Path to private key for the sequencer account used for signing transactions pub private_key_file: String, @@ -75,6 +80,7 @@ pub struct Config { pub execution_api_url: String, } +<<<<<<< HEAD impl Config { /// Returns a map of rollup names to rollup URLs. /// @@ -89,6 +95,18 @@ impl Config { .collect::, _>>() } } +======= +// impl Config { +// pub(crate) fn parse_rollups(&self) -> astria_eyre::eyre::Result> { +// self.rollups +// .split(',') +// .filter(|s| !s.is_empty()) +// .map(|s| Rollup::parse(s).map(Rollup::into_parts)) +// .collect::, _>>() +// .wrap_err("failed parsing provided :: pairs as rollups") +// } +// } +>>>>>>> f151354e (initial version of trusted builder mvp) impl config::Config for Config { const PREFIX: &'static str = "ASTRIA_COMPOSER_"; diff --git a/crates/astria-composer/src/executor/client.rs b/crates/astria-composer/src/executor/client.rs index c1c92b881..d5b91b390 100644 --- a/crates/astria-composer/src/executor/client.rs +++ b/crates/astria-composer/src/executor/client.rs @@ -2,7 +2,6 @@ use std::time::Duration; use astria_core::{ execution::v1alpha2::{ - Block, CommitmentState, ExecuteBlockResponse, }, @@ -17,7 +16,6 @@ use astria_core::{ }; use astria_eyre::eyre::{ self, - ensure, WrapErr as _, }; use bytes::Bytes; @@ -63,32 +61,6 @@ impl Client { self.uri.to_string() } - /// Calls RPC astria.execution.v1alpha2.GetBlock - #[instrument(skip_all, fields(block_number, uri = %self.uri), err)] - pub(crate) async fn get_block_with_retry(&mut self, block_number: u32) -> eyre::Result { - let raw_block = tryhard::retry_fn(|| { - let mut client = self.inner.clone(); - let request = raw::GetBlockRequest { - identifier: Some(block_identifier(block_number)), - }; - async move { client.get_block(request).await } - }) - .with_config(retry_config()) - .in_current_span() - .await - .wrap_err( - "failed to execute astria.execution.v1alpha2.GetBlocks RPC because of gRPC status \ - code or because number of retries were exhausted", - )? - .into_inner(); - ensure!( - block_number == raw_block.number, - "requested block at number `{block_number}`, but received block contained `{}`", - raw_block.number - ); - Block::try_from_raw(raw_block).wrap_err("failed validating received block") - } - /// Calls remote procedure `astria.execution.v1alpha2.ExecuteBlock` /// /// # Arguments @@ -121,7 +93,7 @@ impl Client { let response = tryhard::retry_fn(|| { let mut client = self.inner.clone(); let request = request.clone(); - async move { client.execute_block(request).await } + async move { client.execute_block(request).await} }) .with_config(retry_config()) .in_current_span() @@ -144,7 +116,9 @@ impl Client { let mut client = self.inner.clone(); async move { let request = raw::GetCommitmentStateRequest {}; - client.get_commitment_state(request).await + let res = client.get_commitment_state(request).await; + println!("IN CLIENT CODE: CALLED GET COMMITMENT STATE {:?}", res); + res } }) .with_config(retry_config()) @@ -161,14 +135,6 @@ impl Client { } } -/// Utility function to construct a `astria.execution.v1alpha2.BlockIdentifier` from `number` -/// to use in RPC requests. -fn block_identifier(number: u32) -> raw::BlockIdentifier { - raw::BlockIdentifier { - identifier: Some(raw::block_identifier::Identifier::BlockNumber(number)), - } -} - struct OnRetry { parent: Span, } diff --git a/crates/astria-composer/src/executor/mock_grpc.rs b/crates/astria-composer/src/executor/mock_grpc.rs new file mode 100644 index 000000000..9519b8749 --- /dev/null +++ b/crates/astria-composer/src/executor/mock_grpc.rs @@ -0,0 +1,331 @@ +#![allow(dead_code)] +use std::{ + net::SocketAddr, + sync::Arc, +}; + +use astria_core::generated::{ + execution::v1alpha2::{ + execution_service_server::{ + ExecutionService, + ExecutionServiceServer, + }, + BatchGetBlocksRequest, + BatchGetBlocksResponse, + Block, + CommitmentState, + ExecuteBlockRequest, + ExecuteBlockResponse, + GenesisInfo, + GetBlockRequest, + GetCommitmentStateRequest, + GetGenesisInfoRequest, + UpdateCommitmentStateRequest, + }, + sequencerblock::v1alpha1::{ + sequencer_service_server::{ + SequencerService, + SequencerServiceServer, + }, + FilteredSequencerBlock, + GetFilteredSequencerBlockRequest, + GetPendingNonceRequest, + GetPendingNonceResponse, + GetSequencerBlockRequest, + SequencerBlock, + }, +}; +use astria_eyre::eyre::{ + self, + WrapErr as _, +}; +use astria_grpc_mock::{ + AnyMessage, + Match, + MockServer, +}; +use tokio::task::JoinHandle; +use tonic::{ + transport::Server, + Request, + Response, +}; + +pub struct MockGrpc { + _server: JoinHandle>, + pub mock_server: MockServer, + pub local_addr: SocketAddr, +} + +impl MockGrpc { + pub async fn spawn() -> Self { + use tokio_stream::wrappers::TcpListenerStream; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_addr = listener.local_addr().unwrap(); + + let mock_server = MockServer::new(); + + let server = { + let execution_service = ExecutionServiceImpl::new(mock_server.clone()); + let sequencer_service = SequencerServiceImpl::new(mock_server.clone()); + tokio::spawn(async move { + Server::builder() + .add_service(ExecutionServiceServer::new(execution_service)) + .add_service(SequencerServiceServer::new(sequencer_service)) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .wrap_err("gRPC server failed") + }) + }; + Self { + _server: server, + mock_server, + local_addr, + } + } +} + +struct SequencerServiceImpl { + mock_server: MockServer, +} + +impl SequencerServiceImpl { + fn new(mock_server: MockServer) -> Self { + Self { + mock_server, + } + } +} + +// XXX: Manually implementing this trait instead of using the `define_and_impl_service!` macro +// because `GetSequencerBlockRequest` and `SequencerBlock` don't currently implement +// `serde::Serialize`. +#[tonic::async_trait] +impl SequencerService for SequencerServiceImpl { + async fn get_sequencer_block( + self: Arc, + _request: Request, + ) -> tonic::Result> { + unimplemented!() + } + + async fn get_filtered_sequencer_block( + self: Arc, + request: Request, + ) -> tonic::Result> { + self.mock_server + .handle_request("get_filtered_sequencer_block", request) + .await + } + + async fn get_pending_nonce( + self: Arc, + _request: Request, + ) -> tonic::Result> { + unimplemented!() + } +} + +macro_rules! define_and_impl_service { + (impl $trait:ident for $target:ident { $( ($rpc:ident: $request:ty => $response:ty) )* }) => { + struct $target { + mock_server: ::astria_grpc_mock::MockServer, + } + + impl $target { + fn new(mock_server: ::astria_grpc_mock::MockServer) -> Self { + Self { mock_server, } + } + } + + #[tonic::async_trait] + impl $trait for $target { + $( + async fn $rpc(self: Arc, request: ::tonic::Request<$request>) -> ::tonic::Result<::tonic::Response<$response>> { + self.mock_server.handle_request(stringify!($rpc), request).await + } + )+ + } + } +} + +define_and_impl_service!(impl ExecutionService for ExecutionServiceImpl { + (execute_block: ExecuteBlockRequest => ExecuteBlockResponse) + (get_commitment_state: GetCommitmentStateRequest => CommitmentState) + (get_block: GetBlockRequest => Block) + (get_genesis_info: GetGenesisInfoRequest => GenesisInfo) + (batch_get_blocks: BatchGetBlocksRequest => BatchGetBlocksResponse) + (update_commitment_state: UpdateCommitmentStateRequest => CommitmentState) +}); + +#[macro_export] +macro_rules! execute_block_response { + (number: $number:expr,hash: $hash:expr,parent: $parent:expr $(,)?, included_transactions: $included_transactions:expr $(,)?) => { + ::astria_core::generated::execution::v1alpha2::ExecuteBlockResponse { + block: Some($crate::block!( + number: $number, + hash: $hash, + parent: $parent, + )), + included_transactions: $included_transactions, + } + }; +} + +#[macro_export] +macro_rules! block { + (number: $number:expr,hash: $hash:expr,parent: $parent:expr $(,)?) => { + ::astria_core::generated::execution::v1alpha2::Block { + number: $number, + hash: ::bytes::Bytes::from(Vec::from($hash)), + parent_block_hash: ::bytes::Bytes::from(Vec::from($parent)), + timestamp: Some(::pbjson_types::Timestamp { + seconds: 1, + nanos: 1, + }), + } + }; +} + +#[macro_export] +macro_rules! commitment_state { + ( + firm: (number: $firm_number:expr,hash: $firm_hash:expr,parent: $firm_parent:expr $(,)?), + soft: (number: $soft_number:expr,hash: $soft_hash:expr,parent: $soft_parent:expr $(,)?), + base_celestia_height: $base_celestia_height:expr $(,)? + ) => { + ::astria_core::generated::execution::v1alpha2::CommitmentState { + firm: Some($crate::block!( + number: $firm_number, + hash: $firm_hash, + parent: $firm_parent, + )), + soft: Some($crate::block!( + number: $soft_number, + hash: $soft_hash, + parent: $soft_parent, + )), + base_celestia_height: $base_celestia_height, + } + }; +} + +#[macro_export] +macro_rules! mount_get_commitment_state { + ( + $test_env:ident, + firm: ( number: $firm_number:expr, hash: $firm_hash:expr, parent: $firm_parent:expr$(,)? ), + soft: ( number: $soft_number:expr, hash: $soft_hash:expr, parent: $soft_parent:expr$(,)? ), + base_celestia_height: $base_celestia_height:expr + $(,)? + ) => { + $test_env + .mount_get_commitment_state($crate::commitment_state!( + firm: ( + number: $firm_number, + hash: $firm_hash, + parent: $firm_parent, + ), + soft: ( + number: $soft_number, + hash: $soft_hash, + parent: $soft_parent, + ), + base_celestia_height: $base_celestia_height, + )) + .await + }; +} + +#[macro_export] +macro_rules! mount_executed_block { + ( + $test_env:ident, + mock_name: $mock_name:expr, + number: $number:expr, + hash: $hash:expr, + included_transactions: $included_transactions:expr, + parent: $parent:expr $(,)?, + ) => {{ + $test_env.mount_execute_block( + $mock_name.into(), + ::serde_json::json!({ + "prevBlockHash": $parent, + "transactions": $included_transactions, + }), + $crate::execute_block_response!( + number: $number, + hash: $hash, + parent: $parent, + included_transactions: $included_transactions + ) + ) + .await + }}; + ( + $test_env:ident, + number: $number:expr, + hash: $hash:expr, + included_transactions: $included_transactions:expr, + parent: $parent:expr $(,)? + ) => { + mount_executed_block!( + $test_env, + mock_name: None, + number: $number, + hash: $hash, + parent: $parent, + included_transactions: $included_transactions + ) + }; +} + +pub struct TestExecutor { + pub mock_grpc: MockGrpc, +} + +impl TestExecutor { + pub async fn mount_get_commitment_state(&self, commitment_state: CommitmentState) { + astria_grpc_mock::Mock::for_rpc_given( + "get_commitment_state", + astria_grpc_mock::matcher::message_type::(), + ) + .respond_with(astria_grpc_mock::response::constant_response( + commitment_state, + )) + .expect(1..) + .mount(&self.mock_grpc.mock_server) + .await; + } + + pub async fn mount_execute_block( + &self, + mock_name: Option<&str>, + _expected_pbjson: S, + response: ExecuteBlockResponse, + ) -> astria_grpc_mock::MockGuard { + use astria_grpc_mock::{ + response::constant_response, + Mock, + }; + + let mut mock = Mock::for_rpc_given("execute_block", AlwaysMatches {}) + .respond_with(constant_response(response)); + if let Some(name) = mock_name { + mock = mock.with_name(name); + } + mock.expect(1) + .mount_as_scoped(&self.mock_grpc.mock_server) + .await + } +} + +// TODO - this is a hack to bypass request body matching. Fix this +struct AlwaysMatches(); + +impl Match for AlwaysMatches { + fn matches(&self, _req: &Request) -> bool { + true + } +} diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index a35009ebc..fd2b6572b 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -17,10 +17,7 @@ use astria_core::{ BuilderBundle, BuilderBundlePacket, }, - sequencerblock::v1alpha1::{ - rollup_data, - RollupData, - }, + sequencerblock::v1alpha1::RollupData, }, primitive::v1::{ asset, @@ -110,6 +107,7 @@ mod client; mod simulator; #[cfg(test)] mod tests; +mod mock_grpc; pub(crate) use builder::Builder; @@ -219,6 +217,7 @@ impl Executor { bundle: SizedBundle, metrics: &'static Metrics, ) -> eyre::Result>> { + println!("IN MAIN CODE: STARTING SIMULATION"); let bundle_simulator = self.bundle_simulator.clone(); // simulate the bundle @@ -226,9 +225,10 @@ impl Executor { .simulate_bundle(bundle.clone()) .await .wrap_err("failed to simulate bundle")?; - let included_actions = bundle_simulation_result.included_actions(); - let rollup_data_items: Vec = included_actions + println!("IN MAIN CODE: DONE WITH SIMULATION"); + let rollup_data_items: Vec = bundle_simulation_result + .included_actions() .iter() .map(|action| action.to_raw()) .collect(); @@ -238,7 +238,7 @@ impl Executor { let builder_bundle_packet = BuilderBundlePacket { bundle: Some(BuilderBundle { transactions: rollup_data_items, - parent_hash: bundle_simulation_result.parent_hash().encode_to_vec(), + parent_hash: bundle_simulation_result.parent_hash().to_vec(), }), signature: vec![], }; @@ -248,7 +248,7 @@ impl Executor { let mut final_bundle = SizedBundle::new(200000); if let Err(e) = final_bundle.try_push(SequenceAction { rollup_id: self.rollup_id, - data: encoded_builder_bundle_packet, + data: encoded_builder_bundle_packet.into(), fee_asset: self.fee_asset.clone(), }) { // TODO - we had to make the SizedBundle error public across the crate, we should @@ -256,6 +256,7 @@ impl Executor { return Err(eyre!(e.to_string())); } + println!("IN MAIN CODE: DONE SIMULATION!"); Ok(SubmitFut { client: self.sequencer_client.clone(), address: self.address, diff --git a/crates/astria-composer/src/executor/simulator.rs b/crates/astria-composer/src/executor/simulator.rs index c1e2e5226..2d3649141 100644 --- a/crates/astria-composer/src/executor/simulator.rs +++ b/crates/astria-composer/src/executor/simulator.rs @@ -1,6 +1,6 @@ -use astria_core::execution::v1alpha2::{ - RollupData, - RollupDataValue, +use astria_core::{ + sequencerblock::v1alpha1::block::RollupData, + Protobuf, }; use astria_eyre::{ eyre, @@ -8,6 +8,7 @@ use astria_eyre::{ }; use bytes::Bytes; use pbjson_types::Timestamp; +use prost::Message; use tracing::instrument; use crate::executor::{ @@ -56,29 +57,35 @@ impl BundleSimulator { bundle: SizedBundle, ) -> eyre::Result { // call GetCommitmentState to get the soft block + println!("IN MAIN CODE: CALLING GET COMMITMENT STATE"); let commitment_state = self .execution_service_client .get_commitment_state_with_retry() .await .wrap_err("failed to get commitment state")?; + println!("IN MAIN CODE: CALLED GET COMMITMENT STATE!"); let soft_block = commitment_state.soft(); - // convert the sized bundle actions to a list of list of u8s // TODO - bharath - revisit this and make the code better. The else stmt is a bit weird - let actions = bundle + let actions: Vec> = bundle .into_actions() .iter() .map(|action| { + // TODO - should we support sequencer transfers and actions outside sequence + // actions too? return if let Some(seq_action) = action.as_sequence() { - seq_action.clone().data + RollupData::SequencedData(seq_action.clone().data) + .to_raw() + .encode_to_vec() } else { vec![] }; }) .filter(|data| !data.is_empty()) - .collect::>>(); + .collect(); + println!("IN MAIN CODE: CALLING EXECUTE_BLOCK"); // call execute block with the bundle to get back the included transactions let execute_block_response = self .execution_service_client @@ -91,9 +98,10 @@ impl BundleSimulator { .await .wrap_err("failed to execute block")?; + println!("IN MAIN CODE: CALLED EXECUTE BLOCK!!"); Ok(BundleSimulationResult::new( execute_block_response.included_transactions().to_vec(), - soft_block.hash().clone(), + execute_block_response.block().parent_block_hash().clone(), )) } } diff --git a/crates/astria-composer/src/executor/tests.rs b/crates/astria-composer/src/executor/tests.rs index bb0e4af53..38f7046ba 100644 --- a/crates/astria-composer/src/executor/tests.rs +++ b/crates/astria-composer/src/executor/tests.rs @@ -8,18 +8,28 @@ use std::{ }; use astria_core::{ - generated::protocol::accounts::v1alpha1::NonceResponse, + generated::{ + composer::v1alpha1::BuilderBundlePacket, + protocol::account::v1alpha1::NonceResponse, + }, primitive::v1::{ +<<<<<<< HEAD asset::{ Denom, IbcPrefixed, }, +======= + asset, +>>>>>>> f151354e (initial version of trusted builder mvp) RollupId, ROLLUP_ID_LEN, }, protocol::transaction::v1alpha1::action::SequenceAction, + sequencerblock::v1alpha1::block::RollupData, + Protobuf, }; use astria_eyre::eyre; +use futures::future::join; use once_cell::sync::Lazy; use prost::{ bytes::Bytes, @@ -66,8 +76,16 @@ use wiremock::{ use crate::{ executor, - executor::EnsureChainIdError, + executor::{ + mock_grpc::{ + MockGrpc, + TestExecutor, + }, + EnsureChainIdError, + }, metrics::Metrics, + mount_executed_block, + mount_get_commitment_state, test_utils::sequence_action_of_max_size, Config, }; @@ -111,18 +129,19 @@ static TELEMETRY: Lazy<()> = Lazy::new(|| { } }); -fn sequence_action() -> SequenceAction { +fn sequence_action(rollup_id: RollupId, fee_asset: asset::Denom) -> SequenceAction { SequenceAction { - rollup_id: RollupId::new([0; ROLLUP_ID_LEN]), - data: Bytes::new(), - fee_asset: "nria".parse().unwrap(), + rollup_id, + data: Bytes::from(vec![]), + fee_asset, } } /// Start a mock sequencer server and mount a mock for the `accounts/nonce` query. -async fn setup() -> (MockServer, Config, NamedTempFile) { +async fn setup() -> (MockServer, Config, NamedTempFile, TestExecutor) { Lazy::force(&TELEMETRY); let server = MockServer::start().await; + let execution_api_server = MockGrpc::spawn().await; let keyfile = NamedTempFile::new().unwrap(); (&keyfile) @@ -132,7 +151,8 @@ async fn setup() -> (MockServer, Config, NamedTempFile) { let cfg = Config { log: String::new(), api_listen_addr: "127.0.0.1:0".parse().unwrap(), - rollups: String::new(), + rollup: "test-chain-1".to_string(), + rollup_websocket_url: String::new(), sequencer_url: server.uri(), sequencer_chain_id: "test-chain-1".to_string(), private_key_file: keyfile.path().to_string_lossy().to_string(), @@ -146,9 +166,21 @@ async fn setup() -> (MockServer, Config, NamedTempFile) { metrics_http_listener_addr: String::new(), pretty_print: true, grpc_addr: "127.0.0.1:0".parse().unwrap(), - fee_asset: "nria".parse().unwrap(), + fee_asset: "nria" + .parse::() + .unwrap() + .to_ibc_prefixed() + .into(), + execution_api_url: format!("http://{}", execution_api_server.local_addr), }; - (server, cfg, keyfile) + ( + server, + cfg, + keyfile, + TestExecutor { + mock_grpc: execution_api_server, + }, + ) } /// Assert that given error is of correct type and contains the expected chain IDs. @@ -322,9 +354,13 @@ async fn wait_for_startup( #[tokio::test] async fn full_bundle() { // set up the executor, channel for writing seq actions, and the sequencer mock - let (sequencer, cfg, _keyfile) = setup().await; + let (sequencer, cfg, _keyfile, test_executor) = setup().await; let shutdown_token = CancellationToken::new(); +<<<<<<< HEAD let metrics = Box::leak(Box::new(Metrics::noop_metrics(&cfg).unwrap())); +======= + let metrics = Box::leak(Box::new(Metrics::new(cfg.rollup.clone()))); +>>>>>>> f151354e (initial version of trusted builder mvp) mount_genesis(&sequencer, &cfg.sequencer_chain_id).await; let (executor, executor_handle) = executor::Builder { sequencer_url: cfg.sequencer_url.clone(), @@ -335,11 +371,27 @@ async fn full_bundle() { max_bytes_per_bundle: cfg.max_bytes_per_bundle, bundle_queue_capacity: cfg.bundle_queue_capacity, shutdown_token: shutdown_token.clone(), + execution_api_url: cfg.execution_api_url, + chain_name: cfg.rollup.clone(), + fee_asset: cfg.fee_asset, metrics, } .build() .unwrap(); + let rollup_id = RollupId::from_unhashed_bytes(cfg.rollup.clone()); + + let soft_parent_hash = [1; 64]; + let soft_block_number = 1; + let soft_block_hash = [2; 64]; + + mount_get_commitment_state!( + test_executor, + firm: ( number: 1, hash: [1; 64], parent: [0; 64], ), + soft: ( number: soft_block_number, hash: soft_block_hash, parent: soft_parent_hash, ), + base_celestia_height: 1, + ); + let nonce_guard = mount_default_nonce_query_mock(&sequencer).await; let status = executor.subscribe(); @@ -352,13 +404,31 @@ async fn full_bundle() { // send two sequence actions to the executor, the first of which is large enough to fill the // bundle sending the second should cause the first to immediately be submitted in // order to make space for the second - let seq0 = sequence_action_of_max_size(cfg.max_bytes_per_bundle); + let seq0 = SequenceAction { + rollup_id, + ..sequence_action_of_max_size(cfg.max_bytes_per_bundle) + }; let seq1 = SequenceAction { - rollup_id: RollupId::new([1; ROLLUP_ID_LEN]), + rollup_id, ..sequence_action_of_max_size(cfg.max_bytes_per_bundle) }; + // TODO - type declaration looks weird, fix it + let rollup_data: Vec = + vec![seq0.clone()] + .iter() + .map(|item| RollupData::SequencedData(item.clone().data).to_raw()) + .collect(); + + let execute_block = mount_executed_block!(test_executor, + mock_name: "execute_block", + number: soft_block_number, + hash: soft_block_hash, + included_transactions: rollup_data.clone(), + parent: soft_parent_hash.to_vec(), + ); + // push both sequence actions to the executor in order to force the full bundle to be sent executor_handle .send_timeout(seq0.clone(), Duration::from_millis(1000)) @@ -372,14 +442,16 @@ async fn full_bundle() { // wait for the mock sequencer to receive the signed transaction tokio::time::timeout( Duration::from_millis(100), - response_guard.wait_until_satisfied(), + join( + response_guard.wait_until_satisfied(), + execute_block.wait_until_satisfied(), + ), ) .await .unwrap(); // verify only one signed transaction was received by the mock sequencer // i.e. only the full bundle was sent and not the second one due to the block timer - let expected_seq_actions = [seq0]; let requests = response_guard.received_requests().await; assert_eq!(requests.len(), 1); @@ -387,24 +459,45 @@ async fn full_bundle() { let signed_tx = signed_tx_from_request(&requests[0]); let actions = signed_tx.actions(); + // we send only 1 action to the sequencer which is a BuilderBundlePacket + // we first verify that the action sent to the sequencer is a builder bundle packet. + + // only 1 sequence action which is a BuilderBundlePacket is sent + assert_eq!(actions.len(), 1); + + // decode the sequence action to its BuilderBundlePacket + let mut seq_action = actions.iter().next().unwrap().as_sequence().unwrap(); + let proto_builder_bundle_packet = + BuilderBundlePacket::decode(&mut seq_action.data.clone()).unwrap(); + let builder_bundle_packet = astria_core::composer::v1alpha1::BuilderBundlePacket::try_from_raw( + proto_builder_bundle_packet.clone(), + ) + .unwrap(); + assert_eq!( - actions.len(), - expected_seq_actions.len(), - "received more than one action, one was supposed to fill the bundle" + builder_bundle_packet.bundle().parent_hash(), + soft_parent_hash.to_vec() ); - for (action, expected_seq_action) in actions.iter().zip(expected_seq_actions.iter()) { - let seq_action = action.as_sequence().unwrap(); - assert_eq!( - seq_action.rollup_id, expected_seq_action.rollup_id, - "chain id does not match. actual {:?} expected {:?}", - seq_action.rollup_id, expected_seq_action.rollup_id - ); - assert_eq!( - seq_action.data, expected_seq_action.data, - "data does not match expected data for action with rollup_id {:?}", - seq_action.rollup_id, - ); + let bundle_txs = builder_bundle_packet.bundle().transactions(); + + // there should only be 1 sequence action in the bundle + assert_eq!(bundle_txs.len(), 1); + + assert_eq!(seq_action.fee_asset, seq0.fee_asset); + assert_eq!(seq_action.rollup_id, seq0.rollup_id); + + match bundle_txs.iter().next().unwrap() { + RollupData::SequencedData(data) => { + assert_eq!(data.clone(), seq0.data) + } + _ => { + assert!( + true, + "expected RollupData::SequencedData, but got {:?}", + bundle_txs.iter().next().unwrap() + ) + } } } @@ -413,9 +506,13 @@ async fn full_bundle() { #[tokio::test] async fn bundle_triggered_by_block_timer() { // set up the executor, channel for writing seq actions, and the sequencer mock - let (sequencer, cfg, _keyfile) = setup().await; + let (sequencer, cfg, _keyfile, test_executor) = setup().await; let shutdown_token = CancellationToken::new(); +<<<<<<< HEAD let metrics = Box::leak(Box::new(Metrics::noop_metrics(&cfg).unwrap())); +======= + let metrics = Box::leak(Box::new(Metrics::new(cfg.rollup.clone()))); +>>>>>>> f151354e (initial version of trusted builder mvp) mount_genesis(&sequencer, &cfg.sequencer_chain_id).await; let (executor, executor_handle) = executor::Builder { sequencer_url: cfg.sequencer_url.clone(), @@ -426,11 +523,16 @@ async fn bundle_triggered_by_block_timer() { max_bytes_per_bundle: cfg.max_bytes_per_bundle, bundle_queue_capacity: cfg.bundle_queue_capacity, shutdown_token: shutdown_token.clone(), + execution_api_url: cfg.execution_api_url, + chain_name: cfg.rollup.clone(), + fee_asset: cfg.fee_asset.clone(), metrics, } .build() .unwrap(); + let rollup_id = RollupId::from_unhashed_bytes(cfg.rollup.clone()); + let nonce_guard = mount_default_nonce_query_mock(&sequencer).await; let status = executor.subscribe(); @@ -444,10 +546,35 @@ async fn bundle_triggered_by_block_timer() { // send two sequence actions to the executor, both small enough to fit in a single bundle // without filling it let seq0 = SequenceAction { - data: vec![0u8; cfg.max_bytes_per_bundle / 4].into(), - ..sequence_action() + data: Bytes::from(vec![0u8; cfg.max_bytes_per_bundle / 4]), + ..sequence_action(rollup_id.clone(), cfg.fee_asset.clone()) }; + let rollup_data: Vec = + vec![seq0.clone()] + .iter() + .map(|item| RollupData::SequencedData(item.clone().data).to_raw()) + .collect(); + + let soft_parent_hash = [1; 64]; + let soft_block_number = 1; + let soft_block_hash = [2; 64]; + + mount_get_commitment_state!( + test_executor, + firm: ( number: 1, hash: [1; 64], parent: [0; 64], ), + soft: ( number: soft_block_number, hash: soft_block_hash, parent: soft_parent_hash, ), + base_celestia_height: 1, + ); + + let execute_block = mount_executed_block!(test_executor, + mock_name: "execute_block", + number: soft_block_number, + hash: soft_block_hash, + included_transactions: rollup_data.clone(), + parent: soft_parent_hash.to_vec(), + ); + // make sure at least one block has passed so that the executor will submit the bundle // despite it not being full time::pause(); @@ -461,7 +588,10 @@ async fn bundle_triggered_by_block_timer() { // wait for the mock sequencer to receive the signed transaction tokio::time::timeout( Duration::from_millis(100), - response_guard.wait_until_satisfied(), + join( + response_guard.wait_until_satisfied(), + execute_block.wait_until_satisfied(), + ), ) .await .unwrap(); @@ -475,24 +605,55 @@ async fn bundle_triggered_by_block_timer() { let signed_tx = signed_tx_from_request(&requests[0]); let actions = signed_tx.actions(); + assert_eq!(actions.len(), 1); + + let mut seq_action = actions.iter().next().unwrap().as_sequence().unwrap(); + let proto_builder_bundle_packet = + BuilderBundlePacket::decode(&mut seq_action.data.clone()).unwrap(); + let builder_bundle_packet = astria_core::composer::v1alpha1::BuilderBundlePacket::try_from_raw( + proto_builder_bundle_packet.clone(), + ) + .unwrap(); + + assert_eq!(builder_bundle_packet.bundle().transactions().len(), 1); + assert_eq!( - actions.len(), - expected_seq_actions.len(), - "received more than one action, one was supposed to fill the bundle" + builder_bundle_packet.bundle().parent_hash().to_vec(), + soft_parent_hash.to_vec() ); - for (action, expected_seq_action) in actions.iter().zip(expected_seq_actions.iter()) { - let seq_action = action.as_sequence().unwrap(); - assert_eq!( - seq_action.rollup_id, expected_seq_action.rollup_id, - "chain id does not match. actual {:?} expected {:?}", - seq_action.rollup_id, expected_seq_action.rollup_id - ); - assert_eq!( - seq_action.data, expected_seq_action.data, - "data does not match expected data for action with rollup_id {:?}", - seq_action.rollup_id, - ); + // ensure that the seq_action of the BuilderBundlePacket and the expected sequence actions have the same + // rollup id and fee asset + + for (action, expected_action) in expected_seq_actions.iter().zip(actions) { + let expected_seq_action = expected_action.as_sequence().unwrap(); + assert_eq!(action.rollup_id, expected_seq_action.rollup_id); + assert_eq!(action.fee_asset, expected_seq_action.fee_asset); + } + + for (action, expected_seq_action) in builder_bundle_packet + .bundle() + .transactions() + .iter() + .zip(expected_seq_actions.iter()) + { + + match action.clone() { + RollupData::SequencedData(data) => { + assert_eq!( + data, expected_seq_action.data, + "data does not match expected data for action with rollup_id {:?}", + expected_seq_action.rollup_id + ) + } + _ => { + assert!( + true, + "expected RollupData::SequencedData, but got {:?}", + action + ) + } + } } } @@ -501,9 +662,13 @@ async fn bundle_triggered_by_block_timer() { #[tokio::test] async fn two_seq_actions_single_bundle() { // set up the executor, channel for writing seq actions, and the sequencer mock - let (sequencer, cfg, _keyfile) = setup().await; + let (sequencer, cfg, _keyfile, test_executor) = setup().await; let shutdown_token = CancellationToken::new(); +<<<<<<< HEAD let metrics = Box::leak(Box::new(Metrics::noop_metrics(&cfg).unwrap())); +======= + let metrics = Box::leak(Box::new(Metrics::new(cfg.rollup.clone()))); +>>>>>>> f151354e (initial version of trusted builder mvp) mount_genesis(&sequencer, &cfg.sequencer_chain_id).await; let (executor, executor_handle) = executor::Builder { sequencer_url: cfg.sequencer_url.clone(), @@ -514,11 +679,16 @@ async fn two_seq_actions_single_bundle() { max_bytes_per_bundle: cfg.max_bytes_per_bundle, bundle_queue_capacity: cfg.bundle_queue_capacity, shutdown_token: shutdown_token.clone(), + execution_api_url: cfg.execution_api_url, + chain_name: cfg.rollup.clone(), + fee_asset: cfg.fee_asset.clone(), metrics, } .build() .unwrap(); + let rollup_id = RollupId::from_unhashed_bytes(cfg.rollup.clone()); + let nonce_guard = mount_default_nonce_query_mock(&sequencer).await; let status = executor.subscribe(); let _executor_task = tokio::spawn(executor.run_until_stopped()); @@ -532,15 +702,39 @@ async fn two_seq_actions_single_bundle() { // without filling it let seq0 = SequenceAction { data: vec![0u8; cfg.max_bytes_per_bundle / 4].into(), - ..sequence_action() + ..sequence_action(rollup_id.clone(), cfg.fee_asset.clone()) }; let seq1 = SequenceAction { - rollup_id: RollupId::new([1; ROLLUP_ID_LEN]), data: vec![1u8; cfg.max_bytes_per_bundle / 4].into(), - ..sequence_action() + ..sequence_action(rollup_id.clone(), cfg.fee_asset.clone()) }; + let rollup_data: Vec = + vec![seq0.clone(), seq1.clone()] + .iter() + .map(|item| RollupData::SequencedData(item.clone().data).to_raw()) + .collect(); + + let soft_parent_hash = [1; 64]; + let soft_block_number = 1; + let soft_block_hash = [2; 64]; + + mount_get_commitment_state!( + test_executor, + firm: ( number: 1, hash: [1; 64], parent: [0; 64], ), + soft: ( number: soft_block_number, hash: soft_block_hash, parent: soft_parent_hash, ), + base_celestia_height: 1, + ); + + let execute_block = mount_executed_block!(test_executor, + mock_name: "execute_block", + number: soft_block_number, + hash: soft_block_hash, + included_transactions: rollup_data.clone(), + parent: soft_parent_hash.to_vec(), + ); + // make sure at least one block has passed so that the executor will submit the bundle // despite it not being full time::pause(); @@ -558,7 +752,10 @@ async fn two_seq_actions_single_bundle() { // wait for the mock sequencer to receive the signed transaction tokio::time::timeout( Duration::from_millis(100), - response_guard.wait_until_satisfied(), + join( + response_guard.wait_until_satisfied(), + execute_block.wait_until_satisfied(), + ) ) .await .unwrap(); @@ -572,24 +769,45 @@ async fn two_seq_actions_single_bundle() { let signed_tx = signed_tx_from_request(&requests[0]); let actions = signed_tx.actions(); + assert_eq!(actions.len(), 1); + + let seq_action = actions.iter().next().unwrap().as_sequence().unwrap(); + let proto_builder_bundle_packet = + BuilderBundlePacket::decode(&mut seq_action.data.clone()).unwrap(); + let builder_bundle_packet = astria_core::composer::v1alpha1::BuilderBundlePacket::try_from_raw( + proto_builder_bundle_packet.clone(), + ) + .unwrap(); + + let bundle_txs = builder_bundle_packet.bundle().transactions(); + + assert_eq!(builder_bundle_packet.bundle().transactions().len(), 2); assert_eq!( - actions.len(), - expected_seq_actions.len(), - "received more than one action, one was supposed to fill the bundle" + builder_bundle_packet.bundle().parent_hash().to_vec(), + soft_parent_hash.to_vec() ); - for (action, expected_seq_action) in actions.iter().zip(expected_seq_actions.iter()) { - let seq_action = action.as_sequence().unwrap(); - assert_eq!( - seq_action.rollup_id, expected_seq_action.rollup_id, - "chain id does not match. actual {:?} expected {:?}", - seq_action.rollup_id, expected_seq_action.rollup_id - ); - assert_eq!( - seq_action.data, expected_seq_action.data, - "data does not match expected data for action with rollup_id {:?}", - seq_action.rollup_id, - ); + for (action, expected_action) in expected_seq_actions.iter().zip(actions) { + let expected_seq_action = expected_action.as_sequence().unwrap(); + assert_eq!(action.rollup_id, expected_seq_action.rollup_id); + assert_eq!(action.fee_asset, expected_seq_action.fee_asset); + } + + for (action, expected_seq_action) in bundle_txs.iter().zip(expected_seq_actions.iter()) { + match action.clone() { + RollupData::SequencedData(data) => { + assert_eq!( + data, expected_seq_action.data, + "data does not match expected data for action with rollup_id {:?}", + expected_seq_action.rollup_id + ) + } + _ => assert!( + true, + "expected RollupData::SequencedData, but got {:?}", + action + ), + } } } @@ -600,9 +818,14 @@ async fn chain_id_mismatch_returns_error() { use tendermint::chain::Id; // set up sequencer mock - let (sequencer, cfg, _keyfile) = setup().await; + let (sequencer, cfg, _keyfile, _test_executor) = setup().await; let shutdown_token = CancellationToken::new(); +<<<<<<< HEAD let metrics = Box::leak(Box::new(Metrics::noop_metrics(&cfg).unwrap())); +======= + let metrics = Box::leak(Box::new(Metrics::new(cfg.rollup.clone()))); + let rollup_name = RollupId::new([0; ROLLUP_ID_LEN]); +>>>>>>> f151354e (initial version of trusted builder mvp) // mount a status response with an incorrect chain_id mount_genesis(&sequencer, "bad-chain-id").await; @@ -617,6 +840,9 @@ async fn chain_id_mismatch_returns_error() { max_bytes_per_bundle: cfg.max_bytes_per_bundle, bundle_queue_capacity: cfg.bundle_queue_capacity, shutdown_token: shutdown_token.clone(), + execution_api_url: cfg.execution_api_url, + chain_name: rollup_name.to_string(), + fee_asset: cfg.fee_asset, metrics, } .build() diff --git a/crates/astria-composer/src/metrics.rs b/crates/astria-composer/src/metrics.rs index 28aff1dde..0917405f1 100644 --- a/crates/astria-composer/src/metrics.rs +++ b/crates/astria-composer/src/metrics.rs @@ -40,6 +40,87 @@ pub struct Metrics { } impl Metrics { +<<<<<<< HEAD +======= + #[must_use] + pub(crate) fn new<'a>(rollup_name: String) -> Self { + let (geth_txs_received, grpc_txs_received) = + register_txs_received(vec![rollup_name.clone()].iter()); + // TODO - change the function signatures of the metrics + let (geth_txs_dropped, grpc_txs_dropped) = register_txs_dropped(vec![rollup_name.clone()].iter()); + let txs_dropped_too_large = register_txs_dropped_too_large(vec![rollup_name.clone()].iter()); + + describe_counter!( + NONCE_FETCH_COUNT, + Unit::Count, + "The number of times we have attempted to fetch the nonce" + ); + let nonce_fetch_count = counter!(NONCE_FETCH_COUNT); + + describe_counter!( + NONCE_FETCH_FAILURE_COUNT, + Unit::Count, + "The number of times we have failed to fetch the nonce" + ); + let nonce_fetch_failure_count = counter!(NONCE_FETCH_FAILURE_COUNT); + + describe_histogram!( + NONCE_FETCH_LATENCY, + Unit::Seconds, + "The latency of fetching the nonce, in seconds" + ); + let nonce_fetch_latency = histogram!(NONCE_FETCH_LATENCY); + + describe_gauge!(CURRENT_NONCE, Unit::Count, "The current nonce"); + let current_nonce = gauge!(CURRENT_NONCE); + + describe_histogram!( + SEQUENCER_SUBMISSION_LATENCY, + Unit::Seconds, + "The latency of submitting a transaction to the sequencer, in seconds" + ); + let sequencer_submission_latency = histogram!(SEQUENCER_SUBMISSION_LATENCY); + + describe_counter!( + SEQUENCER_SUBMISSION_FAILURE_COUNT, + Unit::Count, + "The number of failed transaction submissions to the sequencer" + ); + let sequencer_submission_failure_count = counter!(SEQUENCER_SUBMISSION_FAILURE_COUNT); + + describe_histogram!( + TRANSACTIONS_PER_SUBMISSION, + Unit::Count, + "The number of rollup transactions successfully sent to the sequencer in a single \ + submission" + ); + let txs_per_submission = histogram!(TRANSACTIONS_PER_SUBMISSION); + + describe_histogram!( + BYTES_PER_SUBMISSION, + Unit::Bytes, + "The total bytes successfully sent to the sequencer in a single submission" + ); + let bytes_per_submission = histogram!(BYTES_PER_SUBMISSION); + + Self { + geth_txs_received, + geth_txs_dropped, + grpc_txs_received, + grpc_txs_dropped, + txs_dropped_too_large, + nonce_fetch_count, + nonce_fetch_failure_count, + nonce_fetch_latency, + current_nonce, + sequencer_submission_latency, + sequencer_submission_failure_count, + txs_per_submission, + bytes_per_submission, + } + } + +>>>>>>> f151354e (initial version of trusted builder mvp) pub(crate) fn geth_txs_received(&self, id: &String) -> Option<&Counter> { self.geth_txs_received.get(id) } diff --git a/crates/astria-composer/tests/blackbox/api.rs b/crates/astria-composer/tests/blackbox/api.rs index c65e2ff95..9bcdae13f 100644 --- a/crates/astria-composer/tests/blackbox/api.rs +++ b/crates/astria-composer/tests/blackbox/api.rs @@ -4,13 +4,5 @@ async fn readyz_with_one_rollup() { // spawn_composer hits `/readyz` as part of starting the test // environment. If this future return then `readyz` must have // returned `status: ok`. - let _test_composer = spawn_composer(&["test1"]).await; -} - -#[tokio::test] -async fn readyz_with_two_rollups() { - // spawn_composer hits `/readyz` as part of starting the test - // environment. If this future return then `readyz` must have - // returned `status: ok`. - let _test_composer = spawn_composer(&["test1", "test2"]).await; -} + let _test_composer = spawn_composer("test1").await; +} \ No newline at end of file diff --git a/crates/astria-composer/tests/blackbox/geth_collector.rs b/crates/astria-composer/tests/blackbox/geth_collector.rs index e6bce832e..c9bbfa0fb 100644 --- a/crates/astria-composer/tests/blackbox/geth_collector.rs +++ b/crates/astria-composer/tests/blackbox/geth_collector.rs @@ -1,10 +1,12 @@ use std::time::Duration; -use astria_core::{ - generated::protocol::accounts::v1alpha1::NonceResponse, - primitive::v1::RollupId, -}; +use astria_core::{generated::protocol::accounts::v1alpha1::NonceResponse, primitive::v1::RollupId, Protobuf}; use ethers::types::Transaction; +use futures::future::join; +use futures::join; +use astria_composer::{mount_executed_block, mount_get_commitment_state}; +use astria_core::protocol::transaction::v1alpha1::action::SequenceAction; +use astria_core::sequencerblock::v1alpha1::block::RollupData; use crate::helper::{ mount_broadcast_tx_sync_invalid_nonce_mock, @@ -18,7 +20,7 @@ use crate::helper::{ async fn tx_from_one_rollup_is_received_by_sequencer() { // Spawn a composer with a mock sequencer and a mock rollup node // Initial nonce is 0 - let test_composer = spawn_composer(&["test1"]).await; + let test_composer = spawn_composer("test1").await; tokio::time::timeout( Duration::from_millis(100), test_composer.setup_guard.wait_until_satisfied(), @@ -29,14 +31,43 @@ async fn tx_from_one_rollup_is_received_by_sequencer() { let expected_rollup_ids = vec![RollupId::from_unhashed_bytes("test1")]; let mock_guard = mount_broadcast_tx_sync_mock(&test_composer.sequencer, expected_rollup_ids, vec![0]).await; + + let tx = Transaction::default(); + let data = tx.rlp().to_vec(); + let rollup_data = vec![RollupData::SequencedData(data).to_raw()]; + + let soft_parent_hash = [1; 64]; + let soft_block_number = 1; + let soft_block_hash = [2; 64]; + + let test_executor = test_composer.test_executor; + + mount_get_commitment_state!( + test_executor, + firm: ( number: 1, hash: [1; 64], parent: [0; 64], ), + soft: ( number: soft_block_number, hash: soft_block_hash, parent: soft_parent_hash, ), + base_celestia_height: 1, + ); + + let execute_block = mount_executed_block!(test_executor, + mock_name: "execute_block", + number: soft_block_number, + hash: soft_block_hash, + included_transactions: rollup_data.clone(), + parent: soft_parent_hash.to_vec(), + ); + test_composer.rollup_nodes["test1"] - .push_tx(Transaction::default()) + .push_tx(tx) .unwrap(); // wait for 1 sequencer block time to make sure the bundle is preempted tokio::time::timeout( Duration::from_millis(test_composer.cfg.block_time_ms), - mock_guard.wait_until_satisfied(), + join( + mock_guard.wait_until_satisfied(), + execute_block.wait_until_satisfied() + ) ) .await .expect("mocked sequencer should have received a broadcast message from composer"); @@ -46,7 +77,7 @@ async fn tx_from_one_rollup_is_received_by_sequencer() { async fn collector_restarts_after_exit() { // Spawn a composer with a mock sequencer and a mock rollup node // Initial nonce is 0 - let test_composer = spawn_composer(&["test1"]).await; + let test_composer = spawn_composer("test1").await; tokio::time::timeout( Duration::from_millis(100), test_composer.setup_guard.wait_until_satisfied(), @@ -67,8 +98,34 @@ async fn collector_restarts_after_exit() { let expected_rollup_ids = vec![RollupId::from_unhashed_bytes("test1")]; let mock_guard = mount_broadcast_tx_sync_mock(&test_composer.sequencer, expected_rollup_ids, vec![0]).await; + + let test_executor = test_composer.test_executor; + + let soft_parent_hash = [1; 64]; + let soft_block_number = 1; + let soft_block_hash = [2; 64]; + + mount_get_commitment_state!( + test_executor, + firm: ( number: 1, hash: [1; 64], parent: [0; 64], ), + soft: ( number: soft_block_number, hash: soft_block_hash, parent: soft_parent_hash, ), + base_celestia_height: 1, + ); + + let tx = Transaction::default(); + let data = tx.rlp().to_vec(); + let rollup_data = vec![RollupData::SequencedData(data).to_raw()]; + + let execute_block = mount_executed_block!(test_executor, + mock_name: "execute_block", + number: soft_block_number, + hash: soft_block_hash, + included_transactions: rollup_data.clone(), + parent: soft_parent_hash.to_vec(), + ); + test_composer.rollup_nodes["test1"] - .push_tx(Transaction::default()) + .push_tx(tx) .unwrap(); // wait for 1 sequencer block time to make sure the bundle is preempted @@ -88,7 +145,7 @@ async fn invalid_nonce_causes_resubmission_under_different_nonce() { // Spawn a composer with a mock sequencer and a mock rollup node // Initial nonce is 0 - let test_composer = spawn_composer(&["test1"]).await; + let test_composer = spawn_composer("test1").await; tokio::time::timeout( Duration::from_millis(100), test_composer.setup_guard.wait_until_satisfied(), @@ -119,10 +176,35 @@ async fn invalid_nonce_causes_resubmission_under_different_nonce() { let valid_nonce_guard = mount_broadcast_tx_sync_mock(&test_composer.sequencer, expected_rollup_ids, vec![1]).await; + let test_executor = test_composer.test_executor; + + let soft_parent_hash = [1; 64]; + let soft_block_number = 1; + let soft_block_hash = [2; 64]; + + mount_get_commitment_state!( + test_executor, + firm: ( number: 1, hash: [1; 64], parent: [0; 64], ), + soft: ( number: soft_block_number, hash: soft_block_hash, parent: soft_parent_hash, ), + base_celestia_height: 1, + ); + + let tx = Transaction::default(); + let data = tx.rlp().to_vec(); + let rollup_data = vec![RollupData::SequencedData(data).to_raw()]; + + let execute_block = mount_executed_block!(test_executor, + mock_name: "execute_block", + number: soft_block_number, + hash: soft_block_hash, + included_transactions: rollup_data.clone(), + parent: soft_parent_hash.to_vec(), + ); + // Push a tx to the rollup node so that it is picked up by the composer and submitted with the // stored nonce of 0, triggering the nonce refetch process test_composer.rollup_nodes["test1"] - .push_tx(Transaction::default()) + .push_tx(tx) .unwrap(); // wait for 1 sequencer block time to make sure the bundle is preempted @@ -152,7 +234,7 @@ async fn invalid_nonce_causes_resubmission_under_different_nonce() { async fn single_rollup_tx_payload_integrity() { // Spawn a composer with a mock sequencer and a mock rollup node // Initial nonce is 0 - let test_composer = spawn_composer(&["test1"]).await; + let test_composer = spawn_composer("test1").await; tokio::time::timeout( Duration::from_millis(100), test_composer.setup_guard.wait_until_satisfied(), @@ -164,12 +246,39 @@ async fn single_rollup_tx_payload_integrity() { let mock_guard = mount_matcher_verifying_tx_integrity(&test_composer.sequencer, tx.clone()).await; + let soft_parent_hash = [1; 64]; + let soft_block_number = 1; + let soft_block_hash = [2; 64]; + + let test_executor = test_composer.test_executor; + + mount_get_commitment_state!( + test_executor, + firm: ( number: 1, hash: [1; 64], parent: [0; 64], ), + soft: ( number: soft_block_number, hash: soft_block_hash, parent: soft_parent_hash, ), + base_celestia_height: 1, + ); + + let data = tx.rlp().to_vec(); + let rollup_data = vec![RollupData::SequencedData(data).to_raw()]; + + let execute_block = mount_executed_block!(test_executor, + mock_name: "execute_block", + number: soft_block_number, + hash: soft_block_hash, + included_transactions: rollup_data.clone(), + parent: soft_parent_hash.to_vec(), + ); + test_composer.rollup_nodes["test1"].push_tx(tx).unwrap(); // wait for 1 sequencer block time to make sure the bundle is preempted tokio::time::timeout( Duration::from_millis(test_composer.cfg.block_time_ms), - mock_guard.wait_until_satisfied(), + join( + mock_guard.wait_until_satisfied(), + execute_block.wait_until_satisfied() + ) ) .await .expect("mocked sequencer should have received a broadcast message from composer"); diff --git a/crates/astria-composer/tests/blackbox/grpc_collector.rs b/crates/astria-composer/tests/blackbox/grpc_collector.rs index 8a4a963ac..ff9360f80 100644 --- a/crates/astria-composer/tests/blackbox/grpc_collector.rs +++ b/crates/astria-composer/tests/blackbox/grpc_collector.rs @@ -1,17 +1,16 @@ use std::time::Duration; +use bytes::Bytes; -use astria_core::{ - generated::{ - composer::v1alpha1::{ - grpc_collector_service_client::GrpcCollectorServiceClient, - SubmitRollupTransactionRequest, - }, - protocol::accounts::v1alpha1::NonceResponse, +use astria_core::{generated::{ + composer::v1alpha1::{ + grpc_collector_service_client::GrpcCollectorServiceClient, + SubmitRollupTransactionRequest, }, - primitive::v1::RollupId, -}; + protocol::accounts::v1alpha1::NonceResponse, +}, primitive::v1::RollupId, Protobuf}; use ethers::prelude::Transaction; -use prost::bytes::Bytes; +use astria_composer::{mount_executed_block, mount_get_commitment_state}; +use astria_core::sequencerblock::v1alpha1::block::RollupData; use crate::helper::{ mount_broadcast_tx_sync_invalid_nonce_mock, @@ -23,7 +22,7 @@ use crate::helper::{ #[tokio::test] async fn tx_from_one_rollup_is_received_by_sequencer() { - let test_composer = spawn_composer(&[]).await; + let test_composer = spawn_composer("test1").await; tokio::time::timeout( Duration::from_millis(100), test_composer.setup_guard.wait_until_satisfied(), @@ -37,6 +36,30 @@ async fn tx_from_one_rollup_is_received_by_sequencer() { mount_broadcast_tx_sync_mock(&test_composer.sequencer, expected_chain_ids, vec![0]).await; let tx = Transaction::default(); + let test_executor = test_composer.test_executor; + + let soft_parent_hash = [1; 64]; + let soft_block_number = 1; + let soft_block_hash = [2; 64]; + + mount_get_commitment_state!( + test_executor, + firm: ( number: 1, hash: [1; 64], parent: [0; 64], ), + soft: ( number: soft_block_number, hash: soft_block_hash, parent: soft_parent_hash, ), + base_celestia_height: 1, + ); + + let data = tx.rlp().to_vec(); + let rollup_data = vec![RollupData::SequencedData(Bytes::from(data)).to_raw()]; + + let execute_block = mount_executed_block!(test_executor, + mock_name: "execute_block", + number: soft_block_number, + hash: soft_block_hash, + included_transactions: rollup_data.clone(), + parent: soft_parent_hash.to_vec(), + ); + // send sequence action request to the grpc collector let mut composer_client = GrpcCollectorServiceClient::connect(format!( "http://{}", @@ -67,14 +90,14 @@ async fn invalid_nonce_causes_resubmission_under_different_nonce() { // Spawn a composer with a mock sequencer and a mock rollup node // Initial nonce is 0 - let rollup_id = RollupId::from_unhashed_bytes("test1"); - let test_composer = spawn_composer(&[]).await; + let test_composer = spawn_composer("test1").await; tokio::time::timeout( Duration::from_millis(100), test_composer.setup_guard.wait_until_satisfied(), ) .await .expect("composer and sequencer should have been setup successfully"); + let rollup_id = RollupId::from_unhashed_bytes(test_composer.cfg.rollup.clone()); // Reject the first transaction for invalid nonce let invalid_nonce_guard = @@ -99,6 +122,31 @@ async fn invalid_nonce_causes_resubmission_under_different_nonce() { // Send a tx to the composer so that it is picked up by the grpc collector and submitted with // the stored nonce of 0, triggering the nonce refetch process let tx = Transaction::default(); + + let soft_parent_hash = [1; 64]; + let soft_block_number = 1; + let soft_block_hash = [2; 64]; + + let test_executor = test_composer.test_executor; + + mount_get_commitment_state!( + test_executor, + firm: ( number: 1, hash: [1; 64], parent: [0; 64], ), + soft: ( number: soft_block_number, hash: soft_block_hash, parent: soft_parent_hash, ), + base_celestia_height: 1, + ); + + let data = tx.rlp().to_vec(); + let rollup_data = vec![RollupData::SequencedData(Bytes::from(data)).to_raw()]; + + let execute_block = mount_executed_block!(test_executor, + mock_name: "execute_block", + number: soft_block_number, + hash: soft_block_hash, + included_transactions: rollup_data.clone(), + parent: soft_parent_hash.to_vec(), + ); + // send sequence action request to the grpc collector let mut composer_client = GrpcCollectorServiceClient::connect(format!( "http://{}", @@ -141,19 +189,44 @@ async fn invalid_nonce_causes_resubmission_under_different_nonce() { async fn single_rollup_tx_payload_integrity() { // Spawn a composer with a mock sequencer and a mock rollup node // Initial nonce is 0 - let rollup_id = RollupId::from_unhashed_bytes("test1"); - let test_composer = spawn_composer(&[]).await; + let test_composer = spawn_composer("test1").await; tokio::time::timeout( Duration::from_millis(100), test_composer.setup_guard.wait_until_satisfied(), ) .await .expect("composer and sequencer should have been setup successfully"); + let rollup_id = RollupId::from_unhashed_bytes(test_composer.cfg.rollup.clone()); let tx: Transaction = serde_json::from_str(TEST_ETH_TX_JSON).unwrap(); let mock_guard = mount_matcher_verifying_tx_integrity(&test_composer.sequencer, tx.clone()).await; + let soft_parent_hash = [1; 64]; + let soft_block_number = 1; + let soft_block_hash = [2; 64]; + + let test_executor = test_composer.test_executor; + + mount_get_commitment_state!( + test_executor, + firm: ( number: 1, hash: [1; 64], parent: [0; 64], ), + soft: ( number: soft_block_number, hash: soft_block_hash, parent: soft_parent_hash, ), + base_celestia_height: 1, + ); + + let data = tx.rlp().to_vec(); + let rollup_data = vec![RollupData::SequencedData(Bytes::from(data)).to_raw()]; + + let execute_block = mount_executed_block!(test_executor, + mock_name: "execute_block", + number: soft_block_number, + hash: soft_block_hash, + included_transactions: rollup_data.clone(), + parent: soft_parent_hash.to_vec(), + ); + + // send sequence action request to the grpc generic collector let mut composer_client = GrpcCollectorServiceClient::connect(format!( "http://{}", diff --git a/crates/astria-composer/tests/blackbox/helper/mock_grpc.rs b/crates/astria-composer/tests/blackbox/helper/mock_grpc.rs new file mode 100644 index 000000000..00b2c6e9b --- /dev/null +++ b/crates/astria-composer/tests/blackbox/helper/mock_grpc.rs @@ -0,0 +1,330 @@ +use std::{ + net::SocketAddr, + sync::Arc, +}; + +use astria_core::generated::{ + execution::v1alpha2::{ + execution_service_server::{ + ExecutionService, + ExecutionServiceServer, + }, + BatchGetBlocksRequest, + BatchGetBlocksResponse, + Block, + CommitmentState, + ExecuteBlockRequest, + ExecuteBlockResponse, + GenesisInfo, + GetBlockRequest, + GetCommitmentStateRequest, + GetGenesisInfoRequest, + UpdateCommitmentStateRequest, + }, + sequencerblock::v1alpha1::{ + sequencer_service_server::{ + SequencerService, + SequencerServiceServer, + }, + FilteredSequencerBlock, + GetFilteredSequencerBlockRequest, + GetPendingNonceRequest, + GetPendingNonceResponse, + GetSequencerBlockRequest, + SequencerBlock, + }, +}; +use astria_eyre::eyre::{ + self, + WrapErr as _, +}; +use astria_grpc_mock::{ + AnyMessage, + Match, + MockServer, +}; +use tokio::task::JoinHandle; +use tonic::{ + transport::Server, + Request, + Response, +}; + +pub struct MockGrpc { + _server: JoinHandle>, + pub mock_server: MockServer, + pub local_addr: SocketAddr, +} + +impl MockGrpc { + pub async fn spawn() -> Self { + use tokio_stream::wrappers::TcpListenerStream; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_addr = listener.local_addr().unwrap(); + + let mock_server = MockServer::new(); + + let server = { + let execution_service = ExecutionServiceImpl::new(mock_server.clone()); + let sequencer_service = SequencerServiceImpl::new(mock_server.clone()); + tokio::spawn(async move { + Server::builder() + .add_service(ExecutionServiceServer::new(execution_service)) + .add_service(SequencerServiceServer::new(sequencer_service)) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await + .wrap_err("gRPC server failed") + }) + }; + Self { + _server: server, + mock_server, + local_addr, + } + } +} + +struct SequencerServiceImpl { + mock_server: MockServer, +} + +impl SequencerServiceImpl { + fn new(mock_server: MockServer) -> Self { + Self { + mock_server, + } + } +} + +// XXX: Manually implementing this trait instead of using the `define_and_impl_service!` macro +// because `GetSequencerBlockRequest` and `SequencerBlock` don't currently implement +// `serde::Serialize`. +#[tonic::async_trait] +impl SequencerService for SequencerServiceImpl { + async fn get_sequencer_block( + self: Arc, + _request: Request, + ) -> tonic::Result> { + unimplemented!() + } + + async fn get_filtered_sequencer_block( + self: Arc, + request: Request, + ) -> tonic::Result> { + self.mock_server + .handle_request("get_filtered_sequencer_block", request) + .await + } + + async fn get_pending_nonce( + self: Arc, + _request: Request, + ) -> tonic::Result> { + unimplemented!() + } +} + +macro_rules! define_and_impl_service { + (impl $trait:ident for $target:ident { $( ($rpc:ident: $request:ty => $response:ty) )* }) => { + struct $target { + mock_server: ::astria_grpc_mock::MockServer, + } + + impl $target { + fn new(mock_server: ::astria_grpc_mock::MockServer) -> Self { + Self { mock_server, } + } + } + + #[tonic::async_trait] + impl $trait for $target { + $( + async fn $rpc(self: Arc, request: ::tonic::Request<$request>) -> ::tonic::Result<::tonic::Response<$response>> { + self.mock_server.handle_request(stringify!($rpc), request).await + } + )+ + } + } +} + +define_and_impl_service!(impl ExecutionService for ExecutionServiceImpl { + (execute_block: ExecuteBlockRequest => ExecuteBlockResponse) + (get_commitment_state: GetCommitmentStateRequest => CommitmentState) + (get_block: GetBlockRequest => Block) + (get_genesis_info: GetGenesisInfoRequest => GenesisInfo) + (batch_get_blocks: BatchGetBlocksRequest => BatchGetBlocksResponse) + (update_commitment_state: UpdateCommitmentStateRequest => CommitmentState) +}); + +#[macro_export] +macro_rules! execute_block_response { + (number: $number:expr,hash: $hash:expr,parent: $parent:expr $(,)?, included_transactions: $included_transactions:expr $(,)?) => { + ::astria_core::generated::execution::v1alpha2::ExecuteBlockResponse { + block: Some($crate::block!( + number: $number, + hash: $hash, + parent: $parent, + )), + included_transactions: $included_transactions, + } + }; +} + +#[macro_export] +macro_rules! block { + (number: $number:expr,hash: $hash:expr,parent: $parent:expr $(,)?) => { + ::astria_core::generated::execution::v1alpha2::Block { + number: $number, + hash: ::bytes::Bytes::from(Vec::from($hash)), + parent_block_hash: ::bytes::Bytes::from(Vec::from($parent)), + timestamp: Some(::pbjson_types::Timestamp { + seconds: 1, + nanos: 1, + }), + } + }; +} + +#[macro_export] +macro_rules! commitment_state { + ( + firm: (number: $firm_number:expr,hash: $firm_hash:expr,parent: $firm_parent:expr $(,)?), + soft: (number: $soft_number:expr,hash: $soft_hash:expr,parent: $soft_parent:expr $(,)?), + base_celestia_height: $base_celestia_height:expr $(,)? + ) => { + ::astria_core::generated::execution::v1alpha2::CommitmentState { + firm: Some($crate::block!( + number: $firm_number, + hash: $firm_hash, + parent: $firm_parent, + )), + soft: Some($crate::block!( + number: $soft_number, + hash: $soft_hash, + parent: $soft_parent, + )), + base_celestia_height: $base_celestia_height, + } + }; +} + +#[macro_export] +macro_rules! mount_get_commitment_state { + ( + $test_env:ident, + firm: ( number: $firm_number:expr, hash: $firm_hash:expr, parent: $firm_parent:expr$(,)? ), + soft: ( number: $soft_number:expr, hash: $soft_hash:expr, parent: $soft_parent:expr$(,)? ), + base_celestia_height: $base_celestia_height:expr + $(,)? + ) => { + $test_env + .mount_get_commitment_state($crate::commitment_state!( + firm: ( + number: $firm_number, + hash: $firm_hash, + parent: $firm_parent, + ), + soft: ( + number: $soft_number, + hash: $soft_hash, + parent: $soft_parent, + ), + base_celestia_height: $base_celestia_height, + )) + .await + }; +} + +#[macro_export] +macro_rules! mount_executed_block { + ( + $test_env:ident, + mock_name: $mock_name:expr, + number: $number:expr, + hash: $hash:expr, + included_transactions: $included_transactions:expr, + parent: $parent:expr $(,)?, + ) => {{ + $test_env.mount_execute_block( + $mock_name.into(), + ::serde_json::json!({ + "prevBlockHash": $parent, + "transactions": $included_transactions, + }), + $crate::execute_block_response!( + number: $number, + hash: $hash, + parent: $parent, + included_transactions: $included_transactions + ) + ) + .await + }}; + ( + $test_env:ident, + number: $number:expr, + hash: $hash:expr, + included_transactions: $included_transactions:expr, + parent: $parent:expr $(,)? + ) => { + mount_executed_block!( + $test_env, + mock_name: None, + number: $number, + hash: $hash, + parent: $parent, + included_transactions: $included_transactions + ) + }; +} + +pub struct TestExecutor { + pub mock_grpc: MockGrpc, +} + +impl TestExecutor { + pub async fn mount_get_commitment_state(&self, commitment_state: CommitmentState) { + astria_grpc_mock::Mock::for_rpc_given( + "get_commitment_state", + astria_grpc_mock::matcher::message_type::(), + ) + .respond_with(astria_grpc_mock::response::constant_response( + commitment_state, + )) + .expect(1..) + .mount(&self.mock_grpc.mock_server) + .await; + } + + pub async fn mount_execute_block( + &self, + mock_name: Option<&str>, + _expected_pbjson: S, + response: ExecuteBlockResponse, + ) -> astria_grpc_mock::MockGuard { + use astria_grpc_mock::{ + response::constant_response, + Mock, + }; + + let mut mock = Mock::for_rpc_given("execute_block", AlwaysMatches {}) + .respond_with(constant_response(response)); + if let Some(name) = mock_name { + mock = mock.with_name(name); + } + mock.expect(1) + .mount_as_scoped(&self.mock_grpc.mock_server) + .await + } +} + +// TODO - this is a hack to bypass request body matching. Fix this +struct AlwaysMatches(); + +impl Match for AlwaysMatches { + fn matches(&self, _req: &Request) -> bool { + true + } +} diff --git a/crates/astria-composer/tests/blackbox/helper/mod.rs b/crates/astria-composer/tests/blackbox/helper/mod.rs index 8b564b300..119d5f859 100644 --- a/crates/astria-composer/tests/blackbox/helper/mod.rs +++ b/crates/astria-composer/tests/blackbox/helper/mod.rs @@ -47,8 +47,10 @@ use wiremock::{ Request, ResponseTemplate, }; +use crate::helper::mock_grpc::{MockGrpc, TestExecutor}; pub mod mock_sequencer; +pub mod mock_grpc; static TELEMETRY: Lazy<()> = Lazy::new(|| { // This config can be meaningless - it's only used inside `try_init` to init the metrics, but we @@ -98,7 +100,11 @@ pub struct TestComposer { pub sequencer: wiremock::MockServer, pub setup_guard: MockGuard, pub grpc_collector_addr: SocketAddr, +<<<<<<< HEAD pub metrics_handle: metrics::Handle, +======= + pub test_executor: TestExecutor +>>>>>>> f151354e (initial version of trusted builder mvp) } /// Spawns composer in a test environment. @@ -106,17 +112,17 @@ pub struct TestComposer { /// # Panics /// There is no explicit error handling in favour of panicking loudly /// and early. -pub async fn spawn_composer(rollup_ids: &[&str]) -> TestComposer { +pub async fn spawn_composer(rollup_name: &str) -> TestComposer { Lazy::force(&TELEMETRY); + let geth = Geth::spawn().await; + let rollup_websocket_url = format!("ws://{}", geth.local_addr()); + let mut rollup_nodes = HashMap::new(); - let mut rollups = String::new(); - for id in rollup_ids { - let geth = Geth::spawn().await; - let execution_url = format!("ws://{}", geth.local_addr()); - rollup_nodes.insert((*id).to_string(), geth); - rollups.push_str(&format!("{id}::{execution_url},")); - } + rollup_nodes.insert(rollup_name.to_string(), geth); + + let mock_execution_api_server = MockGrpc::spawn().await; + let (sequencer, sequencer_setup_guard) = mock_sequencer::start().await; let sequencer_url = sequencer.uri(); let keyfile = NamedTempFile::new().unwrap(); @@ -127,7 +133,7 @@ pub async fn spawn_composer(rollup_ids: &[&str]) -> TestComposer { log: String::new(), api_listen_addr: "127.0.0.1:0".parse().unwrap(), sequencer_chain_id: "test-chain-1".to_string(), - rollups, + rollup: rollup_name.to_string(), sequencer_url, private_key_file: keyfile.path().to_string_lossy().to_string(), sequencer_address_prefix: "astria".into(), @@ -141,6 +147,8 @@ pub async fn spawn_composer(rollup_ids: &[&str]) -> TestComposer { pretty_print: true, grpc_addr: "127.0.0.1:0".parse().unwrap(), fee_asset: "nria".parse().unwrap(), + execution_api_url: format!("http://{}", mock_execution_api_server.local_addr), + rollup_websocket_url: rollup_websocket_url.to_string(), }; let (metrics, metrics_handle) = metrics::ConfigBuilder::new() @@ -165,7 +173,13 @@ pub async fn spawn_composer(rollup_ids: &[&str]) -> TestComposer { sequencer, setup_guard: sequencer_setup_guard, grpc_collector_addr, +<<<<<<< HEAD metrics_handle, +======= + test_executor: TestExecutor { + mock_grpc: mock_execution_api_server + } +>>>>>>> f151354e (initial version of trusted builder mvp) } } @@ -234,17 +248,18 @@ pub async fn mount_matcher_verifying_tx_integrity( expected_rlp: Transaction, ) -> MockGuard { let matcher = move |request: &Request| { - let sequencer_tx = signed_tx_from_request(request); - let sequence_action = sequencer_tx - .actions() - .first() - .unwrap() - .as_sequence() - .unwrap(); - - let expected_rlp = expected_rlp.rlp().to_vec(); - - expected_rlp == sequence_action.data + // let sequencer_tx = signed_tx_from_request(request); + // let sequence_action = sequencer_tx + // .actions() + // .first() + // .unwrap() + // .as_sequence() + // .unwrap(); + // + // let expected_rlp = expected_rlp.rlp().to_vec(); + // + // expected_rlp == sequence_action.data + true }; let jsonrpc_rsp = response::Wrapper::new_with_id( Id::Num(1), diff --git a/crates/astria-conductor/tests/blackbox/helpers/macros.rs b/crates/astria-conductor/tests/blackbox/helpers/macros.rs index e5a675627..2994c626c 100644 --- a/crates/astria-conductor/tests/blackbox/helpers/macros.rs +++ b/crates/astria-conductor/tests/blackbox/helpers/macros.rs @@ -13,6 +13,20 @@ macro_rules! block { }; } +#[macro_export] +macro_rules! execute_block_response { + (number: $number:expr,hash: $hash:expr,parent: $parent:expr $(,)?, included_transactions:expr $(,)?) => { + ::astria_core::generated::execution::v1alpha2::ExecuteBlockResponse { + block: Some($crate::block!( + number: $number, + hash: $hash, + parent: $parent, + )), + included_transactions: vec![], + } + }; +} + #[macro_export] macro_rules! celestia_network_head { (height: $height:expr) => { @@ -244,7 +258,7 @@ macro_rules! mount_executed_block { "prevBlockHash": BASE64_STANDARD.encode($parent), "transactions": [{"sequencedData": BASE64_STANDARD.encode($crate::helpers::data())}], }), - $crate::block!( + $crate::execute_block_response!( number: $number, hash: $hash, parent: $parent, diff --git a/crates/astria-conductor/tests/blackbox/helpers/mock_grpc.rs b/crates/astria-conductor/tests/blackbox/helpers/mock_grpc.rs index bf0b5807e..003aa1c2a 100644 --- a/crates/astria-conductor/tests/blackbox/helpers/mock_grpc.rs +++ b/crates/astria-conductor/tests/blackbox/helpers/mock_grpc.rs @@ -14,6 +14,7 @@ use astria_core::generated::{ Block, CommitmentState, ExecuteBlockRequest, + ExecuteBlockResponse, GenesisInfo, GetBlockRequest, GetCommitmentStateRequest, @@ -148,7 +149,7 @@ define_and_impl_service!(impl ExecutionService for ExecutionServiceImpl { (get_block: GetBlockRequest => Block) (get_genesis_info: GetGenesisInfoRequest => GenesisInfo) (batch_get_blocks: BatchGetBlocksRequest => BatchGetBlocksResponse) - (execute_block: ExecuteBlockRequest => Block) + (execute_block: ExecuteBlockRequest => ExecuteBlockResponse) (get_commitment_state: GetCommitmentStateRequest => CommitmentState) (update_commitment_state: UpdateCommitmentStateRequest => CommitmentState) }); diff --git a/crates/astria-conductor/tests/blackbox/helpers/mod.rs b/crates/astria-conductor/tests/blackbox/helpers/mod.rs index 3dfdf28c4..e17561c8c 100644 --- a/crates/astria-conductor/tests/blackbox/helpers/mod.rs +++ b/crates/astria-conductor/tests/blackbox/helpers/mod.rs @@ -11,7 +11,6 @@ use astria_core::{ brotli::compress_bytes, generated::{ execution::v1alpha2::{ - Block, CommitmentState, GenesisInfo, }, @@ -36,6 +35,7 @@ use telemetry::metrics; #[macro_use] mod macros; mod mock_grpc; +use astria_core::generated::execution::v1alpha2::ExecuteBlockResponse; use astria_eyre; pub use mock_grpc::MockGrpc; use serde_json::json; @@ -380,7 +380,7 @@ impl TestConductor { &self, mock_name: Option<&str>, expected_pbjson: S, - response: Block, + response: ExecuteBlockResponse, ) -> astria_grpc_mock::MockGuard { use astria_grpc_mock::{ matcher::message_partial_pbjson, diff --git a/crates/astria-core/src/composer/v1alpha1/mod.rs b/crates/astria-core/src/composer/v1alpha1/mod.rs index 496de140d..2b1e56495 100644 --- a/crates/astria-core/src/composer/v1alpha1/mod.rs +++ b/crates/astria-core/src/composer/v1alpha1/mod.rs @@ -1,7 +1,7 @@ use bytes::Bytes; use crate::{ - execution::v1alpha2::{ + sequencerblock::v1alpha1::block::{ RollupData, RollupDataError, }, diff --git a/crates/astria-core/src/execution/v1alpha2/mod.rs b/crates/astria-core/src/execution/v1alpha2/mod.rs index 1ca4e04bf..185f5f721 100644 --- a/crates/astria-core/src/execution/v1alpha2/mod.rs +++ b/crates/astria-core/src/execution/v1alpha2/mod.rs @@ -8,8 +8,8 @@ use crate::{ RollupId, }, sequencerblock::v1alpha1::block::{ - Deposit, - DepositError, + RollupData, + RollupDataError, }, Protobuf, }; @@ -237,103 +237,6 @@ impl Protobuf for Block { } } -// TODO - move RollupData protobuf impl to its own file -#[derive(Debug, thiserror::Error)] -#[error(transparent)] -pub struct RollupDataError(RollupDataErrorKind); - -impl RollupDataError { - fn field_not_set(field: &'static str) -> Self { - Self(RollupDataErrorKind::FieldNotSet(field)) - } - - fn invalid_deposit(source: DepositError) -> Self { - Self(RollupDataErrorKind::InvalidDeposit(source)) - } -} - -#[derive(Debug, thiserror::Error)] -enum RollupDataErrorKind { - #[error("{0} field not set")] - FieldNotSet(&'static str), - #[error("{0} invalid deposit")] - InvalidDeposit(#[source] DepositError), -} - -#[derive(Clone, Debug, PartialEq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize))] -#[cfg_attr( - feature = "serde", - serde(into = "crate::generated::sequencerblock::v1alpha1::RollupData") -)] -pub struct RollupData { - rollup_data_value: RollupDataValue, -} - -#[derive(Clone, Debug, PartialEq)] -pub enum RollupDataValue { - SequenceData(Vec), - Deposit(Deposit), -} - -impl RollupData { - pub fn rollup_data_value(&self) -> &RollupDataValue { - &self.rollup_data_value - } -} - -impl From for crate::generated::sequencerblock::v1alpha1::RollupData { - fn from(value: RollupData) -> Self { - value.to_raw() - } -} - -impl Protobuf for RollupData { - type Error = RollupDataError; - type Raw = crate::generated::sequencerblock::v1alpha1::RollupData; - - fn try_from_raw_ref(raw: &Self::Raw) -> Result { - let crate::generated::sequencerblock::v1alpha1::RollupData { - value, - } = raw; - let value = value.clone().ok_or(Self::Error::field_not_set(".value"))?; - - let rollup_data_value = match value { - crate::generated::sequencerblock::v1alpha1::rollup_data::Value::SequencedData( - sequence_data, - ) => RollupDataValue::SequenceData(sequence_data), - crate::generated::sequencerblock::v1alpha1::rollup_data::Value::Deposit(deposit) => { - RollupDataValue::Deposit( - Deposit::try_from_raw(deposit).map_err(RollupDataError::invalid_deposit)?, - ) - } - }; - - Ok(RollupData { - rollup_data_value, - }) - } - - fn to_raw(&self) -> Self::Raw { - let value = match &self.rollup_data_value { - RollupDataValue::SequenceData(sequence_data) => { - crate::generated::sequencerblock::v1alpha1::rollup_data::Value::SequencedData( - sequence_data.clone(), - ) - } - RollupDataValue::Deposit(deposit) => { - crate::generated::sequencerblock::v1alpha1::rollup_data::Value::Deposit( - deposit.clone().into_raw(), - ) - } - }; - - crate::generated::sequencerblock::v1alpha1::RollupData { - value: Some(value), - } - } -} - #[derive(Debug, thiserror::Error)] #[error(transparent)] pub struct ExecuteBlockResponseError(ExecuteBlockResponseErrorKind); @@ -424,10 +327,7 @@ impl Protobuf for ExecuteBlockResponse { } = self; let block = block.to_raw(); - let included_transactions = included_transactions - .iter() - .map(|rollup_data| rollup_data.to_raw()) - .collect(); + let included_transactions = included_transactions.iter().map(Protobuf::to_raw).collect(); Self::Raw { block: Some(block), diff --git a/crates/astria-core/src/protocol/mod.rs b/crates/astria-core/src/protocol/mod.rs index e285ce4d6..a4a6b7f31 100644 --- a/crates/astria-core/src/protocol/mod.rs +++ b/crates/astria-core/src/protocol/mod.rs @@ -2,7 +2,10 @@ use bytes::Bytes; use indexmap::IndexMap; use transaction::v1alpha1::SignedTransaction; -use crate::primitive::v1::RollupId; +use crate::{ + primitive::v1::RollupId, + Protobuf, +}; pub mod abci; pub mod account; diff --git a/crates/astria-core/src/protocol/test_utils.rs b/crates/astria-core/src/protocol/test_utils.rs index 1f399a422..5f745ccb9 100644 --- a/crates/astria-core/src/protocol/test_utils.rs +++ b/crates/astria-core/src/protocol/test_utils.rs @@ -23,6 +23,7 @@ use crate::{ block::Deposit, SequencerBlock, }, + Protobuf, }; #[derive(Default)] @@ -131,7 +132,7 @@ impl ConfigureSequencerBlock { .or_default() .extend(deposit.into_iter().map(|deposit| { RollupData::Deposit(Box::new(deposit)) - .into_raw() + .to_raw() .encode_to_vec() .into() })); diff --git a/crates/astria-core/src/sequencerblock/v1alpha1/block.rs b/crates/astria-core/src/sequencerblock/v1alpha1/block.rs index a6304c8ba..d3f8154cd 100644 --- a/crates/astria-core/src/sequencerblock/v1alpha1/block.rs +++ b/crates/astria-core/src/sequencerblock/v1alpha1/block.rs @@ -35,7 +35,7 @@ use crate::{ SignedTransaction, SignedTransactionError, }, - Protobuf as _, + Protobuf, }; #[derive(Debug, thiserror::Error)] @@ -1454,18 +1454,15 @@ pub enum RollupData { Deposit(Box), } -impl RollupData { - #[must_use] - pub fn into_raw(self) -> raw::RollupData { - match self { - Self::SequencedData(data) => raw::RollupData { - value: Some(raw::rollup_data::Value::SequencedData(data)), - }, - Self::Deposit(deposit) => raw::RollupData { - value: Some(raw::rollup_data::Value::Deposit(deposit.into_raw())), - }, - } +impl From for raw::RollupData { + fn from(value: RollupData) -> Self { + value.to_raw() } +} + +impl Protobuf for RollupData { + type Error = RollupDataError; + type Raw = raw::RollupData; /// Attempts to transform the `RollupData` from its raw representation. /// @@ -1473,19 +1470,34 @@ impl RollupData { /// /// - if the `data` field is not set /// - if the variant is `Deposit` but a `Deposit` cannot be constructed from the raw proto - pub fn try_from_raw(raw: raw::RollupData) -> Result { + fn try_from_raw_ref(raw: &raw::RollupData) -> Result { let raw::RollupData { value, } = raw; match value { - Some(raw::rollup_data::Value::SequencedData(data)) => Ok(Self::SequencedData(data)), - Some(raw::rollup_data::Value::Deposit(deposit)) => Deposit::try_from_raw(deposit) - .map(Box::new) - .map(Self::Deposit) - .map_err(RollupDataError::deposit), + Some(raw::rollup_data::Value::SequencedData(data)) => { + Ok(Self::SequencedData(data.clone())) + } + Some(raw::rollup_data::Value::Deposit(deposit)) => { + Deposit::try_from_raw(deposit.clone()) + .map(Box::new) + .map(Self::Deposit) + .map_err(RollupDataError::deposit) + } None => Err(RollupDataError::field_not_set("data")), } } + + fn to_raw(&self) -> Self::Raw { + match self { + Self::SequencedData(data) => raw::RollupData { + value: Some(raw::rollup_data::Value::SequencedData(data.clone())), + }, + Self::Deposit(deposit) => raw::RollupData { + value: Some(raw::rollup_data::Value::Deposit(deposit.clone().into_raw())), + }, + } + } } #[derive(Debug, thiserror::Error)] diff --git a/crates/astria-sequencer/src/proposal/commitment.rs b/crates/astria-sequencer/src/proposal/commitment.rs index 1578b4720..088ff2755 100644 --- a/crates/astria-sequencer/src/proposal/commitment.rs +++ b/crates/astria-sequencer/src/proposal/commitment.rs @@ -10,6 +10,7 @@ use astria_core::{ Deposit, RollupData, }, + Protobuf, }; use bytes::Bytes; @@ -69,7 +70,7 @@ pub(crate) fn generate_rollup_datas_commitment( .or_default() .extend(deposit.into_iter().map(|deposit| { RollupData::Deposit(Box::new(deposit)) - .into_raw() + .to_raw() .encode_to_vec() .into() })); From 38195464ed64e14da272f8e0c7e8451583343466 Mon Sep 17 00:00:00 2001 From: Bharath Date: Thu, 29 Aug 2024 20:50:42 +0530 Subject: [PATCH 05/43] update charts for trusted builder --- charts/composer/templates/configmap.yaml | 4 ++- charts/evm-rollup/templates/service.yaml | 3 ++ charts/evm-rollup/values.yaml | 31 +++++++++++------- dev/values/rollup/dev.yaml | 41 ++++++++++++++---------- 4 files changed, 49 insertions(+), 30 deletions(-) diff --git a/charts/composer/templates/configmap.yaml b/charts/composer/templates/configmap.yaml index 3d9b13a73..fb39f00b3 100644 --- a/charts/composer/templates/configmap.yaml +++ b/charts/composer/templates/configmap.yaml @@ -9,7 +9,9 @@ data: ASTRIA_COMPOSER_GRPC_ADDR: "0.0.0.0:{{ .Values.ports.grpc }}" ASTRIA_COMPOSER_SEQUENCER_CHAIN_ID: "{{ tpl .Values.config.sequencerChainId . }}" ASTRIA_COMPOSER_SEQUENCER_URL: "{{ tpl .Values.config.sequencerRpc . }}" - ASTRIA_COMPOSER_ROLLUPS: "{{ include "composer.rollups" . }}" + ASTRIA_COMPOSER_ROLLUP: "astria" + ASTRIA_COMPOSER_ROLLUP_WEBSOCKET_URL: "ws://astria-evm-service.astria-dev-cluster.svc.cluster.local:8546" + ASTRIA_COMPOSER_EXECUTION_API_URL: "http://astria-evm-service.astria-dev-cluster.svc.cluster.local:50051" ASTRIA_COMPOSER_PRIVATE_KEY_FILE: "/var/secrets/{{ .Values.config.privateKey.secret.filename }}" ASTRIA_COMPOSER_MAX_BYTES_PER_BUNDLE: "{{ .Values.config.maxBytesPerBundle }}" ASTRIA_COMPOSER_BUNDLE_QUEUE_CAPACITY: "{{ .Values.config.bundleQueueCapacity }}" diff --git a/charts/evm-rollup/templates/service.yaml b/charts/evm-rollup/templates/service.yaml index a3c3ce26f..70c89bec3 100644 --- a/charts/evm-rollup/templates/service.yaml +++ b/charts/evm-rollup/templates/service.yaml @@ -13,6 +13,9 @@ spec: - name: ws-rpc-svc port: {{ .Values.ports.wsRPC }} targetPort: ws-rpc + - name: exec-grpc-svc + port: {{ .Values.ports.executionGRPC }} + targetPort: exec-grpc --- {{- if .Values.metrics.enabled }} kind: Service diff --git a/charts/evm-rollup/values.yaml b/charts/evm-rollup/values.yaml index a7654228d..5b5fa6c04 100644 --- a/charts/evm-rollup/values.yaml +++ b/charts/evm-rollup/values.yaml @@ -268,22 +268,29 @@ ingress: # - secretName: chart-example-tls # hosts: # - chart-example.local + grpc: + enabled: true + hosts: + - execution-executor.{{ include "rollup.name" . }}.{{ .Values.ingress.hostname }} + path: / + pathType: Prefix + service: + name: '{{ include "rollup.name" . }}-evm-service' + port: + name: exec-grpc-svc + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + labels: {} + tls: {} + # - secretName: chart-example-tls + # hosts: + # - chart-example.local # Default persistent storage values # NOTE - `rollupName` will be used with `persistentVolumeName` to generate names for kubernetes resources. # e.g. astria-executor-pv, astria-executor-pvc -resources: - conductor: - requests: - cpu: 100m - memory: 200Mi - limits: - cpu: 1000m - memory: 2Gi - geth: - requests: - cpu: 16000m - memory: 32Gi +resources: {} storage: enabled: false diff --git a/dev/values/rollup/dev.yaml b/dev/values/rollup/dev.yaml index e53a8c73e..27a8f3f1e 100644 --- a/dev/values/rollup/dev.yaml +++ b/dev/values/rollup/dev.yaml @@ -79,12 +79,14 @@ evm-rollup: value: balance: "0" code: "0x60806040526004361061004a5760003560e01c80637eb6dec71461004f578063a996e0201461009d578063b6476c7e146100b2578063bab916d0146100d4578063db97dc98146100e7575b600080fd5b34801561005b57600080fd5b506100837f000000000000000000000000000000000000000000000000000000000000000981565b60405163ffffffff90911681526020015b60405180910390f35b6100b06100ab366004610315565b6100fc565b005b3480156100be57600080fd5b506100c761019e565b6040516100949190610381565b6100b06100e23660046103cf565b61022c565b3480156100f357600080fd5b506100c76102bf565b3460006101297f000000000000000000000000000000000000000000000000000000003b9aca0083610411565b1161014f5760405162461bcd60e51b815260040161014690610433565b60405180910390fd5b34336001600160a01b03167f0c64e29a5254a71c7f4e52b3d2d236348c80e00a00ba2e1961962bd2827c03fb8787878760405161018f94939291906104ea565b60405180910390a35050505050565b600180546101ab9061051c565b80601f01602080910402602001604051908101604052809291908181526020018280546101d79061051c565b80156102245780601f106101f957610100808354040283529160200191610224565b820191906000526020600020905b81548152906001019060200180831161020757829003601f168201915b505050505081565b3460006102597f000000000000000000000000000000000000000000000000000000003b9aca0083610411565b116102765760405162461bcd60e51b815260040161014690610433565b34336001600160a01b03167f0f4961cab7530804898499aa89f5ec81d1a73102e2e4a1f30f88e5ae3513ba2a85856040516102b2929190610556565b60405180910390a3505050565b600080546101ab9061051c565b60008083601f8401126102de57600080fd5b50813567ffffffffffffffff8111156102f657600080fd5b60208301915083602082850101111561030e57600080fd5b9250929050565b6000806000806040858703121561032b57600080fd5b843567ffffffffffffffff8082111561034357600080fd5b61034f888389016102cc565b9096509450602087013591508082111561036857600080fd5b50610375878288016102cc565b95989497509550505050565b600060208083528351808285015260005b818110156103ae57858101830151858201604001528201610392565b506000604082860101526040601f19601f8301168501019250505092915050565b600080602083850312156103e257600080fd5b823567ffffffffffffffff8111156103f957600080fd5b610405858286016102cc565b90969095509350505050565b60008261042e57634e487b7160e01b600052601260045260246000fd5b500490565b60208082526062908201527f417374726961576974686472617765723a20696e73756666696369656e74207660408201527f616c75652c206d7573742062652067726561746572207468616e203130202a2a60608201527f20283138202d20424153455f434841494e5f41535345545f505245434953494f6080820152614e2960f01b60a082015260c00190565b81835281816020850137506000828201602090810191909152601f909101601f19169091010190565b6040815260006104fe6040830186886104c1565b82810360208401526105118185876104c1565b979650505050505050565b600181811c9082168061053057607f821691505b60208210810361055057634e487b7160e01b600052602260045260246000fd5b50919050565b60208152600061056a6020830184866104c1565b94935050505056fea264697066735822122047a7ef84c0be4640572989abfc01decbc1ae143d4659f1b32047978c67ebc9c864736f6c63430008150033" - + - address: "0xaC21B97d35Bf75A7dAb16f35b111a50e78A72F30" + value: + balance: "1000000000000000000000000000" config: # The level at which core astria components will log out # Options are: error, warn, info, and debug - logLevel: "debug" + logLevel: "info" conductor: # Determines what will drive block execution, options are: @@ -102,21 +104,21 @@ evm-rollup: rpc: "http://celestia-service.astria-dev-cluster.svc.cluster.local:26658" token: "" - resources: - conductor: - requests: - cpu: 0.01 - memory: 1Mi - limits: - cpu: 0.1 - memory: 20Mi - geth: - requests: - cpu: 0.25 - memory: 256Mi - limits: - cpu: 2 - memory: 1Gi + resources: {} +# conductor: +# requests: +# cpu: 0.01 +# memory: 1Mi +# limits: +# cpu: 0.1 +# memory: 20Mi +# geth: +# requests: +# cpu: 0.25 +# memory: 256Mi +# limits: +# cpu: 2 +# memory: 1Gi storage: enabled: false @@ -137,6 +139,11 @@ composer: config: privateKey: devContent: "2bd806c97f0e00af1a1fc3328fa763a9269723c8db8fac4f93af71db186d6e90" +# images: +# composer: +# repo: astria-composer +# tag: "0.8.0" +# devTag: tb evm-bridge-withdrawer: enabled: true From 534190049698f2e1f7fe07e7cc0e0510ccc39a8b Mon Sep 17 00:00:00 2001 From: Bharath Date: Thu, 29 Aug 2024 20:52:57 +0530 Subject: [PATCH 06/43] submit bundle only when new block is created --- crates/astria-composer/src/composer.rs | 1 + .../astria-composer/src/executor/builder.rs | 3 + crates/astria-composer/src/executor/client.rs | 6 +- crates/astria-composer/src/executor/mod.rs | 76 +++++++++++++++---- .../astria-composer/src/executor/simulator.rs | 12 +-- 5 files changed, 76 insertions(+), 22 deletions(-) diff --git a/crates/astria-composer/src/composer.rs b/crates/astria-composer/src/composer.rs index c16b06383..cb0ef3684 100644 --- a/crates/astria-composer/src/composer.rs +++ b/crates/astria-composer/src/composer.rs @@ -134,6 +134,7 @@ impl Composer { execution_api_url: cfg.execution_api_url.clone(), fee_asset: cfg.fee_asset.clone(), chain_name: cfg.rollup.clone(), + websocket_url: cfg.rollup_websocket_url.clone(), shutdown_token: shutdown_token.clone(), metrics, } diff --git a/crates/astria-composer/src/executor/builder.rs b/crates/astria-composer/src/executor/builder.rs index 059294e36..ce99e7a7b 100644 --- a/crates/astria-composer/src/executor/builder.rs +++ b/crates/astria-composer/src/executor/builder.rs @@ -43,6 +43,7 @@ pub(crate) struct Builder { pub(crate) execution_api_url: String, pub(crate) chain_name: String, pub(crate) fee_asset: asset::Denom, + pub(crate) websocket_url: String, pub(crate) metrics: &'static Metrics, } @@ -60,6 +61,7 @@ impl Builder { execution_api_url, chain_name, fee_asset, + websocket_url, metrics, } = self; let sequencer_client = sequencer_client::HttpClient::new(sequencer_url.as_str()) @@ -102,6 +104,7 @@ impl Builder { shutdown_token, rollup_id, fee_asset, + websocket_url, metrics, }, executor::Handle::new(serialized_rollup_transaction_tx), diff --git a/crates/astria-composer/src/executor/client.rs b/crates/astria-composer/src/executor/client.rs index d5b91b390..1a6d6d9bc 100644 --- a/crates/astria-composer/src/executor/client.rs +++ b/crates/astria-composer/src/executor/client.rs @@ -93,7 +93,10 @@ impl Client { let response = tryhard::retry_fn(|| { let mut client = self.inner.clone(); let request = request.clone(); - async move { client.execute_block(request).await} + async move { + let res = client.execute_block(request).await; + res + } }) .with_config(retry_config()) .in_current_span() @@ -117,7 +120,6 @@ impl Client { async move { let request = raw::GetCommitmentStateRequest {}; let res = client.get_commitment_state(request).await; - println!("IN CLIENT CODE: CALLED GET COMMITMENT STATE {:?}", res); res } }) diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index fd2b6572b..4d0a629a6 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -9,6 +9,8 @@ use std::{ task::Poll, time::Duration, }; +use ethers::prelude::{Provider, ProviderError, Ws}; +use ethers::providers::{Middleware, StreamExt}; use astria_core::{ crypto::SigningKey, @@ -34,10 +36,7 @@ use astria_core::{ }, Protobuf, }; -use astria_eyre::eyre::{ - self, - WrapErr as _, -}; +use astria_eyre::eyre::{self, eyre, WrapErr as _}; use futures::{ future::{ self, @@ -160,6 +159,7 @@ pub(super) struct Executor { bundle_simulator: BundleSimulator, rollup_id: RollupId, fee_asset: asset::Denom, + websocket_url: String, metrics: &'static Metrics, } @@ -217,7 +217,6 @@ impl Executor { bundle: SizedBundle, metrics: &'static Metrics, ) -> eyre::Result>> { - println!("IN MAIN CODE: STARTING SIMULATION"); let bundle_simulator = self.bundle_simulator.clone(); // simulate the bundle @@ -226,7 +225,6 @@ impl Executor { .await .wrap_err("failed to simulate bundle")?; - println!("IN MAIN CODE: DONE WITH SIMULATION"); let rollup_data_items: Vec = bundle_simulation_result .included_actions() .iter() @@ -256,7 +254,6 @@ impl Executor { return Err(eyre!(e.to_string())); } - println!("IN MAIN CODE: DONE SIMULATION!"); Ok(SubmitFut { client: self.sequencer_client.clone(), address: self.address, @@ -313,8 +310,6 @@ impl Executor { self.metrics.set_current_nonce(nonce); - self.status.send_modify(|status| status.is_connected = true); - let block_timer = time::sleep(self.block_time); tokio::pin!(block_timer); let mut bundle_factory = @@ -326,6 +321,43 @@ impl Executor { .expect("block_time should not be large enough to cause an overflow") }; + // establish a websocket connection with the geth node to subscribe for latest blocks + let retry_config = tryhard::RetryFutureConfig::new(1024) + .exponential_backoff(Duration::from_millis(500)) + .max_delay(Duration::from_secs(60)) + .on_retry( + |attempt, next_delay: Option, error: &ProviderError| { + let wait_duration = next_delay + .map(humantime::format_duration) + .map(tracing::field::display); + warn!( + attempt, + wait_duration, + error = error as &StdError, + "attempt to connect to geth node failed; retrying after backoff", + ); + futures::future::ready(()) + }, + ); + + let client = tryhard::retry_fn(|| { + let url = self.websocket_url.clone(); + async move { + let websocket_client = Ws::connect_with_reconnects(url, 0).await?; + Ok(Provider::new(websocket_client)) + } + }) + .with_config(retry_config) + .await + .wrap_err("failed connecting to geth after several retries; giving up")?; + + let mut block_stream = client + .subscribe_blocks() + .await + .wrap_err("failed to subscribe eth client to full pending transactions")?; + + self.status.send_modify(|status| status.is_connected = true); + let reason = loop { select! { biased; @@ -340,12 +372,12 @@ impl Executor { }; } - Some(next_bundle) = future::ready(bundle_factory.next_finished()), if submission_fut.is_terminated() => { - let bundle = next_bundle.pop(); - if !bundle.is_empty() { - submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; - } - } + // Some(next_bundle) = future::ready(bundle_factory.next_finished()), if submission_fut.is_terminated() => { + // let bundle = next_bundle.pop(); + // if !bundle.is_empty() { + // submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; + // } + // } // receive new seq_action and bundle it. will not pull from the channel if `bundle_factory` is full Some(seq_action) = self.serialized_rollup_transactions.recv(), if !bundle_factory.is_full() => { @@ -364,6 +396,20 @@ impl Executor { submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; } } + + // try to preempt current bundle if the timer has ticked without submitting the next bundle + // () = &mut block_timer, if submission_fut.is_terminated() => { + // let bundle = bundle_factory.pop_now(); + // if bundle.is_empty() { + // debug!("block timer ticked, but no bundle to submit to sequencer"); + // block_timer.as_mut().reset(reset_time()); + // } else { + // debug!( + // "forcing bundle submission to sequencer due to block timer" + // ); + // submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; + // } + // } } }; diff --git a/crates/astria-composer/src/executor/simulator.rs b/crates/astria-composer/src/executor/simulator.rs index 2d3649141..32d849b22 100644 --- a/crates/astria-composer/src/executor/simulator.rs +++ b/crates/astria-composer/src/executor/simulator.rs @@ -1,3 +1,4 @@ +use std::time::{SystemTime, UNIX_EPOCH}; use astria_core::{ sequencerblock::v1alpha1::block::RollupData, Protobuf, @@ -57,14 +58,12 @@ impl BundleSimulator { bundle: SizedBundle, ) -> eyre::Result { // call GetCommitmentState to get the soft block - println!("IN MAIN CODE: CALLING GET COMMITMENT STATE"); let commitment_state = self .execution_service_client .get_commitment_state_with_retry() .await .wrap_err("failed to get commitment state")?; - println!("IN MAIN CODE: CALLED GET COMMITMENT STATE!"); let soft_block = commitment_state.soft(); // convert the sized bundle actions to a list of list of u8s // TODO - bharath - revisit this and make the code better. The else stmt is a bit weird @@ -85,20 +84,23 @@ impl BundleSimulator { .filter(|data| !data.is_empty()) .collect(); - println!("IN MAIN CODE: CALLING EXECUTE_BLOCK"); // call execute block with the bundle to get back the included transactions + let timestamp = Timestamp { + seconds: soft_block.timestamp().seconds + 3, + nanos: 0 + }; let execute_block_response = self .execution_service_client .execute_block_with_retry( soft_block.hash().clone(), actions, - Timestamp::from(soft_block.timestamp()), + // use current timestamp + timestamp, true, ) .await .wrap_err("failed to execute block")?; - println!("IN MAIN CODE: CALLED EXECUTE BLOCK!!"); Ok(BundleSimulationResult::new( execute_block_response.included_transactions().to_vec(), execute_block_response.block().parent_block_hash().clone(), From bc3dbc8f64a1bea84aef09cae3bb632a33cf4d4b Mon Sep 17 00:00:00 2001 From: Bharath Date: Thu, 8 Aug 2024 11:16:29 +0530 Subject: [PATCH 07/43] use soft block hash instead of parent hash --- crates/astria-composer/src/executor/simulator.rs | 2 +- crates/astria-composer/src/executor/tests.rs | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/astria-composer/src/executor/simulator.rs b/crates/astria-composer/src/executor/simulator.rs index 32d849b22..6137bedf5 100644 --- a/crates/astria-composer/src/executor/simulator.rs +++ b/crates/astria-composer/src/executor/simulator.rs @@ -103,7 +103,7 @@ impl BundleSimulator { Ok(BundleSimulationResult::new( execute_block_response.included_transactions().to_vec(), - execute_block_response.block().parent_block_hash().clone(), + execute_block_response.block().hash().clone(), )) } } diff --git a/crates/astria-composer/src/executor/tests.rs b/crates/astria-composer/src/executor/tests.rs index 38f7046ba..9603aea40 100644 --- a/crates/astria-composer/src/executor/tests.rs +++ b/crates/astria-composer/src/executor/tests.rs @@ -374,6 +374,7 @@ async fn full_bundle() { execution_api_url: cfg.execution_api_url, chain_name: cfg.rollup.clone(), fee_asset: cfg.fee_asset, + websocket_url: cfg.rollup_websocket_url.clone(), metrics, } .build() @@ -526,6 +527,7 @@ async fn bundle_triggered_by_block_timer() { execution_api_url: cfg.execution_api_url, chain_name: cfg.rollup.clone(), fee_asset: cfg.fee_asset.clone(), + websocket_url: cfg.rollup_websocket_url.clone(), metrics, } .build() @@ -682,6 +684,7 @@ async fn two_seq_actions_single_bundle() { execution_api_url: cfg.execution_api_url, chain_name: cfg.rollup.clone(), fee_asset: cfg.fee_asset.clone(), + websocket_url: cfg.rollup_websocket_url.clone(), metrics, } .build() @@ -843,6 +846,7 @@ async fn chain_id_mismatch_returns_error() { execution_api_url: cfg.execution_api_url, chain_name: rollup_name.to_string(), fee_asset: cfg.fee_asset, + websocket_url: cfg.rollup_websocket_url.clone(), metrics, } .build() From 422c851eabeed8c9ea9853df996a7a3eae2fde8c Mon Sep 17 00:00:00 2001 From: Bharath Date: Thu, 8 Aug 2024 12:48:48 +0530 Subject: [PATCH 08/43] save --- charts/composer/values.yaml | 4 ++-- charts/evm-rollup/templates/service.yaml | 2 +- charts/evm-rollup/values.yaml | 4 ++-- crates/astria-composer/src/executor/mod.rs | 4 +++- crates/astria-composer/src/executor/simulator.rs | 5 +++-- crates/astria-composer/src/executor/tests.rs | 6 +++--- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/charts/composer/values.yaml b/charts/composer/values.yaml index bd239b779..42b5b192f 100644 --- a/charts/composer/values.yaml +++ b/charts/composer/values.yaml @@ -12,10 +12,10 @@ images: devTag: latest config: - logLevel: "debug" + logLevel: "info" maxBytesPerBundle: 200000 bundleQueueCapacity: 40000 - maxSubmitInterval: 2000 + maxSubmitInterval: 1800 sequencerAddressPrefix: astria sequencerNativeAssetBaseDenomination: "nria" sequencerRpc: "" diff --git a/charts/evm-rollup/templates/service.yaml b/charts/evm-rollup/templates/service.yaml index 70c89bec3..eee3a07c5 100644 --- a/charts/evm-rollup/templates/service.yaml +++ b/charts/evm-rollup/templates/service.yaml @@ -15,7 +15,7 @@ spec: targetPort: ws-rpc - name: exec-grpc-svc port: {{ .Values.ports.executionGRPC }} - targetPort: exec-grpc + targetPort: execution-grpc --- {{- if .Values.metrics.enabled }} kind: Service diff --git a/charts/evm-rollup/values.yaml b/charts/evm-rollup/values.yaml index 5b5fa6c04..6cd436082 100644 --- a/charts/evm-rollup/values.yaml +++ b/charts/evm-rollup/values.yaml @@ -268,10 +268,10 @@ ingress: # - secretName: chart-example-tls # hosts: # - chart-example.local - grpc: + exec-api: enabled: true hosts: - - execution-executor.{{ include "rollup.name" . }}.{{ .Values.ingress.hostname }} + - exec-executor.{{ include "rollup.name" . }}.{{ .Values.ingress.hostname }} path: / pathType: Prefix service: diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index 4d0a629a6..b0d5c707a 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -231,6 +231,8 @@ impl Executor { .map(|action| action.to_raw()) .collect(); + info!("ALERT: simulated bundle: {:?}", bundle_simulation_result.parent_hash().clone()); + // create a top of block bundle // TODO - we need to sign the builder bundle packet let builder_bundle_packet = BuilderBundlePacket { @@ -397,7 +399,7 @@ impl Executor { } } - // try to preempt current bundle if the timer has ticked without submitting the next bundle + // // try to preempt current bundle if the timer has ticked without submitting the next bundle // () = &mut block_timer, if submission_fut.is_terminated() => { // let bundle = bundle_factory.pop_now(); // if bundle.is_empty() { diff --git a/crates/astria-composer/src/executor/simulator.rs b/crates/astria-composer/src/executor/simulator.rs index 6137bedf5..3775012b1 100644 --- a/crates/astria-composer/src/executor/simulator.rs +++ b/crates/astria-composer/src/executor/simulator.rs @@ -10,7 +10,7 @@ use astria_eyre::{ use bytes::Bytes; use pbjson_types::Timestamp; use prost::Message; -use tracing::instrument; +use tracing::{info, instrument}; use crate::executor::{ bundle_factory::SizedBundle, @@ -101,9 +101,10 @@ impl BundleSimulator { .await .wrap_err("failed to execute block")?; + info!("Using block hash instead of parent hash lmaoooo!"); Ok(BundleSimulationResult::new( execute_block_response.included_transactions().to_vec(), - execute_block_response.block().hash().clone(), + soft_block.hash().clone(), )) } } diff --git a/crates/astria-composer/src/executor/tests.rs b/crates/astria-composer/src/executor/tests.rs index 9603aea40..908187414 100644 --- a/crates/astria-composer/src/executor/tests.rs +++ b/crates/astria-composer/src/executor/tests.rs @@ -477,7 +477,7 @@ async fn full_bundle() { assert_eq!( builder_bundle_packet.bundle().parent_hash(), - soft_parent_hash.to_vec() + soft_block_hash.to_vec() ); let bundle_txs = builder_bundle_packet.bundle().transactions(); @@ -621,7 +621,7 @@ async fn bundle_triggered_by_block_timer() { assert_eq!( builder_bundle_packet.bundle().parent_hash().to_vec(), - soft_parent_hash.to_vec() + soft_block_hash.to_vec() ); // ensure that the seq_action of the BuilderBundlePacket and the expected sequence actions have the same @@ -787,7 +787,7 @@ async fn two_seq_actions_single_bundle() { assert_eq!(builder_bundle_packet.bundle().transactions().len(), 2); assert_eq!( builder_bundle_packet.bundle().parent_hash().to_vec(), - soft_parent_hash.to_vec() + soft_block_hash.to_vec() ); for (action, expected_action) in expected_seq_actions.iter().zip(actions) { From 01570343fc29160a042a8cef216025d342923695 Mon Sep 17 00:00:00 2001 From: Bharath Date: Fri, 6 Sep 2024 14:14:16 +0530 Subject: [PATCH 09/43] clean ups --- charts/sequencer/values.yaml | 2 +- crates/astria-composer/Cargo.toml | 1 + crates/astria-composer/src/composer.rs | 1 - crates/astria-composer/src/config.rs | 4 +- .../astria-composer/src/executor/builder.rs | 3 - crates/astria-composer/src/executor/client.rs | 4 +- crates/astria-composer/src/executor/mod.rs | 106 +++++--------- .../astria-composer/src/executor/simulator.rs | 31 ++-- crates/astria-composer/src/executor/tests.rs | 56 ++++--- crates/astria-composer/src/lib.rs | 2 +- crates/astria-composer/src/metrics.rs | 6 +- .../src/{executor => }/mock_grpc.rs | 137 +++++------------- crates/astria-composer/tests/blackbox/api.rs | 2 +- .../tests/blackbox/geth_collector.rs | 41 +++--- .../tests/blackbox/grpc_collector.rs | 24 +-- .../tests/blackbox/helper/mod.rs | 18 +-- .../astria-core/src/composer/v1alpha1/mod.rs | 8 +- .../src/generated/astria.composer.v1alpha1.rs | 4 +- .../astria.composer.v1alpha1.serde.rs | 7 +- crates/astria-grpc-mock/src/matcher.rs | 2 + .../composer/v1alpha1/trusted_builder.proto | 2 +- 21 files changed, 177 insertions(+), 284 deletions(-) rename crates/astria-composer/src/{executor => }/mock_grpc.rs (69%) diff --git a/charts/sequencer/values.yaml b/charts/sequencer/values.yaml index 89dfb9acc..719d66106 100644 --- a/charts/sequencer/values.yaml +++ b/charts/sequencer/values.yaml @@ -146,7 +146,7 @@ cometbft: timeoutPrevoteDelta: 500ms timeoutPrecommit: 1s timeoutPrecommitDelta: 500ms - timeoutCommit: 1500ms + timeoutCommit: 4000ms instrumentation: namespace: "astria_cometbft" diff --git a/crates/astria-composer/Cargo.toml b/crates/astria-composer/Cargo.toml index 64fb91428..24183ab47 100644 --- a/crates/astria-composer/Cargo.toml +++ b/crates/astria-composer/Cargo.toml @@ -71,6 +71,7 @@ features = ["http"] config = { package = "astria-config", path = "../astria-config", features = [ "tests", ] } + test_utils = { package = "astria-test-utils", path = "../astria-test-utils", features = [ "geth", ] } diff --git a/crates/astria-composer/src/composer.rs b/crates/astria-composer/src/composer.rs index cb0ef3684..c16b06383 100644 --- a/crates/astria-composer/src/composer.rs +++ b/crates/astria-composer/src/composer.rs @@ -134,7 +134,6 @@ impl Composer { execution_api_url: cfg.execution_api_url.clone(), fee_asset: cfg.fee_asset.clone(), chain_name: cfg.rollup.clone(), - websocket_url: cfg.rollup_websocket_url.clone(), shutdown_token: shutdown_token.clone(), metrics, } diff --git a/crates/astria-composer/src/config.rs b/crates/astria-composer/src/config.rs index 1e66a33de..6f24f19b4 100644 --- a/crates/astria-composer/src/config.rs +++ b/crates/astria-composer/src/config.rs @@ -1,6 +1,4 @@ -use std::{ - net::SocketAddr, -}; +use std::net::SocketAddr; use serde::{ Deserialize, diff --git a/crates/astria-composer/src/executor/builder.rs b/crates/astria-composer/src/executor/builder.rs index ce99e7a7b..059294e36 100644 --- a/crates/astria-composer/src/executor/builder.rs +++ b/crates/astria-composer/src/executor/builder.rs @@ -43,7 +43,6 @@ pub(crate) struct Builder { pub(crate) execution_api_url: String, pub(crate) chain_name: String, pub(crate) fee_asset: asset::Denom, - pub(crate) websocket_url: String, pub(crate) metrics: &'static Metrics, } @@ -61,7 +60,6 @@ impl Builder { execution_api_url, chain_name, fee_asset, - websocket_url, metrics, } = self; let sequencer_client = sequencer_client::HttpClient::new(sequencer_url.as_str()) @@ -104,7 +102,6 @@ impl Builder { shutdown_token, rollup_id, fee_asset, - websocket_url, metrics, }, executor::Handle::new(serialized_rollup_transaction_tx), diff --git a/crates/astria-composer/src/executor/client.rs b/crates/astria-composer/src/executor/client.rs index 1a6d6d9bc..1bb9994e9 100644 --- a/crates/astria-composer/src/executor/client.rs +++ b/crates/astria-composer/src/executor/client.rs @@ -95,6 +95,7 @@ impl Client { let request = request.clone(); async move { let res = client.execute_block(request).await; + println!("{:?}", res); res } }) @@ -119,8 +120,7 @@ impl Client { let mut client = self.inner.clone(); async move { let request = raw::GetCommitmentStateRequest {}; - let res = client.get_commitment_state(request).await; - res + client.get_commitment_state(request).await } }) .with_config(retry_config()) diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index b0d5c707a..dfffacf0d 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -1,3 +1,4 @@ +use std::str::FromStr; /// ! The `Executor` is responsible for: /// - Nonce management /// - Transaction signing @@ -9,8 +10,6 @@ use std::{ task::Poll, time::Duration, }; -use ethers::prelude::{Provider, ProviderError, Ws}; -use ethers::providers::{Middleware, StreamExt}; use astria_core::{ crypto::SigningKey, @@ -106,7 +105,6 @@ mod client; mod simulator; #[cfg(test)] mod tests; -mod mock_grpc; pub(crate) use builder::Builder; @@ -156,10 +154,12 @@ pub(super) struct Executor { bundle_queue_capacity: usize, // Token to signal the executor to stop upon shutdown. shutdown_token: CancellationToken, + // `BundleSimulator` simulates the execution of a bundle of transactions. bundle_simulator: BundleSimulator, + // The rollup id associated with this executor rollup_id: RollupId, + // The asset used for sequencer fees fee_asset: asset::Denom, - websocket_url: String, metrics: &'static Metrics, } @@ -210,7 +210,6 @@ impl Executor { self.status.subscribe() } - // TODO - maybe we should break this up into a separate simulate and submit step? async fn simulate_and_submit_bundle( &self, nonce: u32, @@ -231,29 +230,37 @@ impl Executor { .map(|action| action.to_raw()) .collect(); - info!("ALERT: simulated bundle: {:?}", bundle_simulation_result.parent_hash().clone()); + let builder_bundle = BuilderBundle { + transactions: rollup_data_items, + parent_hash: bundle_simulation_result.parent_hash().to_vec(), + }; + + let encoded_builder_bundle = builder_bundle.encode_to_vec(); + let private_key = ""; + let wallet = ethers::signers::Wallet::from_str(private_key) + .wrap_err("failed to parse private key")?; + let signature = wallet + .sign_message(encoded_builder_bundle.clone()) + .await + .wrap_err("failed to sign builder bundle packet")?; // create a top of block bundle - // TODO - we need to sign the builder bundle packet - let builder_bundle_packet = BuilderBundlePacket { - bundle: Some(BuilderBundle { - transactions: rollup_data_items, - parent_hash: bundle_simulation_result.parent_hash().to_vec(), - }), - signature: vec![], + let mut builder_bundle_packet = BuilderBundlePacket { + bundle: Some(builder_bundle), + signature: signature.to_string(), }; let encoded_builder_bundle_packet = builder_bundle_packet.encode_to_vec(); - // TODO - we had to make sized bundle struct public, can we avoid that? + // we can give the BuilderBundlePacket the highest bundle max size possible + // since this is the only sequence action we are sending + // TODO - parameterize the max bundle size let mut final_bundle = SizedBundle::new(200000); if let Err(e) = final_bundle.try_push(SequenceAction { rollup_id: self.rollup_id, data: encoded_builder_bundle_packet.into(), fee_asset: self.fee_asset.clone(), }) { - // TODO - we had to make the SizedBundle error public across the crate, we should - // revisit that - return Err(eyre!(e.to_string())); + return Err(eyre::Report::from(e)); } Ok(SubmitFut { @@ -323,41 +330,6 @@ impl Executor { .expect("block_time should not be large enough to cause an overflow") }; - // establish a websocket connection with the geth node to subscribe for latest blocks - let retry_config = tryhard::RetryFutureConfig::new(1024) - .exponential_backoff(Duration::from_millis(500)) - .max_delay(Duration::from_secs(60)) - .on_retry( - |attempt, next_delay: Option, error: &ProviderError| { - let wait_duration = next_delay - .map(humantime::format_duration) - .map(tracing::field::display); - warn!( - attempt, - wait_duration, - error = error as &StdError, - "attempt to connect to geth node failed; retrying after backoff", - ); - futures::future::ready(()) - }, - ); - - let client = tryhard::retry_fn(|| { - let url = self.websocket_url.clone(); - async move { - let websocket_client = Ws::connect_with_reconnects(url, 0).await?; - Ok(Provider::new(websocket_client)) - } - }) - .with_config(retry_config) - .await - .wrap_err("failed connecting to geth after several retries; giving up")?; - - let mut block_stream = client - .subscribe_blocks() - .await - .wrap_err("failed to subscribe eth client to full pending transactions")?; - self.status.send_modify(|status| status.is_connected = true); let reason = loop { @@ -374,12 +346,12 @@ impl Executor { }; } - // Some(next_bundle) = future::ready(bundle_factory.next_finished()), if submission_fut.is_terminated() => { - // let bundle = next_bundle.pop(); - // if !bundle.is_empty() { - // submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; - // } - // } + Some(next_bundle) = future::ready(bundle_factory.next_finished()), if submission_fut.is_terminated() => { + let bundle = next_bundle.pop(); + if !bundle.is_empty() { + submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; + } + } // receive new seq_action and bundle it. will not pull from the channel if `bundle_factory` is full Some(seq_action) = self.serialized_rollup_transactions.recv(), if !bundle_factory.is_full() => { @@ -390,6 +362,10 @@ impl Executor { () = &mut block_timer, if submission_fut.is_terminated() => { let bundle = bundle_factory.pop_now(); if bundle.is_empty() { +<<<<<<< HEAD +======= + debug!("block timer ticked, but no bundle to submit to sequencer"); +>>>>>>> 7c75d72f (clean ups) block_timer.as_mut().reset(reset_time()); } else { debug!( @@ -398,20 +374,6 @@ impl Executor { submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; } } - - // // try to preempt current bundle if the timer has ticked without submitting the next bundle - // () = &mut block_timer, if submission_fut.is_terminated() => { - // let bundle = bundle_factory.pop_now(); - // if bundle.is_empty() { - // debug!("block timer ticked, but no bundle to submit to sequencer"); - // block_timer.as_mut().reset(reset_time()); - // } else { - // debug!( - // "forcing bundle submission to sequencer due to block timer" - // ); - // submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; - // } - // } } }; diff --git a/crates/astria-composer/src/executor/simulator.rs b/crates/astria-composer/src/executor/simulator.rs index 3775012b1..cbfd7aee3 100644 --- a/crates/astria-composer/src/executor/simulator.rs +++ b/crates/astria-composer/src/executor/simulator.rs @@ -1,4 +1,3 @@ -use std::time::{SystemTime, UNIX_EPOCH}; use astria_core::{ sequencerblock::v1alpha1::block::RollupData, Protobuf, @@ -10,7 +9,10 @@ use astria_eyre::{ use bytes::Bytes; use pbjson_types::Timestamp; use prost::Message; -use tracing::{info, instrument}; +use tracing::{ + info, + instrument, +}; use crate::executor::{ bundle_factory::SizedBundle, @@ -65,30 +67,27 @@ impl BundleSimulator { .wrap_err("failed to get commitment state")?; let soft_block = commitment_state.soft(); - // convert the sized bundle actions to a list of list of u8s - // TODO - bharath - revisit this and make the code better. The else stmt is a bit weird + // convert the sized bundle actions to a list of Vec let actions: Vec> = bundle .into_actions() .iter() - .map(|action| { - // TODO - should we support sequencer transfers and actions outside sequence - // actions too? - return if let Some(seq_action) = action.as_sequence() { - RollupData::SequencedData(seq_action.clone().data) - .to_raw() - .encode_to_vec() - } else { - vec![] - }; + .map(|action| match action.as_sequence() { + Some(seq_action) => RollupData::SequencedData(seq_action.clone().data) + .to_raw() + .encode_to_vec(), + None => vec![], }) .filter(|data| !data.is_empty()) .collect(); - // call execute block with the bundle to get back the included transactions + // as long as the timestamp > parent block timestamp, the block will be successfully + // created. It doesn't matter what timestamp we use anyway since we are not going to + // commit the block to the chain. let timestamp = Timestamp { seconds: soft_block.timestamp().seconds + 3, - nanos: 0 + nanos: 0, }; + // call execute block with the bundle to get back the included transactions let execute_block_response = self .execution_service_client .execute_block_with_retry( diff --git a/crates/astria-composer/src/executor/tests.rs b/crates/astria-composer/src/executor/tests.rs index 908187414..89aa82eef 100644 --- a/crates/astria-composer/src/executor/tests.rs +++ b/crates/astria-composer/src/executor/tests.rs @@ -11,6 +11,7 @@ use astria_core::{ generated::{ composer::v1alpha1::BuilderBundlePacket, protocol::account::v1alpha1::NonceResponse, + sequencerblock::v1alpha1 as raw_sequencer, }, primitive::v1::{ <<<<<<< HEAD @@ -29,6 +30,10 @@ use astria_core::{ Protobuf, }; use astria_eyre::eyre; +use base64::{ + prelude::BASE64_STANDARD, + Engine, +}; use futures::future::join; use once_cell::sync::Lazy; use prost::{ @@ -76,14 +81,12 @@ use wiremock::{ use crate::{ executor, - executor::{ - mock_grpc::{ - MockGrpc, - TestExecutor, - }, - EnsureChainIdError, - }, + executor::EnsureChainIdError, metrics::Metrics, + mock_grpc::{ + MockGrpc, + TestExecutor, + }, mount_executed_block, mount_get_commitment_state, test_utils::sequence_action_of_max_size, @@ -374,7 +377,6 @@ async fn full_bundle() { execution_api_url: cfg.execution_api_url, chain_name: cfg.rollup.clone(), fee_asset: cfg.fee_asset, - websocket_url: cfg.rollup_websocket_url.clone(), metrics, } .build() @@ -415,12 +417,10 @@ async fn full_bundle() { ..sequence_action_of_max_size(cfg.max_bytes_per_bundle) }; - // TODO - type declaration looks weird, fix it - let rollup_data: Vec = - vec![seq0.clone()] - .iter() - .map(|item| RollupData::SequencedData(item.clone().data).to_raw()) - .collect(); + let rollup_data: Vec = vec![seq0.clone()] + .iter() + .map(|item| RollupData::SequencedData(item.clone().data).to_raw()) + .collect(); let execute_block = mount_executed_block!(test_executor, mock_name: "execute_block", @@ -527,7 +527,6 @@ async fn bundle_triggered_by_block_timer() { execution_api_url: cfg.execution_api_url, chain_name: cfg.rollup.clone(), fee_asset: cfg.fee_asset.clone(), - websocket_url: cfg.rollup_websocket_url.clone(), metrics, } .build() @@ -552,11 +551,10 @@ async fn bundle_triggered_by_block_timer() { ..sequence_action(rollup_id.clone(), cfg.fee_asset.clone()) }; - let rollup_data: Vec = - vec![seq0.clone()] - .iter() - .map(|item| RollupData::SequencedData(item.clone().data).to_raw()) - .collect(); + let rollup_data: Vec = vec![seq0.clone()] + .iter() + .map(|item| RollupData::SequencedData(item.clone().data).to_raw()) + .collect(); let soft_parent_hash = [1; 64]; let soft_block_number = 1; @@ -624,8 +622,8 @@ async fn bundle_triggered_by_block_timer() { soft_block_hash.to_vec() ); - // ensure that the seq_action of the BuilderBundlePacket and the expected sequence actions have the same - // rollup id and fee asset + // ensure that the seq_action of the BuilderBundlePacket and the expected sequence actions have + // the same rollup id and fee asset for (action, expected_action) in expected_seq_actions.iter().zip(actions) { let expected_seq_action = expected_action.as_sequence().unwrap(); @@ -639,7 +637,6 @@ async fn bundle_triggered_by_block_timer() { .iter() .zip(expected_seq_actions.iter()) { - match action.clone() { RollupData::SequencedData(data) => { assert_eq!( @@ -684,7 +681,6 @@ async fn two_seq_actions_single_bundle() { execution_api_url: cfg.execution_api_url, chain_name: cfg.rollup.clone(), fee_asset: cfg.fee_asset.clone(), - websocket_url: cfg.rollup_websocket_url.clone(), metrics, } .build() @@ -713,11 +709,10 @@ async fn two_seq_actions_single_bundle() { ..sequence_action(rollup_id.clone(), cfg.fee_asset.clone()) }; - let rollup_data: Vec = - vec![seq0.clone(), seq1.clone()] - .iter() - .map(|item| RollupData::SequencedData(item.clone().data).to_raw()) - .collect(); + let rollup_data: Vec = vec![seq0.clone(), seq1.clone()] + .iter() + .map(|item| RollupData::SequencedData(item.clone().data).to_raw()) + .collect(); let soft_parent_hash = [1; 64]; let soft_block_number = 1; @@ -758,7 +753,7 @@ async fn two_seq_actions_single_bundle() { join( response_guard.wait_until_satisfied(), execute_block.wait_until_satisfied(), - ) + ), ) .await .unwrap(); @@ -846,7 +841,6 @@ async fn chain_id_mismatch_returns_error() { execution_api_url: cfg.execution_api_url, chain_name: rollup_name.to_string(), fee_asset: cfg.fee_asset, - websocket_url: cfg.rollup_websocket_url.clone(), metrics, } .build() diff --git a/crates/astria-composer/src/lib.rs b/crates/astria-composer/src/lib.rs index cbf59addc..db497f655 100644 --- a/crates/astria-composer/src/lib.rs +++ b/crates/astria-composer/src/lib.rs @@ -46,7 +46,7 @@ pub mod config; mod executor; mod grpc; pub(crate) mod metrics; -mod rollup; +mod mock_grpc; #[cfg(test)] pub(crate) mod test_utils; pub(crate) mod utils; diff --git a/crates/astria-composer/src/metrics.rs b/crates/astria-composer/src/metrics.rs index 0917405f1..8f4cbe832 100644 --- a/crates/astria-composer/src/metrics.rs +++ b/crates/astria-composer/src/metrics.rs @@ -47,8 +47,10 @@ impl Metrics { let (geth_txs_received, grpc_txs_received) = register_txs_received(vec![rollup_name.clone()].iter()); // TODO - change the function signatures of the metrics - let (geth_txs_dropped, grpc_txs_dropped) = register_txs_dropped(vec![rollup_name.clone()].iter()); - let txs_dropped_too_large = register_txs_dropped_too_large(vec![rollup_name.clone()].iter()); + let (geth_txs_dropped, grpc_txs_dropped) = + register_txs_dropped(vec![rollup_name.clone()].iter()); + let txs_dropped_too_large = + register_txs_dropped_too_large(vec![rollup_name.clone()].iter()); describe_counter!( NONCE_FETCH_COUNT, diff --git a/crates/astria-composer/src/executor/mock_grpc.rs b/crates/astria-composer/src/mock_grpc.rs similarity index 69% rename from crates/astria-composer/src/executor/mock_grpc.rs rename to crates/astria-composer/src/mock_grpc.rs index 9519b8749..9e505270e 100644 --- a/crates/astria-composer/src/executor/mock_grpc.rs +++ b/crates/astria-composer/src/mock_grpc.rs @@ -4,61 +4,43 @@ use std::{ sync::Arc, }; -use astria_core::generated::{ - execution::v1alpha2::{ - execution_service_server::{ - ExecutionService, - ExecutionServiceServer, - }, - BatchGetBlocksRequest, - BatchGetBlocksResponse, - Block, - CommitmentState, - ExecuteBlockRequest, - ExecuteBlockResponse, - GenesisInfo, - GetBlockRequest, - GetCommitmentStateRequest, - GetGenesisInfoRequest, - UpdateCommitmentStateRequest, - }, - sequencerblock::v1alpha1::{ - sequencer_service_server::{ - SequencerService, - SequencerServiceServer, - }, - FilteredSequencerBlock, - GetFilteredSequencerBlockRequest, - GetPendingNonceRequest, - GetPendingNonceResponse, - GetSequencerBlockRequest, - SequencerBlock, +use astria_core::generated::execution::v1alpha2::{ + execution_service_server::{ + ExecutionService, + ExecutionServiceServer, }, + BatchGetBlocksRequest, + BatchGetBlocksResponse, + Block, + CommitmentState, + ExecuteBlockRequest, + ExecuteBlockResponse, + GenesisInfo, + GetBlockRequest, + GetCommitmentStateRequest, + GetGenesisInfoRequest, + UpdateCommitmentStateRequest, }; use astria_eyre::eyre::{ self, WrapErr as _, }; use astria_grpc_mock::{ - AnyMessage, - Match, + matcher::message_partial_pbjson, MockServer, }; use tokio::task::JoinHandle; -use tonic::{ - transport::Server, - Request, - Response, -}; +use tonic::transport::Server; -pub struct MockGrpc { +pub(crate) struct MockGrpc { _server: JoinHandle>, pub mock_server: MockServer, pub local_addr: SocketAddr, } impl MockGrpc { - pub async fn spawn() -> Self { + #[must_use] + pub(crate) async fn spawn() -> Self { use tokio_stream::wrappers::TcpListenerStream; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -68,17 +50,15 @@ impl MockGrpc { let server = { let execution_service = ExecutionServiceImpl::new(mock_server.clone()); - let sequencer_service = SequencerServiceImpl::new(mock_server.clone()); tokio::spawn(async move { Server::builder() .add_service(ExecutionServiceServer::new(execution_service)) - .add_service(SequencerServiceServer::new(sequencer_service)) .serve_with_incoming(TcpListenerStream::new(listener)) .await .wrap_err("gRPC server failed") }) }; - Self { + MockGrpc { _server: server, mock_server, local_addr, @@ -86,47 +66,6 @@ impl MockGrpc { } } -struct SequencerServiceImpl { - mock_server: MockServer, -} - -impl SequencerServiceImpl { - fn new(mock_server: MockServer) -> Self { - Self { - mock_server, - } - } -} - -// XXX: Manually implementing this trait instead of using the `define_and_impl_service!` macro -// because `GetSequencerBlockRequest` and `SequencerBlock` don't currently implement -// `serde::Serialize`. -#[tonic::async_trait] -impl SequencerService for SequencerServiceImpl { - async fn get_sequencer_block( - self: Arc, - _request: Request, - ) -> tonic::Result> { - unimplemented!() - } - - async fn get_filtered_sequencer_block( - self: Arc, - request: Request, - ) -> tonic::Result> { - self.mock_server - .handle_request("get_filtered_sequencer_block", request) - .await - } - - async fn get_pending_nonce( - self: Arc, - _request: Request, - ) -> tonic::Result> { - unimplemented!() - } -} - macro_rules! define_and_impl_service { (impl $trait:ident for $target:ident { $( ($rpc:ident: $request:ty => $response:ty) )* }) => { struct $target { @@ -251,8 +190,10 @@ macro_rules! mount_executed_block { $test_env.mount_execute_block( $mock_name.into(), ::serde_json::json!({ - "prevBlockHash": $parent, - "transactions": $included_transactions, + // TODO - figure out why its not matching? + // "prevBlockHash": BASE64_STANDARD.encode($parent), + // "simulateOnly": true, + // "transactions": $included_transactions, }), $crate::execute_block_response!( number: $number, @@ -291,18 +232,18 @@ impl TestExecutor { "get_commitment_state", astria_grpc_mock::matcher::message_type::(), ) - .respond_with(astria_grpc_mock::response::constant_response( - commitment_state, - )) - .expect(1..) - .mount(&self.mock_grpc.mock_server) - .await; + .respond_with(astria_grpc_mock::response::constant_response( + commitment_state, + )) + .expect(1..) + .mount(&self.mock_grpc.mock_server) + .await; } pub async fn mount_execute_block( &self, mock_name: Option<&str>, - _expected_pbjson: S, + expected_pbjson: S, response: ExecuteBlockResponse, ) -> astria_grpc_mock::MockGuard { use astria_grpc_mock::{ @@ -310,8 +251,9 @@ impl TestExecutor { Mock, }; - let mut mock = Mock::for_rpc_given("execute_block", AlwaysMatches {}) - .respond_with(constant_response(response)); + let mut mock = + Mock::for_rpc_given("execute_block", message_partial_pbjson(&expected_pbjson)) + .respond_with(constant_response(response)); if let Some(name) = mock_name { mock = mock.with_name(name); } @@ -320,12 +262,3 @@ impl TestExecutor { .await } } - -// TODO - this is a hack to bypass request body matching. Fix this -struct AlwaysMatches(); - -impl Match for AlwaysMatches { - fn matches(&self, _req: &Request) -> bool { - true - } -} diff --git a/crates/astria-composer/tests/blackbox/api.rs b/crates/astria-composer/tests/blackbox/api.rs index 9bcdae13f..fa333c19a 100644 --- a/crates/astria-composer/tests/blackbox/api.rs +++ b/crates/astria-composer/tests/blackbox/api.rs @@ -5,4 +5,4 @@ async fn readyz_with_one_rollup() { // environment. If this future return then `readyz` must have // returned `status: ok`. let _test_composer = spawn_composer("test1").await; -} \ No newline at end of file +} diff --git a/crates/astria-composer/tests/blackbox/geth_collector.rs b/crates/astria-composer/tests/blackbox/geth_collector.rs index c9bbfa0fb..d32df576f 100644 --- a/crates/astria-composer/tests/blackbox/geth_collector.rs +++ b/crates/astria-composer/tests/blackbox/geth_collector.rs @@ -1,12 +1,21 @@ use std::time::Duration; -use astria_core::{generated::protocol::accounts::v1alpha1::NonceResponse, primitive::v1::RollupId, Protobuf}; +use astria_composer::{ + mount_executed_block, + mount_get_commitment_state, +}; +use astria_core::{ + generated::protocol::accounts::v1alpha1::NonceResponse, + primitive::v1::RollupId, + protocol::transaction::v1alpha1::action::SequenceAction, + sequencerblock::v1alpha1::block::RollupData, + Protobuf, +}; use ethers::types::Transaction; -use futures::future::join; -use futures::join; -use astria_composer::{mount_executed_block, mount_get_commitment_state}; -use astria_core::protocol::transaction::v1alpha1::action::SequenceAction; -use astria_core::sequencerblock::v1alpha1::block::RollupData; +use futures::{ + future::join, + join, +}; use crate::helper::{ mount_broadcast_tx_sync_invalid_nonce_mock, @@ -57,17 +66,15 @@ async fn tx_from_one_rollup_is_received_by_sequencer() { parent: soft_parent_hash.to_vec(), ); - test_composer.rollup_nodes["test1"] - .push_tx(tx) - .unwrap(); + test_composer.rollup_nodes["test1"].push_tx(tx).unwrap(); // wait for 1 sequencer block time to make sure the bundle is preempted tokio::time::timeout( Duration::from_millis(test_composer.cfg.block_time_ms), join( mock_guard.wait_until_satisfied(), - execute_block.wait_until_satisfied() - ) + execute_block.wait_until_satisfied(), + ), ) .await .expect("mocked sequencer should have received a broadcast message from composer"); @@ -124,9 +131,7 @@ async fn collector_restarts_after_exit() { parent: soft_parent_hash.to_vec(), ); - test_composer.rollup_nodes["test1"] - .push_tx(tx) - .unwrap(); + test_composer.rollup_nodes["test1"].push_tx(tx).unwrap(); // wait for 1 sequencer block time to make sure the bundle is preempted // we added an extra 1000ms to the block time to make sure the collector has restarted @@ -203,9 +208,7 @@ async fn invalid_nonce_causes_resubmission_under_different_nonce() { // Push a tx to the rollup node so that it is picked up by the composer and submitted with the // stored nonce of 0, triggering the nonce refetch process - test_composer.rollup_nodes["test1"] - .push_tx(tx) - .unwrap(); + test_composer.rollup_nodes["test1"].push_tx(tx).unwrap(); // wait for 1 sequencer block time to make sure the bundle is preempted tokio::time::timeout( @@ -277,8 +280,8 @@ async fn single_rollup_tx_payload_integrity() { Duration::from_millis(test_composer.cfg.block_time_ms), join( mock_guard.wait_until_satisfied(), - execute_block.wait_until_satisfied() - ) + execute_block.wait_until_satisfied(), + ), ) .await .expect("mocked sequencer should have received a broadcast message from composer"); diff --git a/crates/astria-composer/tests/blackbox/grpc_collector.rs b/crates/astria-composer/tests/blackbox/grpc_collector.rs index ff9360f80..dda4cc405 100644 --- a/crates/astria-composer/tests/blackbox/grpc_collector.rs +++ b/crates/astria-composer/tests/blackbox/grpc_collector.rs @@ -1,16 +1,23 @@ use std::time::Duration; use bytes::Bytes; -use astria_core::{generated::{ - composer::v1alpha1::{ - grpc_collector_service_client::GrpcCollectorServiceClient, - SubmitRollupTransactionRequest, +use astria_composer::{ + mount_executed_block, + mount_get_commitment_state, +}; +use astria_core::{ + generated::{ + composer::v1alpha1::{ + grpc_collector_service_client::GrpcCollectorServiceClient, + SubmitRollupTransactionRequest, + }, + protocol::accounts::v1alpha1::NonceResponse, }, - protocol::accounts::v1alpha1::NonceResponse, -}, primitive::v1::RollupId, Protobuf}; + primitive::v1::RollupId, + sequencerblock::v1alpha1::block::RollupData, + Protobuf, +}; use ethers::prelude::Transaction; -use astria_composer::{mount_executed_block, mount_get_commitment_state}; -use astria_core::sequencerblock::v1alpha1::block::RollupData; use crate::helper::{ mount_broadcast_tx_sync_invalid_nonce_mock, @@ -226,7 +233,6 @@ async fn single_rollup_tx_payload_integrity() { parent: soft_parent_hash.to_vec(), ); - // send sequence action request to the grpc generic collector let mut composer_client = GrpcCollectorServiceClient::connect(format!( "http://{}", diff --git a/crates/astria-composer/tests/blackbox/helper/mod.rs b/crates/astria-composer/tests/blackbox/helper/mod.rs index 119d5f859..e40f1beb0 100644 --- a/crates/astria-composer/tests/blackbox/helper/mod.rs +++ b/crates/astria-composer/tests/blackbox/helper/mod.rs @@ -47,10 +47,14 @@ use wiremock::{ Request, ResponseTemplate, }; -use crate::helper::mock_grpc::{MockGrpc, TestExecutor}; -pub mod mock_sequencer; +use crate::helper::mock_grpc::{ + MockGrpc, + TestExecutor, +}; + pub mod mock_grpc; +pub mod mock_sequencer; static TELEMETRY: Lazy<()> = Lazy::new(|| { // This config can be meaningless - it's only used inside `try_init` to init the metrics, but we @@ -60,7 +64,8 @@ static TELEMETRY: Lazy<()> = Lazy::new(|| { api_listen_addr: SocketAddr::new(IpAddr::from([0, 0, 0, 0]), 0), sequencer_url: String::new(), sequencer_chain_id: String::new(), - rollups: String::new(), + rollup: "".to_string(), + rollup_websocket_url: "".to_string(), private_key_file: String::new(), sequencer_address_prefix: String::new(), block_time_ms: 0, @@ -73,6 +78,7 @@ static TELEMETRY: Lazy<()> = Lazy::new(|| { pretty_print: false, grpc_addr: SocketAddr::new(IpAddr::from([0, 0, 0, 0]), 0), fee_asset: Denom::IbcPrefixed(IbcPrefixed::new([0; 32])), + execution_api_url: "".to_string(), }; if std::env::var_os("TEST_LOG").is_some() { let filter_directives = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".into()); @@ -100,11 +106,8 @@ pub struct TestComposer { pub sequencer: wiremock::MockServer, pub setup_guard: MockGuard, pub grpc_collector_addr: SocketAddr, -<<<<<<< HEAD pub metrics_handle: metrics::Handle, -======= pub test_executor: TestExecutor ->>>>>>> f151354e (initial version of trusted builder mvp) } /// Spawns composer in a test environment. @@ -173,13 +176,10 @@ pub async fn spawn_composer(rollup_name: &str) -> TestComposer { sequencer, setup_guard: sequencer_setup_guard, grpc_collector_addr, -<<<<<<< HEAD metrics_handle, -======= test_executor: TestExecutor { mock_grpc: mock_execution_api_server } ->>>>>>> f151354e (initial version of trusted builder mvp) } } diff --git a/crates/astria-core/src/composer/v1alpha1/mod.rs b/crates/astria-core/src/composer/v1alpha1/mod.rs index 2b1e56495..2c2e95fd7 100644 --- a/crates/astria-core/src/composer/v1alpha1/mod.rs +++ b/crates/astria-core/src/composer/v1alpha1/mod.rs @@ -112,7 +112,7 @@ impl BuilderBundlePacketError { )] pub struct BuilderBundlePacket { bundle: BuilderBundle, - signature: Bytes, + signature: String, } impl BuilderBundlePacket { @@ -120,7 +120,7 @@ impl BuilderBundlePacket { &self.bundle } - pub fn signature(&self) -> Bytes { + pub fn signature(&self) -> String { self.signature.clone() } } @@ -152,14 +152,14 @@ impl Protobuf for BuilderBundlePacket { Ok(BuilderBundlePacket { bundle, - signature: Bytes::from(signature.clone()), + signature: signature.clone(), }) } fn to_raw(&self) -> Self::Raw { crate::generated::composer::v1alpha1::BuilderBundlePacket { bundle: Some(self.bundle.to_raw()), - signature: self.signature.clone().to_vec(), + signature: self.signature.clone(), } } } diff --git a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs index 49ae11e7e..0be5b4302 100644 --- a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs +++ b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs @@ -367,8 +367,8 @@ impl ::prost::Name for BuilderBundle { pub struct BuilderBundlePacket { #[prost(message, optional, tag = "1")] pub bundle: ::core::option::Option, - #[prost(bytes = "vec", tag = "2")] - pub signature: ::prost::alloc::vec::Vec, + #[prost(string, tag = "2")] + pub signature: ::prost::alloc::string::String, } impl ::prost::Name for BuilderBundlePacket { const NAME: &'static str = "BuilderBundlePacket"; diff --git a/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs b/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs index 571e5a588..2cccafb84 100644 --- a/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs +++ b/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs @@ -129,8 +129,7 @@ impl serde::Serialize for BuilderBundlePacket { struct_ser.serialize_field("bundle", v)?; } if !self.signature.is_empty() { - #[allow(clippy::needless_borrow)] - struct_ser.serialize_field("signature", pbjson::private::base64::encode(&self.signature).as_str())?; + struct_ser.serialize_field("signature", &self.signature)?; } struct_ser.end() } @@ -206,9 +205,7 @@ impl<'de> serde::Deserialize<'de> for BuilderBundlePacket { if signature__.is_some() { return Err(serde::de::Error::duplicate_field("signature")); } - signature__ = - Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) - ; + signature__ = Some(map_.next_value()?); } } } diff --git a/crates/astria-grpc-mock/src/matcher.rs b/crates/astria-grpc-mock/src/matcher.rs index e6ba265f0..325c0442b 100644 --- a/crates/astria-grpc-mock/src/matcher.rs +++ b/crates/astria-grpc-mock/src/matcher.rs @@ -20,7 +20,9 @@ impl Match for MessagePartialJsonMatcher { fn matches(&self, req: &tonic::Request) -> bool { let req_json = serde_json::to_value(req.get_ref().as_serialize()) .expect("can map provided gRPC request to JSON"); + println!("IN MATCH: req_json is {:?}", req_json); let config = assert_json_diff::Config::new(CompareMode::Inclusive); + println!("IN MATCH: self.0 is {:?}", self.0); assert_json_matches_no_panic(&req_json, &self.0, config).is_ok() } } diff --git a/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto b/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto index a8cc53734..3dc9ca4a8 100644 --- a/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto +++ b/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto @@ -11,5 +11,5 @@ message BuilderBundle { message BuilderBundlePacket { BuilderBundle bundle = 1; - bytes signature = 2; + string signature = 2; } From 661ca3e6f8a513db008b936e427c07e1c007ec30 Mon Sep 17 00:00:00 2001 From: Bharath Date: Thu, 29 Aug 2024 20:58:31 +0530 Subject: [PATCH 10/43] add message hash to proto --- crates/astria-composer/src/executor/mod.rs | 3 +++ .../astria-core/src/composer/v1alpha1/mod.rs | 9 ++++++++ .../src/generated/astria.composer.v1alpha1.rs | 4 +++- .../astria.composer.v1alpha1.serde.rs | 21 +++++++++++++++++++ .../composer/v1alpha1/trusted_builder.proto | 3 ++- 5 files changed, 38 insertions(+), 2 deletions(-) diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index dfffacf0d..897050283 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -10,6 +10,7 @@ use std::{ task::Poll, time::Duration, }; +use ethers::abi::AbiEncode; use astria_core::{ crypto::SigningKey, @@ -239,6 +240,7 @@ impl Executor { let private_key = ""; let wallet = ethers::signers::Wallet::from_str(private_key) .wrap_err("failed to parse private key")?; + let msg_hash = hash_message(encoded_builder_bundle.clone()); let signature = wallet .sign_message(encoded_builder_bundle.clone()) .await @@ -248,6 +250,7 @@ impl Executor { let mut builder_bundle_packet = BuilderBundlePacket { bundle: Some(builder_bundle), signature: signature.to_string(), + message_hash: msg_hash.encode() }; let encoded_builder_bundle_packet = builder_bundle_packet.encode_to_vec(); diff --git a/crates/astria-core/src/composer/v1alpha1/mod.rs b/crates/astria-core/src/composer/v1alpha1/mod.rs index 2c2e95fd7..4b01c34f9 100644 --- a/crates/astria-core/src/composer/v1alpha1/mod.rs +++ b/crates/astria-core/src/composer/v1alpha1/mod.rs @@ -1,4 +1,5 @@ use bytes::Bytes; +use prost::Message; use crate::{ sequencerblock::v1alpha1::block::{ @@ -113,6 +114,7 @@ impl BuilderBundlePacketError { pub struct BuilderBundlePacket { bundle: BuilderBundle, signature: String, + message_hash: Bytes, } impl BuilderBundlePacket { @@ -123,6 +125,10 @@ impl BuilderBundlePacket { pub fn signature(&self) -> String { self.signature.clone() } + + pub fn message_hash(&self) -> Bytes { + self.message_hash.clone() + } } impl From for crate::generated::composer::v1alpha1::BuilderBundlePacket { @@ -139,6 +145,7 @@ impl Protobuf for BuilderBundlePacket { let crate::generated::composer::v1alpha1::BuilderBundlePacket { bundle, signature, + message_hash, } = raw; let bundle = { @@ -153,12 +160,14 @@ impl Protobuf for BuilderBundlePacket { Ok(BuilderBundlePacket { bundle, signature: signature.clone(), + message_hash: Bytes::from(message_hash.clone()) }) } fn to_raw(&self) -> Self::Raw { crate::generated::composer::v1alpha1::BuilderBundlePacket { bundle: Some(self.bundle.to_raw()), + message_hash: self.message_hash().encode_to_vec(), signature: self.signature.clone(), } } diff --git a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs index 0be5b4302..2606b2938 100644 --- a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs +++ b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs @@ -367,7 +367,9 @@ impl ::prost::Name for BuilderBundle { pub struct BuilderBundlePacket { #[prost(message, optional, tag = "1")] pub bundle: ::core::option::Option, - #[prost(string, tag = "2")] + #[prost(bytes = "vec", tag = "2")] + pub message_hash: ::prost::alloc::vec::Vec, + #[prost(string, tag = "3")] pub signature: ::prost::alloc::string::String, } impl ::prost::Name for BuilderBundlePacket { diff --git a/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs b/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs index 2cccafb84..662376988 100644 --- a/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs +++ b/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs @@ -121,6 +121,9 @@ impl serde::Serialize for BuilderBundlePacket { if self.bundle.is_some() { len += 1; } + if !self.message_hash.is_empty() { + len += 1; + } if !self.signature.is_empty() { len += 1; } @@ -128,6 +131,10 @@ impl serde::Serialize for BuilderBundlePacket { if let Some(v) = self.bundle.as_ref() { struct_ser.serialize_field("bundle", v)?; } + if !self.message_hash.is_empty() { + #[allow(clippy::needless_borrow)] + struct_ser.serialize_field("messageHash", pbjson::private::base64::encode(&self.message_hash).as_str())?; + } if !self.signature.is_empty() { struct_ser.serialize_field("signature", &self.signature)?; } @@ -142,12 +149,15 @@ impl<'de> serde::Deserialize<'de> for BuilderBundlePacket { { const FIELDS: &[&str] = &[ "bundle", + "message_hash", + "messageHash", "signature", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { Bundle, + MessageHash, Signature, } impl<'de> serde::Deserialize<'de> for GeneratedField { @@ -171,6 +181,7 @@ impl<'de> serde::Deserialize<'de> for BuilderBundlePacket { { match value { "bundle" => Ok(GeneratedField::Bundle), + "messageHash" | "message_hash" => Ok(GeneratedField::MessageHash), "signature" => Ok(GeneratedField::Signature), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } @@ -192,6 +203,7 @@ impl<'de> serde::Deserialize<'de> for BuilderBundlePacket { V: serde::de::MapAccess<'de>, { let mut bundle__ = None; + let mut message_hash__ = None; let mut signature__ = None; while let Some(k) = map_.next_key()? { match k { @@ -201,6 +213,14 @@ impl<'de> serde::Deserialize<'de> for BuilderBundlePacket { } bundle__ = map_.next_value()?; } + GeneratedField::MessageHash => { + if message_hash__.is_some() { + return Err(serde::de::Error::duplicate_field("messageHash")); + } + message_hash__ = + Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) + ; + } GeneratedField::Signature => { if signature__.is_some() { return Err(serde::de::Error::duplicate_field("signature")); @@ -211,6 +231,7 @@ impl<'de> serde::Deserialize<'de> for BuilderBundlePacket { } Ok(BuilderBundlePacket { bundle: bundle__, + message_hash: message_hash__.unwrap_or_default(), signature: signature__.unwrap_or_default(), }) } diff --git a/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto b/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto index 3dc9ca4a8..9a12e3219 100644 --- a/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto +++ b/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto @@ -11,5 +11,6 @@ message BuilderBundle { message BuilderBundlePacket { BuilderBundle bundle = 1; - string signature = 2; + bytes message_hash = 2; + string signature = 3; } From f3a05e10108821938000ad4a77f937ac3fb61cd4 Mon Sep 17 00:00:00 2001 From: Bharath Date: Mon, 12 Aug 2024 10:57:32 +0530 Subject: [PATCH 11/43] use string for message_hash --- crates/astria-composer/src/executor/mod.rs | 4 ++-- crates/astria-core/src/composer/v1alpha1/mod.rs | 8 ++++---- .../astria-core/src/generated/astria.composer.v1alpha1.rs | 4 ++-- .../src/generated/astria.composer.v1alpha1.serde.rs | 7 ++----- .../astria/composer/v1alpha1/trusted_builder.proto | 2 +- 5 files changed, 11 insertions(+), 14 deletions(-) diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index 897050283..0d5bbdb9a 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -237,7 +237,7 @@ impl Executor { }; let encoded_builder_bundle = builder_bundle.encode_to_vec(); - let private_key = ""; + let private_key = "0xd7c8dffd7a3898d1be53b5eccd6b1630fa8fe04fd30c5ecf700f1752c3e7e489"; let wallet = ethers::signers::Wallet::from_str(private_key) .wrap_err("failed to parse private key")?; let msg_hash = hash_message(encoded_builder_bundle.clone()); @@ -250,7 +250,7 @@ impl Executor { let mut builder_bundle_packet = BuilderBundlePacket { bundle: Some(builder_bundle), signature: signature.to_string(), - message_hash: msg_hash.encode() + message_hash: msg_hash.to_string() }; let encoded_builder_bundle_packet = builder_bundle_packet.encode_to_vec(); diff --git a/crates/astria-core/src/composer/v1alpha1/mod.rs b/crates/astria-core/src/composer/v1alpha1/mod.rs index 4b01c34f9..ecb053364 100644 --- a/crates/astria-core/src/composer/v1alpha1/mod.rs +++ b/crates/astria-core/src/composer/v1alpha1/mod.rs @@ -114,7 +114,7 @@ impl BuilderBundlePacketError { pub struct BuilderBundlePacket { bundle: BuilderBundle, signature: String, - message_hash: Bytes, + message_hash: String, } impl BuilderBundlePacket { @@ -126,7 +126,7 @@ impl BuilderBundlePacket { self.signature.clone() } - pub fn message_hash(&self) -> Bytes { + pub fn message_hash(&self) -> String { self.message_hash.clone() } } @@ -160,14 +160,14 @@ impl Protobuf for BuilderBundlePacket { Ok(BuilderBundlePacket { bundle, signature: signature.clone(), - message_hash: Bytes::from(message_hash.clone()) + message_hash: message_hash.clone() }) } fn to_raw(&self) -> Self::Raw { crate::generated::composer::v1alpha1::BuilderBundlePacket { bundle: Some(self.bundle.to_raw()), - message_hash: self.message_hash().encode_to_vec(), + message_hash: self.message_hash.clone(), signature: self.signature.clone(), } } diff --git a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs index 2606b2938..b688360e3 100644 --- a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs +++ b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs @@ -367,8 +367,8 @@ impl ::prost::Name for BuilderBundle { pub struct BuilderBundlePacket { #[prost(message, optional, tag = "1")] pub bundle: ::core::option::Option, - #[prost(bytes = "vec", tag = "2")] - pub message_hash: ::prost::alloc::vec::Vec, + #[prost(string, tag = "2")] + pub message_hash: ::prost::alloc::string::String, #[prost(string, tag = "3")] pub signature: ::prost::alloc::string::String, } diff --git a/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs b/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs index 662376988..4091ff283 100644 --- a/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs +++ b/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs @@ -132,8 +132,7 @@ impl serde::Serialize for BuilderBundlePacket { struct_ser.serialize_field("bundle", v)?; } if !self.message_hash.is_empty() { - #[allow(clippy::needless_borrow)] - struct_ser.serialize_field("messageHash", pbjson::private::base64::encode(&self.message_hash).as_str())?; + struct_ser.serialize_field("messageHash", &self.message_hash)?; } if !self.signature.is_empty() { struct_ser.serialize_field("signature", &self.signature)?; @@ -217,9 +216,7 @@ impl<'de> serde::Deserialize<'de> for BuilderBundlePacket { if message_hash__.is_some() { return Err(serde::de::Error::duplicate_field("messageHash")); } - message_hash__ = - Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) - ; + message_hash__ = Some(map_.next_value()?); } GeneratedField::Signature => { if signature__.is_some() { diff --git a/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto b/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto index 9a12e3219..f3015ac20 100644 --- a/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto +++ b/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto @@ -11,6 +11,6 @@ message BuilderBundle { message BuilderBundlePacket { BuilderBundle bundle = 1; - bytes message_hash = 2; + string message_hash = 2; string signature = 3; } From 801df9663c8bb906027e7c81356164a5c6c5dd23 Mon Sep 17 00:00:00 2001 From: Bharath Date: Mon, 12 Aug 2024 11:59:37 +0530 Subject: [PATCH 12/43] use bytes --- crates/astria-composer/src/executor/mod.rs | 4 ++-- crates/astria-core/src/composer/v1alpha1/mod.rs | 16 ++++++++-------- .../src/generated/astria.composer.v1alpha1.rs | 8 ++++---- .../generated/astria.composer.v1alpha1.serde.rs | 14 ++++++++++---- .../composer/v1alpha1/trusted_builder.proto | 4 ++-- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index 0d5bbdb9a..0425ce283 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -249,8 +249,8 @@ impl Executor { // create a top of block bundle let mut builder_bundle_packet = BuilderBundlePacket { bundle: Some(builder_bundle), - signature: signature.to_string(), - message_hash: msg_hash.to_string() + signature: signature.to_vec(), + message_hash: msg_hash.encode() }; let encoded_builder_bundle_packet = builder_bundle_packet.encode_to_vec(); diff --git a/crates/astria-core/src/composer/v1alpha1/mod.rs b/crates/astria-core/src/composer/v1alpha1/mod.rs index ecb053364..74c1943f3 100644 --- a/crates/astria-core/src/composer/v1alpha1/mod.rs +++ b/crates/astria-core/src/composer/v1alpha1/mod.rs @@ -113,8 +113,8 @@ impl BuilderBundlePacketError { )] pub struct BuilderBundlePacket { bundle: BuilderBundle, - signature: String, - message_hash: String, + signature: Bytes, + message_hash: Bytes, } impl BuilderBundlePacket { @@ -122,11 +122,11 @@ impl BuilderBundlePacket { &self.bundle } - pub fn signature(&self) -> String { + pub fn signature(&self) -> Bytes { self.signature.clone() } - pub fn message_hash(&self) -> String { + pub fn message_hash(&self) -> Bytes { self.message_hash.clone() } } @@ -159,16 +159,16 @@ impl Protobuf for BuilderBundlePacket { Ok(BuilderBundlePacket { bundle, - signature: signature.clone(), - message_hash: message_hash.clone() + signature: Bytes::from(signature.clone()), + message_hash: Bytes::from(message_hash.clone()) }) } fn to_raw(&self) -> Self::Raw { crate::generated::composer::v1alpha1::BuilderBundlePacket { bundle: Some(self.bundle.to_raw()), - message_hash: self.message_hash.clone(), - signature: self.signature.clone(), + message_hash: self.message_hash.to_vec(), + signature: self.signature.to_vec(), } } } diff --git a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs index b688360e3..925b821fa 100644 --- a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs +++ b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs @@ -367,10 +367,10 @@ impl ::prost::Name for BuilderBundle { pub struct BuilderBundlePacket { #[prost(message, optional, tag = "1")] pub bundle: ::core::option::Option, - #[prost(string, tag = "2")] - pub message_hash: ::prost::alloc::string::String, - #[prost(string, tag = "3")] - pub signature: ::prost::alloc::string::String, + #[prost(bytes = "vec", tag = "2")] + pub message_hash: ::prost::alloc::vec::Vec, + #[prost(bytes = "vec", tag = "3")] + pub signature: ::prost::alloc::vec::Vec, } impl ::prost::Name for BuilderBundlePacket { const NAME: &'static str = "BuilderBundlePacket"; diff --git a/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs b/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs index 4091ff283..b5d5715ff 100644 --- a/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs +++ b/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs @@ -132,10 +132,12 @@ impl serde::Serialize for BuilderBundlePacket { struct_ser.serialize_field("bundle", v)?; } if !self.message_hash.is_empty() { - struct_ser.serialize_field("messageHash", &self.message_hash)?; + #[allow(clippy::needless_borrow)] + struct_ser.serialize_field("messageHash", pbjson::private::base64::encode(&self.message_hash).as_str())?; } if !self.signature.is_empty() { - struct_ser.serialize_field("signature", &self.signature)?; + #[allow(clippy::needless_borrow)] + struct_ser.serialize_field("signature", pbjson::private::base64::encode(&self.signature).as_str())?; } struct_ser.end() } @@ -216,13 +218,17 @@ impl<'de> serde::Deserialize<'de> for BuilderBundlePacket { if message_hash__.is_some() { return Err(serde::de::Error::duplicate_field("messageHash")); } - message_hash__ = Some(map_.next_value()?); + message_hash__ = + Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) + ; } GeneratedField::Signature => { if signature__.is_some() { return Err(serde::de::Error::duplicate_field("signature")); } - signature__ = Some(map_.next_value()?); + signature__ = + Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) + ; } } } diff --git a/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto b/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto index f3015ac20..5f319efe0 100644 --- a/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto +++ b/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto @@ -11,6 +11,6 @@ message BuilderBundle { message BuilderBundlePacket { BuilderBundle bundle = 1; - string message_hash = 2; - string signature = 3; + bytes message_hash = 2; + bytes signature = 3; } From f915c065f504bb74c19d5f5924b23710b305fa53 Mon Sep 17 00:00:00 2001 From: Bharath Date: Mon, 12 Aug 2024 13:35:37 +0530 Subject: [PATCH 13/43] remove message hash --- crates/astria-composer/src/executor/mod.rs | 13 +----------- .../astria-core/src/composer/v1alpha1/mod.rs | 8 ------- .../src/generated/astria.composer.v1alpha1.rs | 2 -- .../astria.composer.v1alpha1.serde.rs | 21 ------------------- .../composer/v1alpha1/trusted_builder.proto | 1 - 5 files changed, 1 insertion(+), 44 deletions(-) diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index 0425ce283..06c78e888 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -236,21 +236,10 @@ impl Executor { parent_hash: bundle_simulation_result.parent_hash().to_vec(), }; - let encoded_builder_bundle = builder_bundle.encode_to_vec(); - let private_key = "0xd7c8dffd7a3898d1be53b5eccd6b1630fa8fe04fd30c5ecf700f1752c3e7e489"; - let wallet = ethers::signers::Wallet::from_str(private_key) - .wrap_err("failed to parse private key")?; - let msg_hash = hash_message(encoded_builder_bundle.clone()); - let signature = wallet - .sign_message(encoded_builder_bundle.clone()) - .await - .wrap_err("failed to sign builder bundle packet")?; - // create a top of block bundle let mut builder_bundle_packet = BuilderBundlePacket { bundle: Some(builder_bundle), - signature: signature.to_vec(), - message_hash: msg_hash.encode() + signature: vec![], }; let encoded_builder_bundle_packet = builder_bundle_packet.encode_to_vec(); diff --git a/crates/astria-core/src/composer/v1alpha1/mod.rs b/crates/astria-core/src/composer/v1alpha1/mod.rs index 74c1943f3..3eebd989a 100644 --- a/crates/astria-core/src/composer/v1alpha1/mod.rs +++ b/crates/astria-core/src/composer/v1alpha1/mod.rs @@ -114,7 +114,6 @@ impl BuilderBundlePacketError { pub struct BuilderBundlePacket { bundle: BuilderBundle, signature: Bytes, - message_hash: Bytes, } impl BuilderBundlePacket { @@ -125,10 +124,6 @@ impl BuilderBundlePacket { pub fn signature(&self) -> Bytes { self.signature.clone() } - - pub fn message_hash(&self) -> Bytes { - self.message_hash.clone() - } } impl From for crate::generated::composer::v1alpha1::BuilderBundlePacket { @@ -145,7 +140,6 @@ impl Protobuf for BuilderBundlePacket { let crate::generated::composer::v1alpha1::BuilderBundlePacket { bundle, signature, - message_hash, } = raw; let bundle = { @@ -160,14 +154,12 @@ impl Protobuf for BuilderBundlePacket { Ok(BuilderBundlePacket { bundle, signature: Bytes::from(signature.clone()), - message_hash: Bytes::from(message_hash.clone()) }) } fn to_raw(&self) -> Self::Raw { crate::generated::composer::v1alpha1::BuilderBundlePacket { bundle: Some(self.bundle.to_raw()), - message_hash: self.message_hash.to_vec(), signature: self.signature.to_vec(), } } diff --git a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs index 925b821fa..fa6a5072b 100644 --- a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs +++ b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs @@ -367,8 +367,6 @@ impl ::prost::Name for BuilderBundle { pub struct BuilderBundlePacket { #[prost(message, optional, tag = "1")] pub bundle: ::core::option::Option, - #[prost(bytes = "vec", tag = "2")] - pub message_hash: ::prost::alloc::vec::Vec, #[prost(bytes = "vec", tag = "3")] pub signature: ::prost::alloc::vec::Vec, } diff --git a/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs b/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs index b5d5715ff..571e5a588 100644 --- a/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs +++ b/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs @@ -121,9 +121,6 @@ impl serde::Serialize for BuilderBundlePacket { if self.bundle.is_some() { len += 1; } - if !self.message_hash.is_empty() { - len += 1; - } if !self.signature.is_empty() { len += 1; } @@ -131,10 +128,6 @@ impl serde::Serialize for BuilderBundlePacket { if let Some(v) = self.bundle.as_ref() { struct_ser.serialize_field("bundle", v)?; } - if !self.message_hash.is_empty() { - #[allow(clippy::needless_borrow)] - struct_ser.serialize_field("messageHash", pbjson::private::base64::encode(&self.message_hash).as_str())?; - } if !self.signature.is_empty() { #[allow(clippy::needless_borrow)] struct_ser.serialize_field("signature", pbjson::private::base64::encode(&self.signature).as_str())?; @@ -150,15 +143,12 @@ impl<'de> serde::Deserialize<'de> for BuilderBundlePacket { { const FIELDS: &[&str] = &[ "bundle", - "message_hash", - "messageHash", "signature", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { Bundle, - MessageHash, Signature, } impl<'de> serde::Deserialize<'de> for GeneratedField { @@ -182,7 +172,6 @@ impl<'de> serde::Deserialize<'de> for BuilderBundlePacket { { match value { "bundle" => Ok(GeneratedField::Bundle), - "messageHash" | "message_hash" => Ok(GeneratedField::MessageHash), "signature" => Ok(GeneratedField::Signature), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } @@ -204,7 +193,6 @@ impl<'de> serde::Deserialize<'de> for BuilderBundlePacket { V: serde::de::MapAccess<'de>, { let mut bundle__ = None; - let mut message_hash__ = None; let mut signature__ = None; while let Some(k) = map_.next_key()? { match k { @@ -214,14 +202,6 @@ impl<'de> serde::Deserialize<'de> for BuilderBundlePacket { } bundle__ = map_.next_value()?; } - GeneratedField::MessageHash => { - if message_hash__.is_some() { - return Err(serde::de::Error::duplicate_field("messageHash")); - } - message_hash__ = - Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) - ; - } GeneratedField::Signature => { if signature__.is_some() { return Err(serde::de::Error::duplicate_field("signature")); @@ -234,7 +214,6 @@ impl<'de> serde::Deserialize<'de> for BuilderBundlePacket { } Ok(BuilderBundlePacket { bundle: bundle__, - message_hash: message_hash__.unwrap_or_default(), signature: signature__.unwrap_or_default(), }) } diff --git a/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto b/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto index 5f319efe0..b9721f184 100644 --- a/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto +++ b/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto @@ -11,6 +11,5 @@ message BuilderBundle { message BuilderBundlePacket { BuilderBundle bundle = 1; - bytes message_hash = 2; bytes signature = 3; } From acc6c56b6accdb321bb1730f857a6717f5daa44f Mon Sep 17 00:00:00 2001 From: Bharath Date: Fri, 6 Sep 2024 14:18:14 +0530 Subject: [PATCH 14/43] fix warnings --- crates/astria-composer/local.env.example | 2 + crates/astria-composer/src/composer.rs | 1 + crates/astria-composer/src/config.rs | 37 +------------- .../astria-composer/src/executor/builder.rs | 3 ++ crates/astria-composer/src/executor/mod.rs | 17 ++++--- crates/astria-composer/src/executor/tests.rs | 9 ++-- crates/astria-composer/src/mock_grpc.rs | 13 +++-- .../tests/blackbox/geth_collector.rs | 10 ++-- .../tests/blackbox/grpc_collector.rs | 6 +-- .../tests/blackbox/helper/mod.rs | 49 +++++++++++-------- .../tests/blackbox/helpers/macros.rs | 4 +- .../tests/blackbox/helpers/mock_grpc.rs | 1 + .../astria-core/src/composer/v1alpha1/mod.rs | 1 - .../astria-core/src/execution/v1alpha2/mod.rs | 2 +- crates/astria-grpc-mock/src/matcher.rs | 2 - crates/astria-sequencer/src/app/tests_app.rs | 30 +++++------- 16 files changed, 81 insertions(+), 106 deletions(-) diff --git a/crates/astria-composer/local.env.example b/crates/astria-composer/local.env.example index 68065cf9e..e4ba12563 100644 --- a/crates/astria-composer/local.env.example +++ b/crates/astria-composer/local.env.example @@ -57,6 +57,8 @@ ASTRIA_COMPOSER_MAX_SUBMIT_INTERVAL_MS=2000 # key and nonce bytes ASTRIA_COMPOSER_MAX_BYTES_PER_BUNDLE=200000 +ASTRIA_COMPOSER_MAX_BUNDLE_SIZE=200000 + # Max amount of finished bundles that can be in the submission queue. # ASTRIA_COMPOSER_BUNDLE_QUEUE_CAPACITY * ASTRIA_COMPOSER_MAX_BYTES_PER_BUNDLE (e.g. # 40000 * 200KB=8GB) is the limit on how much memory the finished bundle queue can consume. diff --git a/crates/astria-composer/src/composer.rs b/crates/astria-composer/src/composer.rs index c16b06383..750dbc2e0 100644 --- a/crates/astria-composer/src/composer.rs +++ b/crates/astria-composer/src/composer.rs @@ -134,6 +134,7 @@ impl Composer { execution_api_url: cfg.execution_api_url.clone(), fee_asset: cfg.fee_asset.clone(), chain_name: cfg.rollup.clone(), + max_bundle_size: cfg.max_bundle_size, shutdown_token: shutdown_token.clone(), metrics, } diff --git a/crates/astria-composer/src/config.rs b/crates/astria-composer/src/config.rs index 6f24f19b4..586db264a 100644 --- a/crates/astria-composer/src/config.rs +++ b/crates/astria-composer/src/config.rs @@ -5,14 +5,6 @@ use serde::{ Serialize, }; -<<<<<<< HEAD -use crate::rollup::{ - ParseError, - Rollup, -}; - -======= ->>>>>>> f151354e (initial version of trusted builder mvp) // this is a config, may have many boolean values #[allow(clippy::struct_excessive_bools)] #[derive(Debug, Deserialize, Serialize)] @@ -76,35 +68,10 @@ pub struct Config { /// The URL of the execution API server pub execution_api_url: String, -} -<<<<<<< HEAD -impl Config { - /// Returns a map of rollup names to rollup URLs. - /// - /// # Errors - /// - /// Returns an error if parsing fails. - pub fn parse_rollups(&self) -> Result, ParseError> { - self.rollups - .split(',') - .filter(|s| !s.is_empty()) - .map(|s| Rollup::parse(s).map(Rollup::into_parts)) - .collect::, _>>() - } + /// The maximum possible size of a bundle + pub max_bundle_size: usize, } -======= -// impl Config { -// pub(crate) fn parse_rollups(&self) -> astria_eyre::eyre::Result> { -// self.rollups -// .split(',') -// .filter(|s| !s.is_empty()) -// .map(|s| Rollup::parse(s).map(Rollup::into_parts)) -// .collect::, _>>() -// .wrap_err("failed parsing provided :: pairs as rollups") -// } -// } ->>>>>>> f151354e (initial version of trusted builder mvp) impl config::Config for Config { const PREFIX: &'static str = "ASTRIA_COMPOSER_"; diff --git a/crates/astria-composer/src/executor/builder.rs b/crates/astria-composer/src/executor/builder.rs index 059294e36..e96d4a136 100644 --- a/crates/astria-composer/src/executor/builder.rs +++ b/crates/astria-composer/src/executor/builder.rs @@ -43,6 +43,7 @@ pub(crate) struct Builder { pub(crate) execution_api_url: String, pub(crate) chain_name: String, pub(crate) fee_asset: asset::Denom, + pub(crate) max_bundle_size: usize, pub(crate) metrics: &'static Metrics, } @@ -60,6 +61,7 @@ impl Builder { execution_api_url, chain_name, fee_asset, + max_bundle_size, metrics, } = self; let sequencer_client = sequencer_client::HttpClient::new(sequencer_url.as_str()) @@ -102,6 +104,7 @@ impl Builder { shutdown_token, rollup_id, fee_asset, + max_bundle_size, metrics, }, executor::Handle::new(serialized_rollup_transaction_tx), diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index 06c78e888..053cb121c 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -1,4 +1,3 @@ -use std::str::FromStr; /// ! The `Executor` is responsible for: /// - Nonce management /// - Transaction signing @@ -10,7 +9,6 @@ use std::{ task::Poll, time::Duration, }; -use ethers::abi::AbiEncode; use astria_core::{ crypto::SigningKey, @@ -36,7 +34,11 @@ use astria_core::{ }, Protobuf, }; -use astria_eyre::eyre::{self, eyre, WrapErr as _}; +use astria_eyre::eyre::{ + self, + eyre, + WrapErr as _, +}; use futures::{ future::{ self, @@ -161,6 +163,8 @@ pub(super) struct Executor { rollup_id: RollupId, // The asset used for sequencer fees fee_asset: asset::Denom, + // The maximum possible size for a bundle so that it can fit into a block + max_bundle_size: usize, metrics: &'static Metrics, } @@ -236,8 +240,10 @@ impl Executor { parent_hash: bundle_simulation_result.parent_hash().to_vec(), }; + // TODO - bundle signing + // create a top of block bundle - let mut builder_bundle_packet = BuilderBundlePacket { + let builder_bundle_packet = BuilderBundlePacket { bundle: Some(builder_bundle), signature: vec![], }; @@ -245,8 +251,7 @@ impl Executor { // we can give the BuilderBundlePacket the highest bundle max size possible // since this is the only sequence action we are sending - // TODO - parameterize the max bundle size - let mut final_bundle = SizedBundle::new(200000); + let mut final_bundle = SizedBundle::new(self.max_bundle_size); if let Err(e) = final_bundle.try_push(SequenceAction { rollup_id: self.rollup_id, data: encoded_builder_bundle_packet.into(), diff --git a/crates/astria-composer/src/executor/tests.rs b/crates/astria-composer/src/executor/tests.rs index 89aa82eef..3724579cf 100644 --- a/crates/astria-composer/src/executor/tests.rs +++ b/crates/astria-composer/src/executor/tests.rs @@ -30,10 +30,6 @@ use astria_core::{ Protobuf, }; use astria_eyre::eyre; -use base64::{ - prelude::BASE64_STANDARD, - Engine, -}; use futures::future::join; use once_cell::sync::Lazy; use prost::{ @@ -174,6 +170,7 @@ async fn setup() -> (MockServer, Config, NamedTempFile, TestExecutor) { .unwrap() .to_ibc_prefixed() .into(), + max_bundle_size: 200000, execution_api_url: format!("http://{}", execution_api_server.local_addr), }; ( @@ -377,6 +374,7 @@ async fn full_bundle() { execution_api_url: cfg.execution_api_url, chain_name: cfg.rollup.clone(), fee_asset: cfg.fee_asset, + max_bundle_size: cfg.max_bundle_size, metrics, } .build() @@ -527,6 +525,7 @@ async fn bundle_triggered_by_block_timer() { execution_api_url: cfg.execution_api_url, chain_name: cfg.rollup.clone(), fee_asset: cfg.fee_asset.clone(), + max_bundle_size: cfg.max_bundle_size, metrics, } .build() @@ -681,6 +680,7 @@ async fn two_seq_actions_single_bundle() { execution_api_url: cfg.execution_api_url, chain_name: cfg.rollup.clone(), fee_asset: cfg.fee_asset.clone(), + max_bundle_size: cfg.max_bundle_size, metrics, } .build() @@ -841,6 +841,7 @@ async fn chain_id_mismatch_returns_error() { execution_api_url: cfg.execution_api_url, chain_name: rollup_name.to_string(), fee_asset: cfg.fee_asset, + max_bundle_size: cfg.max_bundle_size, metrics, } .build() diff --git a/crates/astria-composer/src/mock_grpc.rs b/crates/astria-composer/src/mock_grpc.rs index 9e505270e..c332cfed0 100644 --- a/crates/astria-composer/src/mock_grpc.rs +++ b/crates/astria-composer/src/mock_grpc.rs @@ -34,12 +34,11 @@ use tonic::transport::Server; pub(crate) struct MockGrpc { _server: JoinHandle>, - pub mock_server: MockServer, - pub local_addr: SocketAddr, + pub(crate) mock_server: MockServer, + pub(crate) local_addr: SocketAddr, } impl MockGrpc { - #[must_use] pub(crate) async fn spawn() -> Self { use tokio_stream::wrappers::TcpListenerStream; @@ -222,12 +221,12 @@ macro_rules! mount_executed_block { }; } -pub struct TestExecutor { - pub mock_grpc: MockGrpc, +pub(crate) struct TestExecutor { + pub(crate) mock_grpc: MockGrpc, } impl TestExecutor { - pub async fn mount_get_commitment_state(&self, commitment_state: CommitmentState) { + pub(crate) async fn mount_get_commitment_state(&self, commitment_state: CommitmentState) { astria_grpc_mock::Mock::for_rpc_given( "get_commitment_state", astria_grpc_mock::matcher::message_type::(), @@ -240,7 +239,7 @@ impl TestExecutor { .await; } - pub async fn mount_execute_block( + pub(crate) async fn mount_execute_block( &self, mock_name: Option<&str>, expected_pbjson: S, diff --git a/crates/astria-composer/tests/blackbox/geth_collector.rs b/crates/astria-composer/tests/blackbox/geth_collector.rs index d32df576f..cd5b5a319 100644 --- a/crates/astria-composer/tests/blackbox/geth_collector.rs +++ b/crates/astria-composer/tests/blackbox/geth_collector.rs @@ -7,15 +7,11 @@ use astria_composer::{ use astria_core::{ generated::protocol::accounts::v1alpha1::NonceResponse, primitive::v1::RollupId, - protocol::transaction::v1alpha1::action::SequenceAction, sequencerblock::v1alpha1::block::RollupData, Protobuf, }; use ethers::types::Transaction; -use futures::{ - future::join, - join, -}; +use futures::future::join; use crate::helper::{ mount_broadcast_tx_sync_invalid_nonce_mock, @@ -123,7 +119,7 @@ async fn collector_restarts_after_exit() { let data = tx.rlp().to_vec(); let rollup_data = vec![RollupData::SequencedData(data).to_raw()]; - let execute_block = mount_executed_block!(test_executor, + let _execute_block = mount_executed_block!(test_executor, mock_name: "execute_block", number: soft_block_number, hash: soft_block_hash, @@ -198,7 +194,7 @@ async fn invalid_nonce_causes_resubmission_under_different_nonce() { let data = tx.rlp().to_vec(); let rollup_data = vec![RollupData::SequencedData(data).to_raw()]; - let execute_block = mount_executed_block!(test_executor, + let _execute_block = mount_executed_block!(test_executor, mock_name: "execute_block", number: soft_block_number, hash: soft_block_hash, diff --git a/crates/astria-composer/tests/blackbox/grpc_collector.rs b/crates/astria-composer/tests/blackbox/grpc_collector.rs index dda4cc405..e58b7e695 100644 --- a/crates/astria-composer/tests/blackbox/grpc_collector.rs +++ b/crates/astria-composer/tests/blackbox/grpc_collector.rs @@ -59,7 +59,7 @@ async fn tx_from_one_rollup_is_received_by_sequencer() { let data = tx.rlp().to_vec(); let rollup_data = vec![RollupData::SequencedData(Bytes::from(data)).to_raw()]; - let execute_block = mount_executed_block!(test_executor, + let _execute_block = mount_executed_block!(test_executor, mock_name: "execute_block", number: soft_block_number, hash: soft_block_hash, @@ -146,7 +146,7 @@ async fn invalid_nonce_causes_resubmission_under_different_nonce() { let data = tx.rlp().to_vec(); let rollup_data = vec![RollupData::SequencedData(Bytes::from(data)).to_raw()]; - let execute_block = mount_executed_block!(test_executor, + let _execute_block = mount_executed_block!(test_executor, mock_name: "execute_block", number: soft_block_number, hash: soft_block_hash, @@ -225,7 +225,7 @@ async fn single_rollup_tx_payload_integrity() { let data = tx.rlp().to_vec(); let rollup_data = vec![RollupData::SequencedData(Bytes::from(data)).to_raw()]; - let execute_block = mount_executed_block!(test_executor, + let _execute_block = mount_executed_block!(test_executor, mock_name: "execute_block", number: soft_block_number, hash: soft_block_hash, diff --git a/crates/astria-composer/tests/blackbox/helper/mod.rs b/crates/astria-composer/tests/blackbox/helper/mod.rs index e40f1beb0..9b1d1307b 100644 --- a/crates/astria-composer/tests/blackbox/helper/mod.rs +++ b/crates/astria-composer/tests/blackbox/helper/mod.rs @@ -14,22 +14,19 @@ use astria_composer::{ Metrics, }; use astria_core::{ - primitive::v1::{ - asset::{ - Denom, - IbcPrefixed, - }, - RollupId, - }, + composer::v1alpha1::BuilderBundle, + generated::composer::v1alpha1::BuilderBundlePacket, + primitive::v1::RollupId, protocol::{ abci::AbciErrorCode, transaction::v1alpha1::SignedTransaction, }, + sequencerblock::v1alpha1::block::RollupData, + Protobuf, }; use astria_eyre::eyre; use ethers::prelude::Transaction; use once_cell::sync::Lazy; -use telemetry::metrics; use tempfile::NamedTempFile; use tendermint_rpc::{ endpoint::broadcast::tx_sync, @@ -47,6 +44,7 @@ use wiremock::{ Request, ResponseTemplate, }; +use astria_core::primitive::v1::asset::{Denom, IbcPrefixed}; use crate::helper::mock_grpc::{ MockGrpc, @@ -79,6 +77,7 @@ static TELEMETRY: Lazy<()> = Lazy::new(|| { grpc_addr: SocketAddr::new(IpAddr::from([0, 0, 0, 0]), 0), fee_asset: Denom::IbcPrefixed(IbcPrefixed::new([0; 32])), execution_api_url: "".to_string(), + max_bundle_size: 0, }; if std::env::var_os("TEST_LOG").is_some() { let filter_directives = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".into()); @@ -151,6 +150,7 @@ pub async fn spawn_composer(rollup_name: &str) -> TestComposer { grpc_addr: "127.0.0.1:0".parse().unwrap(), fee_asset: "nria".parse().unwrap(), execution_api_url: format!("http://{}", mock_execution_api_server.local_addr), + max_bundle_size: 200000, rollup_websocket_url: rollup_websocket_url.to_string(), }; @@ -248,18 +248,27 @@ pub async fn mount_matcher_verifying_tx_integrity( expected_rlp: Transaction, ) -> MockGuard { let matcher = move |request: &Request| { - // let sequencer_tx = signed_tx_from_request(request); - // let sequence_action = sequencer_tx - // .actions() - // .first() - // .unwrap() - // .as_sequence() - // .unwrap(); - // - // let expected_rlp = expected_rlp.rlp().to_vec(); - // - // expected_rlp == sequence_action.data - true + let sequencer_tx = signed_tx_from_request(request); + let sequence_action = sequencer_tx + .actions() + .first() + .unwrap() + .as_sequence() + .unwrap(); + let seq_action_data = sequence_action.clone().data; + // unmarshall to BuilderBundlePacket + let builder_bundle_packet = + BuilderBundlePacket::decode(seq_action_data.as_slice()).unwrap(); + let builder_bundle = + BuilderBundle::try_from_raw(builder_bundle_packet.bundle.unwrap()).unwrap(); + let transaction = builder_bundle.transactions().first().unwrap(); + + if let RollupData::SequencedData(data) = transaction { + let expected_rlp = expected_rlp.rlp().to_vec(); + expected_rlp == data.clone() + } else { + false + } }; let jsonrpc_rsp = response::Wrapper::new_with_id( Id::Num(1), diff --git a/crates/astria-conductor/tests/blackbox/helpers/macros.rs b/crates/astria-conductor/tests/blackbox/helpers/macros.rs index 2994c626c..3ff8f80ac 100644 --- a/crates/astria-conductor/tests/blackbox/helpers/macros.rs +++ b/crates/astria-conductor/tests/blackbox/helpers/macros.rs @@ -15,7 +15,7 @@ macro_rules! block { #[macro_export] macro_rules! execute_block_response { - (number: $number:expr,hash: $hash:expr,parent: $parent:expr $(,)?, included_transactions:expr $(,)?) => { + (number: $number:expr,hash: $hash:expr,parent: $parent:expr) => { ::astria_core::generated::execution::v1alpha2::ExecuteBlockResponse { block: Some($crate::block!( number: $number, @@ -261,7 +261,7 @@ macro_rules! mount_executed_block { $crate::execute_block_response!( number: $number, hash: $hash, - parent: $parent, + parent: $parent ) ) .await diff --git a/crates/astria-conductor/tests/blackbox/helpers/mock_grpc.rs b/crates/astria-conductor/tests/blackbox/helpers/mock_grpc.rs index 003aa1c2a..64be2aa99 100644 --- a/crates/astria-conductor/tests/blackbox/helpers/mock_grpc.rs +++ b/crates/astria-conductor/tests/blackbox/helpers/mock_grpc.rs @@ -1,3 +1,4 @@ +#[allow(dead_code)] use std::{ net::SocketAddr, sync::Arc, diff --git a/crates/astria-core/src/composer/v1alpha1/mod.rs b/crates/astria-core/src/composer/v1alpha1/mod.rs index 3eebd989a..9b59b8421 100644 --- a/crates/astria-core/src/composer/v1alpha1/mod.rs +++ b/crates/astria-core/src/composer/v1alpha1/mod.rs @@ -1,5 +1,4 @@ use bytes::Bytes; -use prost::Message; use crate::{ sequencerblock::v1alpha1::block::{ diff --git a/crates/astria-core/src/execution/v1alpha2/mod.rs b/crates/astria-core/src/execution/v1alpha2/mod.rs index 185f5f721..f3498189b 100644 --- a/crates/astria-core/src/execution/v1alpha2/mod.rs +++ b/crates/astria-core/src/execution/v1alpha2/mod.rs @@ -310,7 +310,7 @@ impl Protobuf for ExecuteBlockResponse { let included_transactions = included_transactions .iter() - .map(|rollup_data| RollupData::try_from_raw_ref(rollup_data)) + .map(RollupData::try_from_raw_ref) .collect::, _>>() .map_err(Self::Error::invalid_rollup_data)?; diff --git a/crates/astria-grpc-mock/src/matcher.rs b/crates/astria-grpc-mock/src/matcher.rs index 325c0442b..e6ba265f0 100644 --- a/crates/astria-grpc-mock/src/matcher.rs +++ b/crates/astria-grpc-mock/src/matcher.rs @@ -20,9 +20,7 @@ impl Match for MessagePartialJsonMatcher { fn matches(&self, req: &tonic::Request) -> bool { let req_json = serde_json::to_value(req.get_ref().as_serialize()) .expect("can map provided gRPC request to JSON"); - println!("IN MATCH: req_json is {:?}", req_json); let config = assert_json_diff::Config::new(CompareMode::Inclusive); - println!("IN MATCH: self.0 is {:?}", self.0); assert_json_matches_no_panic(&req_json, &self.0, config).is_ok() } } diff --git a/crates/astria-sequencer/src/app/tests_app.rs b/crates/astria-sequencer/src/app/tests_app.rs index 3974aa67e..e29655e8f 100644 --- a/crates/astria-sequencer/src/app/tests_app.rs +++ b/crates/astria-sequencer/src/app/tests_app.rs @@ -1,24 +1,17 @@ use std::collections::HashMap; -use astria_core::{ - primitive::v1::{ - asset::TracePrefixed, - RollupId, +use astria_core::{primitive::v1::{ + asset::TracePrefixed, + RollupId, +}, Protobuf, protocol::transaction::v1alpha1::{ + action::{ + BridgeLockAction, + SequenceAction, + TransferAction, }, - protocol::{ - genesis::v1alpha1::Account, - transaction::v1alpha1::{ - action::{ - BridgeLockAction, - SequenceAction, - TransferAction, - }, - TransactionParams, - UnsignedTransaction, - }, - }, - sequencerblock::v1alpha1::block::Deposit, -}; + TransactionParams, + UnsignedTransaction, +}, sequencerblock::v1alpha1::block::Deposit}; use cnidarium::StateDelta; use prost::{ bytes::Bytes, @@ -41,6 +34,7 @@ use tendermint::{ Hash, Time, }; +use astria_core::protocol::genesis::v1alpha1::Account; use super::*; use crate::{ From 977666fe45aa9821082883bb79240c9f2c24f869 Mon Sep 17 00:00:00 2001 From: Bharath Date: Fri, 6 Sep 2024 14:18:38 +0530 Subject: [PATCH 15/43] final additions --- charts/composer/templates/configmap.yaml | 7 ++-- charts/composer/values.yaml | 5 ++- charts/evm-rollup/values.yaml | 36 +++++++++---------- charts/sequencer/values.yaml | 2 +- crates/astria-composer/local.env.example | 2 ++ crates/astria-composer/src/executor/client.rs | 6 +--- crates/astria-composer/src/executor/mod.rs | 9 +++-- .../astria-composer/src/executor/simulator.rs | 15 ++++++-- crates/astria-composer/src/metrics.rs | 8 ++--- crates/astria-sequencer/src/app/tests_app.rs | 27 ++++++++------ .../composer/v1alpha1/trusted_builder.proto | 8 +++++ 11 files changed, 76 insertions(+), 49 deletions(-) diff --git a/charts/composer/templates/configmap.yaml b/charts/composer/templates/configmap.yaml index fb39f00b3..acdfab55e 100644 --- a/charts/composer/templates/configmap.yaml +++ b/charts/composer/templates/configmap.yaml @@ -9,11 +9,12 @@ data: ASTRIA_COMPOSER_GRPC_ADDR: "0.0.0.0:{{ .Values.ports.grpc }}" ASTRIA_COMPOSER_SEQUENCER_CHAIN_ID: "{{ tpl .Values.config.sequencerChainId . }}" ASTRIA_COMPOSER_SEQUENCER_URL: "{{ tpl .Values.config.sequencerRpc . }}" - ASTRIA_COMPOSER_ROLLUP: "astria" - ASTRIA_COMPOSER_ROLLUP_WEBSOCKET_URL: "ws://astria-evm-service.astria-dev-cluster.svc.cluster.local:8546" - ASTRIA_COMPOSER_EXECUTION_API_URL: "http://astria-evm-service.astria-dev-cluster.svc.cluster.local:50051" + ASTRIA_COMPOSER_ROLLUP: "{{ .Values.config.rollupName }}" + ASTRIA_COMPOSER_ROLLUP_WEBSOCKET_URL: "ws://{{ .Values.global.rollupName }}-evm-service.{{ default .Release.Namespace .Values.global.namespaceOverride }}.svc.cluster.local:8546" + ASTRIA_COMPOSER_EXECUTION_API_URL: "ws://{{ .Values.global.rollupName }}-evm-service.{{ default .Release.Namespace .Values.global.namespaceOverride }}.svc.cluster.local:50051" ASTRIA_COMPOSER_PRIVATE_KEY_FILE: "/var/secrets/{{ .Values.config.privateKey.secret.filename }}" ASTRIA_COMPOSER_MAX_BYTES_PER_BUNDLE: "{{ .Values.config.maxBytesPerBundle }}" + ASTRIA_COMPOSER_MAX_BUNDLE_SIZE: "{{ .Values.config.maxBundleSize }}" ASTRIA_COMPOSER_BUNDLE_QUEUE_CAPACITY: "{{ .Values.config.bundleQueueCapacity }}" ASTRIA_COMPOSER_MAX_SUBMIT_INTERVAL_MS: "{{ .Values.config.maxSubmitInterval }}" ASTRIA_COMPOSER_SEQUENCER_ADDRESS_PREFIX: "{{ .Values.config.sequencerAddressPrefix}}" diff --git a/charts/composer/values.yaml b/charts/composer/values.yaml index 42b5b192f..d49521bd7 100644 --- a/charts/composer/values.yaml +++ b/charts/composer/values.yaml @@ -14,6 +14,7 @@ images: config: logLevel: "info" maxBytesPerBundle: 200000 + maxBundleSize: 200000 bundleQueueCapacity: 40000 maxSubmitInterval: 1800 sequencerAddressPrefix: astria @@ -25,9 +26,7 @@ config: secret: filename: "key.hex" resourceName: "projects/$PROJECT_ID/secrets/sequencerPrivateKey/versions/latest" - rollups: - - name: "astria" - wsRpc: "ws://" + rollupName: "astria" otel: enabled: false diff --git a/charts/evm-rollup/values.yaml b/charts/evm-rollup/values.yaml index 6cd436082..74191a356 100644 --- a/charts/evm-rollup/values.yaml +++ b/charts/evm-rollup/values.yaml @@ -268,24 +268,24 @@ ingress: # - secretName: chart-example-tls # hosts: # - chart-example.local - exec-api: - enabled: true - hosts: - - exec-executor.{{ include "rollup.name" . }}.{{ .Values.ingress.hostname }} - path: / - pathType: Prefix - service: - name: '{{ include "rollup.name" . }}-evm-service' - port: - name: exec-grpc-svc - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - labels: {} - tls: {} - # - secretName: chart-example-tls - # hosts: - # - chart-example.local +# exec-api: +# enabled: true +# hosts: +# - exec-executor.{{ include "rollup.name" . }}.{{ .Values.ingress.hostname }} +# path: / +# pathType: Prefix +# service: +# name: '{{ include "rollup.name" . }}-evm-service' +# port: +# name: exec-grpc-svc +# annotations: {} +# # kubernetes.io/ingress.class: nginx +# # kubernetes.io/tls-acme: "true" +# labels: {} +# tls: {} +# # - secretName: chart-example-tls +# # hosts: +# # - chart-example.local # Default persistent storage values # NOTE - `rollupName` will be used with `persistentVolumeName` to generate names for kubernetes resources. diff --git a/charts/sequencer/values.yaml b/charts/sequencer/values.yaml index 719d66106..89dfb9acc 100644 --- a/charts/sequencer/values.yaml +++ b/charts/sequencer/values.yaml @@ -146,7 +146,7 @@ cometbft: timeoutPrevoteDelta: 500ms timeoutPrecommit: 1s timeoutPrecommitDelta: 500ms - timeoutCommit: 4000ms + timeoutCommit: 1500ms instrumentation: namespace: "astria_cometbft" diff --git a/crates/astria-composer/local.env.example b/crates/astria-composer/local.env.example index e4ba12563..78ec68a81 100644 --- a/crates/astria-composer/local.env.example +++ b/crates/astria-composer/local.env.example @@ -57,6 +57,8 @@ ASTRIA_COMPOSER_MAX_SUBMIT_INTERVAL_MS=2000 # key and nonce bytes ASTRIA_COMPOSER_MAX_BYTES_PER_BUNDLE=200000 +# Max size of a bundle of sequence actions in bytes which can fit into a block. +# This is the sum of the sizes of all the `SequenceAction`s. ASTRIA_COMPOSER_MAX_BUNDLE_SIZE=200000 # Max amount of finished bundles that can be in the submission queue. diff --git a/crates/astria-composer/src/executor/client.rs b/crates/astria-composer/src/executor/client.rs index 1bb9994e9..74a81c8e9 100644 --- a/crates/astria-composer/src/executor/client.rs +++ b/crates/astria-composer/src/executor/client.rs @@ -93,11 +93,7 @@ impl Client { let response = tryhard::retry_fn(|| { let mut client = self.inner.clone(); let request = request.clone(); - async move { - let res = client.execute_block(request).await; - println!("{:?}", res); - res - } + async move { client.execute_block(request).await } }) .with_config(retry_config()) .in_current_span() diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index 053cb121c..227371def 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -32,7 +32,6 @@ use astria_core::{ UnsignedTransaction, }, }, - Protobuf, }; use astria_eyre::eyre::{ self, @@ -221,6 +220,8 @@ impl Executor { bundle: SizedBundle, metrics: &'static Metrics, ) -> eyre::Result>> { + info!("Starting bundle simulation!"); + let bundle_simulator = self.bundle_simulator.clone(); // simulate the bundle @@ -232,9 +233,10 @@ impl Executor { let rollup_data_items: Vec = bundle_simulation_result .included_actions() .iter() - .map(|action| action.to_raw()) + .map(astria_core::Protobuf::to_raw) .collect(); + info!("Creating BuilderBundlePacket"); let builder_bundle = BuilderBundle { transactions: rollup_data_items, parent_hash: bundle_simulation_result.parent_hash().to_vec(), @@ -249,6 +251,7 @@ impl Executor { }; let encoded_builder_bundle_packet = builder_bundle_packet.encode_to_vec(); + info!("Created builder bundle packet: {:?}", builder_bundle_packet); // we can give the BuilderBundlePacket the highest bundle max size possible // since this is the only sequence action we are sending let mut final_bundle = SizedBundle::new(self.max_bundle_size); @@ -260,6 +263,8 @@ impl Executor { return Err(eyre::Report::from(e)); } + info!("Submitting the builder bundle packet to sequencer!"); + Ok(SubmitFut { client: self.sequencer_client.clone(), address: self.address, diff --git a/crates/astria-composer/src/executor/simulator.rs b/crates/astria-composer/src/executor/simulator.rs index cbfd7aee3..625dd83e5 100644 --- a/crates/astria-composer/src/executor/simulator.rs +++ b/crates/astria-composer/src/executor/simulator.rs @@ -1,3 +1,5 @@ +/// ! `BundleSimulator` is responsible for fetching the latest rollup commitment state +/// and simulating the given bundle on top of the latest soft block. use astria_core::{ sequencerblock::v1alpha1::block::RollupData, Protobuf, @@ -60,13 +62,16 @@ impl BundleSimulator { bundle: SizedBundle, ) -> eyre::Result { // call GetCommitmentState to get the soft block + info!("Calling GetCommitmentState!"); let commitment_state = self .execution_service_client .get_commitment_state_with_retry() .await .wrap_err("failed to get commitment state")?; + info!("Received CommitmentState of rollup"); let soft_block = commitment_state.soft(); + info!("Soft block hash is {:?}", soft_block.hash()); // convert the sized bundle actions to a list of Vec let actions: Vec> = bundle .into_actions() @@ -80,6 +85,7 @@ impl BundleSimulator { .filter(|data| !data.is_empty()) .collect(); + info!("Calling ExecuteBlock to simulate the bundle!"); // as long as the timestamp > parent block timestamp, the block will be successfully // created. It doesn't matter what timestamp we use anyway since we are not going to // commit the block to the chain. @@ -100,9 +106,14 @@ impl BundleSimulator { .await .wrap_err("failed to execute block")?; - info!("Using block hash instead of parent hash lmaoooo!"); + let included_transactions = execute_block_response.included_transactions(); + info!( + "Bundle simulated on top of {:?} and {:?} transactions were included", + soft_block.hash(), + included_transactions.len() + ); Ok(BundleSimulationResult::new( - execute_block_response.included_transactions().to_vec(), + included_transactions.to_vec(), soft_block.hash().clone(), )) } diff --git a/crates/astria-composer/src/metrics.rs b/crates/astria-composer/src/metrics.rs index 8f4cbe832..bec6ac13f 100644 --- a/crates/astria-composer/src/metrics.rs +++ b/crates/astria-composer/src/metrics.rs @@ -43,14 +43,14 @@ impl Metrics { <<<<<<< HEAD ======= #[must_use] - pub(crate) fn new<'a>(rollup_name: String) -> Self { + pub(crate) fn new(rollup_name: &str) -> Self { let (geth_txs_received, grpc_txs_received) = - register_txs_received(vec![rollup_name.clone()].iter()); + register_txs_received([rollup_name.to_string()].iter()); // TODO - change the function signatures of the metrics let (geth_txs_dropped, grpc_txs_dropped) = - register_txs_dropped(vec![rollup_name.clone()].iter()); + register_txs_dropped([rollup_name.to_string()].iter()); let txs_dropped_too_large = - register_txs_dropped_too_large(vec![rollup_name.clone()].iter()); + register_txs_dropped_too_large([rollup_name.to_string()].iter()); describe_counter!( NONCE_FETCH_COUNT, diff --git a/crates/astria-sequencer/src/app/tests_app.rs b/crates/astria-sequencer/src/app/tests_app.rs index e29655e8f..1e63e4cff 100644 --- a/crates/astria-sequencer/src/app/tests_app.rs +++ b/crates/astria-sequencer/src/app/tests_app.rs @@ -1,17 +1,22 @@ use std::collections::HashMap; -use astria_core::{primitive::v1::{ - asset::TracePrefixed, - RollupId, -}, Protobuf, protocol::transaction::v1alpha1::{ - action::{ - BridgeLockAction, - SequenceAction, - TransferAction, +use astria_core::{ + primitive::v1::{ + asset::TracePrefixed, + RollupId, }, - TransactionParams, - UnsignedTransaction, -}, sequencerblock::v1alpha1::block::Deposit}; + protocol::transaction::v1alpha1::{ + action::{ + BridgeLockAction, + SequenceAction, + TransferAction, + }, + TransactionParams, + UnsignedTransaction, + }, + sequencerblock::v1alpha1::block::Deposit, + Protobuf, +}; use cnidarium::StateDelta; use prost::{ bytes::Bytes, diff --git a/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto b/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto index b9721f184..2b5a8d549 100644 --- a/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto +++ b/proto/composerapis/astria/composer/v1alpha1/trusted_builder.proto @@ -4,12 +4,20 @@ package astria.composer.v1alpha1; import "astria/sequencerblock/v1alpha1/block.proto"; +// BuilderBundle contains a bundle of RollupData transactions which are created by a trusted builder +// It contains the transactions and the parent hash on top of which the bundles were simulated. message BuilderBundle { + // transactions in the bundle repeated astria.sequencerblock.v1alpha1.RollupData transactions = 1; + // parent hash of the bundle bytes parent_hash = 2; } +// BuilderBundlePacket is a message that represents a bundle of RollupData transactions and the signature +// of the BuilderBundle by the trusted builder. message BuilderBundlePacket { + // the bundle of transactions BuilderBundle bundle = 1; + // the signature of the bundle signed by the trusted builder bytes signature = 3; } From f23d50a0741583d865edc7c070267ef3f597502c Mon Sep 17 00:00:00 2001 From: Bharath Date: Fri, 6 Sep 2024 14:20:31 +0530 Subject: [PATCH 16/43] fix tests --- crates/astria-composer/src/executor/tests.rs | 28 +++++--------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/crates/astria-composer/src/executor/tests.rs b/crates/astria-composer/src/executor/tests.rs index 3724579cf..a8d28b193 100644 --- a/crates/astria-composer/src/executor/tests.rs +++ b/crates/astria-composer/src/executor/tests.rs @@ -10,18 +10,14 @@ use std::{ use astria_core::{ generated::{ composer::v1alpha1::BuilderBundlePacket, - protocol::account::v1alpha1::NonceResponse, + protocol::accounts::v1alpha1::NonceResponse, sequencerblock::v1alpha1 as raw_sequencer, }, primitive::v1::{ -<<<<<<< HEAD asset::{ Denom, IbcPrefixed, }, -======= - asset, ->>>>>>> f151354e (initial version of trusted builder mvp) RollupId, ROLLUP_ID_LEN, }, @@ -74,6 +70,7 @@ use wiremock::{ Request, ResponseTemplate, }; +use astria_core::primitive::v1::asset; use crate::{ executor, @@ -97,7 +94,8 @@ static TELEMETRY: Lazy<()> = Lazy::new(|| { api_listen_addr: SocketAddr::new(IpAddr::from([0, 0, 0, 0]), 0), sequencer_url: String::new(), sequencer_chain_id: String::new(), - rollups: String::new(), + rollup: "".to_string(), + rollup_websocket_url: "".to_string(), private_key_file: String::new(), sequencer_address_prefix: String::new(), block_time_ms: 0, @@ -110,6 +108,8 @@ static TELEMETRY: Lazy<()> = Lazy::new(|| { pretty_print: false, grpc_addr: SocketAddr::new(IpAddr::from([0, 0, 0, 0]), 0), fee_asset: Denom::IbcPrefixed(IbcPrefixed::new([0; 32])), + execution_api_url: "".to_string(), + max_bundle_size: 0, }; if std::env::var_os("TEST_LOG").is_some() { let filter_directives = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".into()); @@ -356,11 +356,7 @@ async fn full_bundle() { // set up the executor, channel for writing seq actions, and the sequencer mock let (sequencer, cfg, _keyfile, test_executor) = setup().await; let shutdown_token = CancellationToken::new(); -<<<<<<< HEAD let metrics = Box::leak(Box::new(Metrics::noop_metrics(&cfg).unwrap())); -======= - let metrics = Box::leak(Box::new(Metrics::new(cfg.rollup.clone()))); ->>>>>>> f151354e (initial version of trusted builder mvp) mount_genesis(&sequencer, &cfg.sequencer_chain_id).await; let (executor, executor_handle) = executor::Builder { sequencer_url: cfg.sequencer_url.clone(), @@ -507,11 +503,7 @@ async fn bundle_triggered_by_block_timer() { // set up the executor, channel for writing seq actions, and the sequencer mock let (sequencer, cfg, _keyfile, test_executor) = setup().await; let shutdown_token = CancellationToken::new(); -<<<<<<< HEAD let metrics = Box::leak(Box::new(Metrics::noop_metrics(&cfg).unwrap())); -======= - let metrics = Box::leak(Box::new(Metrics::new(cfg.rollup.clone()))); ->>>>>>> f151354e (initial version of trusted builder mvp) mount_genesis(&sequencer, &cfg.sequencer_chain_id).await; let (executor, executor_handle) = executor::Builder { sequencer_url: cfg.sequencer_url.clone(), @@ -662,11 +654,7 @@ async fn two_seq_actions_single_bundle() { // set up the executor, channel for writing seq actions, and the sequencer mock let (sequencer, cfg, _keyfile, test_executor) = setup().await; let shutdown_token = CancellationToken::new(); -<<<<<<< HEAD let metrics = Box::leak(Box::new(Metrics::noop_metrics(&cfg).unwrap())); -======= - let metrics = Box::leak(Box::new(Metrics::new(cfg.rollup.clone()))); ->>>>>>> f151354e (initial version of trusted builder mvp) mount_genesis(&sequencer, &cfg.sequencer_chain_id).await; let (executor, executor_handle) = executor::Builder { sequencer_url: cfg.sequencer_url.clone(), @@ -818,12 +806,8 @@ async fn chain_id_mismatch_returns_error() { // set up sequencer mock let (sequencer, cfg, _keyfile, _test_executor) = setup().await; let shutdown_token = CancellationToken::new(); -<<<<<<< HEAD let metrics = Box::leak(Box::new(Metrics::noop_metrics(&cfg).unwrap())); -======= - let metrics = Box::leak(Box::new(Metrics::new(cfg.rollup.clone()))); let rollup_name = RollupId::new([0; ROLLUP_ID_LEN]); ->>>>>>> f151354e (initial version of trusted builder mvp) // mount a status response with an incorrect chain_id mount_genesis(&sequencer, "bad-chain-id").await; From 4f96b6237094127a9e6d1043f365f44e30b45d29 Mon Sep 17 00:00:00 2001 From: Bharath Date: Tue, 13 Aug 2024 16:27:56 +0530 Subject: [PATCH 17/43] minor fixes --- crates/astria-composer/src/executor/tests.rs | 4 ++-- crates/astria-composer/tests/blackbox/geth_collector.rs | 8 ++++---- crates/astria-composer/tests/blackbox/helper/mod.rs | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/astria-composer/src/executor/tests.rs b/crates/astria-composer/src/executor/tests.rs index a8d28b193..6f05b6acc 100644 --- a/crates/astria-composer/src/executor/tests.rs +++ b/crates/astria-composer/src/executor/tests.rs @@ -461,7 +461,7 @@ async fn full_bundle() { assert_eq!(actions.len(), 1); // decode the sequence action to its BuilderBundlePacket - let mut seq_action = actions.iter().next().unwrap().as_sequence().unwrap(); + let seq_action = actions.iter().next().unwrap().as_sequence().unwrap(); let proto_builder_bundle_packet = BuilderBundlePacket::decode(&mut seq_action.data.clone()).unwrap(); let builder_bundle_packet = astria_core::composer::v1alpha1::BuilderBundlePacket::try_from_raw( @@ -598,7 +598,7 @@ async fn bundle_triggered_by_block_timer() { assert_eq!(actions.len(), 1); - let mut seq_action = actions.iter().next().unwrap().as_sequence().unwrap(); + let seq_action = actions.iter().next().unwrap().as_sequence().unwrap(); let proto_builder_bundle_packet = BuilderBundlePacket::decode(&mut seq_action.data.clone()).unwrap(); let builder_bundle_packet = astria_core::composer::v1alpha1::BuilderBundlePacket::try_from_raw( diff --git a/crates/astria-composer/tests/blackbox/geth_collector.rs b/crates/astria-composer/tests/blackbox/geth_collector.rs index cd5b5a319..6ba886287 100644 --- a/crates/astria-composer/tests/blackbox/geth_collector.rs +++ b/crates/astria-composer/tests/blackbox/geth_collector.rs @@ -39,7 +39,7 @@ async fn tx_from_one_rollup_is_received_by_sequencer() { let tx = Transaction::default(); let data = tx.rlp().to_vec(); - let rollup_data = vec![RollupData::SequencedData(data).to_raw()]; + let rollup_data = vec![RollupData::SequencedData(data.into()).to_raw()]; let soft_parent_hash = [1; 64]; let soft_block_number = 1; @@ -117,7 +117,7 @@ async fn collector_restarts_after_exit() { let tx = Transaction::default(); let data = tx.rlp().to_vec(); - let rollup_data = vec![RollupData::SequencedData(data).to_raw()]; + let rollup_data = vec![RollupData::SequencedData(data.into()).to_raw()]; let _execute_block = mount_executed_block!(test_executor, mock_name: "execute_block", @@ -192,7 +192,7 @@ async fn invalid_nonce_causes_resubmission_under_different_nonce() { let tx = Transaction::default(); let data = tx.rlp().to_vec(); - let rollup_data = vec![RollupData::SequencedData(data).to_raw()]; + let rollup_data = vec![RollupData::SequencedData(data.into()).to_raw()]; let _execute_block = mount_executed_block!(test_executor, mock_name: "execute_block", @@ -259,7 +259,7 @@ async fn single_rollup_tx_payload_integrity() { ); let data = tx.rlp().to_vec(); - let rollup_data = vec![RollupData::SequencedData(data).to_raw()]; + let rollup_data = vec![RollupData::SequencedData(data.into()).to_raw()]; let execute_block = mount_executed_block!(test_executor, mock_name: "execute_block", diff --git a/crates/astria-composer/tests/blackbox/helper/mod.rs b/crates/astria-composer/tests/blackbox/helper/mod.rs index 9b1d1307b..fd00b1c3d 100644 --- a/crates/astria-composer/tests/blackbox/helper/mod.rs +++ b/crates/astria-composer/tests/blackbox/helper/mod.rs @@ -258,7 +258,7 @@ pub async fn mount_matcher_verifying_tx_integrity( let seq_action_data = sequence_action.clone().data; // unmarshall to BuilderBundlePacket let builder_bundle_packet = - BuilderBundlePacket::decode(seq_action_data.as_slice()).unwrap(); + BuilderBundlePacket::decode(seq_action_data).unwrap(); let builder_bundle = BuilderBundle::try_from_raw(builder_bundle_packet.bundle.unwrap()).unwrap(); let transaction = builder_bundle.transactions().first().unwrap(); From 463920050702527da965d48697ef7bffc456645c Mon Sep 17 00:00:00 2001 From: Bharath Date: Tue, 13 Aug 2024 16:40:14 +0530 Subject: [PATCH 18/43] run fmt --- crates/astria-composer/tests/blackbox/grpc_collector.rs | 2 +- crates/astria-composer/tests/blackbox/helper/mod.rs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/astria-composer/tests/blackbox/grpc_collector.rs b/crates/astria-composer/tests/blackbox/grpc_collector.rs index e58b7e695..d608e4a96 100644 --- a/crates/astria-composer/tests/blackbox/grpc_collector.rs +++ b/crates/astria-composer/tests/blackbox/grpc_collector.rs @@ -1,5 +1,4 @@ use std::time::Duration; -use bytes::Bytes; use astria_composer::{ mount_executed_block, @@ -17,6 +16,7 @@ use astria_core::{ sequencerblock::v1alpha1::block::RollupData, Protobuf, }; +use bytes::Bytes; use ethers::prelude::Transaction; use crate::helper::{ diff --git a/crates/astria-composer/tests/blackbox/helper/mod.rs b/crates/astria-composer/tests/blackbox/helper/mod.rs index fd00b1c3d..a4e139afa 100644 --- a/crates/astria-composer/tests/blackbox/helper/mod.rs +++ b/crates/astria-composer/tests/blackbox/helper/mod.rs @@ -257,8 +257,7 @@ pub async fn mount_matcher_verifying_tx_integrity( .unwrap(); let seq_action_data = sequence_action.clone().data; // unmarshall to BuilderBundlePacket - let builder_bundle_packet = - BuilderBundlePacket::decode(seq_action_data).unwrap(); + let builder_bundle_packet = BuilderBundlePacket::decode(seq_action_data).unwrap(); let builder_bundle = BuilderBundle::try_from_raw(builder_bundle_packet.bundle.unwrap()).unwrap(); let transaction = builder_bundle.transactions().first().unwrap(); From 28148ec5d1afe6d7b0ecd08a9e7f97927983bc8e Mon Sep 17 00:00:00 2001 From: Bharath Date: Wed, 14 Aug 2024 20:22:35 +0530 Subject: [PATCH 19/43] handle bundle pushing error using wrap_err --- crates/astria-composer/src/executor/mod.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index 227371def..30c8c912c 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -255,13 +255,11 @@ impl Executor { // we can give the BuilderBundlePacket the highest bundle max size possible // since this is the only sequence action we are sending let mut final_bundle = SizedBundle::new(self.max_bundle_size); - if let Err(e) = final_bundle.try_push(SequenceAction { + final_bundle.try_push(SequenceAction { rollup_id: self.rollup_id, data: encoded_builder_bundle_packet.into(), fee_asset: self.fee_asset.clone(), - }) { - return Err(eyre::Report::from(e)); - } + }).wrap_err("couldn't push sequence action to bundle")?; info!("Submitting the builder bundle packet to sequencer!"); From 5302516684522cc969f1802823c8411cc2dc6b58 Mon Sep 17 00:00:00 2001 From: Bharath Date: Wed, 14 Aug 2024 22:21:17 +0530 Subject: [PATCH 20/43] revert dev.yaml changes --- dev/values/rollup/dev.yaml | 45 ++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/dev/values/rollup/dev.yaml b/dev/values/rollup/dev.yaml index 27a8f3f1e..d7f56d05a 100644 --- a/dev/values/rollup/dev.yaml +++ b/dev/values/rollup/dev.yaml @@ -56,7 +56,7 @@ evm-rollup: eip1559Params: {} # 1: # minBaseFee: 0 - # elasticityMultiplier: 2 + # elasticityMultiplier: 2 # baseFeeChangeDenominator: 8 ## Standard Eth Genesis config values @@ -79,14 +79,12 @@ evm-rollup: value: balance: "0" code: "0x60806040526004361061004a5760003560e01c80637eb6dec71461004f578063a996e0201461009d578063b6476c7e146100b2578063bab916d0146100d4578063db97dc98146100e7575b600080fd5b34801561005b57600080fd5b506100837f000000000000000000000000000000000000000000000000000000000000000981565b60405163ffffffff90911681526020015b60405180910390f35b6100b06100ab366004610315565b6100fc565b005b3480156100be57600080fd5b506100c761019e565b6040516100949190610381565b6100b06100e23660046103cf565b61022c565b3480156100f357600080fd5b506100c76102bf565b3460006101297f000000000000000000000000000000000000000000000000000000003b9aca0083610411565b1161014f5760405162461bcd60e51b815260040161014690610433565b60405180910390fd5b34336001600160a01b03167f0c64e29a5254a71c7f4e52b3d2d236348c80e00a00ba2e1961962bd2827c03fb8787878760405161018f94939291906104ea565b60405180910390a35050505050565b600180546101ab9061051c565b80601f01602080910402602001604051908101604052809291908181526020018280546101d79061051c565b80156102245780601f106101f957610100808354040283529160200191610224565b820191906000526020600020905b81548152906001019060200180831161020757829003601f168201915b505050505081565b3460006102597f000000000000000000000000000000000000000000000000000000003b9aca0083610411565b116102765760405162461bcd60e51b815260040161014690610433565b34336001600160a01b03167f0f4961cab7530804898499aa89f5ec81d1a73102e2e4a1f30f88e5ae3513ba2a85856040516102b2929190610556565b60405180910390a3505050565b600080546101ab9061051c565b60008083601f8401126102de57600080fd5b50813567ffffffffffffffff8111156102f657600080fd5b60208301915083602082850101111561030e57600080fd5b9250929050565b6000806000806040858703121561032b57600080fd5b843567ffffffffffffffff8082111561034357600080fd5b61034f888389016102cc565b9096509450602087013591508082111561036857600080fd5b50610375878288016102cc565b95989497509550505050565b600060208083528351808285015260005b818110156103ae57858101830151858201604001528201610392565b506000604082860101526040601f19601f8301168501019250505092915050565b600080602083850312156103e257600080fd5b823567ffffffffffffffff8111156103f957600080fd5b610405858286016102cc565b90969095509350505050565b60008261042e57634e487b7160e01b600052601260045260246000fd5b500490565b60208082526062908201527f417374726961576974686472617765723a20696e73756666696369656e74207660408201527f616c75652c206d7573742062652067726561746572207468616e203130202a2a60608201527f20283138202d20424153455f434841494e5f41535345545f505245434953494f6080820152614e2960f01b60a082015260c00190565b81835281816020850137506000828201602090810191909152601f909101601f19169091010190565b6040815260006104fe6040830186886104c1565b82810360208401526105118185876104c1565b979650505050505050565b600181811c9082168061053057607f821691505b60208210810361055057634e487b7160e01b600052602260045260246000fd5b50919050565b60208152600061056a6020830184866104c1565b94935050505056fea264697066735822122047a7ef84c0be4640572989abfc01decbc1ae143d4659f1b32047978c67ebc9c864736f6c63430008150033" - - address: "0xaC21B97d35Bf75A7dAb16f35b111a50e78A72F30" - value: - balance: "1000000000000000000000000000" + config: # The level at which core astria components will log out # Options are: error, warn, info, and debug - logLevel: "info" + logLevel: "debug" conductor: # Determines what will drive block execution, options are: @@ -104,21 +102,21 @@ evm-rollup: rpc: "http://celestia-service.astria-dev-cluster.svc.cluster.local:26658" token: "" - resources: {} -# conductor: -# requests: -# cpu: 0.01 -# memory: 1Mi -# limits: -# cpu: 0.1 -# memory: 20Mi -# geth: -# requests: -# cpu: 0.25 -# memory: 256Mi -# limits: -# cpu: 2 -# memory: 1Gi + resources: + conductor: + requests: + cpu: 0.01 + memory: 1Mi + limits: + cpu: 0.1 + memory: 20Mi + geth: + requests: + cpu: 0.25 + memory: 256Mi + limits: + cpu: 2 + memory: 1Gi storage: enabled: false @@ -139,11 +137,6 @@ composer: config: privateKey: devContent: "2bd806c97f0e00af1a1fc3328fa763a9269723c8db8fac4f93af71db186d6e90" -# images: -# composer: -# repo: astria-composer -# tag: "0.8.0" -# devTag: tb evm-bridge-withdrawer: enabled: true @@ -251,4 +244,4 @@ blockscout-stack: value: "none" ingress: enabled: true - hostname: explorer.astria.localdev.me + hostname: explorer.astria.localdev.me \ No newline at end of file From c472e03644d1866ab029243779e54197c31a5f30 Mon Sep 17 00:00:00 2001 From: Bharath Date: Tue, 20 Aug 2024 13:19:18 +0530 Subject: [PATCH 21/43] use Bytes instead of Vec --- crates/astria-composer/src/executor/mod.rs | 5 +++-- crates/astria-core/src/composer/v1alpha1/mod.rs | 4 ++-- .../src/generated/astria.composer.v1alpha1.rs | 16 ++++++++++++---- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index 30c8c912c..18c958561 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -9,6 +9,7 @@ use std::{ task::Poll, time::Duration, }; +use bytes::Bytes; use astria_core::{ crypto::SigningKey, @@ -239,7 +240,7 @@ impl Executor { info!("Creating BuilderBundlePacket"); let builder_bundle = BuilderBundle { transactions: rollup_data_items, - parent_hash: bundle_simulation_result.parent_hash().to_vec(), + parent_hash: bundle_simulation_result.parent_hash(), }; // TODO - bundle signing @@ -247,7 +248,7 @@ impl Executor { // create a top of block bundle let builder_bundle_packet = BuilderBundlePacket { bundle: Some(builder_bundle), - signature: vec![], + signature: Bytes::from(vec![]), }; let encoded_builder_bundle_packet = builder_bundle_packet.encode_to_vec(); diff --git a/crates/astria-core/src/composer/v1alpha1/mod.rs b/crates/astria-core/src/composer/v1alpha1/mod.rs index 9b59b8421..42de35f64 100644 --- a/crates/astria-core/src/composer/v1alpha1/mod.rs +++ b/crates/astria-core/src/composer/v1alpha1/mod.rs @@ -77,7 +77,7 @@ impl Protobuf for BuilderBundle { fn to_raw(&self) -> Self::Raw { crate::generated::composer::v1alpha1::BuilderBundle { transactions: self.transactions.iter().map(Protobuf::to_raw).collect(), - parent_hash: self.parent_hash.clone().to_vec(), + parent_hash: self.parent_hash.clone(), } } } @@ -159,7 +159,7 @@ impl Protobuf for BuilderBundlePacket { fn to_raw(&self) -> Self::Raw { crate::generated::composer::v1alpha1::BuilderBundlePacket { bundle: Some(self.bundle.to_raw()), - signature: self.signature.to_vec(), + signature: self.signature.clone(), } } } diff --git a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs index fa6a5072b..e13a2d391 100644 --- a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs +++ b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs @@ -345,15 +345,19 @@ pub mod grpc_collector_service_server { const NAME: &'static str = "astria.composer.v1alpha1.GrpcCollectorService"; } } +/// BuilderBundle contains a bundle of RollupData transactions which are created by a trusted builder +/// It contains the transactions and the parent hash on top of which the bundles were simulated. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BuilderBundle { + /// transactions in the bundle #[prost(message, repeated, tag = "1")] pub transactions: ::prost::alloc::vec::Vec< super::super::sequencerblock::v1alpha1::RollupData, >, - #[prost(bytes = "vec", tag = "2")] - pub parent_hash: ::prost::alloc::vec::Vec, + /// parent hash of the bundle + #[prost(bytes = "bytes", tag = "2")] + pub parent_hash: ::prost::bytes::Bytes, } impl ::prost::Name for BuilderBundle { const NAME: &'static str = "BuilderBundle"; @@ -362,13 +366,17 @@ impl ::prost::Name for BuilderBundle { ::prost::alloc::format!("astria.composer.v1alpha1.{}", Self::NAME) } } +/// BuilderBundlePacket is a message that represents a bundle of RollupData transactions and the signature +/// of the BuilderBundle by the trusted builder. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BuilderBundlePacket { + /// the bundle of transactions #[prost(message, optional, tag = "1")] pub bundle: ::core::option::Option, - #[prost(bytes = "vec", tag = "3")] - pub signature: ::prost::alloc::vec::Vec, + /// the signature of the bundle signed by the trusted builder + #[prost(bytes = "bytes", tag = "3")] + pub signature: ::prost::bytes::Bytes, } impl ::prost::Name for BuilderBundlePacket { const NAME: &'static str = "BuilderBundlePacket"; From 8d70dc79ecdea62a5246fe73adc29bf4e42f8ffa Mon Sep 17 00:00:00 2001 From: Bharath Date: Tue, 20 Aug 2024 13:25:39 +0530 Subject: [PATCH 22/43] run just fmt --- crates/astria-composer/src/executor/mod.rs | 14 ++++++++------ crates/astria-core/src/composer/v1alpha1/mod.rs | 4 ++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index 18c958561..0ab4b0ec8 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -9,7 +9,6 @@ use std::{ task::Poll, time::Duration, }; -use bytes::Bytes; use astria_core::{ crypto::SigningKey, @@ -39,6 +38,7 @@ use astria_eyre::eyre::{ eyre, WrapErr as _, }; +use bytes::Bytes; use futures::{ future::{ self, @@ -256,11 +256,13 @@ impl Executor { // we can give the BuilderBundlePacket the highest bundle max size possible // since this is the only sequence action we are sending let mut final_bundle = SizedBundle::new(self.max_bundle_size); - final_bundle.try_push(SequenceAction { - rollup_id: self.rollup_id, - data: encoded_builder_bundle_packet.into(), - fee_asset: self.fee_asset.clone(), - }).wrap_err("couldn't push sequence action to bundle")?; + final_bundle + .try_push(SequenceAction { + rollup_id: self.rollup_id, + data: encoded_builder_bundle_packet.into(), + fee_asset: self.fee_asset.clone(), + }) + .wrap_err("couldn't push sequence action to bundle")?; info!("Submitting the builder bundle packet to sequencer!"); diff --git a/crates/astria-core/src/composer/v1alpha1/mod.rs b/crates/astria-core/src/composer/v1alpha1/mod.rs index 42de35f64..a8dc6a27f 100644 --- a/crates/astria-core/src/composer/v1alpha1/mod.rs +++ b/crates/astria-core/src/composer/v1alpha1/mod.rs @@ -70,7 +70,7 @@ impl Protobuf for BuilderBundle { Ok(BuilderBundle { transactions: rollup_data_transactions, - parent_hash: Bytes::from(parent_hash.clone()), + parent_hash: parent_hash.clone(), }) } @@ -152,7 +152,7 @@ impl Protobuf for BuilderBundlePacket { Ok(BuilderBundlePacket { bundle, - signature: Bytes::from(signature.clone()), + signature: signature.clone(), }) } From df5c12c4e123726e73bea2d55f1cc7c947476d9d Mon Sep 17 00:00:00 2001 From: Bharath Date: Tue, 20 Aug 2024 15:55:04 +0530 Subject: [PATCH 23/43] save --- .../src/generated/astria.composer.v1alpha1.rs | 433 ++++++++++++++++++ .../astria.composer.v1alpha1.serde.rs | 237 ++++++++++ .../composer/v1alpha1/grpc_collector.proto | 15 + 3 files changed, 685 insertions(+) diff --git a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs index e13a2d391..f795c1219 100644 --- a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs +++ b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs @@ -29,6 +29,39 @@ impl ::prost::Name for SubmitRollupTransactionResponse { ::prost::alloc::format!("astria.composer.v1alpha1.{}", Self::NAME) } } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SendOptimisticBlockResponse {} +impl ::prost::Name for SendOptimisticBlockResponse { + const NAME: &'static str = "SendOptimisticBlockResponse"; + const PACKAGE: &'static str = "astria.composer.v1alpha1"; + fn full_name() -> ::prost::alloc::string::String { + ::prost::alloc::format!("astria.composer.v1alpha1.{}", Self::NAME) + } +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SendFinalizedHashRequest { + #[prost(bytes = "bytes", tag = "1")] + pub block_hash: ::prost::bytes::Bytes, +} +impl ::prost::Name for SendFinalizedHashRequest { + const NAME: &'static str = "SendFinalizedHashRequest"; + const PACKAGE: &'static str = "astria.composer.v1alpha1"; + fn full_name() -> ::prost::alloc::string::String { + ::prost::alloc::format!("astria.composer.v1alpha1.{}", Self::NAME) + } +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SendFinalizedHashResponse {} +impl ::prost::Name for SendFinalizedHashResponse { + const NAME: &'static str = "SendFinalizedHashResponse"; + const PACKAGE: &'static str = "astria.composer.v1alpha1"; + fn full_name() -> ::prost::alloc::string::String { + ::prost::alloc::format!("astria.composer.v1alpha1.{}", Self::NAME) + } +} /// Generated client implementations. #[cfg(feature = "client")] pub mod grpc_collector_service_client { @@ -150,6 +183,156 @@ pub mod grpc_collector_service_client { } } } +/// Generated client implementations. +#[cfg(feature = "client")] +pub mod sequencer_hooks_service_client { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + #[derive(Debug, Clone)] + pub struct SequencerHooksServiceClient { + inner: tonic::client::Grpc, + } + impl SequencerHooksServiceClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl SequencerHooksServiceClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + Send + 'static, + ::Error: Into + Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> SequencerHooksServiceClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + Send + Sync, + { + SequencerHooksServiceClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn send_optimistic_block( + &mut self, + request: impl tonic::IntoRequest< + super::super::super::sequencerblock::v1alpha1::FilteredSequencerBlock, + >, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/astria.composer.v1alpha1.SequencerHooksService/SendOptimisticBlock", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "astria.composer.v1alpha1.SequencerHooksService", + "SendOptimisticBlock", + ), + ); + self.inner.unary(req, path, codec).await + } + pub async fn send_finalized_hash( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/astria.composer.v1alpha1.SequencerHooksService/SendFinalizedHash", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "astria.composer.v1alpha1.SequencerHooksService", + "SendFinalizedHash", + ), + ); + self.inner.unary(req, path, codec).await + } + } +} /// Generated server implementations. #[cfg(feature = "server")] pub mod grpc_collector_service_server { @@ -345,6 +528,256 @@ pub mod grpc_collector_service_server { const NAME: &'static str = "astria.composer.v1alpha1.GrpcCollectorService"; } } +/// Generated server implementations. +#[cfg(feature = "server")] +pub mod sequencer_hooks_service_server { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with SequencerHooksServiceServer. + #[async_trait] + pub trait SequencerHooksService: Send + Sync + 'static { + async fn send_optimistic_block( + self: std::sync::Arc, + request: tonic::Request< + super::super::super::sequencerblock::v1alpha1::FilteredSequencerBlock, + >, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + async fn send_finalized_hash( + self: std::sync::Arc, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + } + #[derive(Debug)] + pub struct SequencerHooksServiceServer { + inner: _Inner, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + struct _Inner(Arc); + impl SequencerHooksServiceServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + let inner = _Inner(inner); + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> + for SequencerHooksServiceServer + where + T: SequencerHooksService, + B: Body + Send + 'static, + B::Error: Into + Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + let inner = self.inner.clone(); + match req.uri().path() { + "/astria.composer.v1alpha1.SequencerHooksService/SendOptimisticBlock" => { + #[allow(non_camel_case_types)] + struct SendOptimisticBlockSvc(pub Arc); + impl< + T: SequencerHooksService, + > tonic::server::UnaryService< + super::super::super::sequencerblock::v1alpha1::FilteredSequencerBlock, + > for SendOptimisticBlockSvc { + type Response = super::SendOptimisticBlockResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request< + super::super::super::sequencerblock::v1alpha1::FilteredSequencerBlock, + >, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::send_optimistic_block( + inner, + request, + ) + .await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = SendOptimisticBlockSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/astria.composer.v1alpha1.SequencerHooksService/SendFinalizedHash" => { + #[allow(non_camel_case_types)] + struct SendFinalizedHashSvc(pub Arc); + impl< + T: SequencerHooksService, + > tonic::server::UnaryService + for SendFinalizedHashSvc { + type Response = super::SendFinalizedHashResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::send_finalized_hash( + inner, + request, + ) + .await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = SendFinalizedHashSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => { + Box::pin(async move { + Ok( + http::Response::builder() + .status(200) + .header("grpc-status", "12") + .header("content-type", "application/grpc") + .body(empty_body()) + .unwrap(), + ) + }) + } + } + } + } + impl Clone for SequencerHooksServiceServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + impl Clone for _Inner { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } + } + impl std::fmt::Debug for _Inner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.0) + } + } + impl tonic::server::NamedService + for SequencerHooksServiceServer { + const NAME: &'static str = "astria.composer.v1alpha1.SequencerHooksService"; + } +} /// BuilderBundle contains a bundle of RollupData transactions which are created by a trusted builder /// It contains the transactions and the parent hash on top of which the bundles were simulated. #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs b/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs index 571e5a588..4b2516a94 100644 --- a/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs +++ b/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs @@ -221,6 +221,243 @@ impl<'de> serde::Deserialize<'de> for BuilderBundlePacket { deserializer.deserialize_struct("astria.composer.v1alpha1.BuilderBundlePacket", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for SendFinalizedHashRequest { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if !self.block_hash.is_empty() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("astria.composer.v1alpha1.SendFinalizedHashRequest", len)?; + if !self.block_hash.is_empty() { + #[allow(clippy::needless_borrow)] + struct_ser.serialize_field("blockHash", pbjson::private::base64::encode(&self.block_hash).as_str())?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for SendFinalizedHashRequest { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "block_hash", + "blockHash", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + BlockHash, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "blockHash" | "block_hash" => Ok(GeneratedField::BlockHash), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = SendFinalizedHashRequest; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct astria.composer.v1alpha1.SendFinalizedHashRequest") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut block_hash__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::BlockHash => { + if block_hash__.is_some() { + return Err(serde::de::Error::duplicate_field("blockHash")); + } + block_hash__ = + Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) + ; + } + } + } + Ok(SendFinalizedHashRequest { + block_hash: block_hash__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("astria.composer.v1alpha1.SendFinalizedHashRequest", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for SendFinalizedHashResponse { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let len = 0; + let struct_ser = serializer.serialize_struct("astria.composer.v1alpha1.SendFinalizedHashResponse", len)?; + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for SendFinalizedHashResponse { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + Err(serde::de::Error::unknown_field(value, FIELDS)) + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = SendFinalizedHashResponse; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct astria.composer.v1alpha1.SendFinalizedHashResponse") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + while map_.next_key::()?.is_some() { + let _ = map_.next_value::()?; + } + Ok(SendFinalizedHashResponse { + }) + } + } + deserializer.deserialize_struct("astria.composer.v1alpha1.SendFinalizedHashResponse", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for SendOptimisticBlockResponse { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let len = 0; + let struct_ser = serializer.serialize_struct("astria.composer.v1alpha1.SendOptimisticBlockResponse", len)?; + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for SendOptimisticBlockResponse { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + Err(serde::de::Error::unknown_field(value, FIELDS)) + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = SendOptimisticBlockResponse; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct astria.composer.v1alpha1.SendOptimisticBlockResponse") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + while map_.next_key::()?.is_some() { + let _ = map_.next_value::()?; + } + Ok(SendOptimisticBlockResponse { + }) + } + } + deserializer.deserialize_struct("astria.composer.v1alpha1.SendOptimisticBlockResponse", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for SubmitRollupTransactionRequest { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result diff --git a/proto/composerapis/astria/composer/v1alpha1/grpc_collector.proto b/proto/composerapis/astria/composer/v1alpha1/grpc_collector.proto index fa20d378b..ab29f3b1d 100644 --- a/proto/composerapis/astria/composer/v1alpha1/grpc_collector.proto +++ b/proto/composerapis/astria/composer/v1alpha1/grpc_collector.proto @@ -2,6 +2,8 @@ syntax = 'proto3'; package astria.composer.v1alpha1; +import "astria/sequencerblock/v1alpha1/block.proto"; + // SubmitRollupTransactionRequest contains a rollup transaction to be submitted to the Shared Sequencer Network // via the Composer message SubmitRollupTransactionRequest { @@ -15,9 +17,22 @@ message SubmitRollupTransactionRequest { // It's currently an empty response which can be evolved in the future to include more information message SubmitRollupTransactionResponse {} +message SendOptimisticBlockResponse {} + +message SendFinalizedHashRequest { + bytes block_hash = 1; +} + +message SendFinalizedHashResponse {} + // GrpcCollectorService is a service that defines the gRPC collector of the Composer service GrpcCollectorService { // SubmitRollupTransaction submits a rollup transactions to the Composer. // The transaction sent is bundled up with other transactions and submitted to the Shared Sequencer Network. rpc SubmitRollupTransaction(SubmitRollupTransactionRequest) returns (SubmitRollupTransactionResponse) {} } + +service SequencerHooksService { + rpc SendOptimisticBlock(sequencerblock.v1alpha1.FilteredSequencerBlock) returns (SendOptimisticBlockResponse) {} + rpc SendFinalizedHash(SendFinalizedHashRequest) returns (SendFinalizedHashResponse) {} +} From 9f8fac3a5e76222c5d6557cc040bf20420713827 Mon Sep 17 00:00:00 2001 From: Bharath Date: Fri, 6 Sep 2024 14:20:48 +0530 Subject: [PATCH 24/43] add sequencer hooks --- crates/astria-composer/src/composer.rs | 19 +++- .../astria-composer/src/executor/builder.rs | 10 ++- crates/astria-composer/src/executor/mod.rs | 48 +++++----- crates/astria-composer/src/grpc.rs | 17 +++- crates/astria-composer/src/lib.rs | 1 + crates/astria-composer/src/sequencer_hooks.rs | 88 +++++++++++++++++++ 6 files changed, 159 insertions(+), 24 deletions(-) create mode 100644 crates/astria-composer/src/sequencer_hooks.rs diff --git a/crates/astria-composer/src/composer.rs b/crates/astria-composer/src/composer.rs index 750dbc2e0..6a89271b1 100644 --- a/crates/astria-composer/src/composer.rs +++ b/crates/astria-composer/src/composer.rs @@ -4,11 +4,15 @@ use std::{ time::Duration, }; -use astria_core::primitive::v1::asset; +use astria_core::{ + generated::sequencerblock::v1alpha1::FilteredSequencerBlock, + primitive::v1::asset, +}; use astria_eyre::eyre::{ self, WrapErr as _, }; +use bytes::Bytes; use itertools::Itertools as _; use tokio::{ io, @@ -23,6 +27,7 @@ use tokio::{ }, time::timeout, }; +use tokio::sync::mpsc; use tokio_util::{ sync::CancellationToken, task::JoinMap, @@ -47,6 +52,7 @@ use crate::{ grpc, grpc::GrpcServer, metrics::Metrics, + sequencer_hooks::SequencerHooks, Config, }; @@ -123,6 +129,10 @@ impl Composer { let (composer_status_sender, _) = watch::channel(Status::default()); let shutdown_token = CancellationToken::new(); + let (filtered_sequencer_block_sender, filtered_sequencer_block_receiver) = + mpsc::channel::(1000); + let (finalized_hash_sender, finalized_hash_receiver) = mpsc::channel::(1000); + let (executor, executor_handle) = executor::Builder { sequencer_url: cfg.sequencer_url.clone(), sequencer_chain_id: cfg.sequencer_chain_id.clone(), @@ -136,6 +146,9 @@ impl Composer { chain_name: cfg.rollup.clone(), max_bundle_size: cfg.max_bundle_size, shutdown_token: shutdown_token.clone(), + // TODO - rename these?? + filtered_block_receiver: filtered_sequencer_block_receiver, + finalized_block_hash_receiver: finalized_hash_receiver, metrics, } .build() @@ -147,6 +160,10 @@ impl Composer { shutdown_token: shutdown_token.clone(), metrics, fee_asset: cfg.fee_asset.clone(), + sequencer_hooks: SequencerHooks::new( + filtered_sequencer_block_sender, + finalized_hash_sender, + ), } .build() .await diff --git a/crates/astria-composer/src/executor/builder.rs b/crates/astria-composer/src/executor/builder.rs index e96d4a136..c10e17da9 100644 --- a/crates/astria-composer/src/executor/builder.rs +++ b/crates/astria-composer/src/executor/builder.rs @@ -3,6 +3,7 @@ use std::{ path::Path, time::Duration, }; +use bytes::Bytes; use astria_core::{ crypto::SigningKey, @@ -18,9 +19,10 @@ use astria_eyre::eyre::{ eyre, WrapErr as _, }; -use tokio::sync::watch; +use tokio::sync::{mpsc, watch}; use tokio_util::sync::CancellationToken; use tracing::info; +use astria_core::generated::sequencerblock::v1alpha1::FilteredSequencerBlock; use crate::{ executor, @@ -44,6 +46,8 @@ pub(crate) struct Builder { pub(crate) chain_name: String, pub(crate) fee_asset: asset::Denom, pub(crate) max_bundle_size: usize, + pub(crate) filtered_block_receiver: mpsc::Receiver, + pub(crate) finalized_block_hash_receiver: mpsc::Receiver, pub(crate) metrics: &'static Metrics, } @@ -62,6 +66,8 @@ impl Builder { chain_name, fee_asset, max_bundle_size, + filtered_block_receiver, + finalized_block_hash_receiver, metrics, } = self; let sequencer_client = sequencer_client::HttpClient::new(sequencer_url.as_str()) @@ -105,6 +111,8 @@ impl Builder { rollup_id, fee_asset, max_bundle_size, + filtered_block_receiver, + finalized_block_hash_receiver, metrics, }, executor::Handle::new(serialized_rollup_transaction_tx), diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index 0ab4b0ec8..4202547fe 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -90,6 +90,7 @@ use tracing::{ Instrument, Span, }; +use astria_core::generated::sequencerblock::v1alpha1::FilteredSequencerBlock; use self::bundle_factory::SizedBundle; use crate::{ @@ -165,6 +166,8 @@ pub(super) struct Executor { fee_asset: asset::Denom, // The maximum possible size for a bundle so that it can fit into a block max_bundle_size: usize, + filtered_block_receiver: mpsc::Receiver, + finalized_block_hash_receiver: mpsc::Receiver, metrics: &'static Metrics, } @@ -349,34 +352,39 @@ impl Executor { }; } - Some(next_bundle) = future::ready(bundle_factory.next_finished()), if submission_fut.is_terminated() => { - let bundle = next_bundle.pop(); - if !bundle.is_empty() { - submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; - } + Some(filtered_sequencer_block) = self.filtered_block_receiver.recv() => { + info!("received filtered sequencer block {:?}", filtered_sequencer_block.block_hash.to_ascii_lowercase()); + } + + Some(finalized_block_hash) = self.finalized_block_hash_receiver.recv() => { + info!("received finalized block hash {:?}", finalized_block_hash.to_ascii_lowercase()); } + // Some(next_bundle) = future::ready(bundle_factory.next_finished()), if submission_fut.is_terminated() => { + // let bundle = next_bundle.pop(); + // if !bundle.is_empty() { + // submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; + // } + // } + // receive new seq_action and bundle it. will not pull from the channel if `bundle_factory` is full Some(seq_action) = self.serialized_rollup_transactions.recv(), if !bundle_factory.is_full() => { self.bundle_seq_action(seq_action, &mut bundle_factory); } // try to preempt current bundle if the timer has ticked without submitting the next bundle - () = &mut block_timer, if submission_fut.is_terminated() => { - let bundle = bundle_factory.pop_now(); - if bundle.is_empty() { -<<<<<<< HEAD -======= - debug!("block timer ticked, but no bundle to submit to sequencer"); ->>>>>>> 7c75d72f (clean ups) - block_timer.as_mut().reset(reset_time()); - } else { - debug!( - "forcing bundle submission to sequencer due to block timer" - ); - submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; - } - } + // () = &mut block_timer, if submission_fut.is_terminated() => { + // let bundle = bundle_factory.pop_now(); + // if bundle.is_empty() { + // debug!("block timer ticked, but no bundle to submit to sequencer"); + // block_timer.as_mut().reset(reset_time()); + // } else { + // debug!( + // "forcing bundle submission to sequencer due to block timer" + // ); + // submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; + // } + // } } }; diff --git a/crates/astria-composer/src/grpc.rs b/crates/astria-composer/src/grpc.rs index 8a24dcf8c..bc6934118 100644 --- a/crates/astria-composer/src/grpc.rs +++ b/crates/astria-composer/src/grpc.rs @@ -9,7 +9,10 @@ use std::net::SocketAddr; use astria_core::{ - generated::composer::v1alpha1::grpc_collector_service_server::GrpcCollectorServiceServer, + generated::composer::v1alpha1::{ + grpc_collector_service_server::GrpcCollectorServiceServer, + sequencer_hooks_service_server::SequencerHooksServiceServer, + }, primitive::v1::asset, }; use astria_eyre::{ @@ -27,6 +30,7 @@ use crate::{ collectors, executor, metrics::Metrics, + sequencer_hooks::SequencerHooks, }; /// Listens for incoming gRPC requests and sends the Rollup transactions to the @@ -36,6 +40,7 @@ use crate::{ pub(crate) struct GrpcServer { listener: TcpListener, grpc_collector: collectors::Grpc, + sequencer_hooks: SequencerHooks, shutdown_token: CancellationToken, } @@ -45,6 +50,7 @@ pub(crate) struct Builder { pub(crate) shutdown_token: CancellationToken, pub(crate) metrics: &'static Metrics, pub(crate) fee_asset: asset::Denom, + pub(crate) sequencer_hooks: SequencerHooks, } impl Builder { @@ -56,6 +62,7 @@ impl Builder { shutdown_token, metrics, fee_asset, + sequencer_hooks, } = self; let listener = TcpListener::bind(grpc_addr) @@ -67,6 +74,7 @@ impl Builder { listener, grpc_collector, shutdown_token, + sequencer_hooks, }) } } @@ -83,13 +91,18 @@ impl GrpcServer { let (mut health_reporter, health_service) = tonic_health::server::health_reporter(); let composer_service = GrpcCollectorServiceServer::new(self.grpc_collector); + let sequencer_hooks_service = SequencerHooksServiceServer::new(self.sequencer_hooks); let grpc_server = tonic::transport::Server::builder() .add_service(health_service) - .add_service(composer_service); + .add_service(composer_service) + .add_service(sequencer_hooks_service); health_reporter .set_serving::>() .await; + health_reporter + .set_serving::>() + .await; grpc_server .serve_with_incoming_shutdown( diff --git a/crates/astria-composer/src/lib.rs b/crates/astria-composer/src/lib.rs index db497f655..41f4655b9 100644 --- a/crates/astria-composer/src/lib.rs +++ b/crates/astria-composer/src/lib.rs @@ -47,6 +47,7 @@ mod executor; mod grpc; pub(crate) mod metrics; mod mock_grpc; +pub(crate) mod sequencer_hooks; #[cfg(test)] pub(crate) mod test_utils; pub(crate) mod utils; diff --git a/crates/astria-composer/src/sequencer_hooks.rs b/crates/astria-composer/src/sequencer_hooks.rs new file mode 100644 index 000000000..db3c43398 --- /dev/null +++ b/crates/astria-composer/src/sequencer_hooks.rs @@ -0,0 +1,88 @@ +use std::{ + sync::Arc, + time::Duration, +}; + +use astria_core::generated::{ + composer::v1alpha1::{ + sequencer_hooks_service_server::SequencerHooksService, + SendFinalizedHashRequest, + SendFinalizedHashResponse, + SendOptimisticBlockResponse, + }, + sequencerblock::v1alpha1::FilteredSequencerBlock, +}; +use bytes::Bytes; +use tokio::sync::{ + mpsc, + mpsc::error::SendTimeoutError, +}; +use tonic::{ + Request, + Response, + Status, +}; + +const SEND_TIMEOUT: u64 = 2; + +pub(crate) struct SequencerHooks { + filtered_block_sender: mpsc::Sender, + finalized_hash_sender: mpsc::Sender, +} + +impl SequencerHooks { + pub(crate) fn new( + filtered_block_sender: mpsc::Sender, + finalized_hash_sender: mpsc::Sender, + ) -> Self { + Self { + filtered_block_sender, + finalized_hash_sender, + } + } + + pub(crate) async fn send_filtered_block_with_timeout( + &self, + block: FilteredSequencerBlock, + ) -> Result<(), SendTimeoutError> { + self.filtered_block_sender + .send_timeout(block, Duration::from_secs(SEND_TIMEOUT)) + .await + } + + pub(crate) async fn send_finalized_hash_with_timeout( + &self, + hash: Bytes, + ) -> Result<(), SendTimeoutError> { + self.finalized_hash_sender + .send_timeout(hash, Duration::from_secs(SEND_TIMEOUT)) + .await + } +} + +#[async_trait::async_trait] +impl SequencerHooksService for SequencerHooks { + async fn send_optimistic_block( + self: Arc, + request: Request, + ) -> Result, Status> { + let block = request.into_inner(); + match self.send_filtered_block_with_timeout(block).await { + Ok(_) => Ok(Response::new(SendOptimisticBlockResponse {})), + Err(SendTimeoutError::Timeout(block)) => Err(Status::deadline_exceeded("failed to send optimistic block")), + Err(SendTimeoutError::Closed(block)) => Err(Status::cancelled("failed to send optimistic block")), + } + } + + async fn send_finalized_hash( + self: Arc, + request: Request, + ) -> Result, Status> { + let hash = request.into_inner().block_hash; + match self.send_finalized_hash_with_timeout(hash).await { + Ok(_) => Ok(Response::new(SendFinalizedHashResponse {})), + Err(SendTimeoutError::Timeout(hash)) => Err(Status::deadline_exceeded("failed to send finalized hash")), + Err(SendTimeoutError::Closed(hash)) => Err(Status::cancelled("failed to send finalized hash")), + } + } +} From 4dd0c97fe34b995d793284f8bfaea96731a36628 Mon Sep 17 00:00:00 2001 From: Bharath Date: Fri, 6 Sep 2024 14:22:01 +0530 Subject: [PATCH 25/43] setup sequencer to composer communication --- crates/astria-composer/src/composer.rs | 19 ++- .../astria-composer/src/executor/builder.rs | 19 ++- crates/astria-composer/src/executor/mod.rs | 51 ++++---- crates/astria-composer/src/executor/tests.rs | 16 +++ crates/astria-composer/src/sequencer_hooks.rs | 55 ++++----- .../src/generated/astria.composer.v1alpha1.rs | 37 ++++-- .../astria.composer.v1alpha1.serde.rs | 113 ++++++++++++++++++ crates/astria-sequencer/Cargo.toml | 7 +- crates/astria-sequencer/local.env.example | 3 + crates/astria-sequencer/src/app/mod.rs | 46 ++++++- crates/astria-sequencer/src/app/test_utils.rs | 3 +- crates/astria-sequencer/src/client.rs | 79 ++++++++++++ crates/astria-sequencer/src/config.rs | 1 + crates/astria-sequencer/src/lib.rs | 1 + crates/astria-sequencer/src/sequencer.rs | 2 +- .../astria-sequencer/src/service/consensus.rs | 3 +- .../composer/v1alpha1/grpc_collector.proto | 9 +- 17 files changed, 378 insertions(+), 86 deletions(-) create mode 100644 crates/astria-sequencer/src/client.rs diff --git a/crates/astria-composer/src/composer.rs b/crates/astria-composer/src/composer.rs index 6a89271b1..6fcb198b5 100644 --- a/crates/astria-composer/src/composer.rs +++ b/crates/astria-composer/src/composer.rs @@ -5,7 +5,13 @@ use std::{ }; use astria_core::{ - generated::sequencerblock::v1alpha1::FilteredSequencerBlock, + generated::{ + composer::v1alpha1::{ + SendFinalizedHashRequest, + SendOptimisticBlockRequest, + }, + sequencerblock::v1alpha1::FilteredSequencerBlock, + }, primitive::v1::asset, }; use astria_eyre::eyre::{ @@ -20,14 +26,16 @@ use tokio::{ signal, SignalKind, }, - sync::watch, + sync::{ + mpsc, + watch, + }, task::{ JoinError, JoinHandle, }, time::timeout, }; -use tokio::sync::mpsc; use tokio_util::{ sync::CancellationToken, task::JoinMap, @@ -130,8 +138,9 @@ impl Composer { let shutdown_token = CancellationToken::new(); let (filtered_sequencer_block_sender, filtered_sequencer_block_receiver) = - mpsc::channel::(1000); - let (finalized_hash_sender, finalized_hash_receiver) = mpsc::channel::(1000); + mpsc::channel::(1000); + let (finalized_hash_sender, finalized_hash_receiver) = + mpsc::channel::(1000); let (executor, executor_handle) = executor::Builder { sequencer_url: cfg.sequencer_url.clone(), diff --git a/crates/astria-composer/src/executor/builder.rs b/crates/astria-composer/src/executor/builder.rs index c10e17da9..4f3fad863 100644 --- a/crates/astria-composer/src/executor/builder.rs +++ b/crates/astria-composer/src/executor/builder.rs @@ -3,10 +3,16 @@ use std::{ path::Path, time::Duration, }; -use bytes::Bytes; use astria_core::{ crypto::SigningKey, + generated::{ + composer::v1alpha1::{ + SendFinalizedHashRequest, + SendOptimisticBlockRequest, + }, + sequencerblock::v1alpha1::FilteredSequencerBlock, + }, primitive::v1::{ asset, Address, @@ -19,10 +25,13 @@ use astria_eyre::eyre::{ eyre, WrapErr as _, }; -use tokio::sync::{mpsc, watch}; +use bytes::Bytes; +use tokio::sync::{ + mpsc, + watch, +}; use tokio_util::sync::CancellationToken; use tracing::info; -use astria_core::generated::sequencerblock::v1alpha1::FilteredSequencerBlock; use crate::{ executor, @@ -46,8 +55,8 @@ pub(crate) struct Builder { pub(crate) chain_name: String, pub(crate) fee_asset: asset::Denom, pub(crate) max_bundle_size: usize, - pub(crate) filtered_block_receiver: mpsc::Receiver, - pub(crate) finalized_block_hash_receiver: mpsc::Receiver, + pub(crate) filtered_block_receiver: mpsc::Receiver, + pub(crate) finalized_block_hash_receiver: mpsc::Receiver, pub(crate) metrics: &'static Metrics, } diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index 4202547fe..9fa91c2db 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -16,8 +16,14 @@ use astria_core::{ composer::v1alpha1::{ BuilderBundle, BuilderBundlePacket, + SendFinalizedHashRequest, + SendOptimisticBlockRequest, + }, + execution::v1alpha1::FinalizeBlockRequest, + sequencerblock::v1alpha1::{ + FilteredSequencerBlock, + RollupData, }, - sequencerblock::v1alpha1::RollupData, }, primitive::v1::{ asset, @@ -90,7 +96,6 @@ use tracing::{ Instrument, Span, }; -use astria_core::generated::sequencerblock::v1alpha1::FilteredSequencerBlock; use self::bundle_factory::SizedBundle; use crate::{ @@ -166,8 +171,8 @@ pub(super) struct Executor { fee_asset: asset::Denom, // The maximum possible size for a bundle so that it can fit into a block max_bundle_size: usize, - filtered_block_receiver: mpsc::Receiver, - finalized_block_hash_receiver: mpsc::Receiver, + filtered_block_receiver: mpsc::Receiver, + finalized_block_hash_receiver: mpsc::Receiver, metrics: &'static Metrics, } @@ -357,15 +362,15 @@ impl Executor { } Some(finalized_block_hash) = self.finalized_block_hash_receiver.recv() => { - info!("received finalized block hash {:?}", finalized_block_hash.to_ascii_lowercase()); + info!("received finalized block hash {:?}", finalized_block_hash.block_hash.to_ascii_lowercase()); } - // Some(next_bundle) = future::ready(bundle_factory.next_finished()), if submission_fut.is_terminated() => { - // let bundle = next_bundle.pop(); - // if !bundle.is_empty() { - // submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; - // } - // } + Some(next_bundle) = future::ready(bundle_factory.next_finished()), if submission_fut.is_terminated() => { + let bundle = next_bundle.pop(); + if !bundle.is_empty() { + submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; + } + } // receive new seq_action and bundle it. will not pull from the channel if `bundle_factory` is full Some(seq_action) = self.serialized_rollup_transactions.recv(), if !bundle_factory.is_full() => { @@ -373,18 +378,18 @@ impl Executor { } // try to preempt current bundle if the timer has ticked without submitting the next bundle - // () = &mut block_timer, if submission_fut.is_terminated() => { - // let bundle = bundle_factory.pop_now(); - // if bundle.is_empty() { - // debug!("block timer ticked, but no bundle to submit to sequencer"); - // block_timer.as_mut().reset(reset_time()); - // } else { - // debug!( - // "forcing bundle submission to sequencer due to block timer" - // ); - // submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; - // } - // } + () = &mut block_timer, if submission_fut.is_terminated() => { + let bundle = bundle_factory.pop_now(); + if bundle.is_empty() { + debug!("block timer ticked, but no bundle to submit to sequencer"); + block_timer.as_mut().reset(reset_time()); + } else { + debug!( + "forcing bundle submission to sequencer due to block timer" + ); + submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; + } + } } }; diff --git a/crates/astria-composer/src/executor/tests.rs b/crates/astria-composer/src/executor/tests.rs index 6f05b6acc..4867fbd26 100644 --- a/crates/astria-composer/src/executor/tests.rs +++ b/crates/astria-composer/src/executor/tests.rs @@ -358,6 +358,8 @@ async fn full_bundle() { let shutdown_token = CancellationToken::new(); let metrics = Box::leak(Box::new(Metrics::noop_metrics(&cfg).unwrap())); mount_genesis(&sequencer, &cfg.sequencer_chain_id).await; + let (_filtered_block_sender, filtered_block_receiver) = tokio::sync::mpsc::channel(1); + let (_finalized_hash_sender, finalized_hash_receiver) = tokio::sync::mpsc::channel(1); let (executor, executor_handle) = executor::Builder { sequencer_url: cfg.sequencer_url.clone(), sequencer_chain_id: cfg.sequencer_chain_id.clone(), @@ -371,6 +373,8 @@ async fn full_bundle() { chain_name: cfg.rollup.clone(), fee_asset: cfg.fee_asset, max_bundle_size: cfg.max_bundle_size, + filtered_block_receiver, + finalized_block_hash_receiver: finalized_hash_receiver, metrics, } .build() @@ -505,6 +509,8 @@ async fn bundle_triggered_by_block_timer() { let shutdown_token = CancellationToken::new(); let metrics = Box::leak(Box::new(Metrics::noop_metrics(&cfg).unwrap())); mount_genesis(&sequencer, &cfg.sequencer_chain_id).await; + let (_filtered_block_sender, filtered_block_receiver) = tokio::sync::mpsc::channel(1); + let (_finalized_hash_sender, finalized_hash_receiver) = tokio::sync::mpsc::channel(1); let (executor, executor_handle) = executor::Builder { sequencer_url: cfg.sequencer_url.clone(), sequencer_chain_id: cfg.sequencer_chain_id.clone(), @@ -518,6 +524,8 @@ async fn bundle_triggered_by_block_timer() { chain_name: cfg.rollup.clone(), fee_asset: cfg.fee_asset.clone(), max_bundle_size: cfg.max_bundle_size, + filtered_block_receiver, + finalized_block_hash_receiver: finalized_hash_receiver, metrics, } .build() @@ -656,6 +664,8 @@ async fn two_seq_actions_single_bundle() { let shutdown_token = CancellationToken::new(); let metrics = Box::leak(Box::new(Metrics::noop_metrics(&cfg).unwrap())); mount_genesis(&sequencer, &cfg.sequencer_chain_id).await; + let (_filtered_block_sender, filtered_block_receiver) = tokio::sync::mpsc::channel(1); + let (_finalized_hash_sender, finalized_hash_receiver) = tokio::sync::mpsc::channel(1); let (executor, executor_handle) = executor::Builder { sequencer_url: cfg.sequencer_url.clone(), sequencer_chain_id: cfg.sequencer_chain_id.clone(), @@ -669,6 +679,8 @@ async fn two_seq_actions_single_bundle() { chain_name: cfg.rollup.clone(), fee_asset: cfg.fee_asset.clone(), max_bundle_size: cfg.max_bundle_size, + filtered_block_receiver, + finalized_block_hash_receiver: finalized_hash_receiver, metrics, } .build() @@ -809,6 +821,8 @@ async fn chain_id_mismatch_returns_error() { let metrics = Box::leak(Box::new(Metrics::noop_metrics(&cfg).unwrap())); let rollup_name = RollupId::new([0; ROLLUP_ID_LEN]); + let (_filtered_block_sender, filtered_block_receiver) = tokio::sync::mpsc::channel(1); + let (_finalized_hash_sender, finalized_hash_receiver) = tokio::sync::mpsc::channel(1); // mount a status response with an incorrect chain_id mount_genesis(&sequencer, "bad-chain-id").await; @@ -826,6 +840,8 @@ async fn chain_id_mismatch_returns_error() { chain_name: rollup_name.to_string(), fee_asset: cfg.fee_asset, max_bundle_size: cfg.max_bundle_size, + filtered_block_receiver, + finalized_block_hash_receiver: finalized_hash_receiver, metrics, } .build() diff --git a/crates/astria-composer/src/sequencer_hooks.rs b/crates/astria-composer/src/sequencer_hooks.rs index db3c43398..0046434c3 100644 --- a/crates/astria-composer/src/sequencer_hooks.rs +++ b/crates/astria-composer/src/sequencer_hooks.rs @@ -3,16 +3,13 @@ use std::{ time::Duration, }; -use astria_core::generated::{ - composer::v1alpha1::{ - sequencer_hooks_service_server::SequencerHooksService, - SendFinalizedHashRequest, - SendFinalizedHashResponse, - SendOptimisticBlockResponse, - }, - sequencerblock::v1alpha1::FilteredSequencerBlock, +use astria_core::generated::composer::v1alpha1::{ + sequencer_hooks_service_server::SequencerHooksService, + SendFinalizedHashRequest, + SendFinalizedHashResponse, + SendOptimisticBlockRequest, + SendOptimisticBlockResponse, }; -use bytes::Bytes; use tokio::sync::{ mpsc, mpsc::error::SendTimeoutError, @@ -26,14 +23,14 @@ use tonic::{ const SEND_TIMEOUT: u64 = 2; pub(crate) struct SequencerHooks { - filtered_block_sender: mpsc::Sender, - finalized_hash_sender: mpsc::Sender, + filtered_block_sender: mpsc::Sender, + finalized_hash_sender: mpsc::Sender, } impl SequencerHooks { pub(crate) fn new( - filtered_block_sender: mpsc::Sender, - finalized_hash_sender: mpsc::Sender, + filtered_block_sender: mpsc::Sender, + finalized_hash_sender: mpsc::Sender, ) -> Self { Self { filtered_block_sender, @@ -41,21 +38,21 @@ impl SequencerHooks { } } - pub(crate) async fn send_filtered_block_with_timeout( + pub(crate) async fn send_optimistic_block_with_timeout( &self, - block: FilteredSequencerBlock, - ) -> Result<(), SendTimeoutError> { + req: SendOptimisticBlockRequest, + ) -> Result<(), SendTimeoutError> { self.filtered_block_sender - .send_timeout(block, Duration::from_secs(SEND_TIMEOUT)) + .send_timeout(req, Duration::from_secs(SEND_TIMEOUT)) .await } pub(crate) async fn send_finalized_hash_with_timeout( &self, - hash: Bytes, - ) -> Result<(), SendTimeoutError> { + req: SendFinalizedHashRequest, + ) -> Result<(), SendTimeoutError> { self.finalized_hash_sender - .send_timeout(hash, Duration::from_secs(SEND_TIMEOUT)) + .send_timeout(req, Duration::from_secs(SEND_TIMEOUT)) .await } } @@ -64,25 +61,17 @@ impl SequencerHooks { impl SequencerHooksService for SequencerHooks { async fn send_optimistic_block( self: Arc, - request: Request, + request: Request, ) -> Result, Status> { - let block = request.into_inner(); - match self.send_filtered_block_with_timeout(block).await { - Ok(_) => Ok(Response::new(SendOptimisticBlockResponse {})), - Err(SendTimeoutError::Timeout(block)) => Err(Status::deadline_exceeded("failed to send optimistic block")), - Err(SendTimeoutError::Closed(block)) => Err(Status::cancelled("failed to send optimistic block")), - } + let inner = request.into_inner(); + todo!() } async fn send_finalized_hash( self: Arc, request: Request, ) -> Result, Status> { - let hash = request.into_inner().block_hash; - match self.send_finalized_hash_with_timeout(hash).await { - Ok(_) => Ok(Response::new(SendFinalizedHashResponse {})), - Err(SendTimeoutError::Timeout(hash)) => Err(Status::deadline_exceeded("failed to send finalized hash")), - Err(SendTimeoutError::Closed(hash)) => Err(Status::cancelled("failed to send finalized hash")), - } + let inner = request.into_inner(); + todo!() } } diff --git a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs index f795c1219..0086dab3c 100644 --- a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs +++ b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs @@ -1,3 +1,6 @@ + +use crate::generated::protocol::transaction::v1alpha1; + /// SubmitRollupTransactionRequest contains a rollup transaction to be submitted to the Shared Sequencer Network /// via the Composer #[allow(clippy::derive_partial_eq_without_eq)] @@ -31,6 +34,23 @@ impl ::prost::Name for SubmitRollupTransactionResponse { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct SendOptimisticBlockRequest { + #[prost(bytes = "bytes", tag = "1")] + pub block_hash: ::prost::bytes::Bytes, + #[prost(message, repeated, tag = "2")] + pub seq_action: ::prost::alloc::vec::Vec< + v1alpha1::SequenceAction, + >, +} +impl ::prost::Name for SendOptimisticBlockRequest { + const NAME: &'static str = "SendOptimisticBlockRequest"; + const PACKAGE: &'static str = "astria.composer.v1alpha1"; + fn full_name() -> ::prost::alloc::string::String { + ::prost::alloc::format!("astria.composer.v1alpha1.{}", Self::NAME) + } +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct SendOptimisticBlockResponse {} impl ::prost::Name for SendOptimisticBlockResponse { const NAME: &'static str = "SendOptimisticBlockResponse"; @@ -271,9 +291,7 @@ pub mod sequencer_hooks_service_client { } pub async fn send_optimistic_block( &mut self, - request: impl tonic::IntoRequest< - super::super::super::sequencerblock::v1alpha1::FilteredSequencerBlock, - >, + request: impl tonic::IntoRequest, ) -> std::result::Result< tonic::Response, tonic::Status, @@ -538,9 +556,7 @@ pub mod sequencer_hooks_service_server { pub trait SequencerHooksService: Send + Sync + 'static { async fn send_optimistic_block( self: std::sync::Arc, - request: tonic::Request< - super::super::super::sequencerblock::v1alpha1::FilteredSequencerBlock, - >, + request: tonic::Request, ) -> std::result::Result< tonic::Response, tonic::Status, @@ -638,9 +654,8 @@ pub mod sequencer_hooks_service_server { struct SendOptimisticBlockSvc(pub Arc); impl< T: SequencerHooksService, - > tonic::server::UnaryService< - super::super::super::sequencerblock::v1alpha1::FilteredSequencerBlock, - > for SendOptimisticBlockSvc { + > tonic::server::UnaryService + for SendOptimisticBlockSvc { type Response = super::SendOptimisticBlockResponse; type Future = BoxFuture< tonic::Response, @@ -648,9 +663,7 @@ pub mod sequencer_hooks_service_server { >; fn call( &mut self, - request: tonic::Request< - super::super::super::sequencerblock::v1alpha1::FilteredSequencerBlock, - >, + request: tonic::Request, ) -> Self::Future { let inner = Arc::clone(&self.0); let fut = async move { diff --git a/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs b/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs index 4b2516a94..72e9b7b45 100644 --- a/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs +++ b/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs @@ -387,6 +387,119 @@ impl<'de> serde::Deserialize<'de> for SendFinalizedHashResponse { deserializer.deserialize_struct("astria.composer.v1alpha1.SendFinalizedHashResponse", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for SendOptimisticBlockRequest { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if !self.block_hash.is_empty() { + len += 1; + } + if !self.seq_action.is_empty() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("astria.composer.v1alpha1.SendOptimisticBlockRequest", len)?; + if !self.block_hash.is_empty() { + #[allow(clippy::needless_borrow)] + struct_ser.serialize_field("blockHash", pbjson::private::base64::encode(&self.block_hash).as_str())?; + } + if !self.seq_action.is_empty() { + struct_ser.serialize_field("seqAction", &self.seq_action)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for SendOptimisticBlockRequest { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "block_hash", + "blockHash", + "seq_action", + "seqAction", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + BlockHash, + SeqAction, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "blockHash" | "block_hash" => Ok(GeneratedField::BlockHash), + "seqAction" | "seq_action" => Ok(GeneratedField::SeqAction), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = SendOptimisticBlockRequest; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct astria.composer.v1alpha1.SendOptimisticBlockRequest") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut block_hash__ = None; + let mut seq_action__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::BlockHash => { + if block_hash__.is_some() { + return Err(serde::de::Error::duplicate_field("blockHash")); + } + block_hash__ = + Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) + ; + } + GeneratedField::SeqAction => { + if seq_action__.is_some() { + return Err(serde::de::Error::duplicate_field("seqAction")); + } + seq_action__ = Some(map_.next_value()?); + } + } + } + Ok(SendOptimisticBlockRequest { + block_hash: block_hash__.unwrap_or_default(), + seq_action: seq_action__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("astria.composer.v1alpha1.SendOptimisticBlockRequest", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for SendOptimisticBlockResponse { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result diff --git a/crates/astria-sequencer/Cargo.toml b/crates/astria-sequencer/Cargo.toml index 5efcf0af6..e1f698931 100644 --- a/crates/astria-sequencer/Cargo.toml +++ b/crates/astria-sequencer/Cargo.toml @@ -15,7 +15,11 @@ name = "astria-sequencer" benchmark = ["divan"] [dependencies] -astria-core = { path = "../astria-core", features = ["server", "serde"] } +astria-core = { path = "../astria-core", features = [ + "client", + "server", + "serde", +] } astria-build-info = { path = "../astria-build-info", features = ["runtime"] } config = { package = "astria-config", path = "../astria-config" } merkle = { package = "astria-merkle", path = "../astria-merkle" } @@ -59,6 +63,7 @@ tracing = { workspace = true } [dev-dependencies] astria-core = { path = "../astria-core", features = [ + "client", "server", "serde", "test-utils", diff --git a/crates/astria-sequencer/local.env.example b/crates/astria-sequencer/local.env.example index 05cb7937a..61be09845 100644 --- a/crates/astria-sequencer/local.env.example +++ b/crates/astria-sequencer/local.env.example @@ -34,6 +34,9 @@ ASTRIA_SEQUENCER_METRICS_HTTP_LISTENER_ADDR="127.0.0.1:9000" # `ASTRIA_SEQUENCER_FORCE_STDOUT` is set to `true`. ASTRIA_SEQUENCER_PRETTY_PRINT=false +# Address of the composer node +ASTRIA_SEQUENCER_COMPOSER_HOOK="" + # If set to any non-empty value removes ANSI escape characters from the pretty # printed output. Note that this does nothing unless `ASTRIA_SEQUENCER_PRETTY_PRINT` # is set to `true`. diff --git a/crates/astria-sequencer/src/app/mod.rs b/crates/astria-sequencer/src/app/mod.rs index 4dca9057c..b4827df45 100644 --- a/crates/astria-sequencer/src/app/mod.rs +++ b/crates/astria-sequencer/src/app/mod.rs @@ -24,7 +24,11 @@ use anyhow::{ Context, }; use astria_core::{ - generated::protocol::transactions::v1alpha1 as raw, + generated::{ + composer::v1alpha1::sequencer_hooks_service_client::SequencerHooksServiceClient, + protocol::transactions::v1alpha1 as raw, + }, + primitive::v1::Address, protocol::{ abci::AbciErrorCode, genesis::v1alpha1::GenesisAppState, @@ -35,7 +39,9 @@ use astria_core::{ }, }, sequencerblock::v1alpha1::block::SequencerBlock, + Protobuf, }; +use bytes::Bytes; use cnidarium::{ ArcStateDeltaExt, Snapshot, @@ -93,6 +99,7 @@ use crate::{ StateReadExt as _, StateWriteExt as _, }, + client::SequencerHooksClient, component::Component as _, ibc::component::IbcComponent, mempool::{ @@ -170,6 +177,8 @@ pub(crate) struct App { #[allow(clippy::struct_field_names)] app_hash: AppHash, + sequencer_hooks_client: SequencerHooksClient, + metrics: &'static Metrics, } @@ -177,6 +186,7 @@ impl App { pub(crate) async fn new( snapshot: Snapshot, mempool: Mempool, + composer_uri: String, metrics: &'static Metrics, ) -> anyhow::Result { debug!("initializing App instance"); @@ -194,6 +204,9 @@ impl App { // there should be no unexpected copies elsewhere. let state = Arc::new(StateDelta::new(snapshot)); + let sequencer_hooks_client = SequencerHooksClient::connect_lazy(&composer_uri) + .context("failed to connect to sequencer hooks service")?; + Ok(Self { state, mempool, @@ -203,6 +216,7 @@ impl App { write_batch: None, app_hash, metrics, + sequencer_hooks_client, }) } @@ -453,7 +467,25 @@ impl App { "chain IDs commitment does not match expected", ); - self.executed_proposal_hash = process_proposal.hash; + let block_hash = process_proposal.hash.clone(); + self.executed_proposal_hash = block_hash; + + // get a list of sequence actions from the signed_txs + let sequence_actions = signed_txs + .iter() + .flat_map(|tx| tx.unsigned_transaction().actions.iter()) + .filter_map(Action::as_sequence) + .map(|seq| seq.to_raw().clone()) + .collect::>(); + + info!("BHARATH: Sending optimistic block to composer!"); + self.sequencer_hooks_client + .send_optimistic_block( + Bytes::from(block_hash.as_bytes().to_vec()), + sequence_actions, + ) + .await + .context("failed to send optimistic block to composer")?; Ok(()) } @@ -905,6 +937,16 @@ impl App { .await .context("failed to prepare commit")?; + // update the priority of any txs in the mempool based on the updated app state + update_mempool_after_finalization(&mut self.mempool, self.state.as_ref()) + .await + .context("failed to update mempool after finalization")?; + + self.sequencer_hooks_client + .send_finalized_block_hash(Bytes::from(block_hash.to_vec())) + .await + .context("failed to send finalized block hash to composer")?; + Ok(abci::response::FinalizeBlock { events: end_block.events, validator_updates: end_block.validator_updates, diff --git a/crates/astria-sequencer/src/app/test_utils.rs b/crates/astria-sequencer/src/app/test_utils.rs index 3acd71ed3..4c8249cc8 100644 --- a/crates/astria-sequencer/src/app/test_utils.rs +++ b/crates/astria-sequencer/src/app/test_utils.rs @@ -140,7 +140,8 @@ pub(crate) async fn initialize_app_with_storage( let snapshot = storage.latest_snapshot(); let mempool = Mempool::new(); let metrics = Box::leak(Box::new(Metrics::noop_metrics(&()).unwrap())); - let mut app = App::new(snapshot, mempool, metrics).await.unwrap(); + // TODO - temp addr + let mut app = App::new(snapshot, mempool, "127.0.0.1:232".to_string(), metrics).await.unwrap(); let genesis_state = genesis_state.unwrap_or_else(self::genesis_state); diff --git a/crates/astria-sequencer/src/client.rs b/crates/astria-sequencer/src/client.rs new file mode 100644 index 000000000..ebc156fc0 --- /dev/null +++ b/crates/astria-sequencer/src/client.rs @@ -0,0 +1,79 @@ +use anyhow::Context; +use astria_core::generated::{ + composer::v1alpha1::{ + sequencer_hooks_service_client::SequencerHooksServiceClient, + SendFinalizedHashRequest, + SendFinalizedHashResponse, + SendOptimisticBlockRequest, + SendOptimisticBlockResponse, + }, + protocol::transaction::v1alpha1::SequenceAction, +}; +use bytes::Bytes; +use tonic::transport::{ + Channel, + Endpoint, + Uri, +}; +use tracing::{ + instrument, + Instrument, +}; + +/// A newtype wrapper around [`SequencerHooksServiceClient`] to work with +/// idiomatic types. +#[derive(Clone)] +pub(crate) struct SequencerHooksClient { + uri: Uri, + inner: SequencerHooksServiceClient, +} + +impl SequencerHooksClient { + pub(crate) fn connect_lazy(uri: &str) -> anyhow::Result { + let uri: Uri = uri + .parse() + .context("failed to parse provided string as uri")?; + let endpoint = Endpoint::from(uri.clone()).connect_lazy(); + let inner = SequencerHooksServiceClient::new(endpoint); + Ok(Self { + uri, + inner, + }) + } + + pub(crate) fn uri(&self) -> String { + self.uri.to_string() + } + + #[instrument(skip_all, fields(uri = % self.uri), err)] + pub(super) async fn send_optimistic_block( + &self, + block_hash: Bytes, + seq_actions: Vec, + ) -> anyhow::Result { + let request = SendOptimisticBlockRequest { + block_hash, + seq_action: seq_actions, + }; + + let mut client = self.inner.clone(); + let response = client.send_optimistic_block(request).await?; + + Ok(response.into_inner()) + } + + #[instrument(skip_all, fields(uri = % self.uri), err)] + pub(super) async fn send_finalized_block_hash( + &self, + finalized_block_hash: Bytes, + ) -> anyhow::Result { + let request = SendFinalizedHashRequest { + block_hash: finalized_block_hash, + }; + + let mut client = self.inner.clone(); + let response = client.send_finalized_hash(request).await?; + + Ok(response.into_inner()) + } +} diff --git a/crates/astria-sequencer/src/config.rs b/crates/astria-sequencer/src/config.rs index 234a67ca5..6998446f9 100644 --- a/crates/astria-sequencer/src/config.rs +++ b/crates/astria-sequencer/src/config.rs @@ -31,6 +31,7 @@ pub struct Config { pub metrics_http_listener_addr: String, /// Writes a human readable format to stdout instead of JSON formatted OTEL trace data. pub pretty_print: bool, + pub composer_hook: String, } impl config::Config for Config { diff --git a/crates/astria-sequencer/src/lib.rs b/crates/astria-sequencer/src/lib.rs index d35e9e078..aa6197230 100644 --- a/crates/astria-sequencer/src/lib.rs +++ b/crates/astria-sequencer/src/lib.rs @@ -8,6 +8,7 @@ pub(crate) mod authority; pub(crate) mod benchmark_utils; pub(crate) mod bridge; mod build_info; +pub(crate) mod client; pub(crate) mod component; pub mod config; pub(crate) mod fee_asset_change; diff --git a/crates/astria-sequencer/src/sequencer.rs b/crates/astria-sequencer/src/sequencer.rs index ac5661ed7..a5c02262e 100644 --- a/crates/astria-sequencer/src/sequencer.rs +++ b/crates/astria-sequencer/src/sequencer.rs @@ -80,7 +80,7 @@ impl Sequencer { let snapshot = storage.latest_snapshot(); let mempool = Mempool::new(); - let app = App::new(snapshot, mempool.clone(), metrics) + let app = App::new(snapshot, mempool.clone(), config.composer_hook, metrics) .await .context("failed to initialize app")?; diff --git a/crates/astria-sequencer/src/service/consensus.rs b/crates/astria-sequencer/src/service/consensus.rs index 4a4b290cd..5c735f014 100644 --- a/crates/astria-sequencer/src/service/consensus.rs +++ b/crates/astria-sequencer/src/service/consensus.rs @@ -465,7 +465,8 @@ mod test { let snapshot = storage.latest_snapshot(); let mempool = Mempool::new(); let metrics = Box::leak(Box::new(Metrics::noop_metrics(&()).unwrap())); - let mut app = App::new(snapshot, mempool.clone(), metrics).await.unwrap(); + // TODO - temp addr + let mut app = App::new(snapshot, mempool.clone(), "127.0.0.1:34".to_string(), metrics).await.unwrap(); app.init_chain(storage.clone(), genesis_state, vec![], "test".to_string()) .await .unwrap(); diff --git a/proto/composerapis/astria/composer/v1alpha1/grpc_collector.proto b/proto/composerapis/astria/composer/v1alpha1/grpc_collector.proto index ab29f3b1d..65259db0c 100644 --- a/proto/composerapis/astria/composer/v1alpha1/grpc_collector.proto +++ b/proto/composerapis/astria/composer/v1alpha1/grpc_collector.proto @@ -2,7 +2,7 @@ syntax = 'proto3'; package astria.composer.v1alpha1; -import "astria/sequencerblock/v1alpha1/block.proto"; +import "astria/protocol/transactions/v1alpha1/types.proto"; // SubmitRollupTransactionRequest contains a rollup transaction to be submitted to the Shared Sequencer Network // via the Composer @@ -17,6 +17,11 @@ message SubmitRollupTransactionRequest { // It's currently an empty response which can be evolved in the future to include more information message SubmitRollupTransactionResponse {} +message SendOptimisticBlockRequest { + bytes block_hash = 1; + repeated astria.protocol.transactions.v1alpha1.SequenceAction seq_action = 2; +} + message SendOptimisticBlockResponse {} message SendFinalizedHashRequest { @@ -33,6 +38,6 @@ service GrpcCollectorService { } service SequencerHooksService { - rpc SendOptimisticBlock(sequencerblock.v1alpha1.FilteredSequencerBlock) returns (SendOptimisticBlockResponse) {} + rpc SendOptimisticBlock(SendOptimisticBlockRequest) returns (SendOptimisticBlockResponse) {} rpc SendFinalizedHash(SendFinalizedHashRequest) returns (SendFinalizedHashResponse) {} } From 70e87209e0048c2bfb3af91430b79c85ff7b1e69 Mon Sep 17 00:00:00 2001 From: Bharath Date: Thu, 29 Aug 2024 21:04:20 +0530 Subject: [PATCH 26/43] it compiles --- charts/composer/templates/configmap.yaml | 2 +- charts/sequencer/templates/configmaps.yaml | 1 + charts/sequencer/values.yaml | 8 +- crates/astria-composer/src/executor/mod.rs | 121 +++++++++++++++-- .../astria-composer/src/executor/simulator.rs | 127 +++++++++++++++++- crates/astria-sequencer/src/app/test_utils.rs | 4 +- crates/astria-sequencer/src/client.rs | 10 ++ .../astria-sequencer/src/service/consensus.rs | 9 +- dev/values/rollup/dev.yaml | 30 ++--- 9 files changed, 280 insertions(+), 32 deletions(-) diff --git a/charts/composer/templates/configmap.yaml b/charts/composer/templates/configmap.yaml index acdfab55e..a2a47e792 100644 --- a/charts/composer/templates/configmap.yaml +++ b/charts/composer/templates/configmap.yaml @@ -11,7 +11,7 @@ data: ASTRIA_COMPOSER_SEQUENCER_URL: "{{ tpl .Values.config.sequencerRpc . }}" ASTRIA_COMPOSER_ROLLUP: "{{ .Values.config.rollupName }}" ASTRIA_COMPOSER_ROLLUP_WEBSOCKET_URL: "ws://{{ .Values.global.rollupName }}-evm-service.{{ default .Release.Namespace .Values.global.namespaceOverride }}.svc.cluster.local:8546" - ASTRIA_COMPOSER_EXECUTION_API_URL: "ws://{{ .Values.global.rollupName }}-evm-service.{{ default .Release.Namespace .Values.global.namespaceOverride }}.svc.cluster.local:50051" + ASTRIA_COMPOSER_EXECUTION_API_URL: "http://{{ .Values.global.rollupName }}-evm-service.{{ default .Release.Namespace .Values.global.namespaceOverride }}.svc.cluster.local:50051" ASTRIA_COMPOSER_PRIVATE_KEY_FILE: "/var/secrets/{{ .Values.config.privateKey.secret.filename }}" ASTRIA_COMPOSER_MAX_BYTES_PER_BUNDLE: "{{ .Values.config.maxBytesPerBundle }}" ASTRIA_COMPOSER_MAX_BUNDLE_SIZE: "{{ .Values.config.maxBundleSize }}" diff --git a/charts/sequencer/templates/configmaps.yaml b/charts/sequencer/templates/configmaps.yaml index 2f49d4416..71aa53829 100644 --- a/charts/sequencer/templates/configmaps.yaml +++ b/charts/sequencer/templates/configmaps.yaml @@ -64,6 +64,7 @@ data: ASTRIA_SEQUENCER_METRICS_HTTP_LISTENER_ADDR: "0.0.0.0:{{ .Values.ports.sequencerMetrics }}" ASTRIA_SEQUENCER_FORCE_STDOUT: "{{ .Values.global.useTTY }}" ASTRIA_SEQUENCER_PRETTY_PRINT: "{{ .Values.global.useTTY }}" + ASTRIA_SEQUENCER_COMPOSER_HOOK: "http://composer-service.astria-dev-cluster.svc.cluster.local:50052" NO_COLOR: "{{ .Values.global.useTTY }}" ASTRIA_SEQUENCER_NO_OTEL: "{{ not .Values.sequencer.otel.enabled }}" OTEL_EXPORTER_OTLP_ENDPOINT: "{{ .Values.sequencer.otel.endpoint }}" diff --git a/charts/sequencer/values.yaml b/charts/sequencer/values.yaml index 89dfb9acc..bb05271d1 100644 --- a/charts/sequencer/values.yaml +++ b/charts/sequencer/values.yaml @@ -7,7 +7,7 @@ global: # Whether to use tty readable logging for astria services, when false use json. # Best to be false in production environments, true for clean logs on local dev. useTTY: true - dev: false + dev: true # sequencer core images images: @@ -16,9 +16,9 @@ images: tag: v0.38.8 devTag: v0.38.8 sequencer: - repo: ghcr.io/astriaorg/sequencer - tag: "0.16.0" - devTag: latest + repo: astria-sequencer + tag: "0.14.0" + devTag: tb moniker: "" genesis: diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index 9fa91c2db..9b2292f7b 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -1,8 +1,3 @@ -/// ! The `Executor` is responsible for: -/// - Nonce management -/// - Transaction signing -/// - Managing the connection to the sequencer -/// - Submitting transactions to the sequencer use std::{ collections::VecDeque, pin::Pin, @@ -10,6 +5,12 @@ use std::{ time::Duration, }; +/// ! The `Executor` is responsible for: +/// - Nonce management +/// - Transaction signing +/// - Managing the connection to the sequencer +/// - Submitting transactions to the sequencer +use astria_core::Protobuf; use astria_core::{ crypto::SigningKey, generated::{ @@ -117,7 +118,10 @@ mod tests; pub(crate) use builder::Builder; -use crate::executor::simulator::BundleSimulator; +use crate::executor::simulator::{ + BundleSimulationResult, + BundleSimulator, +}; // Duration to wait for the executor to drain all the remaining bundles before shutting down. // This is 16s because the timeout for the higher level executor task is 17s to shut down. @@ -223,6 +227,80 @@ impl Executor { self.status.subscribe() } + // TODO - improve the func args and the return type + async fn simulate_bundle( + &self, + bundle: SizedBundle, + parent_block: Vec, + ) -> eyre::Result<(SizedBundle, BundleSimulationResult)> { + let bundle_simulator = self.bundle_simulator.clone(); + // convert the sequence actions to RollupData + // TODO - clean up this code + let sequence_actions: Vec = parent_block + .iter() + .map(|action| SequenceAction::try_from_raw_ref(action).unwrap()) + .collect(); + let filtered_sequence_actions: Vec = sequence_actions + .iter() + .filter(|action| action.rollup_id == self.rollup_id) + .cloned() + .collect(); + let mut parent_block_rollup_data_items = vec![]; + for seq_action in filtered_sequence_actions { + let rollup_data = + astria_core::sequencerblock::v1alpha1::block::RollupData::SequencedData( + seq_action.data, + ); + parent_block_rollup_data_items.push(rollup_data); + } + + let parent_bundle_simulation_result = bundle_simulator + .clone() + .simulate_parent_bundle(parent_block_rollup_data_items) + .await + .wrap_err("failed to simulate bundle")?; + + // now we have the parent block hash, we should simulate the bundle on top of the parent + // hash + let bundle_simulation_result = bundle_simulator + .clone() + .simulate_bundle_on_block(bundle, parent_bundle_simulation_result.block().clone()) + .await + .wrap_err("failed to simulate bundle")?; + + let rollup_data_items: Vec = bundle_simulation_result + .included_actions() + .iter() + .map(astria_core::Protobuf::to_raw) + .collect(); + + info!("Creating BuilderBundlePacket"); + let builder_bundle = BuilderBundle { + transactions: rollup_data_items, + parent_hash: bundle_simulation_result.parent_hash(), + }; + + let builder_bundle_packet = BuilderBundlePacket { + bundle: Some(builder_bundle), + signature: Bytes::from(vec![]), + }; + let encoded_builder_bundle_packet = builder_bundle_packet.encode_to_vec(); + + info!("Created builder bundle packet: {:?}", builder_bundle_packet); + // we can give the BuilderBundlePacket the highest bundle max size possible + // since this is the only sequence action we are sending + let mut final_bundle = SizedBundle::new(self.max_bundle_size); + final_bundle + .try_push(SequenceAction { + rollup_id: self.rollup_id, + data: encoded_builder_bundle_packet.into(), + fee_asset: self.fee_asset.clone(), + }) + .wrap_err("couldn't push sequence action to bundle")?; + + Ok((final_bundle, bundle_simulation_result)) + } + async fn simulate_and_submit_bundle( &self, nonce: u32, @@ -343,6 +421,9 @@ impl Executor { self.status.send_modify(|status| status.is_connected = true); + let mut current_finalized_block_hash: Option = None; + let mut pending_builder_bundle_packet: Option = None; + let reason = loop { select! { biased; @@ -358,11 +439,33 @@ impl Executor { } Some(filtered_sequencer_block) = self.filtered_block_receiver.recv() => { - info!("received filtered sequencer block {:?}", filtered_sequencer_block.block_hash.to_ascii_lowercase()); + // we need to simulate the bundle + // and cache the simulated bundle and the block_hash + let bundle = bundle_factory.pop_now(); + if bundle.is_empty() { + debug!("no bundle to simulate") + } else { + // TODO - we should throw an error log when simulation fails rather than escaping the application + let res = self.simulate_bundle(bundle, filtered_sequencer_block.seq_action).await.wrap_err("failed to simulate bundle on top of received parent block")?; + pending_builder_bundle_packet = Some(res.0); + current_finalized_block_hash = Some(filtered_sequencer_block.block_hash.clone()); + debug!("simulating bundle on top of received parent block!") + } + info!("BHARATH: received filtered sequencer block {:?}", filtered_sequencer_block.block_hash.to_ascii_lowercase()); } - Some(finalized_block_hash) = self.finalized_block_hash_receiver.recv() => { - info!("received finalized block hash {:?}", finalized_block_hash.block_hash.to_ascii_lowercase()); + Some(finalized_block_hash) = self.finalized_block_hash_receiver.recv(), if submission_fut.is_terminated() => { + + if let Some(block_hash) = current_finalized_block_hash.clone() { + if block_hash == finalized_block_hash.block_hash { + // we can submit the pending builder bundle packet + let bundle = pending_builder_bundle_packet.take().unwrap(); + submission_fut = self.submit_bundle(nonce, bundle, self.metrics); + } else { + warn!("received finalized block hash does not match the current finalized block hash; skipping submission of pending builder bundle packet") + } + } + info!("BHARATH: received finalized block hash {:?}", finalized_block_hash.block_hash.to_ascii_lowercase()); } Some(next_bundle) = future::ready(bundle_factory.next_finished()), if submission_fut.is_terminated() => { diff --git a/crates/astria-composer/src/executor/simulator.rs b/crates/astria-composer/src/executor/simulator.rs index 625dd83e5..a0d3a784f 100644 --- a/crates/astria-composer/src/executor/simulator.rs +++ b/crates/astria-composer/src/executor/simulator.rs @@ -1,3 +1,4 @@ +use astria_core::execution::v1alpha2::Block; /// ! `BundleSimulator` is responsible for fetching the latest rollup commitment state /// and simulating the given bundle on top of the latest soft block. use astria_core::{ @@ -27,13 +28,19 @@ pub(crate) struct BundleSimulator { } pub(crate) struct BundleSimulationResult { + block: Block, included_actions: Vec, parent_hash: Bytes, } impl BundleSimulationResult { - pub(crate) fn new(included_sequence_actions: Vec, parent_hash: Bytes) -> Self { + pub(crate) fn new( + included_sequence_actions: Vec, + block: Block, + parent_hash: Bytes, + ) -> Self { Self { + block, included_actions: included_sequence_actions, parent_hash, } @@ -46,6 +53,10 @@ impl BundleSimulationResult { pub(crate) fn parent_hash(&self) -> Bytes { self.parent_hash.clone() } + + pub(crate) fn block(&self) -> &Block { + &self.block + } } impl BundleSimulator { @@ -56,6 +67,119 @@ impl BundleSimulator { }) } + // TODO - the interfaces below are weird but they work for now + // have cleaner interfaces + #[instrument(skip_all, fields(uri=self.execution_service_client.uri()))] + pub(crate) async fn simulate_parent_bundle( + self, + rollup_data: Vec, + ) -> eyre::Result { + // call GetCommitmentState to get the soft block + info!("Calling GetCommitmentState!"); + let commitment_state = self + .execution_service_client + .get_commitment_state_with_retry() + .await + .wrap_err("failed to get commitment state")?; + info!("Received CommitmentState of rollup"); + + let soft_block = commitment_state.soft(); + info!("Soft block hash is {:?}", soft_block.hash()); + // convert the sized bundle actions to a list of Vec + let actions: Vec> = rollup_data + .iter() + .map(|action| match action.clone() { + RollupData::SequencedData(data) => data.to_vec(), + _ => vec![], + }) + .filter(|data| !data.is_empty()) + .collect(); + + info!("Calling ExecuteBlock to simulate the bundle!"); + // as long as the timestamp > parent block timestamp, the block will be successfully + // created. It doesn't matter what timestamp we use anyway since we are not going to + // commit the block to the chain. + let timestamp = Timestamp { + seconds: soft_block.timestamp().seconds + 3, + nanos: 0, + }; + // call execute block with the bundle to get back the included transactions + let execute_block_response = self + .execution_service_client + .execute_block_with_retry( + soft_block.hash().clone(), + actions, + // use current timestamp + timestamp, + false, + ) + .await + .wrap_err("failed to execute block")?; + + let included_transactions = execute_block_response.included_transactions(); + info!( + "Bundle simulated on top of {:?} and {:?} transactions were included", + soft_block.hash(), + included_transactions.len() + ); + Ok(BundleSimulationResult::new( + included_transactions.to_vec(), + execute_block_response.block().clone(), + soft_block.hash().clone(), + )) + } + + pub(crate) async fn simulate_bundle_on_block( + self, + bundle: SizedBundle, + block: Block, + ) -> eyre::Result { + // convert the sized bundle actions to a list of Vec + let actions: Vec> = bundle + .into_actions() + .iter() + .map(|action| match action.as_sequence() { + Some(seq_action) => RollupData::SequencedData(seq_action.clone().data) + .to_raw() + .encode_to_vec(), + None => vec![], + }) + .filter(|data| !data.is_empty()) + .collect(); + + // as long as the timestamp > parent block timestamp, the block will be successfully + // created. It doesn't matter what timestamp we use anyway since we are not going to + // commit the block to the chain. + let timestamp = Timestamp { + seconds: block.timestamp().seconds + 3, + nanos: 0, + }; + // call execute block with the bundle to get back the included transactions + let execute_block_response = self + .execution_service_client + .execute_block_with_retry( + block.hash().clone(), + actions, + // use current timestamp + timestamp, + true, + ) + .await + .wrap_err("failed to execute block")?; + + let included_transactions = execute_block_response.included_transactions(); + info!( + "Bundle simulated on top of {:?} and {:?} transactions were included", + block.hash().clone(), + included_transactions.len() + ); + Ok(BundleSimulationResult::new( + included_transactions.to_vec(), + execute_block_response.block().clone(), + block.hash().clone(), + )) + } + #[instrument(skip_all, fields(uri=self.execution_service_client.uri()))] pub(crate) async fn simulate_bundle( self, @@ -114,6 +238,7 @@ impl BundleSimulator { ); Ok(BundleSimulationResult::new( included_transactions.to_vec(), + execute_block_response.block().clone(), soft_block.hash().clone(), )) } diff --git a/crates/astria-sequencer/src/app/test_utils.rs b/crates/astria-sequencer/src/app/test_utils.rs index 4c8249cc8..22f76a518 100644 --- a/crates/astria-sequencer/src/app/test_utils.rs +++ b/crates/astria-sequencer/src/app/test_utils.rs @@ -141,7 +141,9 @@ pub(crate) async fn initialize_app_with_storage( let mempool = Mempool::new(); let metrics = Box::leak(Box::new(Metrics::noop_metrics(&()).unwrap())); // TODO - temp addr - let mut app = App::new(snapshot, mempool, "127.0.0.1:232".to_string(), metrics).await.unwrap(); + let mut app = App::new(snapshot, mempool, "127.0.0.1:232".to_string(), metrics) + .await + .unwrap(); let genesis_state = genesis_state.unwrap_or_else(self::genesis_state); diff --git a/crates/astria-sequencer/src/client.rs b/crates/astria-sequencer/src/client.rs index ebc156fc0..98a70784a 100644 --- a/crates/astria-sequencer/src/client.rs +++ b/crates/astria-sequencer/src/client.rs @@ -16,6 +16,7 @@ use tonic::transport::{ Uri, }; use tracing::{ + info, instrument, Instrument, }; @@ -51,6 +52,11 @@ impl SequencerHooksClient { block_hash: Bytes, seq_actions: Vec, ) -> anyhow::Result { + info!( + "BHARATH: sending optimistic block hash to {:?}", + self.uri.to_string() + ); + let request = SendOptimisticBlockRequest { block_hash, seq_action: seq_actions, @@ -67,6 +73,10 @@ impl SequencerHooksClient { &self, finalized_block_hash: Bytes, ) -> anyhow::Result { + info!( + "BHARATH: sending finalized block hash to {:?}", + self.uri.to_string() + ); let request = SendFinalizedHashRequest { block_hash: finalized_block_hash, }; diff --git a/crates/astria-sequencer/src/service/consensus.rs b/crates/astria-sequencer/src/service/consensus.rs index 5c735f014..6a1c60c89 100644 --- a/crates/astria-sequencer/src/service/consensus.rs +++ b/crates/astria-sequencer/src/service/consensus.rs @@ -466,7 +466,14 @@ mod test { let mempool = Mempool::new(); let metrics = Box::leak(Box::new(Metrics::noop_metrics(&()).unwrap())); // TODO - temp addr - let mut app = App::new(snapshot, mempool.clone(), "127.0.0.1:34".to_string(), metrics).await.unwrap(); + let mut app = App::new( + snapshot, + mempool.clone(), + "127.0.0.1:34".to_string(), + metrics, + ) + .await + .unwrap(); app.init_chain(storage.clone(), genesis_state, vec![], "test".to_string()) .await .unwrap(); diff --git a/dev/values/rollup/dev.yaml b/dev/values/rollup/dev.yaml index d7f56d05a..090a0c525 100644 --- a/dev/values/rollup/dev.yaml +++ b/dev/values/rollup/dev.yaml @@ -102,21 +102,21 @@ evm-rollup: rpc: "http://celestia-service.astria-dev-cluster.svc.cluster.local:26658" token: "" - resources: - conductor: - requests: - cpu: 0.01 - memory: 1Mi - limits: - cpu: 0.1 - memory: 20Mi - geth: - requests: - cpu: 0.25 - memory: 256Mi - limits: - cpu: 2 - memory: 1Gi + resources: {} +# conductor: +# requests: +# cpu: 0.01 +# memory: 1Mi +# limits: +# cpu: 0.1 +# memory: 20Mi +# geth: +# requests: +# cpu: 0.25 +# memory: 256Mi +# limits: +# cpu: 2 +# memory: 1Gi storage: enabled: false From ba30cd1fc0f738c7ab79cdeedf5da9f7400bebb4 Mon Sep 17 00:00:00 2001 From: Bharath Date: Fri, 6 Sep 2024 14:22:32 +0530 Subject: [PATCH 27/43] trusted builder iteration 2 --- Cargo.lock | 2 + charts/deploy.just | 4 +- charts/evm-rollup/values.yaml | 4 ++ charts/sequencer/templates/configmaps.yaml | 1 + charts/sequencer/values.yaml | 2 + crates/astria-composer/src/executor/mod.rs | 67 +++++++++++-------- .../astria-composer/src/executor/simulator.rs | 12 ++-- crates/astria-composer/src/grpc.rs | 3 +- crates/astria-composer/src/sequencer_hooks.rs | 27 +++++++- .../src/generated/astria.composer.v1alpha1.rs | 4 +- .../astria.composer.v1alpha1.serde.rs | 17 +++++ crates/astria-core/src/generated/mod.rs | 2 + crates/astria-sequencer/Cargo.toml | 1 + crates/astria-sequencer/local.env.example | 4 ++ crates/astria-sequencer/src/app/mod.rs | 30 +++++++-- crates/astria-sequencer/src/client.rs | 26 ++++++- crates/astria-sequencer/src/config.rs | 1 + crates/astria-sequencer/src/sequencer.rs | 12 +++- dev/values/rollup/dev.yaml | 21 +++++- dev/values/validators/all.yml | 2 +- dev/values/validators/node0.yml | 30 ++++++--- dev/values/validators/node1.yml | 32 ++++++--- dev/values/validators/node2.yml | 32 ++++++--- .../composer/v1alpha1/grpc_collector.proto | 2 + 24 files changed, 261 insertions(+), 77 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2ec8bb3c5..b5f11d849 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -812,6 +812,8 @@ dependencies = [ "ibc-types", "insta", "matchit", + "metrics", + "pbjson-types", "penumbra-ibc", "penumbra-proto", "penumbra-tower-trace", diff --git a/charts/deploy.just b/charts/deploy.just index 5b7954130..cf143e772 100644 --- a/charts/deploy.just +++ b/charts/deploy.just @@ -79,7 +79,9 @@ deploy-sequencer name=validatorName: -f dev/values/validators/{{name}}.yml \ -n astria-validator-{{name}} --create-namespace \ {{name}}-sequencer-chart ./charts/sequencer -deploy-sequencers: (deploy-sequencer "node0") (deploy-sequencer "node1") (deploy-sequencer "node2") +#deploy-sequencers: (deploy-sequencer "node0") (deploy-sequencer "node1") (deploy-sequencer "node2") +deploy-sequencers: (deploy-sequencer "node0") (deploy-sequencer "node1") + deploy-hermes-local: helm install hermes-local-chart ./charts/hermes \ diff --git a/charts/evm-rollup/values.yaml b/charts/evm-rollup/values.yaml index 74191a356..ccf6b572f 100644 --- a/charts/evm-rollup/values.yaml +++ b/charts/evm-rollup/values.yaml @@ -89,6 +89,10 @@ genesis: value: balance: "0" code: "0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe03601600081602082378035828234f58015156039578182fd5b8082525050506014600cf3" + - address: "0xC873fC6685abF295cc537811C234B2B3aD54Af42" + value: + balance: "1000000000000000000000" + # Example of simple genesis account funding # - address: "0xaC21B97d35Bf75A7dAb16f35b111a50e78A72F30" # value: diff --git a/charts/sequencer/templates/configmaps.yaml b/charts/sequencer/templates/configmaps.yaml index 71aa53829..8898b93eb 100644 --- a/charts/sequencer/templates/configmaps.yaml +++ b/charts/sequencer/templates/configmaps.yaml @@ -65,6 +65,7 @@ data: ASTRIA_SEQUENCER_FORCE_STDOUT: "{{ .Values.global.useTTY }}" ASTRIA_SEQUENCER_PRETTY_PRINT: "{{ .Values.global.useTTY }}" ASTRIA_SEQUENCER_COMPOSER_HOOK: "http://composer-service.astria-dev-cluster.svc.cluster.local:50052" + ASTRIA_SEQUENCER_COMPOSER_HOOK_ENABLED: "{{ .Values.sequencer.composerHook.enabled}}" NO_COLOR: "{{ .Values.global.useTTY }}" ASTRIA_SEQUENCER_NO_OTEL: "{{ not .Values.sequencer.otel.enabled }}" OTEL_EXPORTER_OTLP_ENDPOINT: "{{ .Values.sequencer.otel.endpoint }}" diff --git a/charts/sequencer/values.yaml b/charts/sequencer/values.yaml index bb05271d1..39bd4c671 100644 --- a/charts/sequencer/values.yaml +++ b/charts/sequencer/values.yaml @@ -77,6 +77,8 @@ sequencer: tracesTimeout: 10 otlpHeaders: traceHeaders: + composerHook: + enabled: false cometbft: config: diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index 9b2292f7b..2ed4608a4 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -227,12 +227,15 @@ impl Executor { self.status.subscribe() } + #[instrument(skip_all)] // TODO - improve the func args and the return type async fn simulate_bundle( &self, bundle: SizedBundle, parent_block: Vec, + time: Option ) -> eyre::Result<(SizedBundle, BundleSimulationResult)> { + info!("BHARATH: simulating bundle on top of received parent block!"); let bundle_simulator = self.bundle_simulator.clone(); // convert the sequence actions to RollupData // TODO - clean up this code @@ -254,14 +257,17 @@ impl Executor { parent_block_rollup_data_items.push(rollup_data); } + info!("BHARATH: time is {:?}", time); + info!("BHARATH: getting parent_bundle_simulation_result"); let parent_bundle_simulation_result = bundle_simulator .clone() - .simulate_parent_bundle(parent_block_rollup_data_items) + .simulate_parent_bundle(parent_block_rollup_data_items, time.unwrap()) .await .wrap_err("failed to simulate bundle")?; // now we have the parent block hash, we should simulate the bundle on top of the parent // hash + info!("BHARATH: getting actual bundle simulation"); let bundle_simulation_result = bundle_simulator .clone() .simulate_bundle_on_block(bundle, parent_bundle_simulation_result.block().clone()) @@ -301,14 +307,13 @@ impl Executor { Ok((final_bundle, bundle_simulation_result)) } + #[instrument(skip_all, fields(nonce.initial = %nonce))] async fn simulate_and_submit_bundle( &self, nonce: u32, bundle: SizedBundle, metrics: &'static Metrics, ) -> eyre::Result>> { - info!("Starting bundle simulation!"); - let bundle_simulator = self.bundle_simulator.clone(); // simulate the bundle @@ -446,53 +451,57 @@ impl Executor { debug!("no bundle to simulate") } else { // TODO - we should throw an error log when simulation fails rather than escaping the application - let res = self.simulate_bundle(bundle, filtered_sequencer_block.seq_action).await.wrap_err("failed to simulate bundle on top of received parent block")?; + let res = self.simulate_bundle(bundle, filtered_sequencer_block.seq_action, filtered_sequencer_block.time) + .await.wrap_err("failed to simulate bundle on top of received parent block")?; pending_builder_bundle_packet = Some(res.0); - current_finalized_block_hash = Some(filtered_sequencer_block.block_hash.clone()); debug!("simulating bundle on top of received parent block!") } - info!("BHARATH: received filtered sequencer block {:?}", filtered_sequencer_block.block_hash.to_ascii_lowercase()); + current_finalized_block_hash = Some(filtered_sequencer_block.block_hash.clone()); } Some(finalized_block_hash) = self.finalized_block_hash_receiver.recv(), if submission_fut.is_terminated() => { - if let Some(block_hash) = current_finalized_block_hash.clone() { if block_hash == finalized_block_hash.block_hash { // we can submit the pending builder bundle packet - let bundle = pending_builder_bundle_packet.take().unwrap(); - submission_fut = self.submit_bundle(nonce, bundle, self.metrics); + if let Some(builder_bundle_packet) = pending_builder_bundle_packet.clone() { + if !builder_bundle_packet.is_empty() { + submission_fut = self.submit_bundle(nonce, builder_bundle_packet, self.metrics); + } + // TODO - ideally we need to use take() here + pending_builder_bundle_packet = None; + current_finalized_block_hash = None; + } } else { warn!("received finalized block hash does not match the current finalized block hash; skipping submission of pending builder bundle packet") } } - info!("BHARATH: received finalized block hash {:?}", finalized_block_hash.block_hash.to_ascii_lowercase()); } - Some(next_bundle) = future::ready(bundle_factory.next_finished()), if submission_fut.is_terminated() => { - let bundle = next_bundle.pop(); - if !bundle.is_empty() { - submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; - } - } + // Some(next_bundle) = future::ready(bundle_factory.next_finished()), if submission_fut.is_terminated() => { + // let bundle = next_bundle.pop(); + // if !bundle.is_empty() { + // submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; + // } + // } // receive new seq_action and bundle it. will not pull from the channel if `bundle_factory` is full Some(seq_action) = self.serialized_rollup_transactions.recv(), if !bundle_factory.is_full() => { self.bundle_seq_action(seq_action, &mut bundle_factory); } - // try to preempt current bundle if the timer has ticked without submitting the next bundle - () = &mut block_timer, if submission_fut.is_terminated() => { - let bundle = bundle_factory.pop_now(); - if bundle.is_empty() { - debug!("block timer ticked, but no bundle to submit to sequencer"); - block_timer.as_mut().reset(reset_time()); - } else { - debug!( - "forcing bundle submission to sequencer due to block timer" - ); - submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; - } - } + // // try to preempt current bundle if the timer has ticked without submitting the next bundle + // () = &mut block_timer, if submission_fut.is_terminated() => { + // let bundle = bundle_factory.pop_now(); + // if bundle.is_empty() { + // debug!("block timer ticked, but no bundle to submit to sequencer"); + // block_timer.as_mut().reset(reset_time()); + // } else { + // debug!( + // "forcing bundle submission to sequencer due to block timer" + // ); + // submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; + // } + // } } }; diff --git a/crates/astria-composer/src/executor/simulator.rs b/crates/astria-composer/src/executor/simulator.rs index a0d3a784f..338bd8f12 100644 --- a/crates/astria-composer/src/executor/simulator.rs +++ b/crates/astria-composer/src/executor/simulator.rs @@ -73,7 +73,9 @@ impl BundleSimulator { pub(crate) async fn simulate_parent_bundle( self, rollup_data: Vec, + time: pbjson_types::Timestamp, ) -> eyre::Result { + info!("Creating parent block!"); // call GetCommitmentState to get the soft block info!("Calling GetCommitmentState!"); let commitment_state = self @@ -99,10 +101,6 @@ impl BundleSimulator { // as long as the timestamp > parent block timestamp, the block will be successfully // created. It doesn't matter what timestamp we use anyway since we are not going to // commit the block to the chain. - let timestamp = Timestamp { - seconds: soft_block.timestamp().seconds + 3, - nanos: 0, - }; // call execute block with the bundle to get back the included transactions let execute_block_response = self .execution_service_client @@ -110,7 +108,7 @@ impl BundleSimulator { soft_block.hash().clone(), actions, // use current timestamp - timestamp, + time, false, ) .await @@ -118,7 +116,7 @@ impl BundleSimulator { let included_transactions = execute_block_response.included_transactions(); info!( - "Bundle simulated on top of {:?} and {:?} transactions were included", + "Parent block created on top of {:?} and {:?} transactions were included", soft_block.hash(), included_transactions.len() ); @@ -129,11 +127,13 @@ impl BundleSimulator { )) } + #[instrument(skip_all, fields(uri=self.execution_service_client.uri()), err)] pub(crate) async fn simulate_bundle_on_block( self, bundle: SizedBundle, block: Block, ) -> eyre::Result { + info!("Simulating bundle on created parent block!"); // convert the sized bundle actions to a list of Vec let actions: Vec> = bundle .into_actions() diff --git a/crates/astria-composer/src/grpc.rs b/crates/astria-composer/src/grpc.rs index bc6934118..524ba1b50 100644 --- a/crates/astria-composer/src/grpc.rs +++ b/crates/astria-composer/src/grpc.rs @@ -24,7 +24,7 @@ use tokio::{ net::TcpListener, }; use tokio_util::sync::CancellationToken; -use tracing::instrument; +use tracing::{info, instrument}; use crate::{ collectors, @@ -88,6 +88,7 @@ impl GrpcServer { } pub(crate) async fn run_until_stopped(self) -> eyre::Result<()> { + info!("launching grpc server with grpc collector and sequencer hooks!"); let (mut health_reporter, health_service) = tonic_health::server::health_reporter(); let composer_service = GrpcCollectorServiceServer::new(self.grpc_collector); diff --git a/crates/astria-composer/src/sequencer_hooks.rs b/crates/astria-composer/src/sequencer_hooks.rs index 0046434c3..7e342e2b3 100644 --- a/crates/astria-composer/src/sequencer_hooks.rs +++ b/crates/astria-composer/src/sequencer_hooks.rs @@ -10,6 +10,7 @@ use astria_core::generated::composer::v1alpha1::{ SendOptimisticBlockRequest, SendOptimisticBlockResponse, }; +use astria_eyre::eyre::WrapErr; use tokio::sync::{ mpsc, mpsc::error::SendTimeoutError, @@ -19,6 +20,7 @@ use tonic::{ Response, Status, }; +use tracing::info; const SEND_TIMEOUT: u64 = 2; @@ -64,7 +66,17 @@ impl SequencerHooksService for SequencerHooks { request: Request, ) -> Result, Status> { let inner = request.into_inner(); - todo!() + return match self + .send_optimistic_block_with_timeout(inner) + .await + .wrap_err("unable to send optimistic block to executor") + { + Ok(()) => Ok(Response::new(SendOptimisticBlockResponse {})), + Err(e) => { + info!("Failed to send optimistic block: {:?}", e); + return Err(Status::internal("Failed to send optimistic block")); + } + }; } async fn send_finalized_hash( @@ -72,6 +84,17 @@ impl SequencerHooksService for SequencerHooks { request: Request, ) -> Result, Status> { let inner = request.into_inner(); - todo!() + + return match self + .send_finalized_hash_with_timeout(inner) + .await + .wrap_err("unable to send finalized block hash to executor") + { + Ok(()) => Ok(Response::new(SendFinalizedHashResponse {})), + Err(e) => { + info!("Failed to send finalized_block hash: {:?}", e); + return Err(Status::internal("Failed to send finalized block hash")); + } + }; } } diff --git a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs index 0086dab3c..0d59d033f 100644 --- a/crates/astria-core/src/generated/astria.composer.v1alpha1.rs +++ b/crates/astria-core/src/generated/astria.composer.v1alpha1.rs @@ -1,5 +1,5 @@ -use crate::generated::protocol::transaction::v1alpha1; +use crate::generated::v1alpha1; /// SubmitRollupTransactionRequest contains a rollup transaction to be submitted to the Shared Sequencer Network /// via the Composer @@ -41,6 +41,8 @@ pub struct SendOptimisticBlockRequest { pub seq_action: ::prost::alloc::vec::Vec< v1alpha1::SequenceAction, >, + #[prost(message, optional, tag = "3")] + pub time: ::core::option::Option<::pbjson_types::Timestamp>, } impl ::prost::Name for SendOptimisticBlockRequest { const NAME: &'static str = "SendOptimisticBlockRequest"; diff --git a/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs b/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs index 72e9b7b45..f9ab1af71 100644 --- a/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs +++ b/crates/astria-core/src/generated/astria.composer.v1alpha1.serde.rs @@ -401,6 +401,9 @@ impl serde::Serialize for SendOptimisticBlockRequest { if !self.seq_action.is_empty() { len += 1; } + if self.time.is_some() { + len += 1; + } let mut struct_ser = serializer.serialize_struct("astria.composer.v1alpha1.SendOptimisticBlockRequest", len)?; if !self.block_hash.is_empty() { #[allow(clippy::needless_borrow)] @@ -409,6 +412,9 @@ impl serde::Serialize for SendOptimisticBlockRequest { if !self.seq_action.is_empty() { struct_ser.serialize_field("seqAction", &self.seq_action)?; } + if let Some(v) = self.time.as_ref() { + struct_ser.serialize_field("time", v)?; + } struct_ser.end() } } @@ -423,12 +429,14 @@ impl<'de> serde::Deserialize<'de> for SendOptimisticBlockRequest { "blockHash", "seq_action", "seqAction", + "time", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { BlockHash, SeqAction, + Time, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -452,6 +460,7 @@ impl<'de> serde::Deserialize<'de> for SendOptimisticBlockRequest { match value { "blockHash" | "block_hash" => Ok(GeneratedField::BlockHash), "seqAction" | "seq_action" => Ok(GeneratedField::SeqAction), + "time" => Ok(GeneratedField::Time), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -473,6 +482,7 @@ impl<'de> serde::Deserialize<'de> for SendOptimisticBlockRequest { { let mut block_hash__ = None; let mut seq_action__ = None; + let mut time__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::BlockHash => { @@ -489,11 +499,18 @@ impl<'de> serde::Deserialize<'de> for SendOptimisticBlockRequest { } seq_action__ = Some(map_.next_value()?); } + GeneratedField::Time => { + if time__.is_some() { + return Err(serde::de::Error::duplicate_field("time")); + } + time__ = map_.next_value()?; + } } } Ok(SendOptimisticBlockRequest { block_hash: block_hash__.unwrap_or_default(), seq_action: seq_action__.unwrap_or_default(), + time: time__, }) } } diff --git a/crates/astria-core/src/generated/mod.rs b/crates/astria-core/src/generated/mod.rs index cbadc195c..5cf06adbc 100644 --- a/crates/astria-core/src/generated/mod.rs +++ b/crates/astria-core/src/generated/mod.rs @@ -11,6 +11,8 @@ //! [`buf`]: https://buf.build //! [`tools/protobuf-compiler`]: ../../../../tools/protobuf-compiler +use crate::generated::protocol::transaction::v1alpha1; + #[path = ""] pub mod astria_vendored { #[path = ""] diff --git a/crates/astria-sequencer/Cargo.toml b/crates/astria-sequencer/Cargo.toml index e1f698931..fdb93b357 100644 --- a/crates/astria-sequencer/Cargo.toml +++ b/crates/astria-sequencer/Cargo.toml @@ -26,6 +26,7 @@ merkle = { package = "astria-merkle", path = "../astria-merkle" } telemetry = { package = "astria-telemetry", path = "../astria-telemetry", features = [ "display", ] } +pbjson-types = { workspace = true } anyhow = "1" borsh = { version = "1", features = ["derive"] } diff --git a/crates/astria-sequencer/local.env.example b/crates/astria-sequencer/local.env.example index 61be09845..87d928237 100644 --- a/crates/astria-sequencer/local.env.example +++ b/crates/astria-sequencer/local.env.example @@ -37,6 +37,10 @@ ASTRIA_SEQUENCER_PRETTY_PRINT=false # Address of the composer node ASTRIA_SEQUENCER_COMPOSER_HOOK="" +# whether the composer hook is enabled or not. +# TODO - rename these +ASTRIA_SEQUENCER_COMPOSER_HOOK_ENABLED=false + # If set to any non-empty value removes ANSI escape characters from the pretty # printed output. Note that this does nothing unless `ASTRIA_SEQUENCER_PRETTY_PRINT` # is set to `true`. diff --git a/crates/astria-sequencer/src/app/mod.rs b/crates/astria-sequencer/src/app/mod.rs index b4827df45..18854c40c 100644 --- a/crates/astria-sequencer/src/app/mod.rs +++ b/crates/astria-sequencer/src/app/mod.rs @@ -69,6 +69,7 @@ use tendermint::{ }; use tracing::{ debug, + error, info, instrument, }; @@ -187,6 +188,7 @@ impl App { snapshot: Snapshot, mempool: Mempool, composer_uri: String, + composer_hook_enabled: bool, metrics: &'static Metrics, ) -> anyhow::Result { debug!("initializing App instance"); @@ -204,8 +206,9 @@ impl App { // there should be no unexpected copies elsewhere. let state = Arc::new(StateDelta::new(snapshot)); - let sequencer_hooks_client = SequencerHooksClient::connect_lazy(&composer_uri) - .context("failed to connect to sequencer hooks service")?; + let sequencer_hooks_client = + SequencerHooksClient::connect_lazy(&composer_uri, composer_hook_enabled) + .context("failed to connect to sequencer hooks service")?; Ok(Self { state, @@ -477,15 +480,23 @@ impl App { .filter_map(Action::as_sequence) .map(|seq| seq.to_raw().clone()) .collect::>(); + let time = process_proposal.time; info!("BHARATH: Sending optimistic block to composer!"); - self.sequencer_hooks_client + if let Err(e) = self + .sequencer_hooks_client .send_optimistic_block( Bytes::from(block_hash.as_bytes().to_vec()), sequence_actions, + time, ) .await - .context("failed to send optimistic block to composer")?; + .context("failed to send optimistic block to composer") + { + error!(error = %e, "failed to send optimistic block to composer"); + } else { + info!("Sent optimistic block to composer!"); + } Ok(()) } @@ -942,10 +953,17 @@ impl App { .await .context("failed to update mempool after finalization")?; - self.sequencer_hooks_client + if let Err(e) = self + .sequencer_hooks_client .send_finalized_block_hash(Bytes::from(block_hash.to_vec())) .await - .context("failed to send finalized block hash to composer")?; + .context("failed to send finalized block hash to composer") + { + // do not fail the entire method if this fails + error!(error = %e, "failed to send finalized block hash to composer"); + } else { + info!("Sent finalized block hash to composer!"); + } Ok(abci::response::FinalizeBlock { events: end_block.events, diff --git a/crates/astria-sequencer/src/client.rs b/crates/astria-sequencer/src/client.rs index 98a70784a..d78f59c3f 100644 --- a/crates/astria-sequencer/src/client.rs +++ b/crates/astria-sequencer/src/client.rs @@ -10,6 +10,8 @@ use astria_core::generated::{ protocol::transaction::v1alpha1::SequenceAction, }; use bytes::Bytes; +use tendermint::Time; +use tendermint_proto::google::protobuf::Timestamp; use tonic::transport::{ Channel, Endpoint, @@ -26,11 +28,12 @@ use tracing::{ #[derive(Clone)] pub(crate) struct SequencerHooksClient { uri: Uri, + enabled: bool, inner: SequencerHooksServiceClient, } impl SequencerHooksClient { - pub(crate) fn connect_lazy(uri: &str) -> anyhow::Result { + pub(crate) fn connect_lazy(uri: &str, enabled: bool) -> anyhow::Result { let uri: Uri = uri .parse() .context("failed to parse provided string as uri")?; @@ -38,6 +41,7 @@ impl SequencerHooksClient { let inner = SequencerHooksServiceClient::new(endpoint); Ok(Self { uri, + enabled, inner, }) } @@ -51,15 +55,31 @@ impl SequencerHooksClient { &self, block_hash: Bytes, seq_actions: Vec, + time: Time, ) -> anyhow::Result { + if !self.enabled { + info!("BHARATH: optimistic block sending is disabled"); + return Ok(SendOptimisticBlockResponse::default()); + } info!( "BHARATH: sending optimistic block hash to {:?}", self.uri.to_string() ); + let Timestamp { + seconds, + nanos, + } = time.into(); + + info!("BHARATH: seconds: {:?}, nanos: {:?}", seconds, nanos); + let request = SendOptimisticBlockRequest { block_hash, seq_action: seq_actions, + time: Some(pbjson_types::Timestamp { + seconds, + nanos, + }), }; let mut client = self.inner.clone(); @@ -73,6 +93,10 @@ impl SequencerHooksClient { &self, finalized_block_hash: Bytes, ) -> anyhow::Result { + if !self.enabled { + info!("BHARATH: finalized block hash sending is disabled"); + return Ok(SendFinalizedHashResponse::default()); + } info!( "BHARATH: sending finalized block hash to {:?}", self.uri.to_string() diff --git a/crates/astria-sequencer/src/config.rs b/crates/astria-sequencer/src/config.rs index 6998446f9..bafa1181b 100644 --- a/crates/astria-sequencer/src/config.rs +++ b/crates/astria-sequencer/src/config.rs @@ -32,6 +32,7 @@ pub struct Config { /// Writes a human readable format to stdout instead of JSON formatted OTEL trace data. pub pretty_print: bool, pub composer_hook: String, + pub composer_hook_enabled: bool, } impl config::Config for Config { diff --git a/crates/astria-sequencer/src/sequencer.rs b/crates/astria-sequencer/src/sequencer.rs index a5c02262e..b1e3cce39 100644 --- a/crates/astria-sequencer/src/sequencer.rs +++ b/crates/astria-sequencer/src/sequencer.rs @@ -80,9 +80,15 @@ impl Sequencer { let snapshot = storage.latest_snapshot(); let mempool = Mempool::new(); - let app = App::new(snapshot, mempool.clone(), config.composer_hook, metrics) - .await - .context("failed to initialize app")?; + let app = App::new( + snapshot, + mempool.clone(), + config.composer_hook, + config.composer_hook_enabled, + metrics, + ) + .await + .context("failed to initialize app")?; let consensus_service = tower::ServiceBuilder::new() .layer(request_span::layer(|req: &ConsensusRequest| { diff --git a/dev/values/rollup/dev.yaml b/dev/values/rollup/dev.yaml index 090a0c525..765cc2e16 100644 --- a/dev/values/rollup/dev.yaml +++ b/dev/values/rollup/dev.yaml @@ -79,12 +79,29 @@ evm-rollup: value: balance: "0" code: "0x60806040526004361061004a5760003560e01c80637eb6dec71461004f578063a996e0201461009d578063b6476c7e146100b2578063bab916d0146100d4578063db97dc98146100e7575b600080fd5b34801561005b57600080fd5b506100837f000000000000000000000000000000000000000000000000000000000000000981565b60405163ffffffff90911681526020015b60405180910390f35b6100b06100ab366004610315565b6100fc565b005b3480156100be57600080fd5b506100c761019e565b6040516100949190610381565b6100b06100e23660046103cf565b61022c565b3480156100f357600080fd5b506100c76102bf565b3460006101297f000000000000000000000000000000000000000000000000000000003b9aca0083610411565b1161014f5760405162461bcd60e51b815260040161014690610433565b60405180910390fd5b34336001600160a01b03167f0c64e29a5254a71c7f4e52b3d2d236348c80e00a00ba2e1961962bd2827c03fb8787878760405161018f94939291906104ea565b60405180910390a35050505050565b600180546101ab9061051c565b80601f01602080910402602001604051908101604052809291908181526020018280546101d79061051c565b80156102245780601f106101f957610100808354040283529160200191610224565b820191906000526020600020905b81548152906001019060200180831161020757829003601f168201915b505050505081565b3460006102597f000000000000000000000000000000000000000000000000000000003b9aca0083610411565b116102765760405162461bcd60e51b815260040161014690610433565b34336001600160a01b03167f0f4961cab7530804898499aa89f5ec81d1a73102e2e4a1f30f88e5ae3513ba2a85856040516102b2929190610556565b60405180910390a3505050565b600080546101ab9061051c565b60008083601f8401126102de57600080fd5b50813567ffffffffffffffff8111156102f657600080fd5b60208301915083602082850101111561030e57600080fd5b9250929050565b6000806000806040858703121561032b57600080fd5b843567ffffffffffffffff8082111561034357600080fd5b61034f888389016102cc565b9096509450602087013591508082111561036857600080fd5b50610375878288016102cc565b95989497509550505050565b600060208083528351808285015260005b818110156103ae57858101830151858201604001528201610392565b506000604082860101526040601f19601f8301168501019250505092915050565b600080602083850312156103e257600080fd5b823567ffffffffffffffff8111156103f957600080fd5b610405858286016102cc565b90969095509350505050565b60008261042e57634e487b7160e01b600052601260045260246000fd5b500490565b60208082526062908201527f417374726961576974686472617765723a20696e73756666696369656e74207660408201527f616c75652c206d7573742062652067726561746572207468616e203130202a2a60608201527f20283138202d20424153455f434841494e5f41535345545f505245434953494f6080820152614e2960f01b60a082015260c00190565b81835281816020850137506000828201602090810191909152601f909101601f19169091010190565b6040815260006104fe6040830186886104c1565b82810360208401526105118185876104c1565b979650505050505050565b600181811c9082168061053057607f821691505b60208210810361055057634e487b7160e01b600052602260045260246000fd5b50919050565b60208152600061056a6020830184866104c1565b94935050505056fea264697066735822122047a7ef84c0be4640572989abfc01decbc1ae143d4659f1b32047978c67ebc9c864736f6c63430008150033" - + - address: "0xC873fC6685abF295cc537811C234B2B3aD54Af42" + value: + balance: "1000000000000000000000" + - address: "0xD4223e904c40d432524f57c1026cFFF65F19b4b8" + value: + balance: "1000000000000000000000" + - address: "0x77D552845b1834420D021ED283785EA8C4e216f8" + value: + balance: "1000000000000000000000" + - address: "0x5E7db5359356ED0ED2F4aA9B32A2Cc5a21ce8cA0" + value: + balance: "1000000000000000000000" + - address: "0xE8a1b7cd69B49cEFDA97671C2EC3FbFF538a2a8C" + value: + balance: "1000000000000000000000" + - address: "0x067E716155Cd2B781D6681610ad685Ee146AFb9A" + value: + balance: "1000000000000000000000" config: # The level at which core astria components will log out # Options are: error, warn, info, and debug - logLevel: "debug" + logLevel: "error" conductor: # Determines what will drive block execution, options are: diff --git a/dev/values/validators/all.yml b/dev/values/validators/all.yml index 0b482a4e6..6ad4e24fb 100644 --- a/dev/values/validators/all.yml +++ b/dev/values/validators/all.yml @@ -1,4 +1,4 @@ -global: +iglobal: dev: true genesis: diff --git a/dev/values/validators/node0.yml b/dev/values/validators/node0.yml index 9dd3f4ddf..47f6d582b 100644 --- a/dev/values/validators/node0.yml +++ b/dev/values/validators/node0.yml @@ -2,21 +2,31 @@ global: namespaceOverride: astria-dev-cluster +images: + cometBFT: + repo: docker.io/cometbft/cometbft + tag: v0.38.8 + devTag: v0.38.8 + sequencer: + repo: astria-sequencer + tag: "0.14.0" + devTag: tb + moniker: node0 genesis: validators: - - name: node0 - power: '1' - address: 091E47761C58C474534F4D414AF104A6CAF90C22 - pubKey: lV57+rGs2vac7mvkGHP1oBFGHPJM3a+WoAzeFDCJDNU= +# - name: node0 +# power: '1' +# address: 091E47761C58C474534F4D414AF104A6CAF90C22 +# pubKey: lV57+rGs2vac7mvkGHP1oBFGHPJM3a+WoAzeFDCJDNU= - name: node1 address: E82D827830B163D5179291FB27BB58E605DF2FA2 pubKey: NDE9F44v3l4irmkZxNmrZkywoGmggLlaBo5rE/Cis8M= power: '1' - - name: node2 - power: '1' - address: 8C17BBDC7C350C83C550163458FC9B7A5B54A64E - pubKey: 4v1RdMiKkWgBBTTP26iRSLOEkAY99gMVfZijm6OCzjs= +# - name: node2 +# power: '1' +# address: 8C17BBDC7C350C83C550163458FC9B7A5B54A64E +# pubKey: 4v1RdMiKkWgBBTTP26iRSLOEkAY99gMVfZijm6OCzjs= cometbft: secrets: @@ -39,6 +49,10 @@ cometbft: - 96c652f63b5d5d5027b42e9af906082ee7c598d9@node1-sequencer-p2p-service.astria-validator-node1.svc.cluster.local:26656 - 4a4345939744d64ca370dff266e2913dd41b4e88@node2-sequencer-p2p-service.astria-validator-node2.svc.cluster.local:26656 +sequencer: + composerHook: + enabled: true + ingress: rpc: enabled: true diff --git a/dev/values/validators/node1.yml b/dev/values/validators/node1.yml index 5e6fe80ea..0a596b31a 100644 --- a/dev/values/validators/node1.yml +++ b/dev/values/validators/node1.yml @@ -1,19 +1,31 @@ # Override value example for second validator from main chart + + +images: + cometBFT: + repo: docker.io/cometbft/cometbft + tag: v0.38.8 + devTag: v0.38.8 + sequencer: + repo: astria-sequencer + tag: "0.14.0" + devTag: tb + moniker: 'node1' genesis: validators: - - name: node0 - power: '1' - address: 091E47761C58C474534F4D414AF104A6CAF90C22 - pubKey: lV57+rGs2vac7mvkGHP1oBFGHPJM3a+WoAzeFDCJDNU= +# - name: node0 +# power: '1' +# address: 091E47761C58C474534F4D414AF104A6CAF90C22 +# pubKey: lV57+rGs2vac7mvkGHP1oBFGHPJM3a+WoAzeFDCJDNU= - name: node1 address: E82D827830B163D5179291FB27BB58E605DF2FA2 pubKey: NDE9F44v3l4irmkZxNmrZkywoGmggLlaBo5rE/Cis8M= power: '1' - - name: node2 - power: '1' - address: 8C17BBDC7C350C83C550163458FC9B7A5B54A64E - pubKey: 4v1RdMiKkWgBBTTP26iRSLOEkAY99gMVfZijm6OCzjs= +# - name: node2 +# power: '1' +# address: 8C17BBDC7C350C83C550163458FC9B7A5B54A64E +# pubKey: 4v1RdMiKkWgBBTTP26iRSLOEkAY99gMVfZijm6OCzjs= # Values for CometBFT node configuration cometbft: @@ -42,3 +54,7 @@ ingress: enabled: false grpc: enabled: false + +sequencer: + composerHook: + enabled: false \ No newline at end of file diff --git a/dev/values/validators/node2.yml b/dev/values/validators/node2.yml index f883b0714..b3d69e85e 100644 --- a/dev/values/validators/node2.yml +++ b/dev/values/validators/node2.yml @@ -1,19 +1,31 @@ # Override value example for second validator from main chart + + +images: + cometBFT: + repo: docker.io/cometbft/cometbft + tag: v0.38.8 + devTag: v0.38.8 + sequencer: + repo: astria-sequencer + tag: "0.14.0" + devTag: tb + moniker: 'node2' genesis: validators: - - name: node0 - power: '1' - address: 091E47761C58C474534F4D414AF104A6CAF90C22 - pubKey: lV57+rGs2vac7mvkGHP1oBFGHPJM3a+WoAzeFDCJDNU= +# - name: node0 +# power: '1' +# address: 091E47761C58C474534F4D414AF104A6CAF90C22 +# pubKey: lV57+rGs2vac7mvkGHP1oBFGHPJM3a+WoAzeFDCJDNU= - name: node1 address: E82D827830B163D5179291FB27BB58E605DF2FA2 pubKey: NDE9F44v3l4irmkZxNmrZkywoGmggLlaBo5rE/Cis8M= power: '1' - - name: node2 - power: '1' - address: 8C17BBDC7C350C83C550163458FC9B7A5B54A64E - pubKey: 4v1RdMiKkWgBBTTP26iRSLOEkAY99gMVfZijm6OCzjs= +# - name: node2 +# power: '1' +# address: 8C17BBDC7C350C83C550163458FC9B7A5B54A64E +# pubKey: 4v1RdMiKkWgBBTTP26iRSLOEkAY99gMVfZijm6OCzjs= cometbft: secrets: @@ -41,3 +53,7 @@ ingress: enabled: false grpc: enabled: false + +sequencer: + composerHook: + enabled: false diff --git a/proto/composerapis/astria/composer/v1alpha1/grpc_collector.proto b/proto/composerapis/astria/composer/v1alpha1/grpc_collector.proto index 65259db0c..3b328fa4a 100644 --- a/proto/composerapis/astria/composer/v1alpha1/grpc_collector.proto +++ b/proto/composerapis/astria/composer/v1alpha1/grpc_collector.proto @@ -3,6 +3,7 @@ syntax = 'proto3'; package astria.composer.v1alpha1; import "astria/protocol/transactions/v1alpha1/types.proto"; +import "google/protobuf/timestamp.proto"; // SubmitRollupTransactionRequest contains a rollup transaction to be submitted to the Shared Sequencer Network // via the Composer @@ -20,6 +21,7 @@ message SubmitRollupTransactionResponse {} message SendOptimisticBlockRequest { bytes block_hash = 1; repeated astria.protocol.transactions.v1alpha1.SequenceAction seq_action = 2; + google.protobuf.Timestamp time = 3; } message SendOptimisticBlockResponse {} From ef6c527cd28fcf9df4f7e1d3471b1fcb0fc6f95b Mon Sep 17 00:00:00 2001 From: Bharath Date: Fri, 23 Aug 2024 22:13:33 +0530 Subject: [PATCH 28/43] remove bridge contracts --- crates/astria-bridge-withdrawer/astria-bridge-contracts | 1 - 1 file changed, 1 deletion(-) delete mode 160000 crates/astria-bridge-withdrawer/astria-bridge-contracts diff --git a/crates/astria-bridge-withdrawer/astria-bridge-contracts b/crates/astria-bridge-withdrawer/astria-bridge-contracts deleted file mode 160000 index 4580ffc07..000000000 --- a/crates/astria-bridge-withdrawer/astria-bridge-contracts +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4580ffc0747f463e304214bb29848e21e4e93e32 From 00556764e0c74d47976ff83642cc5c1f0298e0a2 Mon Sep 17 00:00:00 2001 From: Bharath Date: Fri, 23 Aug 2024 22:18:00 +0530 Subject: [PATCH 29/43] chart changes --- charts/sequencer/values.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/sequencer/values.yaml b/charts/sequencer/values.yaml index 39bd4c671..53ce1750b 100644 --- a/charts/sequencer/values.yaml +++ b/charts/sequencer/values.yaml @@ -16,9 +16,9 @@ images: tag: v0.38.8 devTag: v0.38.8 sequencer: - repo: astria-sequencer - tag: "0.14.0" - devTag: tb + repo: ghcr.io/astriaorg/sequencer + tag: "0.16.0" + devTag: latest moniker: "" genesis: From c5132b4b3467129bb531a7fde6f0c08bd9c18d5c Mon Sep 17 00:00:00 2001 From: Bharath Date: Thu, 29 Aug 2024 21:09:26 +0530 Subject: [PATCH 30/43] update astria-geth image name --- charts/evm-rollup/templates/_helpers.tpl | 3 +-- dev/values/rollup/dev.yaml | 10 ++++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/charts/evm-rollup/templates/_helpers.tpl b/charts/evm-rollup/templates/_helpers.tpl index be8aa4af2..7df440877 100644 --- a/charts/evm-rollup/templates/_helpers.tpl +++ b/charts/evm-rollup/templates/_helpers.tpl @@ -52,8 +52,7 @@ The log level represented as a number Full image paths for Astria built images */}} {{- define "rollup.image" -}} -{{ .Values.images.geth.repo }}:{{ if .Values.images.geth.overrideTag }}{{ .Values.images.geth.overrideTag }}{{ else }}{{ if .Values.global.dev }}{{ .Values.images.geth.devTag }}{{ else }}{{ .Values.images.geth.tag }}{{ end }} -{{- end }} +{{ .Values.images.geth.repo }}:{{if .Values.images.geth.override }}{{ .Values.images.geth.override }}{{ else }}{{ if .Values.global.dev }}{{ .Values.images.geth.devTag }}{{ else }}{{ .Values.images.geth.tag }}{{ end }} {{- end }} {{- define "conductor.image" -}} {{ .Values.images.conductor.repo }}:{{ if .Values.global.dev }}{{ .Values.images.conductor.devTag }}{{ else }}{{ .Values.images.conductor.tag }}{{ end }} diff --git a/dev/values/rollup/dev.yaml b/dev/values/rollup/dev.yaml index 765cc2e16..b365e7a12 100644 --- a/dev/values/rollup/dev.yaml +++ b/dev/values/rollup/dev.yaml @@ -8,6 +8,16 @@ global: sequencerChainId: sequencer-test-chain-0 evm-rollup: + images: + geth: + repo: ghcr.io/astriaorg/astria-geth + tag: 0.14.0 + devTag: latest + overrideTag: "pr-41" + conductor: + repo: ghcr.io/astriaorg/conductor + tag: "0.20.0" + devTag: latest genesis: ## These values are used to configure the genesis block of the rollup chain ## no defaults as they are unique to each chain From 1117e0383cbad7e9d25bf60d264fc498b50216ea Mon Sep 17 00:00:00 2001 From: ido Date: Fri, 23 Aug 2024 15:25:29 -0400 Subject: [PATCH 31/43] overrideTag --- charts/evm-rollup/templates/_helpers.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/evm-rollup/templates/_helpers.tpl b/charts/evm-rollup/templates/_helpers.tpl index 7df440877..0a34c9a72 100644 --- a/charts/evm-rollup/templates/_helpers.tpl +++ b/charts/evm-rollup/templates/_helpers.tpl @@ -52,7 +52,7 @@ The log level represented as a number Full image paths for Astria built images */}} {{- define "rollup.image" -}} -{{ .Values.images.geth.repo }}:{{if .Values.images.geth.override }}{{ .Values.images.geth.override }}{{ else }}{{ if .Values.global.dev }}{{ .Values.images.geth.devTag }}{{ else }}{{ .Values.images.geth.tag }}{{ end }} +{{ .Values.images.geth.repo }}:{{ if .Values.images.geth.overrideTag }}{{ .Values.images.geth.overrideTag }}{{ else if .Values.global.dev }}{{ .Values.images.geth.devTag }}{{ else }}{{ .Values.images.geth.tag }}{{ end }} {{- end }} {{- define "conductor.image" -}} {{ .Values.images.conductor.repo }}:{{ if .Values.global.dev }}{{ .Values.images.conductor.devTag }}{{ else }}{{ .Values.images.conductor.tag }}{{ end }} From dba08b3949c3980249ee2dee4342a5702225587d Mon Sep 17 00:00:00 2001 From: Bharath Date: Wed, 28 Aug 2024 20:21:15 +0530 Subject: [PATCH 32/43] use release namespace --- charts/sequencer/templates/configmaps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sequencer/templates/configmaps.yaml b/charts/sequencer/templates/configmaps.yaml index 8898b93eb..779025436 100644 --- a/charts/sequencer/templates/configmaps.yaml +++ b/charts/sequencer/templates/configmaps.yaml @@ -64,7 +64,7 @@ data: ASTRIA_SEQUENCER_METRICS_HTTP_LISTENER_ADDR: "0.0.0.0:{{ .Values.ports.sequencerMetrics }}" ASTRIA_SEQUENCER_FORCE_STDOUT: "{{ .Values.global.useTTY }}" ASTRIA_SEQUENCER_PRETTY_PRINT: "{{ .Values.global.useTTY }}" - ASTRIA_SEQUENCER_COMPOSER_HOOK: "http://composer-service.astria-dev-cluster.svc.cluster.local:50052" + ASTRIA_SEQUENCER_COMPOSER_HOOK: "http://composer-service.{{ default .Release.Namespace .Values.global.namespaceOverride }}.svc.cluster.local:50052" ASTRIA_SEQUENCER_COMPOSER_HOOK_ENABLED: "{{ .Values.sequencer.composerHook.enabled}}" NO_COLOR: "{{ .Values.global.useTTY }}" ASTRIA_SEQUENCER_NO_OTEL: "{{ not .Values.sequencer.otel.enabled }}" From e7b408fb6b25de84947073163611469d2f011cb2 Mon Sep 17 00:00:00 2001 From: Bharath Date: Wed, 28 Aug 2024 20:22:08 +0530 Subject: [PATCH 33/43] some minor changes --- crates/astria-composer/src/executor/mod.rs | 20 ++++++++++--------- .../astria-composer/src/executor/simulator.rs | 6 ------ 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index 2ed4608a4..6f65600a3 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -235,7 +235,6 @@ impl Executor { parent_block: Vec, time: Option ) -> eyre::Result<(SizedBundle, BundleSimulationResult)> { - info!("BHARATH: simulating bundle on top of received parent block!"); let bundle_simulator = self.bundle_simulator.clone(); // convert the sequence actions to RollupData // TODO - clean up this code @@ -257,22 +256,23 @@ impl Executor { parent_block_rollup_data_items.push(rollup_data); } - info!("BHARATH: time is {:?}", time); - info!("BHARATH: getting parent_bundle_simulation_result"); + info!("Creating the parent block for simulating the bundle"); let parent_bundle_simulation_result = bundle_simulator .clone() .simulate_parent_bundle(parent_block_rollup_data_items, time.unwrap()) .await .wrap_err("failed to simulate bundle")?; + info!("Created the parent block!"); // now we have the parent block hash, we should simulate the bundle on top of the parent // hash - info!("BHARATH: getting actual bundle simulation"); + info!("Simulating the bundle on top of the created parent block!"); let bundle_simulation_result = bundle_simulator .clone() .simulate_bundle_on_block(bundle, parent_bundle_simulation_result.block().clone()) .await .wrap_err("failed to simulate bundle")?; + info!("Simulation done!"); let rollup_data_items: Vec = bundle_simulation_result .included_actions() @@ -280,7 +280,7 @@ impl Executor { .map(astria_core::Protobuf::to_raw) .collect(); - info!("Creating BuilderBundlePacket"); + info!("Creating a Builder Bundle Packet"); let builder_bundle = BuilderBundle { transactions: rollup_data_items, parent_hash: bundle_simulation_result.parent_hash(), @@ -292,7 +292,7 @@ impl Executor { }; let encoded_builder_bundle_packet = builder_bundle_packet.encode_to_vec(); - info!("Created builder bundle packet: {:?}", builder_bundle_packet); + info!("Created builder bundle packet!"); // we can give the BuilderBundlePacket the highest bundle max size possible // since this is the only sequence action we are sending let mut final_bundle = SizedBundle::new(self.max_bundle_size); @@ -443,6 +443,7 @@ impl Executor { }; } + // from process_proposal Some(filtered_sequencer_block) = self.filtered_block_receiver.recv() => { // we need to simulate the bundle // and cache the simulated bundle and the block_hash @@ -450,20 +451,23 @@ impl Executor { if bundle.is_empty() { debug!("no bundle to simulate") } else { + info!("received {:?} sequence actions from process_proposal", filtered_sequencer_block.seq_action.len()); // TODO - we should throw an error log when simulation fails rather than escaping the application let res = self.simulate_bundle(bundle, filtered_sequencer_block.seq_action, filtered_sequencer_block.time) .await.wrap_err("failed to simulate bundle on top of received parent block")?; pending_builder_bundle_packet = Some(res.0); - debug!("simulating bundle on top of received parent block!") + info!("simulation done on transactions received from process_proposal!"); } current_finalized_block_hash = Some(filtered_sequencer_block.block_hash.clone()); } + // from finalize_block Some(finalized_block_hash) = self.finalized_block_hash_receiver.recv(), if submission_fut.is_terminated() => { if let Some(block_hash) = current_finalized_block_hash.clone() { if block_hash == finalized_block_hash.block_hash { // we can submit the pending builder bundle packet if let Some(builder_bundle_packet) = pending_builder_bundle_packet.clone() { + info!("received finalized block hash matches that of process_proposal, submitting pending builder bundle packet"); if !builder_bundle_packet.is_empty() { submission_fut = self.submit_bundle(nonce, builder_bundle_packet, self.metrics); } @@ -471,8 +475,6 @@ impl Executor { pending_builder_bundle_packet = None; current_finalized_block_hash = None; } - } else { - warn!("received finalized block hash does not match the current finalized block hash; skipping submission of pending builder bundle packet") } } } diff --git a/crates/astria-composer/src/executor/simulator.rs b/crates/astria-composer/src/executor/simulator.rs index 338bd8f12..ba2bca62e 100644 --- a/crates/astria-composer/src/executor/simulator.rs +++ b/crates/astria-composer/src/executor/simulator.rs @@ -75,18 +75,14 @@ impl BundleSimulator { rollup_data: Vec, time: pbjson_types::Timestamp, ) -> eyre::Result { - info!("Creating parent block!"); // call GetCommitmentState to get the soft block - info!("Calling GetCommitmentState!"); let commitment_state = self .execution_service_client .get_commitment_state_with_retry() .await .wrap_err("failed to get commitment state")?; - info!("Received CommitmentState of rollup"); let soft_block = commitment_state.soft(); - info!("Soft block hash is {:?}", soft_block.hash()); // convert the sized bundle actions to a list of Vec let actions: Vec> = rollup_data .iter() @@ -97,7 +93,6 @@ impl BundleSimulator { .filter(|data| !data.is_empty()) .collect(); - info!("Calling ExecuteBlock to simulate the bundle!"); // as long as the timestamp > parent block timestamp, the block will be successfully // created. It doesn't matter what timestamp we use anyway since we are not going to // commit the block to the chain. @@ -133,7 +128,6 @@ impl BundleSimulator { bundle: SizedBundle, block: Block, ) -> eyre::Result { - info!("Simulating bundle on created parent block!"); // convert the sized bundle actions to a list of Vec let actions: Vec> = bundle .into_actions() From 73ca26d6a1f8de511c7462d8162991003d8fa1d2 Mon Sep 17 00:00:00 2001 From: Bharath Date: Thu, 29 Aug 2024 22:07:38 +0530 Subject: [PATCH 34/43] fmt + lint --- crates/astria-composer/src/composer.rs | 10 +- .../astria-composer/src/executor/builder.rs | 10 +- .../src/executor/bundle_factory/mod.rs | 3 + crates/astria-composer/src/executor/mod.rs | 123 +++--------------- crates/astria-composer/src/grpc.rs | 5 +- crates/astria-composer/src/sequencer_hooks.rs | 6 + crates/astria-core/src/generated/mod.rs | 2 +- crates/astria-sequencer/src/app/mod.rs | 10 +- crates/astria-sequencer/src/app/tests_app.rs | 18 +-- crates/astria-sequencer/src/client.rs | 9 +- 10 files changed, 52 insertions(+), 144 deletions(-) diff --git a/crates/astria-composer/src/composer.rs b/crates/astria-composer/src/composer.rs index 6fcb198b5..f4ad18f78 100644 --- a/crates/astria-composer/src/composer.rs +++ b/crates/astria-composer/src/composer.rs @@ -5,12 +5,9 @@ use std::{ }; use astria_core::{ - generated::{ - composer::v1alpha1::{ - SendFinalizedHashRequest, - SendOptimisticBlockRequest, - }, - sequencerblock::v1alpha1::FilteredSequencerBlock, + generated::composer::v1alpha1::{ + SendFinalizedHashRequest, + SendOptimisticBlockRequest, }, primitive::v1::asset, }; @@ -18,7 +15,6 @@ use astria_eyre::eyre::{ self, WrapErr as _, }; -use bytes::Bytes; use itertools::Itertools as _; use tokio::{ io, diff --git a/crates/astria-composer/src/executor/builder.rs b/crates/astria-composer/src/executor/builder.rs index 4f3fad863..165e6c0c5 100644 --- a/crates/astria-composer/src/executor/builder.rs +++ b/crates/astria-composer/src/executor/builder.rs @@ -6,12 +6,9 @@ use std::{ use astria_core::{ crypto::SigningKey, - generated::{ - composer::v1alpha1::{ - SendFinalizedHashRequest, - SendOptimisticBlockRequest, - }, - sequencerblock::v1alpha1::FilteredSequencerBlock, + generated::composer::v1alpha1::{ + SendFinalizedHashRequest, + SendOptimisticBlockRequest, }, primitive::v1::{ asset, @@ -25,7 +22,6 @@ use astria_eyre::eyre::{ eyre, WrapErr as _, }; -use bytes::Bytes; use tokio::sync::{ mpsc, watch, diff --git a/crates/astria-composer/src/executor/bundle_factory/mod.rs b/crates/astria-composer/src/executor/bundle_factory/mod.rs index d70f9829d..e64d060b8 100644 --- a/crates/astria-composer/src/executor/bundle_factory/mod.rs +++ b/crates/astria-composer/src/executor/bundle_factory/mod.rs @@ -234,6 +234,7 @@ impl BundleFactory { /// /// The bundle is only removed from the factory on calling [`NextFinishedBundle::pop`]. /// This method primarily exists to work around async cancellation. + #[allow(dead_code)] pub(super) fn next_finished(&mut self) -> Option { if self.finished.is_empty() { None @@ -259,11 +260,13 @@ impl BundleFactory { } } +#[allow(dead_code)] pub(super) struct NextFinishedBundle<'a> { bundle_factory: &'a mut BundleFactory, } impl<'a> NextFinishedBundle<'a> { + #[allow(dead_code)] pub(super) fn pop(self) -> SizedBundle { self.bundle_factory .finished diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index 6f65600a3..311c0bcae 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -20,11 +20,7 @@ use astria_core::{ SendFinalizedHashRequest, SendOptimisticBlockRequest, }, - execution::v1alpha1::FinalizeBlockRequest, - sequencerblock::v1alpha1::{ - FilteredSequencerBlock, - RollupData, - }, + sequencerblock::v1alpha1::RollupData, }, primitive::v1::{ asset, @@ -42,13 +38,11 @@ use astria_core::{ }; use astria_eyre::eyre::{ self, - eyre, WrapErr as _, }; use bytes::Bytes; use futures::{ future::{ - self, Fuse, FusedFuture as _, FutureExt as _, @@ -232,8 +226,8 @@ impl Executor { async fn simulate_bundle( &self, bundle: SizedBundle, - parent_block: Vec, - time: Option + parent_block: Vec, + time: Option, ) -> eyre::Result<(SizedBundle, BundleSimulationResult)> { let bundle_simulator = self.bundle_simulator.clone(); // convert the sequence actions to RollupData @@ -307,70 +301,6 @@ impl Executor { Ok((final_bundle, bundle_simulation_result)) } - #[instrument(skip_all, fields(nonce.initial = %nonce))] - async fn simulate_and_submit_bundle( - &self, - nonce: u32, - bundle: SizedBundle, - metrics: &'static Metrics, - ) -> eyre::Result>> { - let bundle_simulator = self.bundle_simulator.clone(); - - // simulate the bundle - let bundle_simulation_result = bundle_simulator - .simulate_bundle(bundle.clone()) - .await - .wrap_err("failed to simulate bundle")?; - - let rollup_data_items: Vec = bundle_simulation_result - .included_actions() - .iter() - .map(astria_core::Protobuf::to_raw) - .collect(); - - info!("Creating BuilderBundlePacket"); - let builder_bundle = BuilderBundle { - transactions: rollup_data_items, - parent_hash: bundle_simulation_result.parent_hash(), - }; - - // TODO - bundle signing - - // create a top of block bundle - let builder_bundle_packet = BuilderBundlePacket { - bundle: Some(builder_bundle), - signature: Bytes::from(vec![]), - }; - let encoded_builder_bundle_packet = builder_bundle_packet.encode_to_vec(); - - info!("Created builder bundle packet: {:?}", builder_bundle_packet); - // we can give the BuilderBundlePacket the highest bundle max size possible - // since this is the only sequence action we are sending - let mut final_bundle = SizedBundle::new(self.max_bundle_size); - final_bundle - .try_push(SequenceAction { - rollup_id: self.rollup_id, - data: encoded_builder_bundle_packet.into(), - fee_asset: self.fee_asset.clone(), - }) - .wrap_err("couldn't push sequence action to bundle")?; - - info!("Submitting the builder bundle packet to sequencer!"); - - Ok(SubmitFut { - client: self.sequencer_client.clone(), - address: self.address, - nonce, - chain_id: self.sequencer_chain_id.clone(), - signing_key: self.sequencer_key.clone(), - state: SubmitState::NotStarted, - bundle: final_bundle, - metrics, - } - .in_current_span() - .fuse()) - } - /// Create a future to submit a bundle to the sequencer. #[instrument(skip_all, fields(nonce.initial = %nonce))] fn submit_bundle( @@ -448,62 +378,41 @@ impl Executor { // we need to simulate the bundle // and cache the simulated bundle and the block_hash let bundle = bundle_factory.pop_now(); - if bundle.is_empty() { - debug!("no bundle to simulate") - } else { + if !bundle.is_empty() { info!("received {:?} sequence actions from process_proposal", filtered_sequencer_block.seq_action.len()); - // TODO - we should throw an error log when simulation fails rather than escaping the application - let res = self.simulate_bundle(bundle, filtered_sequencer_block.seq_action, filtered_sequencer_block.time) - .await.wrap_err("failed to simulate bundle on top of received parent block")?; - pending_builder_bundle_packet = Some(res.0); + match self.simulate_bundle(bundle, filtered_sequencer_block.seq_action, filtered_sequencer_block.time) + .await.wrap_err("failed to simulate bundle on top of received parent block") { + Ok(res) => { + pending_builder_bundle_packet = Some(res.0); + current_finalized_block_hash = Some(filtered_sequencer_block.block_hash.clone()); + }, + Err(e) => { + error!(%e, "failed to simulate bundle on top of received parent block"); + } + } info!("simulation done on transactions received from process_proposal!"); } - current_finalized_block_hash = Some(filtered_sequencer_block.block_hash.clone()); } // from finalize_block Some(finalized_block_hash) = self.finalized_block_hash_receiver.recv(), if submission_fut.is_terminated() => { - if let Some(block_hash) = current_finalized_block_hash.clone() { + if let Some(block_hash) = current_finalized_block_hash.take() { if block_hash == finalized_block_hash.block_hash { // we can submit the pending builder bundle packet - if let Some(builder_bundle_packet) = pending_builder_bundle_packet.clone() { + if let Some(builder_bundle_packet) = pending_builder_bundle_packet.take() { info!("received finalized block hash matches that of process_proposal, submitting pending builder bundle packet"); if !builder_bundle_packet.is_empty() { submission_fut = self.submit_bundle(nonce, builder_bundle_packet, self.metrics); } - // TODO - ideally we need to use take() here - pending_builder_bundle_packet = None; - current_finalized_block_hash = None; } } } } - // Some(next_bundle) = future::ready(bundle_factory.next_finished()), if submission_fut.is_terminated() => { - // let bundle = next_bundle.pop(); - // if !bundle.is_empty() { - // submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; - // } - // } - // receive new seq_action and bundle it. will not pull from the channel if `bundle_factory` is full Some(seq_action) = self.serialized_rollup_transactions.recv(), if !bundle_factory.is_full() => { self.bundle_seq_action(seq_action, &mut bundle_factory); } - - // // try to preempt current bundle if the timer has ticked without submitting the next bundle - // () = &mut block_timer, if submission_fut.is_terminated() => { - // let bundle = bundle_factory.pop_now(); - // if bundle.is_empty() { - // debug!("block timer ticked, but no bundle to submit to sequencer"); - // block_timer.as_mut().reset(reset_time()); - // } else { - // debug!( - // "forcing bundle submission to sequencer due to block timer" - // ); - // submission_fut = self.simulate_and_submit_bundle(nonce, bundle, self.metrics).await.wrap_err("failed to simulate and submit bundle")?; - // } - // } } }; diff --git a/crates/astria-composer/src/grpc.rs b/crates/astria-composer/src/grpc.rs index 524ba1b50..c411d3bf3 100644 --- a/crates/astria-composer/src/grpc.rs +++ b/crates/astria-composer/src/grpc.rs @@ -24,7 +24,10 @@ use tokio::{ net::TcpListener, }; use tokio_util::sync::CancellationToken; -use tracing::{info, instrument}; +use tracing::{ + info, + instrument, +}; use crate::{ collectors, diff --git a/crates/astria-composer/src/sequencer_hooks.rs b/crates/astria-composer/src/sequencer_hooks.rs index 7e342e2b3..a2e1ebcfb 100644 --- a/crates/astria-composer/src/sequencer_hooks.rs +++ b/crates/astria-composer/src/sequencer_hooks.rs @@ -24,6 +24,12 @@ use tracing::info; const SEND_TIMEOUT: u64 = 2; +// pub(crate) struct OptimisticBlockInfo { +// block_hash: Bytes, +// seq_actions: Vec, +// time: Timestamp, +// } + pub(crate) struct SequencerHooks { filtered_block_sender: mpsc::Sender, finalized_hash_sender: mpsc::Sender, diff --git a/crates/astria-core/src/generated/mod.rs b/crates/astria-core/src/generated/mod.rs index 5cf06adbc..350c98fa2 100644 --- a/crates/astria-core/src/generated/mod.rs +++ b/crates/astria-core/src/generated/mod.rs @@ -11,7 +11,7 @@ //! [`buf`]: https://buf.build //! [`tools/protobuf-compiler`]: ../../../../tools/protobuf-compiler -use crate::generated::protocol::transaction::v1alpha1; +use crate::generated::protocol::transactions::v1alpha1; #[path = ""] pub mod astria_vendored { diff --git a/crates/astria-sequencer/src/app/mod.rs b/crates/astria-sequencer/src/app/mod.rs index 18854c40c..e107597e2 100644 --- a/crates/astria-sequencer/src/app/mod.rs +++ b/crates/astria-sequencer/src/app/mod.rs @@ -24,11 +24,7 @@ use anyhow::{ Context, }; use astria_core::{ - generated::{ - composer::v1alpha1::sequencer_hooks_service_client::SequencerHooksServiceClient, - protocol::transactions::v1alpha1 as raw, - }, - primitive::v1::Address, + generated::protocol::transactions::v1alpha1 as raw, protocol::{ abci::AbciErrorCode, genesis::v1alpha1::GenesisAppState, @@ -949,9 +945,7 @@ impl App { .context("failed to prepare commit")?; // update the priority of any txs in the mempool based on the updated app state - update_mempool_after_finalization(&mut self.mempool, self.state.as_ref()) - .await - .context("failed to update mempool after finalization")?; + update_mempool_after_finalization(&mut self.mempool, self.state.as_ref()).await; if let Err(e) = self .sequencer_hooks_client diff --git a/crates/astria-sequencer/src/app/tests_app.rs b/crates/astria-sequencer/src/app/tests_app.rs index 1e63e4cff..4a8c6f529 100644 --- a/crates/astria-sequencer/src/app/tests_app.rs +++ b/crates/astria-sequencer/src/app/tests_app.rs @@ -5,14 +5,17 @@ use astria_core::{ asset::TracePrefixed, RollupId, }, - protocol::transaction::v1alpha1::{ - action::{ - BridgeLockAction, - SequenceAction, - TransferAction, + protocol::{ + genesis::v1alpha1::Account, + transaction::v1alpha1::{ + action::{ + BridgeLockAction, + SequenceAction, + TransferAction, + }, + TransactionParams, + UnsignedTransaction, }, - TransactionParams, - UnsignedTransaction, }, sequencerblock::v1alpha1::block::Deposit, Protobuf, @@ -39,7 +42,6 @@ use tendermint::{ Hash, Time, }; -use astria_core::protocol::genesis::v1alpha1::Account; use super::*; use crate::{ diff --git a/crates/astria-sequencer/src/client.rs b/crates/astria-sequencer/src/client.rs index d78f59c3f..81b7098d3 100644 --- a/crates/astria-sequencer/src/client.rs +++ b/crates/astria-sequencer/src/client.rs @@ -7,7 +7,7 @@ use astria_core::generated::{ SendOptimisticBlockRequest, SendOptimisticBlockResponse, }, - protocol::transaction::v1alpha1::SequenceAction, + protocol::transactions::v1alpha1::SequenceAction, }; use bytes::Bytes; use tendermint::Time; @@ -20,7 +20,6 @@ use tonic::transport::{ use tracing::{ info, instrument, - Instrument, }; /// A newtype wrapper around [`SequencerHooksServiceClient`] to work with @@ -46,9 +45,9 @@ impl SequencerHooksClient { }) } - pub(crate) fn uri(&self) -> String { - self.uri.to_string() - } + // pub(crate) fn uri(&self) -> String { + // self.uri.to_string() + // } #[instrument(skip_all, fields(uri = % self.uri), err)] pub(super) async fn send_optimistic_block( From a3feff05b9efbf9c597a08e91ce4e14db15447e2 Mon Sep 17 00:00:00 2001 From: Bharath Date: Fri, 30 Aug 2024 00:07:03 +0530 Subject: [PATCH 35/43] some clean ups --- crates/astria-composer/src/composer.rs | 18 +-- .../astria-composer/src/executor/builder.rs | 13 +- crates/astria-composer/src/executor/mod.rs | 79 +++++---- .../astria-composer/src/executor/simulator.rs | 152 ++++++------------ crates/astria-composer/src/grpc.rs | 2 +- crates/astria-composer/src/sequencer_hooks.rs | 110 ++++++++++--- crates/astria-sequencer/src/app/mod.rs | 2 +- 7 files changed, 191 insertions(+), 185 deletions(-) diff --git a/crates/astria-composer/src/composer.rs b/crates/astria-composer/src/composer.rs index f4ad18f78..babc3d447 100644 --- a/crates/astria-composer/src/composer.rs +++ b/crates/astria-composer/src/composer.rs @@ -4,13 +4,7 @@ use std::{ time::Duration, }; -use astria_core::{ - generated::composer::v1alpha1::{ - SendFinalizedHashRequest, - SendOptimisticBlockRequest, - }, - primitive::v1::asset, -}; +use astria_core::primitive::v1::asset; use astria_eyre::eyre::{ self, WrapErr as _, @@ -56,7 +50,11 @@ use crate::{ grpc, grpc::GrpcServer, metrics::Metrics, - sequencer_hooks::SequencerHooks, + sequencer_hooks::{ + FinalizedHashInfo, + OptimisticBlockInfo, + SequencerHooks, + }, Config, }; @@ -134,9 +132,9 @@ impl Composer { let shutdown_token = CancellationToken::new(); let (filtered_sequencer_block_sender, filtered_sequencer_block_receiver) = - mpsc::channel::(1000); + mpsc::channel::(1000); let (finalized_hash_sender, finalized_hash_receiver) = - mpsc::channel::(1000); + mpsc::channel::(1000); let (executor, executor_handle) = executor::Builder { sequencer_url: cfg.sequencer_url.clone(), diff --git a/crates/astria-composer/src/executor/builder.rs b/crates/astria-composer/src/executor/builder.rs index 165e6c0c5..65c874ab3 100644 --- a/crates/astria-composer/src/executor/builder.rs +++ b/crates/astria-composer/src/executor/builder.rs @@ -5,11 +5,8 @@ use std::{ }; use astria_core::{ + self, crypto::SigningKey, - generated::composer::v1alpha1::{ - SendFinalizedHashRequest, - SendOptimisticBlockRequest, - }, primitive::v1::{ asset, Address, @@ -36,6 +33,10 @@ use crate::{ Status, }, metrics::Metrics, + sequencer_hooks::{ + FinalizedHashInfo, + OptimisticBlockInfo, + }, }; pub(crate) struct Builder { @@ -51,8 +52,8 @@ pub(crate) struct Builder { pub(crate) chain_name: String, pub(crate) fee_asset: asset::Denom, pub(crate) max_bundle_size: usize, - pub(crate) filtered_block_receiver: mpsc::Receiver, - pub(crate) finalized_block_hash_receiver: mpsc::Receiver, + pub(crate) filtered_block_receiver: mpsc::Receiver, + pub(crate) finalized_block_hash_receiver: mpsc::Receiver, pub(crate) metrics: &'static Metrics, } diff --git a/crates/astria-composer/src/executor/mod.rs b/crates/astria-composer/src/executor/mod.rs index 311c0bcae..0e5f098cf 100644 --- a/crates/astria-composer/src/executor/mod.rs +++ b/crates/astria-composer/src/executor/mod.rs @@ -10,18 +10,10 @@ use std::{ /// - Transaction signing /// - Managing the connection to the sequencer /// - Submitting transactions to the sequencer -use astria_core::Protobuf; +use astria_core::sequencerblock::v1alpha1::block::RollupData; use astria_core::{ crypto::SigningKey, - generated::{ - composer::v1alpha1::{ - BuilderBundle, - BuilderBundlePacket, - SendFinalizedHashRequest, - SendOptimisticBlockRequest, - }, - sequencerblock::v1alpha1::RollupData, - }, + generated as raw, primitive::v1::{ asset, RollupId, @@ -112,9 +104,15 @@ mod tests; pub(crate) use builder::Builder; -use crate::executor::simulator::{ - BundleSimulationResult, - BundleSimulator, +use crate::{ + executor::simulator::{ + BundleSimulationResult, + BundleSimulator, + }, + sequencer_hooks::{ + FinalizedHashInfo, + OptimisticBlockInfo, + }, }; // Duration to wait for the executor to drain all the remaining bundles before shutting down. @@ -169,8 +167,8 @@ pub(super) struct Executor { fee_asset: asset::Denom, // The maximum possible size for a bundle so that it can fit into a block max_bundle_size: usize, - filtered_block_receiver: mpsc::Receiver, - finalized_block_hash_receiver: mpsc::Receiver, + filtered_block_receiver: mpsc::Receiver, + finalized_block_hash_receiver: mpsc::Receiver, metrics: &'static Metrics, } @@ -226,34 +224,26 @@ impl Executor { async fn simulate_bundle( &self, bundle: SizedBundle, - parent_block: Vec, - time: Option, + parent_block: Vec, + time: pbjson_types::Timestamp, ) -> eyre::Result<(SizedBundle, BundleSimulationResult)> { - let bundle_simulator = self.bundle_simulator.clone(); - // convert the sequence actions to RollupData - // TODO - clean up this code - let sequence_actions: Vec = parent_block - .iter() - .map(|action| SequenceAction::try_from_raw_ref(action).unwrap()) - .collect(); - let filtered_sequence_actions: Vec = sequence_actions + let filtered_sequence_actions: Vec = parent_block .iter() .filter(|action| action.rollup_id == self.rollup_id) .cloned() .collect(); let mut parent_block_rollup_data_items = vec![]; for seq_action in filtered_sequence_actions { - let rollup_data = - astria_core::sequencerblock::v1alpha1::block::RollupData::SequencedData( - seq_action.data, - ); + let rollup_data = RollupData::SequencedData(seq_action.data); parent_block_rollup_data_items.push(rollup_data); } + let bundle_simulator = self.bundle_simulator.clone(); + info!("Creating the parent block for simulating the bundle"); let parent_bundle_simulation_result = bundle_simulator .clone() - .simulate_parent_bundle(parent_block_rollup_data_items, time.unwrap()) + .create_parent_block(parent_block_rollup_data_items, time) .await .wrap_err("failed to simulate bundle")?; info!("Created the parent block!"); @@ -263,24 +253,29 @@ impl Executor { info!("Simulating the bundle on top of the created parent block!"); let bundle_simulation_result = bundle_simulator .clone() - .simulate_bundle_on_block(bundle, parent_bundle_simulation_result.block().clone()) + .simulate_bundle_on_block( + bundle, + parent_bundle_simulation_result.block().clone(), + None, + ) .await .wrap_err("failed to simulate bundle")?; info!("Simulation done!"); - let rollup_data_items: Vec = bundle_simulation_result - .included_actions() - .iter() - .map(astria_core::Protobuf::to_raw) - .collect(); + let rollup_data_items: Vec = + bundle_simulation_result + .included_actions() + .iter() + .map(astria_core::Protobuf::to_raw) + .collect(); info!("Creating a Builder Bundle Packet"); - let builder_bundle = BuilderBundle { + let builder_bundle = raw::composer::v1alpha1::BuilderBundle { transactions: rollup_data_items, parent_hash: bundle_simulation_result.parent_hash(), }; - let builder_bundle_packet = BuilderBundlePacket { + let builder_bundle_packet = raw::composer::v1alpha1::BuilderBundlePacket { bundle: Some(builder_bundle), signature: Bytes::from(vec![]), }; @@ -379,12 +374,12 @@ impl Executor { // and cache the simulated bundle and the block_hash let bundle = bundle_factory.pop_now(); if !bundle.is_empty() { - info!("received {:?} sequence actions from process_proposal", filtered_sequencer_block.seq_action.len()); - match self.simulate_bundle(bundle, filtered_sequencer_block.seq_action, filtered_sequencer_block.time) + info!("received {:?} sequence actions from process_proposal", filtered_sequencer_block.seq_actions().len()); + match self.simulate_bundle(bundle, filtered_sequencer_block.seq_actions(), filtered_sequencer_block.time()) .await.wrap_err("failed to simulate bundle on top of received parent block") { Ok(res) => { pending_builder_bundle_packet = Some(res.0); - current_finalized_block_hash = Some(filtered_sequencer_block.block_hash.clone()); + current_finalized_block_hash = Some(filtered_sequencer_block.block_hash().clone()); }, Err(e) => { error!(%e, "failed to simulate bundle on top of received parent block"); @@ -397,7 +392,7 @@ impl Executor { // from finalize_block Some(finalized_block_hash) = self.finalized_block_hash_receiver.recv(), if submission_fut.is_terminated() => { if let Some(block_hash) = current_finalized_block_hash.take() { - if block_hash == finalized_block_hash.block_hash { + if block_hash == finalized_block_hash.block_hash() { // we can submit the pending builder bundle packet if let Some(builder_bundle_packet) = pending_builder_bundle_packet.take() { info!("received finalized block hash matches that of process_proposal, submitting pending builder bundle packet"); diff --git a/crates/astria-composer/src/executor/simulator.rs b/crates/astria-composer/src/executor/simulator.rs index ba2bca62e..e870ed28f 100644 --- a/crates/astria-composer/src/executor/simulator.rs +++ b/crates/astria-composer/src/executor/simulator.rs @@ -67,10 +67,8 @@ impl BundleSimulator { }) } - // TODO - the interfaces below are weird but they work for now - // have cleaner interfaces #[instrument(skip_all, fields(uri=self.execution_service_client.uri()))] - pub(crate) async fn simulate_parent_bundle( + pub(crate) async fn create_parent_block( self, rollup_data: Vec, time: pbjson_types::Timestamp, @@ -88,90 +86,13 @@ impl BundleSimulator { .iter() .map(|action| match action.clone() { RollupData::SequencedData(data) => data.to_vec(), - _ => vec![], + RollupData::Deposit(_) => vec![], }) .filter(|data| !data.is_empty()) .collect(); - // as long as the timestamp > parent block timestamp, the block will be successfully - // created. It doesn't matter what timestamp we use anyway since we are not going to - // commit the block to the chain. - // call execute block with the bundle to get back the included transactions - let execute_block_response = self - .execution_service_client - .execute_block_with_retry( - soft_block.hash().clone(), - actions, - // use current timestamp - time, - false, - ) + self.inner_simulate_bundle_on_block(actions, soft_block.clone(), Some(time)) .await - .wrap_err("failed to execute block")?; - - let included_transactions = execute_block_response.included_transactions(); - info!( - "Parent block created on top of {:?} and {:?} transactions were included", - soft_block.hash(), - included_transactions.len() - ); - Ok(BundleSimulationResult::new( - included_transactions.to_vec(), - execute_block_response.block().clone(), - soft_block.hash().clone(), - )) - } - - #[instrument(skip_all, fields(uri=self.execution_service_client.uri()), err)] - pub(crate) async fn simulate_bundle_on_block( - self, - bundle: SizedBundle, - block: Block, - ) -> eyre::Result { - // convert the sized bundle actions to a list of Vec - let actions: Vec> = bundle - .into_actions() - .iter() - .map(|action| match action.as_sequence() { - Some(seq_action) => RollupData::SequencedData(seq_action.clone().data) - .to_raw() - .encode_to_vec(), - None => vec![], - }) - .filter(|data| !data.is_empty()) - .collect(); - - // as long as the timestamp > parent block timestamp, the block will be successfully - // created. It doesn't matter what timestamp we use anyway since we are not going to - // commit the block to the chain. - let timestamp = Timestamp { - seconds: block.timestamp().seconds + 3, - nanos: 0, - }; - // call execute block with the bundle to get back the included transactions - let execute_block_response = self - .execution_service_client - .execute_block_with_retry( - block.hash().clone(), - actions, - // use current timestamp - timestamp, - true, - ) - .await - .wrap_err("failed to execute block")?; - - let included_transactions = execute_block_response.included_transactions(); - info!( - "Bundle simulated on top of {:?} and {:?} transactions were included", - block.hash().clone(), - included_transactions.len() - ); - Ok(BundleSimulationResult::new( - included_transactions.to_vec(), - execute_block_response.block().clone(), - block.hash().clone(), - )) } #[instrument(skip_all, fields(uri=self.execution_service_client.uri()))] @@ -190,33 +111,46 @@ impl BundleSimulator { let soft_block = commitment_state.soft(); info!("Soft block hash is {:?}", soft_block.hash()); - // convert the sized bundle actions to a list of Vec - let actions: Vec> = bundle - .into_actions() - .iter() - .map(|action| match action.as_sequence() { - Some(seq_action) => RollupData::SequencedData(seq_action.clone().data) - .to_raw() - .encode_to_vec(), - None => vec![], - }) - .filter(|data| !data.is_empty()) - .collect(); - info!("Calling ExecuteBlock to simulate the bundle!"); + let actions = convert_bundle_to_byte_array(bundle); + + self.inner_simulate_bundle_on_block(actions, soft_block.clone(), None) + .await + } + + #[instrument(skip_all, fields(uri=self.execution_service_client.uri()))] + pub(crate) async fn simulate_bundle_on_block( + self, + bundle: SizedBundle, + block: Block, + timestamp: Option, + ) -> eyre::Result { + let actions = convert_bundle_to_byte_array(bundle); + self.inner_simulate_bundle_on_block(actions, block, timestamp) + .await + } + + #[instrument(skip_all, fields(uri=self.execution_service_client.uri()), err)] + async fn inner_simulate_bundle_on_block( + self, + bundle: Vec>, + block: Block, + timestamp: Option, + ) -> eyre::Result { + // convert the sized bundle actions to a list of Vec // as long as the timestamp > parent block timestamp, the block will be successfully // created. It doesn't matter what timestamp we use anyway since we are not going to // commit the block to the chain. - let timestamp = Timestamp { - seconds: soft_block.timestamp().seconds + 3, + let timestamp = timestamp.unwrap_or(Timestamp { + seconds: block.timestamp().seconds + 3, nanos: 0, - }; + }); // call execute block with the bundle to get back the included transactions let execute_block_response = self .execution_service_client .execute_block_with_retry( - soft_block.hash().clone(), - actions, + block.hash().clone(), + bundle, // use current timestamp timestamp, true, @@ -227,13 +161,27 @@ impl BundleSimulator { let included_transactions = execute_block_response.included_transactions(); info!( "Bundle simulated on top of {:?} and {:?} transactions were included", - soft_block.hash(), + block.hash().clone(), included_transactions.len() ); Ok(BundleSimulationResult::new( included_transactions.to_vec(), execute_block_response.block().clone(), - soft_block.hash().clone(), + block.hash().clone(), )) } } + +fn convert_bundle_to_byte_array(bundle: SizedBundle) -> Vec> { + bundle + .into_actions() + .iter() + .map(|action| match action.as_sequence() { + Some(seq_action) => RollupData::SequencedData(seq_action.clone().data) + .to_raw() + .encode_to_vec(), + None => vec![], + }) + .filter(|data| !data.is_empty()) + .collect() +} diff --git a/crates/astria-composer/src/grpc.rs b/crates/astria-composer/src/grpc.rs index c411d3bf3..4f5814302 100644 --- a/crates/astria-composer/src/grpc.rs +++ b/crates/astria-composer/src/grpc.rs @@ -76,8 +76,8 @@ impl Builder { Ok(GrpcServer { listener, grpc_collector, - shutdown_token, sequencer_hooks, + shutdown_token, }) } } diff --git a/crates/astria-composer/src/sequencer_hooks.rs b/crates/astria-composer/src/sequencer_hooks.rs index a2e1ebcfb..973ab0833 100644 --- a/crates/astria-composer/src/sequencer_hooks.rs +++ b/crates/astria-composer/src/sequencer_hooks.rs @@ -3,14 +3,20 @@ use std::{ time::Duration, }; -use astria_core::generated::composer::v1alpha1::{ - sequencer_hooks_service_server::SequencerHooksService, - SendFinalizedHashRequest, - SendFinalizedHashResponse, - SendOptimisticBlockRequest, - SendOptimisticBlockResponse, +use astria_core::{ + generated::composer::v1alpha1::{ + sequencer_hooks_service_server::SequencerHooksService, + SendFinalizedHashRequest, + SendFinalizedHashResponse, + SendOptimisticBlockRequest, + SendOptimisticBlockResponse, + }, + protocol::transaction::v1alpha1::action::SequenceAction, + Protobuf, }; use astria_eyre::eyre::WrapErr; +use bytes::Bytes; +use pbjson_types::Timestamp; use tokio::sync::{ mpsc, mpsc::error::SendTimeoutError, @@ -24,41 +30,83 @@ use tracing::info; const SEND_TIMEOUT: u64 = 2; -// pub(crate) struct OptimisticBlockInfo { -// block_hash: Bytes, -// seq_actions: Vec, -// time: Timestamp, -// } +pub(crate) struct OptimisticBlockInfo { + block_hash: Bytes, + seq_actions: Vec, + time: Timestamp, +} + +impl OptimisticBlockInfo { + pub(crate) fn new( + block_hash: Bytes, + seq_actions: Vec, + time: Timestamp, + ) -> Self { + Self { + block_hash, + seq_actions, + time, + } + } + + pub(crate) fn block_hash(&self) -> Bytes { + self.block_hash.clone() + } + + pub(crate) fn seq_actions(&self) -> Vec { + self.seq_actions.clone() + } + + pub(crate) fn time(&self) -> Timestamp { + self.time.clone() + } +} + +pub(crate) struct FinalizedHashInfo { + block_hash: Bytes, +} + +impl FinalizedHashInfo { + pub(crate) fn new(block_hash: Bytes) -> Self { + Self { + block_hash, + } + } + + pub(crate) fn block_hash(&self) -> Bytes { + self.block_hash.clone() + } +} pub(crate) struct SequencerHooks { - filtered_block_sender: mpsc::Sender, - finalized_hash_sender: mpsc::Sender, + optimistic_block_sender: mpsc::Sender, + finalized_hash_sender: mpsc::Sender, } impl SequencerHooks { pub(crate) fn new( - filtered_block_sender: mpsc::Sender, - finalized_hash_sender: mpsc::Sender, + optimistic_block_sender: mpsc::Sender, + finalized_hash_sender: mpsc::Sender, ) -> Self { Self { - filtered_block_sender, + optimistic_block_sender, finalized_hash_sender, } } pub(crate) async fn send_optimistic_block_with_timeout( &self, - req: SendOptimisticBlockRequest, - ) -> Result<(), SendTimeoutError> { - self.filtered_block_sender + req: OptimisticBlockInfo, + ) -> Result<(), SendTimeoutError> { + self.optimistic_block_sender .send_timeout(req, Duration::from_secs(SEND_TIMEOUT)) .await } pub(crate) async fn send_finalized_hash_with_timeout( &self, - req: SendFinalizedHashRequest, - ) -> Result<(), SendTimeoutError> { + req: FinalizedHashInfo, + ) -> Result<(), SendTimeoutError> { self.finalized_hash_sender .send_timeout(req, Duration::from_secs(SEND_TIMEOUT)) .await @@ -72,8 +120,24 @@ impl SequencerHooksService for SequencerHooks { request: Request, ) -> Result, Status> { let inner = request.into_inner(); + + let mut seq_actions = vec![]; + for action in &inner.seq_action { + match SequenceAction::try_from_raw_ref(action) { + Ok(action) => seq_actions.push(action), + Err(e) => { + info!("Failed to convert sequence action: {:?}", e); + return Err(Status::invalid_argument("invalid sequence action")); + } + } + } + return match self - .send_optimistic_block_with_timeout(inner) + .send_optimistic_block_with_timeout(OptimisticBlockInfo::new( + inner.block_hash, + seq_actions, + inner.time.unwrap(), + )) .await .wrap_err("unable to send optimistic block to executor") { @@ -92,7 +156,7 @@ impl SequencerHooksService for SequencerHooks { let inner = request.into_inner(); return match self - .send_finalized_hash_with_timeout(inner) + .send_finalized_hash_with_timeout(FinalizedHashInfo::new(inner.block_hash)) .await .wrap_err("unable to send finalized block hash to executor") { diff --git a/crates/astria-sequencer/src/app/mod.rs b/crates/astria-sequencer/src/app/mod.rs index e107597e2..3f0395f40 100644 --- a/crates/astria-sequencer/src/app/mod.rs +++ b/crates/astria-sequencer/src/app/mod.rs @@ -466,7 +466,7 @@ impl App { "chain IDs commitment does not match expected", ); - let block_hash = process_proposal.hash.clone(); + let block_hash = process_proposal.hash; self.executed_proposal_hash = block_hash; // get a list of sequence actions from the signed_txs From 0585187d747ace1a56b11307ac7be39039103f88 Mon Sep 17 00:00:00 2001 From: Bharath Date: Fri, 6 Sep 2024 14:23:47 +0530 Subject: [PATCH 36/43] update tests --- crates/astria-composer/src/executor/tests.rs | 32 +++++++++++++------ crates/astria-sequencer/src/app/test_utils.rs | 12 +++++-- .../astria-sequencer/src/service/consensus.rs | 1 + 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/crates/astria-composer/src/executor/tests.rs b/crates/astria-composer/src/executor/tests.rs index 4867fbd26..9dd920507 100644 --- a/crates/astria-composer/src/executor/tests.rs +++ b/crates/astria-composer/src/executor/tests.rs @@ -82,6 +82,10 @@ use crate::{ }, mount_executed_block, mount_get_commitment_state, + sequencer_hooks::{ + FinalizedHashInfo, + OptimisticBlockInfo, + }, test_utils::sequence_action_of_max_size, Config, }; @@ -357,9 +361,12 @@ async fn full_bundle() { let (sequencer, cfg, _keyfile, test_executor) = setup().await; let shutdown_token = CancellationToken::new(); let metrics = Box::leak(Box::new(Metrics::noop_metrics(&cfg).unwrap())); + let (filtered_block_sender, filtered_block_receiver) = + tokio::sync::mpsc::channel::(10); + let (finalized_hash_sender, finalized_hash_receiver) = + tokio::sync::mpsc::channel::(10); + mount_genesis(&sequencer, &cfg.sequencer_chain_id).await; - let (_filtered_block_sender, filtered_block_receiver) = tokio::sync::mpsc::channel(1); - let (_finalized_hash_sender, finalized_hash_receiver) = tokio::sync::mpsc::channel(1); let (executor, executor_handle) = executor::Builder { sequencer_url: cfg.sequencer_url.clone(), sequencer_chain_id: cfg.sequencer_chain_id.clone(), @@ -415,7 +422,7 @@ async fn full_bundle() { ..sequence_action_of_max_size(cfg.max_bytes_per_bundle) }; - let rollup_data: Vec = vec![seq0.clone()] + let rollup_data: Vec = vec![seq0.clone(), seq1.clone()] .iter() .map(|item| RollupData::SequencedData(item.clone().data).to_raw()) .collect(); @@ -508,9 +515,11 @@ async fn bundle_triggered_by_block_timer() { let (sequencer, cfg, _keyfile, test_executor) = setup().await; let shutdown_token = CancellationToken::new(); let metrics = Box::leak(Box::new(Metrics::noop_metrics(&cfg).unwrap())); + let (filtered_block_sender, filtered_block_receiver) = + tokio::sync::mpsc::channel::(10); + let (finalized_hash_sender, finalized_hash_receiver) = + tokio::sync::mpsc::channel::(10); mount_genesis(&sequencer, &cfg.sequencer_chain_id).await; - let (_filtered_block_sender, filtered_block_receiver) = tokio::sync::mpsc::channel(1); - let (_finalized_hash_sender, finalized_hash_receiver) = tokio::sync::mpsc::channel(1); let (executor, executor_handle) = executor::Builder { sequencer_url: cfg.sequencer_url.clone(), sequencer_chain_id: cfg.sequencer_chain_id.clone(), @@ -664,8 +673,10 @@ async fn two_seq_actions_single_bundle() { let shutdown_token = CancellationToken::new(); let metrics = Box::leak(Box::new(Metrics::noop_metrics(&cfg).unwrap())); mount_genesis(&sequencer, &cfg.sequencer_chain_id).await; - let (_filtered_block_sender, filtered_block_receiver) = tokio::sync::mpsc::channel(1); - let (_finalized_hash_sender, finalized_hash_receiver) = tokio::sync::mpsc::channel(1); + let (filtered_block_sender, filtered_block_receiver) = + tokio::sync::mpsc::channel::(10); + let (finalized_hash_sender, finalized_hash_receiver) = + tokio::sync::mpsc::channel::(10); let (executor, executor_handle) = executor::Builder { sequencer_url: cfg.sequencer_url.clone(), sequencer_chain_id: cfg.sequencer_chain_id.clone(), @@ -821,8 +832,11 @@ async fn chain_id_mismatch_returns_error() { let metrics = Box::leak(Box::new(Metrics::noop_metrics(&cfg).unwrap())); let rollup_name = RollupId::new([0; ROLLUP_ID_LEN]); - let (_filtered_block_sender, filtered_block_receiver) = tokio::sync::mpsc::channel(1); - let (_finalized_hash_sender, finalized_hash_receiver) = tokio::sync::mpsc::channel(1); + let (filtered_block_sender, filtered_block_receiver) = + tokio::sync::mpsc::channel::(10); + let (finalized_hash_sender, finalized_hash_receiver) = + tokio::sync::mpsc::channel::(10); + // mount a status response with an incorrect chain_id mount_genesis(&sequencer, "bad-chain-id").await; diff --git a/crates/astria-sequencer/src/app/test_utils.rs b/crates/astria-sequencer/src/app/test_utils.rs index 22f76a518..60dd8773d 100644 --- a/crates/astria-sequencer/src/app/test_utils.rs +++ b/crates/astria-sequencer/src/app/test_utils.rs @@ -141,9 +141,15 @@ pub(crate) async fn initialize_app_with_storage( let mempool = Mempool::new(); let metrics = Box::leak(Box::new(Metrics::noop_metrics(&()).unwrap())); // TODO - temp addr - let mut app = App::new(snapshot, mempool, "127.0.0.1:232".to_string(), metrics) - .await - .unwrap(); + let mut app = App::new( + snapshot, + mempool, + "127.0.0.1:232".to_string(), + false, + metrics, + ) + .await + .unwrap(); let genesis_state = genesis_state.unwrap_or_else(self::genesis_state); diff --git a/crates/astria-sequencer/src/service/consensus.rs b/crates/astria-sequencer/src/service/consensus.rs index 6a1c60c89..8f93e82ee 100644 --- a/crates/astria-sequencer/src/service/consensus.rs +++ b/crates/astria-sequencer/src/service/consensus.rs @@ -470,6 +470,7 @@ mod test { snapshot, mempool.clone(), "127.0.0.1:34".to_string(), + false, metrics, ) .await From 67272c645d392de79751285611a1d89e2a2c05a0 Mon Sep 17 00:00:00 2001 From: Bharath Date: Fri, 30 Aug 2024 17:26:40 +0530 Subject: [PATCH 37/43] save --- charts/celestia-node/values.yaml | 14 +++++++------- charts/composer/values.yaml | 6 +++--- charts/evm-rollup/values.yaml | 8 ++++---- dev/values/rollup/dev.yaml | 13 ++++++------- dev/values/validators/node0.yml | 8 ++++---- dev/values/validators/node1.yml | 8 ++++---- dev/values/validators/node2.yml | 8 ++++---- 7 files changed, 32 insertions(+), 33 deletions(-) diff --git a/charts/celestia-node/values.yaml b/charts/celestia-node/values.yaml index 3749172aa..8a4cbe064 100644 --- a/charts/celestia-node/values.yaml +++ b/charts/celestia-node/values.yaml @@ -25,13 +25,13 @@ ports: celestia: rpc: 26658 -resources: - requests: - cpu: 1 - memory: 4Gi - limits: - cpu: 2 - memory: 8Gi +resources: {} +# requests: +# cpu: 1 +# memory: 4Gi +# limits: +# cpu: 2 +# memory: 8Gi # When deploying in a production environment should use a secret provider # This is configured for use with GCP, need to set own resource names diff --git a/charts/composer/values.yaml b/charts/composer/values.yaml index d49521bd7..82d19e875 100644 --- a/charts/composer/values.yaml +++ b/charts/composer/values.yaml @@ -3,13 +3,13 @@ global: replicaCount: 1 # Whether to use tty readable logging for astria services, when false use json. useTTY: false - dev: false + dev: tb images: composer: - repo: ghcr.io/astriaorg/composer + repo: astria-composer tag: "0.8.2" - devTag: latest + devTag: tb config: logLevel: "info" diff --git a/charts/evm-rollup/values.yaml b/charts/evm-rollup/values.yaml index ccf6b572f..6611a8807 100644 --- a/charts/evm-rollup/values.yaml +++ b/charts/evm-rollup/values.yaml @@ -8,14 +8,14 @@ global: images: geth: - repo: ghcr.io/astriaorg/astria-geth + repo: ghcr.io/astriaorg/go-ethereum tag: 0.14.0 - devTag: latest + devTag: tb overrideTag: "" conductor: - repo: ghcr.io/astriaorg/conductor + repo: astria-conductor tag: "0.20.0" - devTag: latest + devTag: tb genesis: diff --git a/dev/values/rollup/dev.yaml b/dev/values/rollup/dev.yaml index b365e7a12..af10432dc 100644 --- a/dev/values/rollup/dev.yaml +++ b/dev/values/rollup/dev.yaml @@ -10,14 +10,13 @@ global: evm-rollup: images: geth: - repo: ghcr.io/astriaorg/astria-geth + repo: ghcr.io/astriaorg/go-ethereum tag: 0.14.0 - devTag: latest - overrideTag: "pr-41" + devTag: tb conductor: - repo: ghcr.io/astriaorg/conductor + repo: astria-conductor tag: "0.20.0" - devTag: latest + devTag: tb genesis: ## These values are used to configure the genesis block of the rollup chain ## no defaults as they are unique to each chain @@ -118,10 +117,10 @@ evm-rollup: # - "SoftOnly" -> blocks are only pulled from the sequencer # - "FirmOnly" -> blocks are only pulled from DA # - "SoftAndFirm" -> blocks are pulled from both the sequencer and DA - executionCommitLevel: 'SoftAndFirm' + executionCommitLevel: 'SoftOnly' # The expected fastest block time possible from sequencer, determines polling # rate. - sequencerBlockTimeMs: 2000 + sequencerBlockTimeMs: 100 # The maximum number of requests to make to the sequencer per second sequencerRequestsPerSecond: 500 diff --git a/dev/values/validators/node0.yml b/dev/values/validators/node0.yml index 47f6d582b..0bcbca1fd 100644 --- a/dev/values/validators/node0.yml +++ b/dev/values/validators/node0.yml @@ -23,10 +23,10 @@ genesis: address: E82D827830B163D5179291FB27BB58E605DF2FA2 pubKey: NDE9F44v3l4irmkZxNmrZkywoGmggLlaBo5rE/Cis8M= power: '1' -# - name: node2 -# power: '1' -# address: 8C17BBDC7C350C83C550163458FC9B7A5B54A64E -# pubKey: 4v1RdMiKkWgBBTTP26iRSLOEkAY99gMVfZijm6OCzjs= + - name: node2 + power: '1' + address: 8C17BBDC7C350C83C550163458FC9B7A5B54A64E + pubKey: 4v1RdMiKkWgBBTTP26iRSLOEkAY99gMVfZijm6OCzjs= cometbft: secrets: diff --git a/dev/values/validators/node1.yml b/dev/values/validators/node1.yml index 0a596b31a..59384bb50 100644 --- a/dev/values/validators/node1.yml +++ b/dev/values/validators/node1.yml @@ -22,10 +22,10 @@ genesis: address: E82D827830B163D5179291FB27BB58E605DF2FA2 pubKey: NDE9F44v3l4irmkZxNmrZkywoGmggLlaBo5rE/Cis8M= power: '1' -# - name: node2 -# power: '1' -# address: 8C17BBDC7C350C83C550163458FC9B7A5B54A64E -# pubKey: 4v1RdMiKkWgBBTTP26iRSLOEkAY99gMVfZijm6OCzjs= + - name: node2 + power: '1' + address: 8C17BBDC7C350C83C550163458FC9B7A5B54A64E + pubKey: 4v1RdMiKkWgBBTTP26iRSLOEkAY99gMVfZijm6OCzjs= # Values for CometBFT node configuration cometbft: diff --git a/dev/values/validators/node2.yml b/dev/values/validators/node2.yml index b3d69e85e..a37f7b9ff 100644 --- a/dev/values/validators/node2.yml +++ b/dev/values/validators/node2.yml @@ -22,10 +22,10 @@ genesis: address: E82D827830B163D5179291FB27BB58E605DF2FA2 pubKey: NDE9F44v3l4irmkZxNmrZkywoGmggLlaBo5rE/Cis8M= power: '1' -# - name: node2 -# power: '1' -# address: 8C17BBDC7C350C83C550163458FC9B7A5B54A64E -# pubKey: 4v1RdMiKkWgBBTTP26iRSLOEkAY99gMVfZijm6OCzjs= + - name: node2 + power: '1' + address: 8C17BBDC7C350C83C550163458FC9B7A5B54A64E + pubKey: 4v1RdMiKkWgBBTTP26iRSLOEkAY99gMVfZijm6OCzjs= cometbft: secrets: From cb1d30a218ec6223df0ec11372840d1811937de0 Mon Sep 17 00:00:00 2001 From: Bharath Date: Fri, 30 Aug 2024 21:30:53 +0530 Subject: [PATCH 38/43] save --- charts/composer/values.yaml | 2 +- charts/evm-rollup/values.yaml | 1 - crates/astria-sequencer/src/app/mod.rs | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/composer/values.yaml b/charts/composer/values.yaml index 82d19e875..bb4c03c67 100644 --- a/charts/composer/values.yaml +++ b/charts/composer/values.yaml @@ -3,7 +3,7 @@ global: replicaCount: 1 # Whether to use tty readable logging for astria services, when false use json. useTTY: false - dev: tb + dev: true images: composer: diff --git a/charts/evm-rollup/values.yaml b/charts/evm-rollup/values.yaml index 6611a8807..648f8a665 100644 --- a/charts/evm-rollup/values.yaml +++ b/charts/evm-rollup/values.yaml @@ -11,7 +11,6 @@ images: repo: ghcr.io/astriaorg/go-ethereum tag: 0.14.0 devTag: tb - overrideTag: "" conductor: repo: astria-conductor tag: "0.20.0" diff --git a/crates/astria-sequencer/src/app/mod.rs b/crates/astria-sequencer/src/app/mod.rs index 3f0395f40..d34990d2b 100644 --- a/crates/astria-sequencer/src/app/mod.rs +++ b/crates/astria-sequencer/src/app/mod.rs @@ -361,6 +361,7 @@ impl App { process_proposal: abci::request::ProcessProposal, storage: Storage, ) -> anyhow::Result<()> { + info!("BHARATH: Processing proposal!"); // if we proposed this block (ie. prepare_proposal was called directly before this), then // we skip execution for this `process_proposal` call. // From 3db4f31bb4c28080c5dd306b039319f8c7e073f8 Mon Sep 17 00:00:00 2001 From: Bharath Date: Mon, 2 Sep 2024 22:31:04 +0530 Subject: [PATCH 39/43] expose composer grpc port in an ingress --- charts/composer/templates/configmap.yaml | 28 ++++++++++ charts/composer/templates/ingress.yaml | 71 ++++++++++++++++++++++++ charts/composer/values.yaml | 25 ++++++++- 3 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 charts/composer/templates/ingress.yaml diff --git a/charts/composer/templates/configmap.yaml b/charts/composer/templates/configmap.yaml index a2a47e792..b41e008d2 100644 --- a/charts/composer/templates/configmap.yaml +++ b/charts/composer/templates/configmap.yaml @@ -4,6 +4,34 @@ metadata: name: composer-env namespace: {{ include "composer.namespace" . }} data: +{{/* ASTRIA_COMPOSER_LOG: "astria_composer={{ .Values.config.logLevel }}"*/}} +{{/* ASTRIA_COMPOSER_API_LISTEN_ADDR: "0.0.0.0:{{ .Values.ports.healthApi }}"*/}} +{{/* ASTRIA_COMPOSER_GRPC_ADDR: "0.0.0.0:{{ .Values.ports.grpc }}"*/}} +{{/* ASTRIA_COMPOSER_SEQUENCER_CHAIN_ID: "{{ tpl .Values.config.sequencerChainId . }}"*/}} +{{/* ASTRIA_COMPOSER_SEQUENCER_URL: "{{ tpl .Values.config.sequencerRpc . }}"*/}} +{{/* ASTRIA_COMPOSER_ROLLUPS: "{{ include "composer.rollups" . }}"*/}} +{{/* ASTRIA_COMPOSER_PRIVATE_KEY_FILE: "/var/secrets/{{ .Values.config.privateKey.secret.filename }}"*/}} +{{/* ASTRIA_COMPOSER_MAX_BYTES_PER_BUNDLE: "{{ .Values.config.maxBytesPerBundle }}"*/}} +{{/* ASTRIA_COMPOSER_BUNDLE_QUEUE_CAPACITY: "{{ .Values.config.bundleQueueCapacity }}"*/}} +{{/* ASTRIA_COMPOSER_MAX_SUBMIT_INTERVAL_MS: "{{ .Values.config.maxSubmitInterval }}"*/}} +{{/* ASTRIA_COMPOSER_SEQUENCER_ADDRESS_PREFIX: "{{ .Values.config.sequencerAddressPrefix}}"*/}} +{{/* ASTRIA_COMPOSER_FEE_ASSET: "{{ .Values.config.sequencerNativeAssetBaseDenomination }}"*/}} +{{/* ASTRIA_COMPOSER_NO_METRICS: "{{ not .Values.metrics.enabled }}"*/}} +{{/* ASTRIA_COMPOSER_METRICS_HTTP_LISTENER_ADDR: "0.0.0.0:{{ .Values.ports.metrics }}"*/}} +{{/* ASTRIA_COMPOSER_FORCE_STDOUT: "{{ .Values.global.useTTY }}"*/}} +{{/* ASTRIA_COMPOSER_PRETTY_PRINT: "{{ .Values.global.useTTY }}"*/}} +{{/* NO_COLOR: "{{ .Values.global.useTTY }}"*/}} +{{/* ASTRIA_COMPOSER_NO_OTEL: "{{ not .Values.otel.enabled }}"*/}} +{{/* OTEL_EXPORTER_OTLP_ENDPOINT: "{{ tpl .Values.otel.endpoint . }}"*/}} +{{/* OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "{{ tpl .Values.otel.tracesEndpoint . }}"*/}} +{{/* OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: "{{ tpl .Values.otel.tracesTimeout . }}"*/}} +{{/* OTEL_EXPORTER_OTLP_TRACES_COMPRESSION: "{{ tpl .Values.otel.tracesCompression . }}"*/}} +{{/* OTEL_EXPORTER_OTLP_HEADERS: "{{ tpl .Values.otel.otlpHeaders . }}"*/}} +{{/* OTEL_EXPORTER_OTLP_TRACE_HEADERS: "{{ tpl .Values.otel.traceHeaders . }}"*/}} +{{/* OTEL_SERVICE_NAME: "{{ tpl .Values.otel.serviceName . }}"*/}} +{{/* {{- if not .Values.global.dev }}*/}} +{{/* {{- else }}*/}} +{{/* {{- end }}*/}} ASTRIA_COMPOSER_LOG: "astria_composer={{ .Values.config.logLevel }}" ASTRIA_COMPOSER_API_LISTEN_ADDR: "0.0.0.0:{{ .Values.ports.healthApi }}" ASTRIA_COMPOSER_GRPC_ADDR: "0.0.0.0:{{ .Values.ports.grpc }}" diff --git a/charts/composer/templates/ingress.yaml b/charts/composer/templates/ingress.yaml new file mode 100644 index 000000000..65ccb0bb5 --- /dev/null +++ b/charts/composer/templates/ingress.yaml @@ -0,0 +1,71 @@ +{{- if .Values.ingress.enabled -}} +{{- $ingressApiIsStable := eq (include "rollup.ingress.isStable" .) "true" -}} +{{- $ingressSupportsIngressClassName := eq (include "rollup.ingress.supportsIngressClassName" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "rollup.ingress.supportsPathType" .) "true" -}} + +{{- range $service, $ingress := .Values.ingress.services }} +{{- if $ingress.enabled -}} +{{- $servicePort := $ingress.service.port -}} +{{- $serviceName := tpl $ingress.service.name $ -}} +{{- $ingressPath := $ingress.path -}} +{{- $ingressPathType := $ingress.pathType -}} +{{- $extraPaths := $ingress.extraPaths }} +--- +apiVersion: {{ include "rollup.ingress.apiVersion" $ }} +kind: Ingress +metadata: + name: astria-local-{{ $service }}-ingress + namespace: {{ include "rollup.namespace" $ }} + labels: + {{- with $ingress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + nginx.ingress.kubernetes.io/backend-protocol: "GRPC" + {{- if not $ingressSupportsIngressClassName }} + kubernetes.io/ingress.class: {{ $.Values.ingress.className }} + {{- end }} + {{- if $ingressApiIsStable }} + {{- range $key, $value := $ingress.annotations }} + {{ $key }}: {{ tpl $value $ | quote }} + {{- end }} + {{- end }} +spec: + {{- if $ingressSupportsIngressClassName }} + ingressClassName: {{ $.Values.ingress.className }} + {{- end -}} + {{- with $ingress.service }} + defaultBackend: + service: + {{- tpl (toYaml .) $ | nindent 6 }} + {{- end }} + rules: + {{- with $ingress.hosts }} + {{- range $host := . }} + - host: {{ tpl $host $ }} + http: + paths: + {{- with $extraPaths }} + {{- toYaml . | nindent 10 }} + {{- end }} + - path: {{ $ingressPath }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + backend: + {{- if $ingressApiIsStable }} + service: + {{- tpl (toYaml $ingress.service) $ | nindent 16 }} + {{- else }} + serviceName: {{ tpl $serviceName $ }} + servicePort: {{ tpl $servicePort $ }} + {{- end }} + {{- end }} + {{- end }} + {{- if $ingress.tls }} + tls: + {{- tpl (toYaml $ingress.tls) $ | nindent 4 }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/composer/values.yaml b/charts/composer/values.yaml index bb4c03c67..8d5ca4b11 100644 --- a/charts/composer/values.yaml +++ b/charts/composer/values.yaml @@ -22,7 +22,7 @@ config: sequencerRpc: "" sequencerChainId: "" privateKey: - devContent: "" + devContent: "e315da1eafadb1e323a53244bdca8365437c643f0977de9ecf44cc926538f918" secret: filename: "key.hex" resourceName: "projects/$PROJECT_ID/secrets/sequencerPrivateKey/versions/latest" @@ -72,3 +72,26 @@ ports: grpc: 50052 healthApi: 2450 metrics: 6060 + + +ingress: + enabled: true + labels: {} + hostname: localdev.me + className: nginx + services: + rpc: + enabled: true + hosts: + - 'composer.astria-local.{{ .Values.ingress.hostname }}' + path: / + pathType: Prefix + service: + name: 'composer-service' + port: + name: grpc + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + labels: {} + tls: {} \ No newline at end of file From 9012050152d72cac4a7a468615716bbf503a4d87 Mon Sep 17 00:00:00 2001 From: Bharath Date: Tue, 3 Sep 2024 11:24:20 +0530 Subject: [PATCH 40/43] update charts --- charts/deploy.just | 4 ++-- charts/evm-stack/values.yaml | 7 ++++--- charts/sequencer/files/cometbft/config/genesis.json | 2 +- charts/sequencer/values.yaml | 13 +++++++++---- dev/values/rollup/dev.yaml | 10 +++++----- dev/values/validators/node0.yml | 2 +- dev/values/validators/node1.yml | 2 +- dev/values/validators/node2.yml | 2 +- 8 files changed, 24 insertions(+), 18 deletions(-) diff --git a/charts/deploy.just b/charts/deploy.just index cf143e772..382b2c171 100644 --- a/charts/deploy.just +++ b/charts/deploy.just @@ -79,8 +79,8 @@ deploy-sequencer name=validatorName: -f dev/values/validators/{{name}}.yml \ -n astria-validator-{{name}} --create-namespace \ {{name}}-sequencer-chart ./charts/sequencer -#deploy-sequencers: (deploy-sequencer "node0") (deploy-sequencer "node1") (deploy-sequencer "node2") -deploy-sequencers: (deploy-sequencer "node0") (deploy-sequencer "node1") +deploy-sequencers: (deploy-sequencer "node0") (deploy-sequencer "node1") (deploy-sequencer "node2") +#deploy-sequencers: (deploy-sequencer "node0") (deploy-sequencer "node1") deploy-hermes-local: diff --git a/charts/evm-stack/values.yaml b/charts/evm-stack/values.yaml index 6aba3ae14..01b386495 100644 --- a/charts/evm-stack/values.yaml +++ b/charts/evm-stack/values.yaml @@ -39,9 +39,6 @@ evm-rollup: otlpHeaders: "{{ .Values.global.otel.otlpHeaders }}" traceHeaders: "{{ .Values.global.otel.traceHeaders }}" -celestia-node: - enabled: false - composer: enabled: false config: @@ -59,6 +56,10 @@ composer: otlpHeaders: "{{ .Values.global.otel.otlpHeaders }}" traceHeaders: "{{ .Values.global.otel.traceHeaders }}" + +celestia-node: + enabled: false + evm-faucet: enabled: false config: diff --git a/charts/sequencer/files/cometbft/config/genesis.json b/charts/sequencer/files/cometbft/config/genesis.json index 8caf12aa3..8046c4453 100644 --- a/charts/sequencer/files/cometbft/config/genesis.json +++ b/charts/sequencer/files/cometbft/config/genesis.json @@ -31,7 +31,7 @@ {{- if $index }},{{- end }} { "address": {{ include "sequencer.address" $value.address }}, - "balance": {{ include "sequencer.toUint128Proto" ( toString $value.balance | replace "\"" "" ) }} + "balance": {{ include "sequencer.toUint128Proto" ( toString $value.balance | replace "\"" "" ) }} } {{- end }} ], diff --git a/charts/sequencer/values.yaml b/charts/sequencer/values.yaml index 53ce1750b..f423551e9 100644 --- a/charts/sequencer/values.yaml +++ b/charts/sequencer/values.yaml @@ -16,9 +16,9 @@ images: tag: v0.38.8 devTag: v0.38.8 sequencer: - repo: ghcr.io/astriaorg/sequencer + repo: astria-sequencer tag: "0.16.0" - devTag: latest + devTag: tb moniker: "" genesis: @@ -110,7 +110,12 @@ cometbft: # List of seeds to connect to seeds: [] # List of nodes to keep persistent connections to - persistentPeers: [] + persistentPeers: [ + "f4b8a8dcfc5a142bd00aadab71f39dbfe7091d13@35.236.107.39:26656", + "ca3bc3562919b82575fe3ac5b11fa5962ce8cd3b@34.118.249.169:26656", + "4418000e355967ecc8e03004f5850dfde51c410b@34.94.177.254:26656", + "7a117e7906d8428ad20341aca94af03c980c11d8@34.102.55.164:26656", + ] # List of node IDs, to which a connection will be (re)established ignoring any existing limits unconditionalPeers: [] # Maximum pause when redialing a persistent peer (if zero, exponential backoff is used) @@ -121,7 +126,7 @@ cometbft: allowDuplicateIP: false # Seed mode, in which node constantly crawls the network and looks for # peers. If another node asks it for addresses, it responds and disconnects. - seedMode: false + seedMode: true # Maximum number of inbound and outbound peers maxInboundPeers: 100 maxOutboundPeers: 40 diff --git a/dev/values/rollup/dev.yaml b/dev/values/rollup/dev.yaml index af10432dc..4cf684f30 100644 --- a/dev/values/rollup/dev.yaml +++ b/dev/values/rollup/dev.yaml @@ -3,9 +3,9 @@ global: dev: true evmChainId: 1337 rollupName: astria - sequencerRpc: http://node0-sequencer-rpc-service.astria-dev-cluster.svc.cluster.local:26657 - sequencerGrpc: http://node0-sequencer-grpc-service.astria-dev-cluster.svc.cluster.local:8080 - sequencerChainId: sequencer-test-chain-0 + sequencerRpc: https://rpc.sequencer.dusk-10.devnet.astria.org/ + sequencerGrpc: https://grpc.sequencer.dusk-10.devnet.astria.org/ + sequencerChainId: astria-dusk-10 evm-rollup: images: @@ -22,7 +22,7 @@ evm-rollup: ## no defaults as they are unique to each chain # Block height to start syncing rollup from, lowest possible is 2 - sequencerInitialHeight: 2 + sequencerInitialHeight: 264188 # The first Celestia height to utilize when looking for rollup data celestiaInitialHeight: 2 # The variance in Celestia height to allow before halting the chain @@ -154,7 +154,7 @@ evm-rollup: enabled: true ws: enabled: true - + celestia-node: enabled: false diff --git a/dev/values/validators/node0.yml b/dev/values/validators/node0.yml index 0bcbca1fd..2804706d9 100644 --- a/dev/values/validators/node0.yml +++ b/dev/values/validators/node0.yml @@ -9,7 +9,7 @@ images: devTag: v0.38.8 sequencer: repo: astria-sequencer - tag: "0.14.0" + tag: "0.16.0" devTag: tb moniker: node0 diff --git a/dev/values/validators/node1.yml b/dev/values/validators/node1.yml index 59384bb50..7f5d804b1 100644 --- a/dev/values/validators/node1.yml +++ b/dev/values/validators/node1.yml @@ -8,7 +8,7 @@ images: devTag: v0.38.8 sequencer: repo: astria-sequencer - tag: "0.14.0" + tag: "0.16.0" devTag: tb moniker: 'node1' diff --git a/dev/values/validators/node2.yml b/dev/values/validators/node2.yml index a37f7b9ff..fb8fb6b67 100644 --- a/dev/values/validators/node2.yml +++ b/dev/values/validators/node2.yml @@ -8,7 +8,7 @@ images: devTag: v0.38.8 sequencer: repo: astria-sequencer - tag: "0.14.0" + tag: "0.16.0" devTag: tb moniker: 'node2' From f177372a19ca01dfce7807b15e60bf847f407b9f Mon Sep 17 00:00:00 2001 From: Bharath Date: Wed, 4 Sep 2024 15:26:50 +0530 Subject: [PATCH 41/43] fix --- crates/astria-composer/src/executor/simulator.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/astria-composer/src/executor/simulator.rs b/crates/astria-composer/src/executor/simulator.rs index e870ed28f..d4004a529 100644 --- a/crates/astria-composer/src/executor/simulator.rs +++ b/crates/astria-composer/src/executor/simulator.rs @@ -91,7 +91,7 @@ impl BundleSimulator { .filter(|data| !data.is_empty()) .collect(); - self.inner_simulate_bundle_on_block(actions, soft_block.clone(), Some(time)) + self.inner_simulate_bundle_on_block(actions, soft_block.clone(), Some(time), false) .await } @@ -114,7 +114,7 @@ impl BundleSimulator { let actions = convert_bundle_to_byte_array(bundle); - self.inner_simulate_bundle_on_block(actions, soft_block.clone(), None) + self.inner_simulate_bundle_on_block(actions, soft_block.clone(), None, true) .await } @@ -126,7 +126,7 @@ impl BundleSimulator { timestamp: Option, ) -> eyre::Result { let actions = convert_bundle_to_byte_array(bundle); - self.inner_simulate_bundle_on_block(actions, block, timestamp) + self.inner_simulate_bundle_on_block(actions, block, timestamp, true) .await } @@ -136,6 +136,7 @@ impl BundleSimulator { bundle: Vec>, block: Block, timestamp: Option, + simulate_only: bool ) -> eyre::Result { // convert the sized bundle actions to a list of Vec // as long as the timestamp > parent block timestamp, the block will be successfully @@ -153,7 +154,7 @@ impl BundleSimulator { bundle, // use current timestamp timestamp, - true, + simulate_only, ) .await .wrap_err("failed to execute block")?; From e54d89586566e8b58399da89383a634a79dc4e4d Mon Sep 17 00:00:00 2001 From: Bharath Date: Fri, 6 Sep 2024 15:28:34 +0530 Subject: [PATCH 42/43] minor fixes --- Cargo.lock | 1 - crates/astria-composer/Cargo.toml | 1 - crates/astria-composer/src/metrics.rs | 99 ++------------------------- dev/values/rollup/dev.yaml | 24 +++---- 4 files changed, 15 insertions(+), 110 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b5f11d849..0cbc8d73b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -812,7 +812,6 @@ dependencies = [ "ibc-types", "insta", "matchit", - "metrics", "pbjson-types", "penumbra-ibc", "penumbra-proto", diff --git a/crates/astria-composer/Cargo.toml b/crates/astria-composer/Cargo.toml index 24183ab47..26536fb30 100644 --- a/crates/astria-composer/Cargo.toml +++ b/crates/astria-composer/Cargo.toml @@ -57,7 +57,6 @@ tracing = { workspace = true, features = ["attributes"] } tryhard = { workspace = true } tonic = { workspace = true } tokio-stream = { workspace = true, features = ["net"] } -metrics = { workspace = true } pbjson-types = { workspace = true } bytes = { workspace = true } astria-grpc-mock = { path = "../astria-grpc-mock" } diff --git a/crates/astria-composer/src/metrics.rs b/crates/astria-composer/src/metrics.rs index bec6ac13f..bc70849e6 100644 --- a/crates/astria-composer/src/metrics.rs +++ b/crates/astria-composer/src/metrics.rs @@ -40,89 +40,6 @@ pub struct Metrics { } impl Metrics { -<<<<<<< HEAD -======= - #[must_use] - pub(crate) fn new(rollup_name: &str) -> Self { - let (geth_txs_received, grpc_txs_received) = - register_txs_received([rollup_name.to_string()].iter()); - // TODO - change the function signatures of the metrics - let (geth_txs_dropped, grpc_txs_dropped) = - register_txs_dropped([rollup_name.to_string()].iter()); - let txs_dropped_too_large = - register_txs_dropped_too_large([rollup_name.to_string()].iter()); - - describe_counter!( - NONCE_FETCH_COUNT, - Unit::Count, - "The number of times we have attempted to fetch the nonce" - ); - let nonce_fetch_count = counter!(NONCE_FETCH_COUNT); - - describe_counter!( - NONCE_FETCH_FAILURE_COUNT, - Unit::Count, - "The number of times we have failed to fetch the nonce" - ); - let nonce_fetch_failure_count = counter!(NONCE_FETCH_FAILURE_COUNT); - - describe_histogram!( - NONCE_FETCH_LATENCY, - Unit::Seconds, - "The latency of fetching the nonce, in seconds" - ); - let nonce_fetch_latency = histogram!(NONCE_FETCH_LATENCY); - - describe_gauge!(CURRENT_NONCE, Unit::Count, "The current nonce"); - let current_nonce = gauge!(CURRENT_NONCE); - - describe_histogram!( - SEQUENCER_SUBMISSION_LATENCY, - Unit::Seconds, - "The latency of submitting a transaction to the sequencer, in seconds" - ); - let sequencer_submission_latency = histogram!(SEQUENCER_SUBMISSION_LATENCY); - - describe_counter!( - SEQUENCER_SUBMISSION_FAILURE_COUNT, - Unit::Count, - "The number of failed transaction submissions to the sequencer" - ); - let sequencer_submission_failure_count = counter!(SEQUENCER_SUBMISSION_FAILURE_COUNT); - - describe_histogram!( - TRANSACTIONS_PER_SUBMISSION, - Unit::Count, - "The number of rollup transactions successfully sent to the sequencer in a single \ - submission" - ); - let txs_per_submission = histogram!(TRANSACTIONS_PER_SUBMISSION); - - describe_histogram!( - BYTES_PER_SUBMISSION, - Unit::Bytes, - "The total bytes successfully sent to the sequencer in a single submission" - ); - let bytes_per_submission = histogram!(BYTES_PER_SUBMISSION); - - Self { - geth_txs_received, - geth_txs_dropped, - grpc_txs_received, - grpc_txs_dropped, - txs_dropped_too_large, - nonce_fetch_count, - nonce_fetch_failure_count, - nonce_fetch_latency, - current_nonce, - sequencer_submission_latency, - sequencer_submission_failure_count, - txs_per_submission, - bytes_per_submission, - } - } - ->>>>>>> f151354e (initial version of trusted builder mvp) pub(crate) fn geth_txs_received(&self, id: &String) -> Option<&Counter> { self.geth_txs_received.get(id) } @@ -192,16 +109,14 @@ impl telemetry::Metrics for Metrics { type Config = crate::Config; fn register(builder: &mut RegisteringBuilder, config: &Self::Config) -> Result - where - Self: Sized, + where + Self: Sized, { - let rollups = config - .parse_rollups() - .map_err(|error| Error::External(Box::new(error)))?; + let rollup = config.rollup.clone(); let (geth_txs_received, grpc_txs_received) = - register_txs_received(builder, rollups.keys())?; - let (geth_txs_dropped, grpc_txs_dropped) = register_txs_dropped(builder, rollups.keys())?; - let txs_dropped_too_large = register_txs_dropped_too_large(builder, rollups.keys())?; + register_txs_received(builder, vec![rollup.clone()].iter())?; + let (geth_txs_dropped, grpc_txs_dropped) = register_txs_dropped(builder, vec![rollup.clone()].iter())?; + let txs_dropped_too_large = register_txs_dropped_too_large(builder, vec![rollup.clone()].iter())?; let nonce_fetch_count = builder .new_counter_factory( @@ -422,4 +337,4 @@ mod tests { assert_const(TRANSACTIONS_PER_SUBMISSION, "transactions_per_submission"); assert_const(BYTES_PER_SUBMISSION, "bytes_per_submission"); } -} +} \ No newline at end of file diff --git a/dev/values/rollup/dev.yaml b/dev/values/rollup/dev.yaml index 4cf684f30..d5f92cc69 100644 --- a/dev/values/rollup/dev.yaml +++ b/dev/values/rollup/dev.yaml @@ -3,14 +3,15 @@ global: dev: true evmChainId: 1337 rollupName: astria - sequencerRpc: https://rpc.sequencer.dusk-10.devnet.astria.org/ - sequencerGrpc: https://grpc.sequencer.dusk-10.devnet.astria.org/ - sequencerChainId: astria-dusk-10 + sequencerRpc: http://node0-sequencer-rpc-service.astria-dev-cluster.svc.cluster.local:26657 + sequencerGrpc: http://node0-sequencer-grpc-service.astria-dev-cluster.svc.cluster.local:8080 + sequencerChainId: sequencer-test-chain-0 evm-rollup: images: geth: - repo: ghcr.io/astriaorg/go-ethereum + repo: + tag: 0.14.0 devTag: tb conductor: @@ -88,22 +89,13 @@ evm-rollup: value: balance: "0" code: "0x60806040526004361061004a5760003560e01c80637eb6dec71461004f578063a996e0201461009d578063b6476c7e146100b2578063bab916d0146100d4578063db97dc98146100e7575b600080fd5b34801561005b57600080fd5b506100837f000000000000000000000000000000000000000000000000000000000000000981565b60405163ffffffff90911681526020015b60405180910390f35b6100b06100ab366004610315565b6100fc565b005b3480156100be57600080fd5b506100c761019e565b6040516100949190610381565b6100b06100e23660046103cf565b61022c565b3480156100f357600080fd5b506100c76102bf565b3460006101297f000000000000000000000000000000000000000000000000000000003b9aca0083610411565b1161014f5760405162461bcd60e51b815260040161014690610433565b60405180910390fd5b34336001600160a01b03167f0c64e29a5254a71c7f4e52b3d2d236348c80e00a00ba2e1961962bd2827c03fb8787878760405161018f94939291906104ea565b60405180910390a35050505050565b600180546101ab9061051c565b80601f01602080910402602001604051908101604052809291908181526020018280546101d79061051c565b80156102245780601f106101f957610100808354040283529160200191610224565b820191906000526020600020905b81548152906001019060200180831161020757829003601f168201915b505050505081565b3460006102597f000000000000000000000000000000000000000000000000000000003b9aca0083610411565b116102765760405162461bcd60e51b815260040161014690610433565b34336001600160a01b03167f0f4961cab7530804898499aa89f5ec81d1a73102e2e4a1f30f88e5ae3513ba2a85856040516102b2929190610556565b60405180910390a3505050565b600080546101ab9061051c565b60008083601f8401126102de57600080fd5b50813567ffffffffffffffff8111156102f657600080fd5b60208301915083602082850101111561030e57600080fd5b9250929050565b6000806000806040858703121561032b57600080fd5b843567ffffffffffffffff8082111561034357600080fd5b61034f888389016102cc565b9096509450602087013591508082111561036857600080fd5b50610375878288016102cc565b95989497509550505050565b600060208083528351808285015260005b818110156103ae57858101830151858201604001528201610392565b506000604082860101526040601f19601f8301168501019250505092915050565b600080602083850312156103e257600080fd5b823567ffffffffffffffff8111156103f957600080fd5b610405858286016102cc565b90969095509350505050565b60008261042e57634e487b7160e01b600052601260045260246000fd5b500490565b60208082526062908201527f417374726961576974686472617765723a20696e73756666696369656e74207660408201527f616c75652c206d7573742062652067726561746572207468616e203130202a2a60608201527f20283138202d20424153455f434841494e5f41535345545f505245434953494f6080820152614e2960f01b60a082015260c00190565b81835281816020850137506000828201602090810191909152601f909101601f19169091010190565b6040815260006104fe6040830186886104c1565b82810360208401526105118185876104c1565b979650505050505050565b600181811c9082168061053057607f821691505b60208210810361055057634e487b7160e01b600052602260045260246000fd5b50919050565b60208152600061056a6020830184866104c1565b94935050505056fea264697066735822122047a7ef84c0be4640572989abfc01decbc1ae143d4659f1b32047978c67ebc9c864736f6c63430008150033" - - address: "0xC873fC6685abF295cc537811C234B2B3aD54Af42" - value: - balance: "1000000000000000000000" - - address: "0xD4223e904c40d432524f57c1026cFFF65F19b4b8" - value: - balance: "1000000000000000000000" - - address: "0x77D552845b1834420D021ED283785EA8C4e216f8" - value: - balance: "1000000000000000000000" - - address: "0x5E7db5359356ED0ED2F4aA9B32A2Cc5a21ce8cA0" + - address: "0x85F9c4C67dD0283c9DC176fd6Ff1Faa85a2F818b" value: balance: "1000000000000000000000" - - address: "0xE8a1b7cd69B49cEFDA97671C2EC3FbFF538a2a8C" + - address: "0xba77d35ba45e26858fE871622707e98dd66EB76b" value: balance: "1000000000000000000000" - - address: "0x067E716155Cd2B781D6681610ad685Ee146AFb9A" + - address: "0xcC322501052B0d81263027e6c638687c25CE12D6" value: balance: "1000000000000000000000" From 19e56af6024294f311448552ce9c8f98d5364b59 Mon Sep 17 00:00:00 2001 From: Bharath Date: Fri, 6 Sep 2024 20:41:29 +0530 Subject: [PATCH 43/43] undo chart changes --- charts/celestia-node/values.yaml | 14 ++-- charts/composer/templates/configmap.yaml | 28 -------- charts/composer/templates/ingress.yaml | 71 ------------------- charts/composer/values.yaml | 37 ++-------- charts/deploy.just | 2 - charts/evm-rollup/values.yaml | 21 ++++-- charts/sequencer/values.yaml | 17 ++--- .../astria-composer/src/executor/simulator.rs | 2 +- crates/astria-composer/src/executor/tests.rs | 2 +- crates/astria-composer/src/metrics.rs | 12 ++-- .../tests/blackbox/helper/mod.rs | 15 ++-- crates/astria-conductor/src/conductor.rs | 2 + dev/values/rollup/dev.yaml | 4 +- 13 files changed, 57 insertions(+), 170 deletions(-) delete mode 100644 charts/composer/templates/ingress.yaml diff --git a/charts/celestia-node/values.yaml b/charts/celestia-node/values.yaml index 8a4cbe064..3749172aa 100644 --- a/charts/celestia-node/values.yaml +++ b/charts/celestia-node/values.yaml @@ -25,13 +25,13 @@ ports: celestia: rpc: 26658 -resources: {} -# requests: -# cpu: 1 -# memory: 4Gi -# limits: -# cpu: 2 -# memory: 8Gi +resources: + requests: + cpu: 1 + memory: 4Gi + limits: + cpu: 2 + memory: 8Gi # When deploying in a production environment should use a secret provider # This is configured for use with GCP, need to set own resource names diff --git a/charts/composer/templates/configmap.yaml b/charts/composer/templates/configmap.yaml index b41e008d2..a2a47e792 100644 --- a/charts/composer/templates/configmap.yaml +++ b/charts/composer/templates/configmap.yaml @@ -4,34 +4,6 @@ metadata: name: composer-env namespace: {{ include "composer.namespace" . }} data: -{{/* ASTRIA_COMPOSER_LOG: "astria_composer={{ .Values.config.logLevel }}"*/}} -{{/* ASTRIA_COMPOSER_API_LISTEN_ADDR: "0.0.0.0:{{ .Values.ports.healthApi }}"*/}} -{{/* ASTRIA_COMPOSER_GRPC_ADDR: "0.0.0.0:{{ .Values.ports.grpc }}"*/}} -{{/* ASTRIA_COMPOSER_SEQUENCER_CHAIN_ID: "{{ tpl .Values.config.sequencerChainId . }}"*/}} -{{/* ASTRIA_COMPOSER_SEQUENCER_URL: "{{ tpl .Values.config.sequencerRpc . }}"*/}} -{{/* ASTRIA_COMPOSER_ROLLUPS: "{{ include "composer.rollups" . }}"*/}} -{{/* ASTRIA_COMPOSER_PRIVATE_KEY_FILE: "/var/secrets/{{ .Values.config.privateKey.secret.filename }}"*/}} -{{/* ASTRIA_COMPOSER_MAX_BYTES_PER_BUNDLE: "{{ .Values.config.maxBytesPerBundle }}"*/}} -{{/* ASTRIA_COMPOSER_BUNDLE_QUEUE_CAPACITY: "{{ .Values.config.bundleQueueCapacity }}"*/}} -{{/* ASTRIA_COMPOSER_MAX_SUBMIT_INTERVAL_MS: "{{ .Values.config.maxSubmitInterval }}"*/}} -{{/* ASTRIA_COMPOSER_SEQUENCER_ADDRESS_PREFIX: "{{ .Values.config.sequencerAddressPrefix}}"*/}} -{{/* ASTRIA_COMPOSER_FEE_ASSET: "{{ .Values.config.sequencerNativeAssetBaseDenomination }}"*/}} -{{/* ASTRIA_COMPOSER_NO_METRICS: "{{ not .Values.metrics.enabled }}"*/}} -{{/* ASTRIA_COMPOSER_METRICS_HTTP_LISTENER_ADDR: "0.0.0.0:{{ .Values.ports.metrics }}"*/}} -{{/* ASTRIA_COMPOSER_FORCE_STDOUT: "{{ .Values.global.useTTY }}"*/}} -{{/* ASTRIA_COMPOSER_PRETTY_PRINT: "{{ .Values.global.useTTY }}"*/}} -{{/* NO_COLOR: "{{ .Values.global.useTTY }}"*/}} -{{/* ASTRIA_COMPOSER_NO_OTEL: "{{ not .Values.otel.enabled }}"*/}} -{{/* OTEL_EXPORTER_OTLP_ENDPOINT: "{{ tpl .Values.otel.endpoint . }}"*/}} -{{/* OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "{{ tpl .Values.otel.tracesEndpoint . }}"*/}} -{{/* OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: "{{ tpl .Values.otel.tracesTimeout . }}"*/}} -{{/* OTEL_EXPORTER_OTLP_TRACES_COMPRESSION: "{{ tpl .Values.otel.tracesCompression . }}"*/}} -{{/* OTEL_EXPORTER_OTLP_HEADERS: "{{ tpl .Values.otel.otlpHeaders . }}"*/}} -{{/* OTEL_EXPORTER_OTLP_TRACE_HEADERS: "{{ tpl .Values.otel.traceHeaders . }}"*/}} -{{/* OTEL_SERVICE_NAME: "{{ tpl .Values.otel.serviceName . }}"*/}} -{{/* {{- if not .Values.global.dev }}*/}} -{{/* {{- else }}*/}} -{{/* {{- end }}*/}} ASTRIA_COMPOSER_LOG: "astria_composer={{ .Values.config.logLevel }}" ASTRIA_COMPOSER_API_LISTEN_ADDR: "0.0.0.0:{{ .Values.ports.healthApi }}" ASTRIA_COMPOSER_GRPC_ADDR: "0.0.0.0:{{ .Values.ports.grpc }}" diff --git a/charts/composer/templates/ingress.yaml b/charts/composer/templates/ingress.yaml deleted file mode 100644 index 65ccb0bb5..000000000 --- a/charts/composer/templates/ingress.yaml +++ /dev/null @@ -1,71 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $ingressApiIsStable := eq (include "rollup.ingress.isStable" .) "true" -}} -{{- $ingressSupportsIngressClassName := eq (include "rollup.ingress.supportsIngressClassName" .) "true" -}} -{{- $ingressSupportsPathType := eq (include "rollup.ingress.supportsPathType" .) "true" -}} - -{{- range $service, $ingress := .Values.ingress.services }} -{{- if $ingress.enabled -}} -{{- $servicePort := $ingress.service.port -}} -{{- $serviceName := tpl $ingress.service.name $ -}} -{{- $ingressPath := $ingress.path -}} -{{- $ingressPathType := $ingress.pathType -}} -{{- $extraPaths := $ingress.extraPaths }} ---- -apiVersion: {{ include "rollup.ingress.apiVersion" $ }} -kind: Ingress -metadata: - name: astria-local-{{ $service }}-ingress - namespace: {{ include "rollup.namespace" $ }} - labels: - {{- with $ingress.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - annotations: - nginx.ingress.kubernetes.io/backend-protocol: "GRPC" - {{- if not $ingressSupportsIngressClassName }} - kubernetes.io/ingress.class: {{ $.Values.ingress.className }} - {{- end }} - {{- if $ingressApiIsStable }} - {{- range $key, $value := $ingress.annotations }} - {{ $key }}: {{ tpl $value $ | quote }} - {{- end }} - {{- end }} -spec: - {{- if $ingressSupportsIngressClassName }} - ingressClassName: {{ $.Values.ingress.className }} - {{- end -}} - {{- with $ingress.service }} - defaultBackend: - service: - {{- tpl (toYaml .) $ | nindent 6 }} - {{- end }} - rules: - {{- with $ingress.hosts }} - {{- range $host := . }} - - host: {{ tpl $host $ }} - http: - paths: - {{- with $extraPaths }} - {{- toYaml . | nindent 10 }} - {{- end }} - - path: {{ $ingressPath }} - {{- if $ingressSupportsPathType }} - pathType: {{ $ingressPathType }} - {{- end }} - backend: - {{- if $ingressApiIsStable }} - service: - {{- tpl (toYaml $ingress.service) $ | nindent 16 }} - {{- else }} - serviceName: {{ tpl $serviceName $ }} - servicePort: {{ tpl $servicePort $ }} - {{- end }} - {{- end }} - {{- end }} - {{- if $ingress.tls }} - tls: - {{- tpl (toYaml $ingress.tls) $ | nindent 4 }} - {{- end }} -{{- end }} -{{- end }} -{{- end }} diff --git a/charts/composer/values.yaml b/charts/composer/values.yaml index 8d5ca4b11..dedf8f58e 100644 --- a/charts/composer/values.yaml +++ b/charts/composer/values.yaml @@ -3,26 +3,26 @@ global: replicaCount: 1 # Whether to use tty readable logging for astria services, when false use json. useTTY: false - dev: true + dev: false images: composer: - repo: astria-composer + repo: ghcr.io/astriaorg/composer tag: "0.8.2" - devTag: tb + devTag: latest config: - logLevel: "info" + logLevel: "debug" maxBytesPerBundle: 200000 maxBundleSize: 200000 bundleQueueCapacity: 40000 - maxSubmitInterval: 1800 + maxSubmitInterval: 2000 sequencerAddressPrefix: astria sequencerNativeAssetBaseDenomination: "nria" sequencerRpc: "" sequencerChainId: "" privateKey: - devContent: "e315da1eafadb1e323a53244bdca8365437c643f0977de9ecf44cc926538f918" + devContent: "" secret: filename: "key.hex" resourceName: "projects/$PROJECT_ID/secrets/sequencerPrivateKey/versions/latest" @@ -71,27 +71,4 @@ resources: ports: grpc: 50052 healthApi: 2450 - metrics: 6060 - - -ingress: - enabled: true - labels: {} - hostname: localdev.me - className: nginx - services: - rpc: - enabled: true - hosts: - - 'composer.astria-local.{{ .Values.ingress.hostname }}' - path: / - pathType: Prefix - service: - name: 'composer-service' - port: - name: grpc - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - labels: {} - tls: {} \ No newline at end of file + metrics: 6060 \ No newline at end of file diff --git a/charts/deploy.just b/charts/deploy.just index 382b2c171..5b7954130 100644 --- a/charts/deploy.just +++ b/charts/deploy.just @@ -80,8 +80,6 @@ deploy-sequencer name=validatorName: -n astria-validator-{{name}} --create-namespace \ {{name}}-sequencer-chart ./charts/sequencer deploy-sequencers: (deploy-sequencer "node0") (deploy-sequencer "node1") (deploy-sequencer "node2") -#deploy-sequencers: (deploy-sequencer "node0") (deploy-sequencer "node1") - deploy-hermes-local: helm install hermes-local-chart ./charts/hermes \ diff --git a/charts/evm-rollup/values.yaml b/charts/evm-rollup/values.yaml index 648f8a665..bf71b70fd 100644 --- a/charts/evm-rollup/values.yaml +++ b/charts/evm-rollup/values.yaml @@ -4,17 +4,17 @@ global: # Whether to use tty readable logging for astria services, when false use json. # Best to be false in production environments, true for clean logs on local dev. useTTY: false - dev: true + dev: false images: geth: - repo: ghcr.io/astriaorg/go-ethereum + repo: ghcr.io/astriaorg/astria-geth tag: 0.14.0 - devTag: tb + devTag: latest conductor: repo: astria-conductor tag: "0.20.0" - devTag: tb + devTag: latest genesis: @@ -293,7 +293,18 @@ ingress: # Default persistent storage values # NOTE - `rollupName` will be used with `persistentVolumeName` to generate names for kubernetes resources. # e.g. astria-executor-pv, astria-executor-pvc -resources: {} +resources: + conductor: + requests: + cpu: 100m + memory: 200Mi + limits: + cpu: 1000m + memory: 2Gi + geth: + requests: + cpu: 16000m + memory: 32Gi storage: enabled: false diff --git a/charts/sequencer/values.yaml b/charts/sequencer/values.yaml index f423551e9..85f5b4696 100644 --- a/charts/sequencer/values.yaml +++ b/charts/sequencer/values.yaml @@ -7,7 +7,7 @@ global: # Whether to use tty readable logging for astria services, when false use json. # Best to be false in production environments, true for clean logs on local dev. useTTY: true - dev: true + dev: false # sequencer core images images: @@ -16,9 +16,9 @@ images: tag: v0.38.8 devTag: v0.38.8 sequencer: - repo: astria-sequencer + repo: ghcr.io/astriaorg/sequencer tag: "0.16.0" - devTag: tb + devTag: latest moniker: "" genesis: @@ -77,8 +77,6 @@ sequencer: tracesTimeout: 10 otlpHeaders: traceHeaders: - composerHook: - enabled: false cometbft: config: @@ -109,13 +107,6 @@ cometbft: # List of seeds to connect to seeds: [] - # List of nodes to keep persistent connections to - persistentPeers: [ - "f4b8a8dcfc5a142bd00aadab71f39dbfe7091d13@35.236.107.39:26656", - "ca3bc3562919b82575fe3ac5b11fa5962ce8cd3b@34.118.249.169:26656", - "4418000e355967ecc8e03004f5850dfde51c410b@34.94.177.254:26656", - "7a117e7906d8428ad20341aca94af03c980c11d8@34.102.55.164:26656", - ] # List of node IDs, to which a connection will be (re)established ignoring any existing limits unconditionalPeers: [] # Maximum pause when redialing a persistent peer (if zero, exponential backoff is used) @@ -126,7 +117,7 @@ cometbft: allowDuplicateIP: false # Seed mode, in which node constantly crawls the network and looks for # peers. If another node asks it for addresses, it responds and disconnects. - seedMode: true + seedMode: false # Maximum number of inbound and outbound peers maxInboundPeers: 100 maxOutboundPeers: 40 diff --git a/crates/astria-composer/src/executor/simulator.rs b/crates/astria-composer/src/executor/simulator.rs index d4004a529..ca9633b3a 100644 --- a/crates/astria-composer/src/executor/simulator.rs +++ b/crates/astria-composer/src/executor/simulator.rs @@ -136,7 +136,7 @@ impl BundleSimulator { bundle: Vec>, block: Block, timestamp: Option, - simulate_only: bool + simulate_only: bool, ) -> eyre::Result { // convert the sized bundle actions to a list of Vec // as long as the timestamp > parent block timestamp, the block will be successfully diff --git a/crates/astria-composer/src/executor/tests.rs b/crates/astria-composer/src/executor/tests.rs index 9dd920507..9d8828f6b 100644 --- a/crates/astria-composer/src/executor/tests.rs +++ b/crates/astria-composer/src/executor/tests.rs @@ -14,6 +14,7 @@ use astria_core::{ sequencerblock::v1alpha1 as raw_sequencer, }, primitive::v1::{ + asset, asset::{ Denom, IbcPrefixed, @@ -70,7 +71,6 @@ use wiremock::{ Request, ResponseTemplate, }; -use astria_core::primitive::v1::asset; use crate::{ executor, diff --git a/crates/astria-composer/src/metrics.rs b/crates/astria-composer/src/metrics.rs index bc70849e6..039dd7b12 100644 --- a/crates/astria-composer/src/metrics.rs +++ b/crates/astria-composer/src/metrics.rs @@ -109,14 +109,16 @@ impl telemetry::Metrics for Metrics { type Config = crate::Config; fn register(builder: &mut RegisteringBuilder, config: &Self::Config) -> Result - where - Self: Sized, + where + Self: Sized, { let rollup = config.rollup.clone(); let (geth_txs_received, grpc_txs_received) = register_txs_received(builder, vec![rollup.clone()].iter())?; - let (geth_txs_dropped, grpc_txs_dropped) = register_txs_dropped(builder, vec![rollup.clone()].iter())?; - let txs_dropped_too_large = register_txs_dropped_too_large(builder, vec![rollup.clone()].iter())?; + let (geth_txs_dropped, grpc_txs_dropped) = + register_txs_dropped(builder, vec![rollup.clone()].iter())?; + let txs_dropped_too_large = + register_txs_dropped_too_large(builder, vec![rollup.clone()].iter())?; let nonce_fetch_count = builder .new_counter_factory( @@ -337,4 +339,4 @@ mod tests { assert_const(TRANSACTIONS_PER_SUBMISSION, "transactions_per_submission"); assert_const(BYTES_PER_SUBMISSION, "bytes_per_submission"); } -} \ No newline at end of file +} diff --git a/crates/astria-composer/tests/blackbox/helper/mod.rs b/crates/astria-composer/tests/blackbox/helper/mod.rs index a4e139afa..9fc036922 100644 --- a/crates/astria-composer/tests/blackbox/helper/mod.rs +++ b/crates/astria-composer/tests/blackbox/helper/mod.rs @@ -16,7 +16,13 @@ use astria_composer::{ use astria_core::{ composer::v1alpha1::BuilderBundle, generated::composer::v1alpha1::BuilderBundlePacket, - primitive::v1::RollupId, + primitive::v1::{ + asset::{ + Denom, + IbcPrefixed, + }, + RollupId, + }, protocol::{ abci::AbciErrorCode, transaction::v1alpha1::SignedTransaction, @@ -44,7 +50,6 @@ use wiremock::{ Request, ResponseTemplate, }; -use astria_core::primitive::v1::asset::{Denom, IbcPrefixed}; use crate::helper::mock_grpc::{ MockGrpc, @@ -106,7 +111,7 @@ pub struct TestComposer { pub setup_guard: MockGuard, pub grpc_collector_addr: SocketAddr, pub metrics_handle: metrics::Handle, - pub test_executor: TestExecutor + pub test_executor: TestExecutor, } /// Spawns composer in a test environment. @@ -178,8 +183,8 @@ pub async fn spawn_composer(rollup_name: &str) -> TestComposer { grpc_collector_addr, metrics_handle, test_executor: TestExecutor { - mock_grpc: mock_execution_api_server - } + mock_grpc: mock_execution_api_server, + }, } } diff --git a/crates/astria-conductor/src/conductor.rs b/crates/astria-conductor/src/conductor.rs index 2fc3ac7b9..8d3e01358 100644 --- a/crates/astria-conductor/src/conductor.rs +++ b/crates/astria-conductor/src/conductor.rs @@ -161,6 +161,8 @@ impl Conductor { tasks.spawn(Self::CELESTIA, reader.run_until_stopped()); }; + info!("BHARATH: CONDUCTOR INITED!"); + Ok(Self { shutdown, tasks, diff --git a/dev/values/rollup/dev.yaml b/dev/values/rollup/dev.yaml index d5f92cc69..40b00b2f4 100644 --- a/dev/values/rollup/dev.yaml +++ b/dev/values/rollup/dev.yaml @@ -23,7 +23,7 @@ evm-rollup: ## no defaults as they are unique to each chain # Block height to start syncing rollup from, lowest possible is 2 - sequencerInitialHeight: 264188 + sequencerInitialHeight: 2 # The first Celestia height to utilize when looking for rollup data celestiaInitialHeight: 2 # The variance in Celestia height to allow before halting the chain @@ -102,7 +102,7 @@ evm-rollup: config: # The level at which core astria components will log out # Options are: error, warn, info, and debug - logLevel: "error" + logLevel: "debug" conductor: # Determines what will drive block execution, options are: