diff --git a/.github/workflows/memcheck.yml b/.github/workflows/memcheck.yml index 4853442936..4fae7db742 100644 --- a/.github/workflows/memcheck.yml +++ b/.github/workflows/memcheck.yml @@ -98,14 +98,14 @@ jobs: ls -l $ANT_DATA_PATH/client_first/logs mkdir $ANT_DATA_PATH/client ls -l $ANT_DATA_PATH - cp ./the-test-data.zip ./the-test-data_1.zip - ./target/release/ant --log-output-dest=data-dir file upload "./the-test-data_1.zip" > ./second_upload 2>&1 + ./target/release/ant --log-output-dest=data-dir file upload --public "./the-test-data.zip" > ./upload_output_second 2>&1 + rg 'Total cost: 0 AttoTokens' ./upload_output_second -c --stats env: ANT_LOG: "all" timeout-minutes: 25 - name: showing the second upload terminal output - run: cat second_upload + run: cat upload_output_second shell: bash if: always() diff --git a/Cargo.lock b/Cargo.lock index f220d37552..1af17e51ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1078,6 +1078,7 @@ dependencies = [ "hex", "lazy_static", "libp2p", + "prometheus-client", "prost 0.9.0", "rand 0.8.5", "rmp-serde", diff --git a/ant-networking/src/cmd.rs b/ant-networking/src/cmd.rs index c5191cda41..1475f97740 100644 --- a/ant-networking/src/cmd.rs +++ b/ant-networking/src/cmd.rs @@ -17,7 +17,7 @@ use ant_evm::{PaymentQuote, QuotingMetrics, U256}; use ant_protocol::{ convert_distance_to_u256, messages::{Cmd, Request, Response}, - storage::{RecordHeader, RecordKind, ValidationType}, + storage::{DataTypes, RecordHeader, RecordKind, ValidationType}, NetworkAddress, PrettyPrintRecordKey, }; use libp2p::{ @@ -661,19 +661,12 @@ impl SwarmDriver { let record_type = match RecordHeader::from_record(&record) { Ok(record_header) => { match record_header.kind { - RecordKind::Chunk => ValidationType::Chunk, - RecordKind::GraphEntry - | RecordKind::Pointer - | RecordKind::Register - | RecordKind::Scratchpad => { + RecordKind::DataOnly(DataTypes::Chunk) => ValidationType::Chunk, + RecordKind::DataOnly(_) => { let content_hash = XorName::from_content(&record.value); ValidationType::NonChunk(content_hash) } - RecordKind::ChunkWithPayment - | RecordKind::RegisterWithPayment - | RecordKind::PointerWithPayment - | RecordKind::GraphEntryWithPayment - | RecordKind::ScratchpadWithPayment => { + RecordKind::DataWithPayment(_) => { error!("Record {record_key:?} with payment shall not be stored locally."); return Err(NetworkError::InCorrectRecordHeader); } diff --git a/ant-networking/src/event/kad.rs b/ant-networking/src/event/kad.rs index 6dcf286cdf..8cd0735fcc 100644 --- a/ant-networking/src/event/kad.rs +++ b/ant-networking/src/event/kad.rs @@ -11,7 +11,7 @@ use crate::{ GetRecordCfg, GetRecordError, NetworkError, Result, SwarmDriver, CLOSE_GROUP_SIZE, }; use ant_protocol::{ - storage::{try_serialize_record, GraphEntry, RecordKind}, + storage::{try_serialize_record, DataTypes, GraphEntry, RecordKind}, NetworkAddress, PrettyPrintRecordKey, }; use itertools::Itertools; @@ -415,7 +415,7 @@ impl SwarmDriver { let bytes = try_serialize_record( &accumulated_transactions, - RecordKind::GraphEntry, + RecordKind::DataOnly(DataTypes::GraphEntry), )?; let new_accumulated_record = Record { diff --git a/ant-networking/src/graph.rs b/ant-networking/src/graph.rs index d58e77599c..d38c56de03 100644 --- a/ant-networking/src/graph.rs +++ b/ant-networking/src/graph.rs @@ -7,7 +7,7 @@ // permissions and limitations relating to use of the SAFE Network Software. use crate::{driver::GetRecordCfg, Network, NetworkError, Result}; -use ant_protocol::storage::{GraphEntry, GraphEntryAddress}; +use ant_protocol::storage::{DataTypes, GraphEntry, GraphEntryAddress}; use ant_protocol::{ storage::{try_deserialize_record, RecordHeader, RecordKind, RetryStrategy}, NetworkAddress, PrettyPrintRecordKey, @@ -37,7 +37,7 @@ impl Network { pub fn get_graph_entry_from_record(record: &Record) -> Result> { let header = RecordHeader::from_record(record)?; - if let RecordKind::GraphEntry = header.kind { + if let RecordKind::DataOnly(DataTypes::GraphEntry) = header.kind { let transactions = try_deserialize_record::>(record)?; Ok(transactions) } else { @@ -45,6 +45,8 @@ pub fn get_graph_entry_from_record(record: &Record) -> Result> { "RecordKind mismatch while trying to retrieve graph_entry from record {:?}", PrettyPrintRecordKey::from(&record.key) ); - Err(NetworkError::RecordKindMismatch(RecordKind::GraphEntry)) + Err(NetworkError::RecordKindMismatch(RecordKind::DataOnly( + DataTypes::GraphEntry, + ))) } } diff --git a/ant-networking/src/lib.rs b/ant-networking/src/lib.rs index cb4f761655..413c7eb730 100644 --- a/ant-networking/src/lib.rs +++ b/ant-networking/src/lib.rs @@ -52,7 +52,7 @@ use ant_evm::{PaymentQuote, QuotingMetrics}; use ant_protocol::{ error::Error as ProtocolError, messages::{ChunkProof, Nonce, Query, QueryResponse, Request, Response}, - storage::{Pointer, RetryStrategy, Scratchpad, ValidationType}, + storage::{DataTypes, Pointer, RetryStrategy, Scratchpad, ValidationType}, NetworkAddress, PrettyPrintKBucketKey, PrettyPrintRecordKey, CLOSE_GROUP_SIZE, }; use futures::future::select_all; @@ -632,16 +632,11 @@ impl Network { } match kind { - RecordKind::Chunk - | RecordKind::ChunkWithPayment - | RecordKind::GraphEntryWithPayment - | RecordKind::RegisterWithPayment - | RecordKind::PointerWithPayment - | RecordKind::ScratchpadWithPayment => { + RecordKind::DataOnly(DataTypes::Chunk) | RecordKind::DataWithPayment(_) => { error!("Encountered a split record for {pretty_key:?} with unexpected RecordKind {kind:?}, skipping."); continue; } - RecordKind::GraphEntry => { + RecordKind::DataOnly(DataTypes::GraphEntry) => { info!("For record {pretty_key:?}, we have a split record for a transaction attempt. Accumulating transactions"); match get_graph_entry_from_record(record) { @@ -653,7 +648,7 @@ impl Network { } } } - RecordKind::Register => { + RecordKind::DataOnly(DataTypes::Register) => { info!("For record {pretty_key:?}, we have a split record for a register. Accumulating registers"); let Ok(register) = try_deserialize_record::(record) else { error!( @@ -675,7 +670,7 @@ impl Network { } } } - RecordKind::Pointer => { + RecordKind::DataOnly(DataTypes::Pointer) => { info!("For record {pretty_key:?}, we have a split record for a pointer. Selecting the one with the highest count"); let Ok(pointer) = try_deserialize_record::(record) else { error!( @@ -697,7 +692,7 @@ impl Network { } valid_pointer = Some(pointer); } - RecordKind::Scratchpad => { + RecordKind::DataOnly(DataTypes::Scratchpad) => { info!("For record {pretty_key:?}, we have a split record for a scratchpad. Selecting the one with the highest count"); let Ok(scratchpad) = try_deserialize_record::(record) else { error!( @@ -733,7 +728,7 @@ impl Network { .collect::>(); let record = Record { key: key.clone(), - value: try_serialize_record(&accumulated_transactions, RecordKind::GraphEntry) + value: try_serialize_record(&accumulated_transactions, RecordKind::DataOnly(DataTypes::GraphEntry)) .map_err(|err| { error!( "Error while serializing the accumulated transactions for {pretty_key:?}: {err:?}" @@ -754,14 +749,15 @@ impl Network { acc }); - let record_value = try_serialize_record(&signed_register, RecordKind::Register) - .map_err(|err| { - error!( + let record_value = + try_serialize_record(&signed_register, RecordKind::DataOnly(DataTypes::Register)) + .map_err(|err| { + error!( "Error while serializing the merged register for {pretty_key:?}: {err:?}" ); - NetworkError::from(err) - })? - .to_vec(); + NetworkError::from(err) + })? + .to_vec(); let record = Record { key: key.clone(), @@ -772,12 +768,13 @@ impl Network { return Ok(Some(record)); } else if let Some(pointer) = valid_pointer { info!("For record {pretty_key:?} task found a valid pointer, returning it."); - let record_value = try_serialize_record(&pointer, RecordKind::Pointer) - .map_err(|err| { - error!("Error while serializing the pointer for {pretty_key:?}: {err:?}"); - NetworkError::from(err) - })? - .to_vec(); + let record_value = + try_serialize_record(&pointer, RecordKind::DataOnly(DataTypes::Pointer)) + .map_err(|err| { + error!("Error while serializing the pointer for {pretty_key:?}: {err:?}"); + NetworkError::from(err) + })? + .to_vec(); let record = Record { key: key.clone(), @@ -788,12 +785,15 @@ impl Network { return Ok(Some(record)); } else if let Some(scratchpad) = valid_scratchpad { info!("For record {pretty_key:?} task found a valid scratchpad, returning it."); - let record_value = try_serialize_record(&scratchpad, RecordKind::Scratchpad) - .map_err(|err| { - error!("Error while serializing the scratchpad for {pretty_key:?}: {err:?}"); - NetworkError::from(err) - })? - .to_vec(); + let record_value = + try_serialize_record(&scratchpad, RecordKind::DataOnly(DataTypes::Scratchpad)) + .map_err(|err| { + error!( + "Error while serializing the scratchpad for {pretty_key:?}: {err:?}" + ); + NetworkError::from(err) + })? + .to_vec(); let record = Record { key: key.clone(), diff --git a/ant-networking/src/record_store.rs b/ant-networking/src/record_store.rs index e9e1d2886c..ef32b98381 100644 --- a/ant-networking/src/record_store.rs +++ b/ant-networking/src/record_store.rs @@ -820,19 +820,11 @@ impl RecordStore for NodeRecordStore { match RecordHeader::from_record(&record) { Ok(record_header) => { match record_header.kind { - RecordKind::ChunkWithPayment - | RecordKind::GraphEntryWithPayment - | RecordKind::PointerWithPayment - | RecordKind::RegisterWithPayment - | RecordKind::ScratchpadWithPayment => { + RecordKind::DataWithPayment(_) => { debug!("Record {record_key:?} with payment shall always be processed."); } // Shall not use wildcard, to avoid mis-match during enum update. - RecordKind::Chunk - | RecordKind::GraphEntry - | RecordKind::Pointer - | RecordKind::Register - | RecordKind::Scratchpad => { + RecordKind::DataOnly(_) => { // Chunk with existing key do not to be stored again. // Others with same content_hash do not to be stored again, // otherwise shall be passed further to allow different version of nonchunk @@ -1003,7 +995,7 @@ mod tests { use ant_protocol::convert_distance_to_u256; use ant_protocol::storage::{ - try_deserialize_record, try_serialize_record, Chunk, ChunkAddress, Scratchpad, + try_deserialize_record, try_serialize_record, Chunk, ChunkAddress, DataTypes, Scratchpad, }; use assert_fs::{ fixture::{PathChild, PathCreateDir}, @@ -1036,7 +1028,7 @@ mod tests { fn arbitrary(g: &mut Gen) -> ArbitraryRecord { let value = match try_serialize_record( &(0..50).map(|_| rand::random::()).collect::(), - RecordKind::Chunk, + RecordKind::DataOnly(DataTypes::Chunk), ) { Ok(value) => value.to_vec(), Err(err) => panic!("Cannot generate record value {err:?}"), @@ -1162,7 +1154,7 @@ mod tests { // Create a record from the chunk let record = Record { key: NetworkAddress::ChunkAddress(chunk_address).to_record_key(), - value: try_serialize_record(&chunk, RecordKind::Chunk)?.to_vec(), + value: try_serialize_record(&chunk, RecordKind::DataOnly(DataTypes::Chunk))?.to_vec(), expires: None, publisher: None, }; @@ -1334,7 +1326,8 @@ mod tests { // Create a record from the scratchpad let record = Record { key: NetworkAddress::ScratchpadAddress(scratchpad_address).to_record_key(), - value: try_serialize_record(&scratchpad, RecordKind::Scratchpad)?.to_vec(), + value: try_serialize_record(&scratchpad, RecordKind::DataOnly(DataTypes::Scratchpad))? + .to_vec(), expires: None, publisher: None, }; @@ -1424,7 +1417,7 @@ mod tests { let record_key = NetworkAddress::from_peer(PeerId::random()).to_record_key(); let value = match try_serialize_record( &(0..50).map(|_| rand::random::()).collect::(), - RecordKind::Chunk, + RecordKind::DataOnly(DataTypes::Chunk), ) { Ok(value) => value.to_vec(), Err(err) => panic!("Cannot generate record value {err:?}"), @@ -1547,7 +1540,7 @@ mod tests { &(0..max_records) .map(|_| rand::random::()) .collect::(), - RecordKind::Chunk, + RecordKind::DataOnly(DataTypes::Chunk), ) { Ok(value) => value.to_vec(), Err(err) => panic!("Cannot generate record value {err:?}"), diff --git a/ant-node/src/metrics.rs b/ant-node/src/metrics.rs index 53c7641db1..f441742570 100644 --- a/ant-node/src/metrics.rs +++ b/ant-node/src/metrics.rs @@ -10,8 +10,9 @@ use crate::Marker; use ant_networking::time::Instant; #[cfg(feature = "open-metrics")] use ant_networking::MetricsRegistries; +use ant_protocol::storage::DataTypes; use prometheus_client::{ - encoding::{EncodeLabelSet, EncodeLabelValue}, + encoding::EncodeLabelSet, metrics::{ counter::Counter, family::Family, @@ -47,14 +48,7 @@ pub(crate) struct NodeMetricsRecorder { #[derive(EncodeLabelSet, Hash, Clone, Eq, PartialEq, Debug)] struct PutRecordOk { - record_type: RecordType, -} - -#[derive(EncodeLabelValue, Hash, Clone, Eq, PartialEq, Debug)] -enum RecordType { - Chunk, - Register, - Spend, + record_type: DataTypes, } impl NodeMetricsRecorder { @@ -157,7 +151,7 @@ impl NodeMetricsRecorder { let _ = self .put_record_ok .get_or_create(&PutRecordOk { - record_type: RecordType::Chunk, + record_type: DataTypes::Chunk, }) .inc(); } @@ -166,7 +160,7 @@ impl NodeMetricsRecorder { let _ = self .put_record_ok .get_or_create(&PutRecordOk { - record_type: RecordType::Register, + record_type: DataTypes::Register, }) .inc(); } @@ -175,7 +169,7 @@ impl NodeMetricsRecorder { let _ = self .put_record_ok .get_or_create(&PutRecordOk { - record_type: RecordType::Spend, + record_type: DataTypes::GraphEntry, }) .inc(); } diff --git a/ant-node/src/put_validation.rs b/ant-node/src/put_validation.rs index 0925c3d1f6..074d9f6ab3 100644 --- a/ant-node/src/put_validation.rs +++ b/ant-node/src/put_validation.rs @@ -15,7 +15,7 @@ use ant_networking::NetworkError; use ant_protocol::storage::GraphEntry; use ant_protocol::{ storage::{ - try_deserialize_record, try_serialize_record, Chunk, GraphEntryAddress, Pointer, + try_deserialize_record, try_serialize_record, Chunk, DataTypes, GraphEntryAddress, Pointer, RecordHeader, RecordKind, Scratchpad, ValidationType, }, NetworkAddress, PrettyPrintRecordKey, @@ -30,7 +30,7 @@ impl Node { let record_header = RecordHeader::from_record(&record)?; match record_header.kind { - RecordKind::ChunkWithPayment => { + RecordKind::DataWithPayment(DataTypes::Chunk) => { let record_key = record.key.clone(); let (payment, chunk) = try_deserialize_record::<(ProofOfPayment, Chunk)>(&record)?; let already_exists = self @@ -87,13 +87,13 @@ impl Node { store_chunk_result } - RecordKind::Chunk => { + RecordKind::DataOnly(DataTypes::Chunk) => { error!("Chunk should not be validated at this point"); Err(Error::InvalidPutWithoutPayment( PrettyPrintRecordKey::from(&record.key).into_owned(), )) } - RecordKind::ScratchpadWithPayment => { + RecordKind::DataWithPayment(DataTypes::Scratchpad) => { let record_key = record.key.clone(); let (payment, scratchpad) = try_deserialize_record::<(ProofOfPayment, Scratchpad)>(&record)?; @@ -148,7 +148,7 @@ impl Node { store_scratchpad_result } - RecordKind::Scratchpad => { + RecordKind::DataOnly(DataTypes::Scratchpad) => { // make sure we already have this scratchpad locally, else reject it as first time upload needs payment let key = record.key.clone(); let scratchpad = try_deserialize_record::(&record)?; @@ -166,14 +166,14 @@ impl Node { self.validate_and_store_scratchpad_record(scratchpad, key, false) .await } - RecordKind::GraphEntry => { + RecordKind::DataOnly(DataTypes::GraphEntry) => { // Transactions should always be paid for error!("Transaction should not be validated at this point"); Err(Error::InvalidPutWithoutPayment( PrettyPrintRecordKey::from(&record.key).into_owned(), )) } - RecordKind::GraphEntryWithPayment => { + RecordKind::DataWithPayment(DataTypes::GraphEntry) => { let (payment, transaction) = try_deserialize_record::<(ProofOfPayment, GraphEntry)>(&record)?; @@ -228,7 +228,7 @@ impl Node { } res } - RecordKind::Register => { + RecordKind::DataOnly(DataTypes::Register) => { let register = try_deserialize_record::(&record)?; // make sure we already have this register locally @@ -267,7 +267,7 @@ impl Node { } result } - RecordKind::RegisterWithPayment => { + RecordKind::DataWithPayment(DataTypes::Register) => { let (payment, register) = try_deserialize_record::<(ProofOfPayment, SignedRegister)>(&record)?; @@ -314,14 +314,14 @@ impl Node { } res } - RecordKind::Pointer => { + RecordKind::DataOnly(DataTypes::Pointer) => { // Pointers should always be paid for error!("Pointer should not be validated at this point"); Err(Error::InvalidPutWithoutPayment( PrettyPrintRecordKey::from(&record.key).into_owned(), )) } - RecordKind::PointerWithPayment => { + RecordKind::DataWithPayment(DataTypes::Pointer) => { let (payment, pointer) = try_deserialize_record::<(ProofOfPayment, Pointer)>(&record)?; @@ -378,18 +378,14 @@ impl Node { debug!("Storing record which was replicated to us {:?}", record.key); let record_header = RecordHeader::from_record(&record)?; match record_header.kind { - // A separate flow handles payment for chunks and registers - RecordKind::ChunkWithPayment - | RecordKind::GraphEntryWithPayment - | RecordKind::RegisterWithPayment - | RecordKind::ScratchpadWithPayment - | RecordKind::PointerWithPayment => { + // A separate flow handles record with payment + RecordKind::DataWithPayment(_) => { warn!("Prepaid record came with Payment, which should be handled in another flow"); Err(Error::UnexpectedRecordWithPayment( PrettyPrintRecordKey::from(&record.key).into_owned(), )) } - RecordKind::Chunk => { + RecordKind::DataOnly(DataTypes::Chunk) => { let chunk = try_deserialize_record::(&record)?; let record_key = record.key.clone(); @@ -406,19 +402,19 @@ impl Node { self.store_chunk(&chunk) } - RecordKind::Scratchpad => { + RecordKind::DataOnly(DataTypes::Scratchpad) => { let key = record.key.clone(); let scratchpad = try_deserialize_record::(&record)?; self.validate_and_store_scratchpad_record(scratchpad, key, false) .await } - RecordKind::GraphEntry => { + RecordKind::DataOnly(DataTypes::GraphEntry) => { let record_key = record.key.clone(); let transactions = try_deserialize_record::>(&record)?; self.validate_merge_and_store_transactions(transactions, &record_key) .await } - RecordKind::Register => { + RecordKind::DataOnly(DataTypes::Register) => { let register = try_deserialize_record::(&record)?; // check if the deserialized value's RegisterAddress matches the record's key @@ -432,7 +428,7 @@ impl Node { } self.validate_and_store_register(register, false).await } - RecordKind::Pointer => { + RecordKind::DataOnly(DataTypes::Pointer) => { let pointer = try_deserialize_record::(&record)?; let key = record.key.clone(); self.validate_and_store_pointer_record(pointer, key) @@ -487,7 +483,7 @@ impl Node { let record = Record { key, - value: try_serialize_record(&chunk, RecordKind::Chunk)?.to_vec(), + value: try_serialize_record(&chunk, RecordKind::DataOnly(DataTypes::Chunk))?.to_vec(), publisher: None, expires: None, }; @@ -551,7 +547,8 @@ impl Node { let record = Record { key: scratchpad_key.clone(), - value: try_serialize_record(&scratchpad, RecordKind::Scratchpad)?.to_vec(), + value: try_serialize_record(&scratchpad, RecordKind::DataOnly(DataTypes::Scratchpad))? + .to_vec(), publisher: None, expires: None, }; @@ -600,7 +597,11 @@ impl Node { // store in kad let record = Record { key: key.clone(), - value: try_serialize_record(&updated_register, RecordKind::Register)?.to_vec(), + value: try_serialize_record( + &updated_register, + RecordKind::DataOnly(DataTypes::Register), + )? + .to_vec(), publisher: None, expires: None, }; @@ -682,7 +683,11 @@ impl Node { // store the record into the local storage let record = Record { key: record_key.clone(), - value: try_serialize_record(&validated_transactions, RecordKind::GraphEntry)?.to_vec(), + value: try_serialize_record( + &validated_transactions, + RecordKind::DataOnly(DataTypes::GraphEntry), + )? + .to_vec(), publisher: None, expires: None, }; @@ -848,9 +853,12 @@ impl Node { // deserialize the record and get the transactions let local_header = RecordHeader::from_record(&local_record)?; let record_kind = local_header.kind; - if !matches!(record_kind, RecordKind::GraphEntry) { + if !matches!(record_kind, RecordKind::DataOnly(DataTypes::GraphEntry)) { error!("Found a {record_kind} when expecting to find Spend at {addr:?}"); - return Err(NetworkError::RecordKindMismatch(RecordKind::GraphEntry).into()); + return Err(NetworkError::RecordKindMismatch(RecordKind::DataOnly( + DataTypes::GraphEntry, + )) + .into()); } let local_transactions: Vec = try_deserialize_record(&local_record)?; Ok(local_transactions) @@ -878,7 +886,8 @@ impl Node { // Store the pointer let record = Record { key: key.clone(), - value: try_serialize_record(&pointer, RecordKind::Pointer)?.to_vec(), + value: try_serialize_record(&pointer, RecordKind::DataOnly(DataTypes::Pointer))? + .to_vec(), publisher: None, expires: None, }; diff --git a/ant-protocol/Cargo.toml b/ant-protocol/Cargo.toml index 4a4011d906..1f83cfd8fb 100644 --- a/ant-protocol/Cargo.toml +++ b/ant-protocol/Cargo.toml @@ -27,6 +27,7 @@ exponential-backoff = "2.0.0" hex = "~0.4.3" lazy_static = "1.4.0" libp2p = { version = "0.54.1", features = ["identify", "kad"] } +prometheus-client = { version = "0.22" } prost = { version = "0.9", optional = true } rand = "0.8" rmp-serde = "1.1.1" diff --git a/ant-protocol/src/storage/header.rs b/ant-protocol/src/storage/header.rs index a67274d5be..4ec619b965 100644 --- a/ant-protocol/src/storage/header.rs +++ b/ant-protocol/src/storage/header.rs @@ -10,11 +10,45 @@ use crate::error::Error; use crate::PrettyPrintRecordKey; use bytes::{BufMut, Bytes, BytesMut}; use libp2p::kad::Record; +use prometheus_client::encoding::EncodeLabelValue; use rmp_serde::Serializer; use serde::{Deserialize, Serialize}; use std::fmt::Display; use xor_name::XorName; +/// Data types that natively suppported by autonomi network. +#[derive(EncodeLabelValue, Debug, Serialize, Deserialize, Clone, Copy, Eq, PartialEq, Hash)] +pub enum DataTypes { + Chunk, + GraphEntry, + Pointer, + Register, + Scratchpad, +} + +impl DataTypes { + pub fn get_index(&self) -> u32 { + match self { + Self::Chunk => 0, + Self::GraphEntry => 1, + Self::Pointer => 2, + Self::Register => 3, + Self::Scratchpad => 4, + } + } + + pub fn from_index(index: u32) -> Option { + match index { + 0 => Some(Self::Chunk), + 1 => Some(Self::GraphEntry), + 2 => Some(Self::Pointer), + 3 => Some(Self::Register), + 4 => Some(Self::Scratchpad), + _ => None, + } + } +} + /// Indicates the type of the record content. /// This is to be only used within the node instance to reflect different content version. /// Hence, only need to have two entries: Chunk and NonChunk. @@ -29,37 +63,28 @@ pub struct RecordHeader { pub kind: RecordKind, } +/// To be used between client and nodes, hence need to indicate whehter payment info involved. #[derive(Debug, Eq, PartialEq, Clone, Copy)] pub enum RecordKind { - Chunk, - ChunkWithPayment, - GraphEntry, - GraphEntryWithPayment, - Register, - RegisterWithPayment, - Scratchpad, - ScratchpadWithPayment, - Pointer, - PointerWithPayment, + DataOnly(DataTypes), + DataWithPayment(DataTypes), } +/// Allowing 10 data types to be defined, leaving margin for future. +pub const RECORD_KIND_PAYMENT_STARTING_INDEX: u32 = 10; + impl Serialize for RecordKind { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { - match *self { - Self::ChunkWithPayment => serializer.serialize_u32(0), - Self::Chunk => serializer.serialize_u32(1), - Self::GraphEntry => serializer.serialize_u32(2), - Self::Register => serializer.serialize_u32(3), - Self::RegisterWithPayment => serializer.serialize_u32(4), - Self::Scratchpad => serializer.serialize_u32(5), - Self::ScratchpadWithPayment => serializer.serialize_u32(6), - Self::GraphEntryWithPayment => serializer.serialize_u32(7), - Self::Pointer => serializer.serialize_u32(8), - Self::PointerWithPayment => serializer.serialize_u32(9), - } + let index = match self { + Self::DataOnly(ref data_types) => data_types.get_index(), + Self::DataWithPayment(ref data_types) => { + RECORD_KIND_PAYMENT_STARTING_INDEX + data_types.get_index() + } + }; + serializer.serialize_u32(index) } } @@ -69,20 +94,22 @@ impl<'de> Deserialize<'de> for RecordKind { D: serde::Deserializer<'de>, { let num = u32::deserialize(deserializer)?; - match num { - 0 => Ok(Self::ChunkWithPayment), - 1 => Ok(Self::Chunk), - 2 => Ok(Self::GraphEntry), - 3 => Ok(Self::Register), - 4 => Ok(Self::RegisterWithPayment), - 5 => Ok(Self::Scratchpad), - 6 => Ok(Self::ScratchpadWithPayment), - 7 => Ok(Self::GraphEntryWithPayment), - 8 => Ok(Self::Pointer), - 9 => Ok(Self::PointerWithPayment), - _ => Err(serde::de::Error::custom( - "Unexpected integer for RecordKind variant", - )), + let data_type_index = if num < RECORD_KIND_PAYMENT_STARTING_INDEX { + num + } else { + num - RECORD_KIND_PAYMENT_STARTING_INDEX + }; + + if let Some(data_type) = DataTypes::from_index(data_type_index) { + if num < RECORD_KIND_PAYMENT_STARTING_INDEX { + Ok(Self::DataOnly(data_type)) + } else { + Ok(Self::DataWithPayment(data_type)) + } + } else { + Err(serde::de::Error::custom( + "Unexpected index {num} for RecordKind variant", + )) } } } @@ -126,7 +153,7 @@ impl RecordHeader { pub fn is_record_of_type_chunk(record: &Record) -> Result { let kind = Self::from_record(record)?.kind; - Ok(kind == RecordKind::Chunk) + Ok(kind == RecordKind::DataOnly(DataTypes::Chunk)) } } @@ -165,61 +192,61 @@ pub fn try_serialize_record( #[cfg(test)] mod tests { - use super::{RecordHeader, RecordKind}; + use super::*; use crate::error::Result; #[test] fn verify_record_header_encoded_size() -> Result<()> { let chunk_with_payment = RecordHeader { - kind: RecordKind::ChunkWithPayment, + kind: RecordKind::DataWithPayment(DataTypes::Chunk), } .try_serialize()?; assert_eq!(chunk_with_payment.len(), RecordHeader::SIZE); let reg_with_payment = RecordHeader { - kind: RecordKind::RegisterWithPayment, + kind: RecordKind::DataWithPayment(DataTypes::Register), } .try_serialize()?; assert_eq!(reg_with_payment.len(), RecordHeader::SIZE); let chunk = RecordHeader { - kind: RecordKind::Chunk, + kind: RecordKind::DataOnly(DataTypes::Chunk), } .try_serialize()?; assert_eq!(chunk.len(), RecordHeader::SIZE); let transaction = RecordHeader { - kind: RecordKind::GraphEntry, + kind: RecordKind::DataOnly(DataTypes::GraphEntry), } .try_serialize()?; assert_eq!(transaction.len(), RecordHeader::SIZE); let register = RecordHeader { - kind: RecordKind::Register, + kind: RecordKind::DataOnly(DataTypes::Register), } .try_serialize()?; assert_eq!(register.len(), RecordHeader::SIZE); let scratchpad = RecordHeader { - kind: RecordKind::Scratchpad, + kind: RecordKind::DataOnly(DataTypes::Scratchpad), } .try_serialize()?; assert_eq!(scratchpad.len(), RecordHeader::SIZE); let scratchpad_with_payment = RecordHeader { - kind: RecordKind::ScratchpadWithPayment, + kind: RecordKind::DataWithPayment(DataTypes::Scratchpad), } .try_serialize()?; assert_eq!(scratchpad_with_payment.len(), RecordHeader::SIZE); let pointer = RecordHeader { - kind: RecordKind::Pointer, + kind: RecordKind::DataOnly(DataTypes::Pointer), } .try_serialize()?; assert_eq!(pointer.len(), RecordHeader::SIZE); let pointer_with_payment = RecordHeader { - kind: RecordKind::PointerWithPayment, + kind: RecordKind::DataWithPayment(DataTypes::Pointer), } .try_serialize()?; assert_eq!(pointer_with_payment.len(), RecordHeader::SIZE); @@ -230,16 +257,16 @@ mod tests { #[test] fn test_record_kind_serialization() -> Result<()> { let kinds = vec![ - RecordKind::Chunk, - RecordKind::ChunkWithPayment, - RecordKind::GraphEntry, - RecordKind::GraphEntryWithPayment, - RecordKind::Register, - RecordKind::RegisterWithPayment, - RecordKind::Scratchpad, - RecordKind::ScratchpadWithPayment, - RecordKind::Pointer, - RecordKind::PointerWithPayment, + RecordKind::DataOnly(DataTypes::Chunk), + RecordKind::DataWithPayment(DataTypes::Chunk), + RecordKind::DataOnly(DataTypes::GraphEntry), + RecordKind::DataWithPayment(DataTypes::GraphEntry), + RecordKind::DataOnly(DataTypes::Register), + RecordKind::DataWithPayment(DataTypes::Register), + RecordKind::DataOnly(DataTypes::Scratchpad), + RecordKind::DataWithPayment(DataTypes::Scratchpad), + RecordKind::DataOnly(DataTypes::Pointer), + RecordKind::DataWithPayment(DataTypes::Pointer), ]; for kind in kinds { diff --git a/ant-protocol/src/storage/mod.rs b/ant-protocol/src/storage/mod.rs index 033aaab757..f23d9bca7e 100644 --- a/ant-protocol/src/storage/mod.rs +++ b/ant-protocol/src/storage/mod.rs @@ -23,7 +23,8 @@ pub use self::{ chunks::Chunk, graph::GraphEntry, header::{ - try_deserialize_record, try_serialize_record, RecordHeader, RecordKind, ValidationType, + try_deserialize_record, try_serialize_record, DataTypes, RecordHeader, RecordKind, + ValidationType, }, scratchpad::Scratchpad, }; diff --git a/autonomi/Cargo.toml b/autonomi/Cargo.toml index 3487dad0aa..e8fd6f6cd0 100644 --- a/autonomi/Cargo.toml +++ b/autonomi/Cargo.toml @@ -70,3 +70,5 @@ workspace = true [package.metadata.docs.rs] all-features = true rustdoc-args = ["--cfg", "docsrs"] +# Adds snippets from the `examples` dir to items if relevant +cargo-args = ["-Zunstable-options", "-Zrustdoc-scrape-examples"] diff --git a/autonomi/src/client/data/public.rs b/autonomi/src/client/data/public.rs index eff9f99f89..8b434c5a98 100644 --- a/autonomi/src/client/data/public.rs +++ b/autonomi/src/client/data/public.rs @@ -17,7 +17,7 @@ use crate::{self_encryption::encrypt, Client}; use ant_evm::{Amount, AttoTokens}; use ant_networking::{GetRecordCfg, NetworkError}; use ant_protocol::{ - storage::{try_deserialize_record, Chunk, ChunkAddress, RecordHeader, RecordKind}, + storage::{try_deserialize_record, Chunk, ChunkAddress, DataTypes, RecordHeader, RecordKind}, NetworkAddress, }; @@ -129,7 +129,7 @@ impl Client { .inspect_err(|err| error!("Error fetching chunk: {err:?}"))?; let header = RecordHeader::from_record(&record)?; - if let RecordKind::Chunk = header.kind { + if let Ok(true) = RecordHeader::is_record_of_type_chunk(&record) { let chunk: Chunk = try_deserialize_record(&record)?; Ok(chunk) } else { @@ -137,7 +137,7 @@ impl Client { "Record kind mismatch: expected Chunk, got {:?}", header.kind ); - Err(NetworkError::RecordKindMismatch(RecordKind::Chunk).into()) + Err(NetworkError::RecordKindMismatch(RecordKind::DataOnly(DataTypes::Chunk)).into()) } } diff --git a/autonomi/src/client/files/archive.rs b/autonomi/src/client/files/archive.rs index 03a82d423a..18ecd1c735 100644 --- a/autonomi/src/client/files/archive.rs +++ b/autonomi/src/client/files/archive.rs @@ -7,7 +7,7 @@ // permissions and limitations relating to use of the SAFE Network Software. use std::{ - collections::HashMap, + collections::BTreeMap, path::{Path, PathBuf}, }; @@ -36,8 +36,6 @@ pub enum RenameError { /// Metadata for a file in an archive. Time values are UNIX timestamps. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct Metadata { - /// When the file was (last) uploaded to the network. - pub uploaded: u64, /// File creation time on local file system. See [`std::fs::Metadata::created`] for details per OS. pub created: u64, /// Last file modification time taken from local file system. See [`std::fs::Metadata::modified`] for details per OS. @@ -55,7 +53,6 @@ impl Metadata { .as_secs(); Self { - uploaded: now, created: now, modified: now, size, @@ -68,7 +65,7 @@ impl Metadata { /// The data maps are stored within this structure instead of uploading them to the network, keeping the data private. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] pub struct PrivateArchive { - map: HashMap, + map: BTreeMap, } impl PrivateArchive { @@ -76,7 +73,7 @@ impl PrivateArchive { /// Note that this does not upload the archive to the network pub fn new() -> Self { Self { - map: HashMap::new(), + map: BTreeMap::new(), } } @@ -130,7 +127,7 @@ impl PrivateArchive { } /// Get the underlying map - pub fn map(&self) -> &HashMap { + pub fn map(&self) -> &BTreeMap { &self.map } diff --git a/autonomi/src/client/files/archive_public.rs b/autonomi/src/client/files/archive_public.rs index 19f1756b8b..dd7cb046e2 100644 --- a/autonomi/src/client/files/archive_public.rs +++ b/autonomi/src/client/files/archive_public.rs @@ -7,7 +7,7 @@ // permissions and limitations relating to use of the SAFE Network Software. use std::{ - collections::HashMap, + collections::BTreeMap, path::{Path, PathBuf}, }; @@ -34,7 +34,7 @@ pub type ArchiveAddr = XorName; /// to the network, of which the addresses are stored in this archive. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] pub struct PublicArchive { - map: HashMap, + map: BTreeMap, } impl PublicArchive { @@ -42,7 +42,7 @@ impl PublicArchive { /// Note that this does not upload the archive to the network pub fn new() -> Self { Self { - map: HashMap::new(), + map: BTreeMap::new(), } } @@ -92,7 +92,7 @@ impl PublicArchive { } /// Get the underlying map - pub fn map(&self) -> &HashMap { + pub fn map(&self) -> &BTreeMap { &self.map } diff --git a/autonomi/src/client/files/fs_public.rs b/autonomi/src/client/files/fs_public.rs index 60f13d0cb1..92e5e5455b 100644 --- a/autonomi/src/client/files/fs_public.rs +++ b/autonomi/src/client/files/fs_public.rs @@ -194,7 +194,6 @@ pub(crate) fn metadata_from_entry(entry: &walkdir::DirEntry) -> Metadata { entry.path().display() ); return Metadata { - uploaded: 0, created: 0, modified: 0, size: 0, @@ -224,10 +223,6 @@ pub(crate) fn metadata_from_entry(entry: &walkdir::DirEntry) -> Metadata { let modified = unix_time("modified", fs_metadata.modified()); Metadata { - uploaded: SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap_or(Duration::from_secs(0)) - .as_secs(), created, modified, size: fs_metadata.len(), diff --git a/autonomi/src/client/graph.rs b/autonomi/src/client/graph.rs index 71749b3289..cf78903209 100644 --- a/autonomi/src/client/graph.rs +++ b/autonomi/src/client/graph.rs @@ -20,7 +20,7 @@ pub use bls::SecretKey; use ant_evm::{EvmWallet, EvmWalletError}; use ant_networking::{GetRecordCfg, NetworkError, PutRecordCfg, VerificationKind}; use ant_protocol::{ - storage::{try_serialize_record, RecordKind, RetryStrategy}, + storage::{try_serialize_record, DataTypes, RecordKind, RetryStrategy}, NetworkAddress, }; use libp2p::kad::{Quorum, Record}; @@ -89,9 +89,12 @@ impl Client { let payees = proof.payees(); let record = Record { key: NetworkAddress::from_graph_entry_address(address).to_record_key(), - value: try_serialize_record(&(proof, &transaction), RecordKind::GraphEntryWithPayment) - .map_err(|_| GraphError::Serialization)? - .to_vec(), + value: try_serialize_record( + &(proof, &transaction), + RecordKind::DataWithPayment(DataTypes::GraphEntry), + ) + .map_err(|_| GraphError::Serialization)? + .to_vec(), publisher: None, expires: None, }; diff --git a/autonomi/src/client/mod.rs b/autonomi/src/client/mod.rs index 8384eb58cf..326059fa52 100644 --- a/autonomi/src/client/mod.rs +++ b/autonomi/src/client/mod.rs @@ -203,7 +203,7 @@ impl Client { Ok(Self { network, client_event_sender: Arc::new(None), - evm_network: Default::default(), + evm_network: config.evm_network, }) } diff --git a/autonomi/src/client/pointer.rs b/autonomi/src/client/pointer.rs index dd759209f5..a5f95e18f8 100644 --- a/autonomi/src/client/pointer.rs +++ b/autonomi/src/client/pointer.rs @@ -5,7 +5,9 @@ use tracing::{debug, error, trace}; use ant_evm::{Amount, AttoTokens, EvmWallet, EvmWalletError}; use ant_networking::{GetRecordCfg, NetworkError, PutRecordCfg, VerificationKind}; use ant_protocol::{ - storage::{try_serialize_record, Pointer, PointerAddress, RecordKind, RetryStrategy}, + storage::{ + try_serialize_record, DataTypes, Pointer, PointerAddress, RecordKind, RetryStrategy, + }, NetworkAddress, }; use bls::SecretKey; @@ -80,9 +82,12 @@ impl Client { let record = Record { key: NetworkAddress::from_pointer_address(address).to_record_key(), - value: try_serialize_record(&(proof, &pointer), RecordKind::PointerWithPayment) - .map_err(|_| PointerError::Serialization)? - .to_vec(), + value: try_serialize_record( + &(proof, &pointer), + RecordKind::DataWithPayment(DataTypes::Pointer), + ) + .map_err(|_| PointerError::Serialization)? + .to_vec(), publisher: None, expires: None, }; diff --git a/autonomi/src/client/registers.rs b/autonomi/src/client/registers.rs index dc56e37b45..d28fbacf2f 100644 --- a/autonomi/src/client/registers.rs +++ b/autonomi/src/client/registers.rs @@ -19,7 +19,7 @@ pub use bls::SecretKey as RegisterSecretKey; use ant_evm::{Amount, AttoTokens, EvmWallet, EvmWalletError}; use ant_networking::{GetRecordCfg, GetRecordError, NetworkError, PutRecordCfg, VerificationKind}; use ant_protocol::{ - storage::{try_deserialize_record, try_serialize_record, RecordKind, RetryStrategy}, + storage::{try_deserialize_record, try_serialize_record, DataTypes, RecordKind, RetryStrategy}, NetworkAddress, }; use ant_registers::Register as BaseRegister; @@ -204,9 +204,12 @@ impl Client { // Prepare the record for network storage let record = Record { key: NetworkAddress::from_register_address(*register.address()).to_record_key(), - value: try_serialize_record(&signed_register, RecordKind::Register) - .map_err(|_| RegisterError::Serialization)? - .to_vec(), + value: try_serialize_record( + &signed_register, + RecordKind::DataOnly(DataTypes::Register), + ) + .map_err(|_| RegisterError::Serialization)? + .to_vec(), publisher: None, expires: None, }; @@ -337,7 +340,7 @@ impl Client { key: NetworkAddress::from_register_address(*address).to_record_key(), value: try_serialize_record( &(proof, &signed_register), - RecordKind::RegisterWithPayment, + RecordKind::DataWithPayment(DataTypes::Register), ) .map_err(|_| RegisterError::Serialization)? .to_vec(), diff --git a/autonomi/src/client/utils.rs b/autonomi/src/client/utils.rs index ad2aeececb..2a8eb70e3e 100644 --- a/autonomi/src/client/utils.rs +++ b/autonomi/src/client/utils.rs @@ -11,7 +11,7 @@ use ant_evm::{EvmWallet, ProofOfPayment}; use ant_networking::{GetRecordCfg, PutRecordCfg, VerificationKind}; use ant_protocol::{ messages::ChunkProof, - storage::{try_serialize_record, Chunk, RecordKind, RetryStrategy}, + storage::{try_serialize_record, Chunk, DataTypes, RecordKind, RetryStrategy}, }; use bytes::Bytes; use futures::stream::{FuturesUnordered, StreamExt}; @@ -110,7 +110,7 @@ impl Client { let key = chunk.network_address().to_record_key(); - let record_kind = RecordKind::ChunkWithPayment; + let record_kind = RecordKind::DataWithPayment(DataTypes::Chunk); let record = Record { key: key.clone(), value: try_serialize_record(&(payment, chunk.clone()), record_kind) @@ -133,9 +133,12 @@ impl Client { is_register: false, }; - let stored_on_node = try_serialize_record(&chunk, RecordKind::Chunk) - .map_err(|e| PutError::Serialization(format!("Failed to serialize chunk: {e:?}")))? - .to_vec(); + let stored_on_node = + try_serialize_record(&chunk, RecordKind::DataOnly(DataTypes::Chunk)) + .map_err(|e| { + PutError::Serialization(format!("Failed to serialize chunk: {e:?}")) + })? + .to_vec(); let random_nonce = thread_rng().gen::(); let expected_proof = ChunkProof::new(&stored_on_node, random_nonce); diff --git a/autonomi/src/client/vault.rs b/autonomi/src/client/vault.rs index 462e2a4cb0..d57c197fc6 100644 --- a/autonomi/src/client/vault.rs +++ b/autonomi/src/client/vault.rs @@ -19,7 +19,7 @@ use crate::client::Client; use ant_evm::{Amount, AttoTokens}; use ant_networking::{GetRecordCfg, GetRecordError, NetworkError, PutRecordCfg, VerificationKind}; use ant_protocol::storage::{ - try_serialize_record, RecordKind, RetryStrategy, Scratchpad, ScratchpadAddress, + try_serialize_record, DataTypes, RecordKind, RetryStrategy, Scratchpad, ScratchpadAddress, }; use ant_protocol::Bytes; use ant_protocol::{storage::try_deserialize_record, NetworkAddress}; @@ -208,20 +208,23 @@ impl Client { Record { key: scratch_key, - value: try_serialize_record(&(proof, scratch), RecordKind::ScratchpadWithPayment) - .map_err(|_| { - PutError::Serialization( - "Failed to serialize scratchpad with payment".to_string(), - ) - })? - .to_vec(), + value: try_serialize_record( + &(proof, scratch), + RecordKind::DataWithPayment(DataTypes::Scratchpad), + ) + .map_err(|_| { + PutError::Serialization( + "Failed to serialize scratchpad with payment".to_string(), + ) + })? + .to_vec(), publisher: None, expires: None, } } else { Record { key: scratch_key, - value: try_serialize_record(&scratch, RecordKind::Scratchpad) + value: try_serialize_record(&scratch, RecordKind::DataOnly(DataTypes::Scratchpad)) .map_err(|_| { PutError::Serialization("Failed to serialize scratchpad".to_string()) })?