diff --git a/.github/workflows/python-publish-client.yml b/.github/workflows/python-publish-client.yml index 5714ec7c22..2f4ee166cb 100644 --- a/.github/workflows/python-publish-client.yml +++ b/.github/workflows/python-publish-client.yml @@ -3,8 +3,9 @@ name: Build and Publish Python Client Package on: push: tags: - - 'xxx' + - 'autonomi-v*' +# Add top-level permissions block permissions: id-token: write contents: read @@ -12,6 +13,7 @@ permissions: jobs: macos: runs-on: macos-latest + # Add permissions to job permissions: id-token: write contents: read @@ -21,43 +23,26 @@ jobs: target: [x86_64, aarch64] steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - - name: Create Python module structure - run: | - mkdir -p autonomi/python/autonomi_client - cat > autonomi/python/autonomi_client/__init__.py << EOL - from .autonomi_client import * - __version__ = "${{ github.ref_name }}" - EOL - name: Build wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} - args: --release --out dist --find-interpreter --compatibility manylinux2014 + args: --release --out dist -i python${{ matrix.python-version }} sccache: 'true' working-directory: ./autonomi - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.python-version }}-${{ matrix.target }} - path: autonomi/dist/*.whl + name: wheels-macos-${{ matrix.target }}-py${{ matrix.python-version }} + path: ./autonomi/dist/*.whl if-no-files-found: error - retention-days: 1 - compression-level: 9 - continue-on-error: true - timeout-minutes: 10 - env: - ACTIONS_STEP_DEBUG: true - ACTIONS_RUNNER_DEBUG: true windows: runs-on: windows-latest + # Add permissions to job permissions: id-token: write contents: read @@ -67,39 +52,22 @@ jobs: target: [x64] steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} architecture: ${{ matrix.target }} - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - - name: Create Python module structure - shell: cmd - run: | - if not exist "autonomi\python\autonomi_client" mkdir autonomi\python\autonomi_client - echo from .autonomi_client import * > autonomi\python\autonomi_client\__init__.py - echo __version__ = "${{ github.ref_name }}" >> autonomi\python\autonomi_client\__init__.py - name: Build wheels uses: PyO3/maturin-action@v1 with: - args: --release --out dist --find-interpreter --compatibility manylinux2014 + args: --release --out dist sccache: 'true' working-directory: ./autonomi - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.python-version }}-${{ matrix.target }} - path: autonomi/dist/*.whl + name: wheels-windows-${{ matrix.target }}-py${{ matrix.python-version }} + path: ./autonomi/dist/*.whl if-no-files-found: error - retention-days: 1 - compression-level: 9 - continue-on-error: true - timeout-minutes: 10 - env: - ACTIONS_STEP_DEBUG: true - ACTIONS_RUNNER_DEBUG: true linux: runs-on: ubuntu-latest @@ -108,131 +76,188 @@ jobs: contents: read strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] target: [x86_64] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - target: x86_64-unknown-linux-gnu - - name: Install dependencies - run: | - python -m pip install --user cffi - python -m pip install --user patchelf - rustup component add rustfmt - - name: Create Python module structure - run: | - mkdir -p autonomi/python/autonomi_client - cat > autonomi/python/autonomi_client/__init__.py << EOL - from .autonomi_client import * - __version__ = "${{ github.ref_name }}" - EOL + architecture: x64 - name: Build wheels uses: PyO3/maturin-action@v1 + env: + PYTHON_VERSION: ${{ matrix.python-version }} with: target: ${{ matrix.target }} - manylinux: "2014" - args: --release --out dist --find-interpreter - sccache: 'true' - working-directory: ./autonomi + manylinux: auto before-script-linux: | - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - source $HOME/.cargo/env + rustup default stable rustup component add rustfmt + args: --release --out dist -i python${{ matrix.python-version }} + sccache: 'true' + working-directory: ./autonomi - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.python-version }}-${{ matrix.target }} - path: autonomi/dist/*.whl + name: wheels-linux-${{ matrix.target }}-py${{ matrix.python-version }} + path: ./autonomi/dist/*.whl if-no-files-found: error - retention-days: 1 - compression-level: 9 - continue-on-error: true - timeout-minutes: 10 + + musllinux: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + strategy: + matrix: + target: + - x86_64-unknown-linux-musl + - aarch64-unknown-linux-musl + - armv7-unknown-linux-musleabihf + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + - name: Build wheels + uses: PyO3/maturin-action@v1 env: - ACTIONS_STEP_DEBUG: true - ACTIONS_RUNNER_DEBUG: true + PYO3_CROSS_PYTHON_VERSION: ${{ matrix.python-version }} + PYO3_CROSS: "1" + with: + target: ${{ matrix.target }} + manylinux: musllinux_1_2 + args: --release --out dist -i python${{ matrix.python-version }} + sccache: 'true' + working-directory: ./autonomi + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-musllinux-${{ matrix.target }}-py${{ matrix.python-version }} + path: ./autonomi/dist/*.whl + if-no-files-found: error sdist: runs-on: ubuntu-latest + # Add permissions to job permissions: id-token: write contents: read steps: - uses: actions/checkout@v4 - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - - name: Create Python module structure + - name: Prepare standalone package run: | - mkdir -p autonomi/python/autonomi_client - cat > autonomi/python/autonomi_client/__init__.py << EOL - from .autonomi_client import * - __version__ = "${{ github.ref_name }}" + # Create build directory structure + mkdir -p build/autonomi + cp -r autonomi/* build/autonomi/ + + # First, copy all workspace members + for dir in ant-* test-utils evmlib; do + if [ -d "$dir" ]; then + echo "Copying $dir to build directory" + cp -r "$dir" "build/$dir" + fi + done + + # Create a new workspace Cargo.toml in the build directory + cat > build/Cargo.toml << EOL + [workspace] + resolver = "2" + members = [ + "ant-bootstrap", + "ant-build-info", + "ant-cli", + "ant-evm", + "ant-logging", + "ant-metrics", + "ant-networking", + "ant-node", + "ant-node-manager", + "ant-node-rpc-client", + "ant-protocol", + "ant-registers", + "ant-service-management", + "ant-token-supplies", + "autonomi", + "evmlib", + "test-utils" + ] + + [workspace.lints.rust] + arithmetic_overflow = "forbid" + mutable_transmutes = "forbid" + no_mangle_const_items = "forbid" + trivial_casts = "warn" + trivial_numeric_casts = "warn" + unsafe_code = "warn" + unknown_crate_types = "forbid" + unused_extern_crates = "warn" + unused_import_braces = "warn" + + [workspace.lints.clippy] + clone_on_ref_ptr = "warn" + unicode_not_nfc = "warn" + uninlined_format_args = "warn" + unused_async = "warn" + unwrap_used = "warn" + + [profile.dev] + debug = 0 + strip = "debuginfo" + + [workspace.metadata.release] + pre-release-commit-message = "chore(release): release commit, tags, deps and changelog updates" + publish = false + push = false + tag = false + + [workspace.dependencies] + backtrace = "=0.3.71" EOL + + # Update all dependency paths to be absolute + find build -name "Cargo.toml" -exec sed -i "s|path = \"\.\./|path = \"/home/runner/work/autonomi/autonomi/build/|g" {} \; + + # Display directory structure for debugging + echo "Contents of build directory:" + ls -la build/ + echo "Contents of workspace Cargo.toml:" + cat build/Cargo.toml - name: Build sdist uses: PyO3/maturin-action@v1 with: command: sdist args: --out dist - working-directory: ./autonomi + working-directory: build/autonomi - name: Upload sdist uses: actions/upload-artifact@v4 with: - name: sdist - path: autonomi/dist/*.tar.gz + name: wheels + path: build/autonomi/dist/*.tar.gz if-no-files-found: error - retention-days: 1 - compression-level: 9 - continue-on-error: true - timeout-minutes: 10 - env: - ACTIONS_STEP_DEBUG: true - ACTIONS_RUNNER_DEBUG: true release: name: Release runs-on: ubuntu-latest - needs: [macos, windows, linux, sdist] + needs: [macos, windows, linux, musllinux, sdist] + # Keep existing permissions permissions: id-token: write contents: read steps: - - name: Create dist directory - run: mkdir -p dist - - # Download all artifacts at once - - name: Download all artifacts - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v4 with: + name: wheels path: dist - - - name: Prepare dist directory - run: | - find dist -type f -name "*.whl" -exec mv {} dist/ \; - find dist -type f -name "*.tar.gz" -exec mv {} dist/ \; - rm -rf dist/*/ - echo "Final dist directory contents:" - ls -la dist/ - - - name: Check if version exists - run: | - VERSION="${{ github.ref_name }}" - VERSION="${VERSION#v}" # Remove 'v' prefix if present - if pip index versions autonomi-client | grep -q "${VERSION}"; then - echo "Version ${VERSION} already exists on PyPI" - exit 1 - fi - + merge-multiple: true + - name: Display structure of downloaded files + run: ls -R dist - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: dist/ verbose: true - print-hash: true + print-hash: true \ No newline at end of file diff --git a/.github/workflows/python-publish-node.yml b/.github/workflows/python-publish-node.yml index e369bd2296..dd28be6866 100644 --- a/.github/workflows/python-publish-node.yml +++ b/.github/workflows/python-publish-node.yml @@ -3,8 +3,9 @@ name: Build and Publish Python Node Package on: push: tags: - - 'xxx' + - 'ant-node-v*' +# Add top-level permissions block permissions: id-token: write contents: read @@ -12,6 +13,7 @@ permissions: jobs: macos: runs-on: macos-latest + # Add permissions to job permissions: id-token: write contents: read @@ -21,43 +23,26 @@ jobs: target: [x86_64, aarch64] steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - - name: Create Python module structure - run: | - mkdir -p ant_node/python/antnode - cat > ant_node/python/antnode/__init__.py << EOL - from ._antnode import * - __version__ = "${{ github.ref_name }}" - EOL - name: Build wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} - args: --release --out dist + args: --release --out dist -i python${{ matrix.python-version }} sccache: 'true' - working-directory: ./ant_node + working-directory: ./ant-node - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.python-version }}-${{ matrix.target }} - path: ant_node/dist/*.whl + name: wheels-macos-${{ matrix.target }}-py${{ matrix.python-version }} + path: ./ant-node/dist/*.whl if-no-files-found: error - retention-days: 1 - compression-level: 9 - continue-on-error: true - timeout-minutes: 10 - env: - ACTIONS_STEP_DEBUG: true - ACTIONS_RUNNER_DEBUG: true windows: runs-on: windows-latest + # Add permissions to job permissions: id-token: write contents: read @@ -67,39 +52,22 @@ jobs: target: [x64] steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} architecture: ${{ matrix.target }} - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - - name: Create Python module structure - shell: cmd - run: | - if not exist "ant_node\python\antnode" mkdir ant_node\python\antnode - echo from ._antnode import * > ant_node\python\antnode\__init__.py - echo __version__ = "${{ github.ref_name }}" >> ant_node\python\antnode\__init__.py - name: Build wheels uses: PyO3/maturin-action@v1 with: args: --release --out dist sccache: 'true' - working-directory: ./ant_node + working-directory: ./ant-node - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.python-version }}-${{ matrix.target }} - path: ant_node/dist/*.whl + name: wheels-windows-${{ matrix.target }}-py${{ matrix.python-version }} + path: ./ant-node/dist/*.whl if-no-files-found: error - retention-days: 1 - compression-level: 9 - continue-on-error: true - timeout-minutes: 10 - env: - ACTIONS_STEP_DEBUG: true - ACTIONS_RUNNER_DEBUG: true linux: runs-on: ubuntu-latest @@ -108,55 +76,69 @@ jobs: contents: read strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] target: [x86_64] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - target: x86_64-unknown-linux-gnu - - name: Install dependencies - run: | - python -m pip install --user cffi - python -m pip install --user patchelf - rustup component add rustfmt - - name: Create Python module structure - run: | - mkdir -p ant_node/python/antnode - cat > ant_node/python/antnode/__init__.py << EOL - from ._antnode import * - __version__ = "${{ github.ref_name }}" - EOL + architecture: x64 - name: Build wheels uses: PyO3/maturin-action@v1 + env: + PYTHON_VERSION: ${{ matrix.python-version }} with: target: ${{ matrix.target }} manylinux: auto - args: --release --out dist - sccache: 'true' - working-directory: ./ant_node before-script-linux: | - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - source $HOME/.cargo/env + rustup default stable rustup component add rustfmt + args: --release --out dist -i python${{ matrix.python-version }} + sccache: false + working-directory: ./ant-node - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.python-version }}-${{ matrix.target }} - path: ant_node/dist/*.whl + name: wheels-linux-${{ matrix.target }}-py${{ matrix.python-version }} + path: ./ant-node/dist/*.whl if-no-files-found: error - retention-days: 1 - compression-level: 9 - continue-on-error: true - timeout-minutes: 10 + + musllinux: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + strategy: + matrix: + target: + - x86_64-unknown-linux-musl + - aarch64-unknown-linux-musl + - armv7-unknown-linux-musleabihf + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + - name: Build wheels + uses: PyO3/maturin-action@v1 env: - ACTIONS_STEP_DEBUG: true - ACTIONS_RUNNER_DEBUG: true + PYO3_CROSS_PYTHON_VERSION: ${{ matrix.python-version }} + PYO3_CROSS: "1" + with: + target: ${{ matrix.target }} + manylinux: musllinux_1_2 + args: --release --out dist -i python${{ matrix.python-version }} + sccache: false + working-directory: ./ant-node + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-musllinux-${{ matrix.target }}-py${{ matrix.python-version }} + path: ./ant-node/dist/*.whl + if-no-files-found: error sdist: runs-on: ubuntu-latest @@ -171,8 +153,8 @@ jobs: components: rustfmt - name: Create Python module structure run: | - mkdir -p ant_node/python/antnode - cat > ant_node/python/antnode/__init__.py << EOL + mkdir -p ant-node/python/antnode + cat > ant-node/python/antnode/__init__.py << EOL from ._antnode import * __version__ = "${{ github.ref_name }}" EOL @@ -181,12 +163,12 @@ jobs: with: command: sdist args: --out dist - working-directory: ./ant_node + working-directory: ./ant-node - name: Upload sdist uses: actions/upload-artifact@v4 with: name: sdist - path: ant_node/dist/*.tar.gz + path: ant-node/dist/*.tar.gz if-no-files-found: error retention-days: 1 compression-level: 9 @@ -199,7 +181,7 @@ jobs: release: name: Release runs-on: ubuntu-latest - needs: [macos, windows, linux, sdist] + needs: [macos, windows, linux, musllinux, sdist] permissions: id-token: write contents: read @@ -226,4 +208,4 @@ jobs: with: packages-dir: dist/ verbose: true - print-hash: true + print-hash: true \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index a3bedeb41f..ba48c53005 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -944,7 +944,7 @@ dependencies = [ [[package]] name = "ant-node" -version = "0.3.1" +version = "0.3.2" dependencies = [ "ant-bootstrap", "ant-build-info", @@ -1074,6 +1074,7 @@ dependencies = [ "ant-build-info", "ant-evm", "ant-registers", + "bincode", "blsttc", "bytes", "color-eyre", @@ -1085,6 +1086,7 @@ dependencies = [ "lazy_static", "libp2p", "prost 0.9.0", + "rand 0.8.5", "rmp-serde", "serde", "serde_json", diff --git a/ant-networking/Cargo.toml b/ant-networking/Cargo.toml index 74c5f0b8da..da438d95aa 100644 --- a/ant-networking/Cargo.toml +++ b/ant-networking/Cargo.toml @@ -26,6 +26,7 @@ ant-evm = { path = "../ant-evm", version = "0.1.6" } ant-protocol = { path = "../ant-protocol", version = "0.3.1" } ant-registers = { path = "../ant-registers", version = "0.4.5" } async-trait = "0.1" +bls = { package = "blsttc", version = "8.0.2" } bytes = { version = "1.0.1", features = ["serde"] } custom_debug = "~0.6.1" futures = "~0.3.13" @@ -75,7 +76,6 @@ xor_name = "5.0.0" [dev-dependencies] assert_fs = "1.0.0" -bls = { package = "blsttc", version = "8.0.1" } eyre = "0.6.8" # add rand to libp2p libp2p-identity = { version = "0.2.7", features = ["rand"] } diff --git a/ant-networking/src/cmd.rs b/ant-networking/src/cmd.rs index 9a694f0650..8d4352e976 100644 --- a/ant-networking/src/cmd.rs +++ b/ant-networking/src/cmd.rs @@ -663,13 +663,15 @@ impl SwarmDriver { match record_header.kind { RecordKind::Chunk => RecordType::Chunk, RecordKind::Scratchpad => RecordType::Scratchpad, - RecordKind::Transaction | RecordKind::Register => { + RecordKind::Pointer => RecordType::Pointer, + RecordKind::LinkedList | RecordKind::Register => { let content_hash = XorName::from_content(&record.value); RecordType::NonChunk(content_hash) } RecordKind::ChunkWithPayment | RecordKind::RegisterWithPayment - | RecordKind::TransactionWithPayment + | RecordKind::PointerWithPayment + | RecordKind::LinkedListWithPayment | RecordKind::ScratchpadWithPayment => { error!("Record {record_key:?} with payment shall not be stored locally."); return Err(NetworkError::InCorrectRecordHeader); diff --git a/ant-networking/src/error.rs b/ant-networking/src/error.rs index c683ff4432..8af5915c8b 100644 --- a/ant-networking/src/error.rs +++ b/ant-networking/src/error.rs @@ -6,7 +6,7 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use ant_protocol::storage::TransactionAddress; +use ant_protocol::storage::LinkedListAddress; use ant_protocol::{messages::Response, storage::RecordKind, NetworkAddress, PrettyPrintRecordKey}; use libp2p::{ kad::{self, QueryId, Record}, @@ -123,19 +123,14 @@ pub enum NetworkError { #[error("Record header is incorrect")] InCorrectRecordHeader, - // ---------- Transfer Errors - #[error("Failed to get transaction: {0}")] - FailedToGetSpend(String), - #[error("Transfer is invalid: {0}")] - InvalidTransfer(String), // ---------- Chunk Errors #[error("Failed to verify the ChunkProof with the provided quorum")] FailedToVerifyChunkProof(NetworkAddress), - // ---------- Transaction Errors - #[error("Transaction not found: {0:?}")] - NoTransactionFoundInsideRecord(TransactionAddress), + // ---------- LinkedList Errors + #[error("Linked list not found: {0:?}")] + NoLinkedListFoundInsideRecord(LinkedListAddress), // ---------- Store Error #[error("No Store Cost Responses")] diff --git a/ant-networking/src/event/kad.rs b/ant-networking/src/event/kad.rs index 1af95f9d1d..d0c1f6e91f 100644 --- a/ant-networking/src/event/kad.rs +++ b/ant-networking/src/event/kad.rs @@ -7,12 +7,12 @@ // permissions and limitations relating to use of the SAFE Network Software. use crate::{ - driver::PendingGetClosestType, get_quorum_value, get_transactions_from_record, + driver::PendingGetClosestType, get_linked_list_from_record, get_quorum_value, target_arch::Instant, GetRecordCfg, GetRecordError, NetworkError, Result, SwarmDriver, CLOSE_GROUP_SIZE, }; use ant_protocol::{ - storage::{try_serialize_record, RecordKind, Transaction}, + storage::{try_serialize_record, LinkedList, RecordKind}, NetworkAddress, PrettyPrintRecordKey, }; use itertools::Itertools; @@ -399,7 +399,7 @@ impl SwarmDriver { debug!("For record {pretty_key:?} task {query_id:?}, fetch completed with split record"); let mut accumulated_transactions = BTreeSet::new(); for (record, _) in result_map.values() { - match get_transactions_from_record(record) { + match get_linked_list_from_record(record) { Ok(transactions) => { accumulated_transactions.extend(transactions); } @@ -412,11 +412,11 @@ impl SwarmDriver { info!("For record {pretty_key:?} task {query_id:?}, found split record for a transaction, accumulated and sending them as a single record"); let accumulated_transactions = accumulated_transactions .into_iter() - .collect::>(); + .collect::>(); let bytes = try_serialize_record( &accumulated_transactions, - RecordKind::Transaction, + RecordKind::LinkedList, )?; let new_accumulated_record = Record { diff --git a/ant-networking/src/lib.rs b/ant-networking/src/lib.rs index fca47f18d0..4d165ef4d8 100644 --- a/ant-networking/src/lib.rs +++ b/ant-networking/src/lib.rs @@ -17,6 +17,7 @@ mod error; mod event; mod external_address; mod fifo_register; +mod linked_list; mod log_markers; #[cfg(feature = "open-metrics")] mod metrics; @@ -26,7 +27,6 @@ mod record_store_api; mod relay_manager; mod replication_fetcher; pub mod target_arch; -mod transactions; mod transport; use cmd::LocalSwarmCmd; @@ -40,8 +40,8 @@ pub use self::{ }, error::{GetRecordError, NetworkError}, event::{MsgResponder, NetworkEvent}, + linked_list::get_linked_list_from_record, record_store::NodeRecordStore, - transactions::get_transactions_from_record, }; #[cfg(feature = "open-metrics")] pub use metrics::service::MetricsRegistries; @@ -52,7 +52,7 @@ use ant_evm::{PaymentQuote, QuotingMetrics}; use ant_protocol::{ error::Error as ProtocolError, messages::{ChunkProof, Nonce, Query, QueryResponse, Request, Response}, - storage::{RecordType, RetryStrategy, Scratchpad}, + storage::{Pointer, RecordType, RetryStrategy, Scratchpad}, NetworkAddress, PrettyPrintKBucketKey, PrettyPrintRecordKey, CLOSE_GROUP_SIZE, }; use futures::future::select_all; @@ -75,7 +75,7 @@ use tokio::sync::{ }; use tokio::time::Duration; use { - ant_protocol::storage::Transaction, + ant_protocol::storage::LinkedList, ant_protocol::storage::{ try_deserialize_record, try_serialize_record, RecordHeader, RecordKind, }, @@ -614,6 +614,7 @@ impl Network { let mut accumulated_transactions = HashSet::new(); let mut collected_registers = Vec::new(); let mut valid_scratchpad: Option = None; + let mut valid_pointer: Option = None; if results_count > 1 { let mut record_kind = None; @@ -633,16 +634,17 @@ impl Network { match kind { RecordKind::Chunk | RecordKind::ChunkWithPayment - | RecordKind::TransactionWithPayment + | RecordKind::LinkedListWithPayment | RecordKind::RegisterWithPayment + | RecordKind::PointerWithPayment | RecordKind::ScratchpadWithPayment => { error!("Encountered a split record for {pretty_key:?} with unexpected RecordKind {kind:?}, skipping."); continue; } - RecordKind::Transaction => { + RecordKind::LinkedList => { info!("For record {pretty_key:?}, we have a split record for a transaction attempt. Accumulating transactions"); - match get_transactions_from_record(record) { + match get_linked_list_from_record(record) { Ok(transactions) => { accumulated_transactions.extend(transactions); } @@ -673,6 +675,28 @@ impl Network { } } } + RecordKind::Pointer => { + info!("For record {pretty_key:?}, we have a split record for a pointer. Selecting the one with the highest count"); + let Ok(pointer) = try_deserialize_record::(record) else { + error!( + "Failed to deserialize pointer {pretty_key}. Skipping accumulation" + ); + continue; + }; + + if !pointer.verify() { + warn!("Rejecting Pointer for {pretty_key} PUT with invalid signature"); + continue; + } + + if let Some(old) = &valid_pointer { + if old.count() >= pointer.count() { + info!("Rejecting Pointer for {pretty_key} with lower count than the previous one"); + continue; + } + } + valid_pointer = Some(pointer); + } RecordKind::Scratchpad => { info!("For record {pretty_key:?}, we have a split record for a scratchpad. Selecting the one with the highest count"); let Ok(scratchpad) = try_deserialize_record::(record) else { @@ -684,23 +708,18 @@ impl Network { if !scratchpad.is_valid() { warn!( - "Rejecting Scratchpad for {pretty_key} PUT with invalid signature during split record error" + "Rejecting Scratchpad for {pretty_key} PUT with invalid signature" ); continue; } if let Some(old) = &valid_scratchpad { if old.count() >= scratchpad.count() { - info!( - "Rejecting Scratchpad for {pretty_key} with lower count than the previous one" - ); + info!("Rejecting Scratchpad for {pretty_key} with lower count than the previous one"); continue; - } else { - valid_scratchpad = Some(scratchpad); } - } else { - valid_scratchpad = Some(scratchpad); } + valid_scratchpad = Some(scratchpad); } } } @@ -711,10 +730,10 @@ impl Network { info!("For record {pretty_key:?} task found split record for a transaction, accumulated and sending them as a single record"); let accumulated_transactions = accumulated_transactions .into_iter() - .collect::>(); + .collect::>(); let record = Record { key: key.clone(), - value: try_serialize_record(&accumulated_transactions, RecordKind::Transaction) + value: try_serialize_record(&accumulated_transactions, RecordKind::LinkedList) .map_err(|err| { error!( "Error while serializing the accumulated transactions for {pretty_key:?}: {err:?}" @@ -744,6 +763,22 @@ impl Network { })? .to_vec(); + let record = Record { + key: key.clone(), + value: record_value, + publisher: None, + expires: None, + }; + return Ok(Some(record)); + } else if let Some(pointer) = valid_pointer { + info!("For record {pretty_key:?} task found a valid pointer, returning it."); + let record_value = try_serialize_record(&pointer, RecordKind::Pointer) + .map_err(|err| { + error!("Error while serializing the pointer for {pretty_key:?}: {err:?}"); + NetworkError::from(err) + })? + .to_vec(); + let record = Record { key: key.clone(), value: record_value, @@ -752,17 +787,17 @@ impl Network { }; return Ok(Some(record)); } else if let Some(scratchpad) = valid_scratchpad { - info!("Found a valid scratchpad for {pretty_key:?}, returning it"); + info!("For record {pretty_key:?} task found a valid scratchpad, returning it."); + let record_value = try_serialize_record(&scratchpad, RecordKind::Scratchpad) + .map_err(|err| { + error!("Error while serializing the scratchpad for {pretty_key:?}: {err:?}"); + NetworkError::from(err) + })? + .to_vec(); + let record = Record { key: key.clone(), - value: try_serialize_record(&scratchpad, RecordKind::Scratchpad) - .map_err(|err| { - error!( - "Error while serializing valid scratchpad for {pretty_key:?}: {err:?}" - ); - NetworkError::from(err) - })? - .to_vec(), + value: record_value, publisher: None, expires: None, }; diff --git a/ant-networking/src/transactions.rs b/ant-networking/src/linked_list.rs similarity index 67% rename from ant-networking/src/transactions.rs rename to ant-networking/src/linked_list.rs index d4ab960971..2834cf9ddc 100644 --- a/ant-networking/src/transactions.rs +++ b/ant-networking/src/linked_list.rs @@ -7,7 +7,7 @@ // permissions and limitations relating to use of the SAFE Network Software. use crate::{driver::GetRecordCfg, Network, NetworkError, Result}; -use ant_protocol::storage::{Transaction, TransactionAddress}; +use ant_protocol::storage::{LinkedList, LinkedListAddress}; use ant_protocol::{ storage::{try_deserialize_record, RecordHeader, RecordKind, RetryStrategy}, NetworkAddress, PrettyPrintRecordKey, @@ -15,9 +15,9 @@ use ant_protocol::{ use libp2p::kad::{Quorum, Record}; impl Network { - /// Gets Transactions at TransactionAddress from the Network. - pub async fn get_transactions(&self, address: TransactionAddress) -> Result> { - let key = NetworkAddress::from_transaction_address(address).to_record_key(); + /// Gets LinkedList at LinkedListAddress from the Network. + pub async fn get_linked_list(&self, address: LinkedListAddress) -> Result> { + let key = NetworkAddress::from_linked_list_address(address).to_record_key(); let get_cfg = GetRecordCfg { get_quorum: Quorum::All, retry_strategy: Some(RetryStrategy::Quick), @@ -31,20 +31,20 @@ impl Network { PrettyPrintRecordKey::from(&record.key) ); - get_transactions_from_record(&record) + get_linked_list_from_record(&record) } } -pub fn get_transactions_from_record(record: &Record) -> Result> { +pub fn get_linked_list_from_record(record: &Record) -> Result> { let header = RecordHeader::from_record(record)?; - if let RecordKind::Transaction = header.kind { - let transactions = try_deserialize_record::>(record)?; + if let RecordKind::LinkedList = header.kind { + let transactions = try_deserialize_record::>(record)?; Ok(transactions) } else { warn!( - "RecordKind mismatch while trying to retrieve transactions from record {:?}", + "RecordKind mismatch while trying to retrieve linked_list from record {:?}", PrettyPrintRecordKey::from(&record.key) ); - Err(NetworkError::RecordKindMismatch(RecordKind::Transaction)) + Err(NetworkError::RecordKindMismatch(RecordKind::LinkedList)) } } diff --git a/ant-node/pyproject.toml b/ant-node/pyproject.toml index 8eda49b80d..77a5ab38a2 100644 --- a/ant-node/pyproject.toml +++ b/ant-node/pyproject.toml @@ -18,4 +18,4 @@ module-name = "antnode._antnode" python-source = "python" bindings = "pyo3" manifest-path = "Cargo.toml" -sdist-include = ["python/antnode/*"] +sdist-include = ["python/antnode/*"] \ No newline at end of file diff --git a/ant-node/python/antnode/README.md b/ant-node/python/antnode/README.md new file mode 100644 index 0000000000..91cace7bc0 --- /dev/null +++ b/ant-node/python/antnode/README.md @@ -0,0 +1,122 @@ +# AntNode Python Bindings + +This document describes the Python bindings for the AntNode Rust implementation. + +## Installation + +The AntNode Python package is built using [maturin](https://github.com/PyO3/maturin) and requires Python 3.8 or later. We recommend using `uv` for Python environment management: + +```bash +uv venv +uv pip install maturin +maturin develop +``` + +## Usage + +```python +from antnode import AntNode + +# Create a new node instance +node = AntNode() + +# Start the node with configuration +node.run( + rewards_address="0x1234567890123456789012345678901234567890", + evm_network="arbitrum_sepolia", # or "arbitrum_one" + ip="0.0.0.0", + port=12000, + initial_peers=[], # List of multiaddresses for initial peers + local=True, # Run in local mode + root_dir=None, # Custom root directory (optional) + home_network=False # Run on home network +) +``` + +## API Reference + +### Constructor + +#### `AntNode()` +Creates a new instance of the AntNode. + +### Node Operations + +#### `run(rewards_address: str, evm_network: str, ip: str = "0.0.0.0", port: int = 0, initial_peers: List[str] = [], local: bool = False, root_dir: Optional[str] = None, home_network: bool = False) -> None` +Start the node with the given configuration. + +- **Parameters:** + - `rewards_address`: Ethereum address for rewards (hex string starting with "0x") + - `evm_network`: Either "arbitrum_one" or "arbitrum_sepolia" + - `ip`: IP address to bind to (default: "0.0.0.0") + - `port`: Port number to use (default: 0 for random port) + - `initial_peers`: List of multiaddresses for initial peers + - `local`: Run in local mode + - `root_dir`: Custom root directory path (optional) + - `home_network`: Run on home network + +#### `peer_id() -> str` +Get the node's PeerId as a string. + +#### `get_rewards_address() -> str` +Get the node's rewards/wallet address as a hex string. + +#### `set_rewards_address(address: str) -> None` +Set a new rewards/wallet address for the node. +- `address`: Hex string starting with "0x" + +### Storage Operations + +#### `store_record(key: str, value: bytes, record_type: str) -> None` +Store a record in the node's storage. +- `key`: Record key +- `value`: Record data as bytes +- `record_type`: Type of record + +#### `get_record(key: str) -> Optional[bytes]` +Get a record from the node's storage. +- Returns `None` if record not found + +#### `delete_record(key: str) -> bool` +Delete a record from the node's storage. +- Returns `True` if record was deleted + +#### `get_stored_records_size() -> int` +Get the total size of stored records in bytes. + +#### `get_all_record_addresses() -> List[str]` +Get all record addresses stored by the node. + +### Network Operations + +#### `get_kbuckets() -> List[Tuple[int, List[str]]]` +Get the node's kbuckets information. +- Returns list of tuples containing (distance, list of peer IDs) + +### Directory Management + +#### `get_root_dir() -> str` +Get the current root directory path for node data. + +#### `get_default_root_dir(peer_id: Optional[str] = None) -> str` +Get the default root directory path for the given peer ID. +- Platform specific paths: + - Linux: `$HOME/.local/share/autonomi/node/` + - macOS: `$HOME/Library/Application Support/autonomi/node/` + - Windows: `C:\Users\\AppData\Roaming\autonomi\node\` + +#### `get_logs_dir() -> str` +Get the logs directory path. + +#### `get_data_dir() -> str` +Get the data directory path where records are stored. + +## Error Handling + +The bindings use Python exceptions to handle errors: +- `ValueError`: For invalid input parameters +- `RuntimeError`: For operational errors + +## Example + +See [example.py](../example.py) for a complete example of using the AntNode Python bindings. diff --git a/ant-node/python/antnode/__init__.py b/ant-node/python/antnode/__init__.py new file mode 100644 index 0000000000..949716f09f --- /dev/null +++ b/ant-node/python/antnode/__init__.py @@ -0,0 +1,21 @@ +"""AntNode Python Bindings + +This module provides Python bindings for the AntNode Rust implementation, +allowing you to run and manage AntNode instances from Python code. + +For detailed documentation, see the README.md file in this directory. + +Example: + >>> from antnode import AntNode + >>> node = AntNode() + >>> node.run( + ... rewards_address="0x1234567890123456789012345678901234567890", + ... evm_network="arbitrum_sepolia", + ... ip="0.0.0.0", + ... port=12000 + ... ) +""" + +from ._antnode import AntNode + +__all__ = ["AntNode"] diff --git a/ant-node/src/error.rs b/ant-node/src/error.rs index 6cc7f3baf1..364215a15d 100644 --- a/ant-node/src/error.rs +++ b/ant-node/src/error.rs @@ -48,6 +48,9 @@ pub enum Error { #[error("Scratchpad signature is invalid over the counter + content hash")] InvalidScratchpadSignature, + #[error("Invalid signature")] + InvalidSignature, + // ---------- Payment Errors #[error("The content of the payment quote is invalid")] InvalidQuoteContent, diff --git a/ant-node/src/log_markers.rs b/ant-node/src/log_markers.rs index 23f7c0829e..fe333fe899 100644 --- a/ant-node/src/log_markers.rs +++ b/ant-node/src/log_markers.rs @@ -55,6 +55,9 @@ pub enum Marker<'a> { /// Valid scratchpad stored ValidScratchpadRecordPutFromClient(&'a PrettyPrintRecordKey<'a>), + /// Valid paid to us and royalty paid pointer stored + ValidPointerPutFromClient(&'a PrettyPrintRecordKey<'a>), + /// Record rejected RecordRejected(&'a PrettyPrintRecordKey<'a>, &'a Error), diff --git a/ant-node/src/put_validation.rs b/ant-node/src/put_validation.rs index 67a01b275b..1a5311c303 100644 --- a/ant-node/src/put_validation.rs +++ b/ant-node/src/put_validation.rs @@ -12,11 +12,11 @@ use crate::{node::Node, Error, Marker, Result}; use ant_evm::payment_vault::verify_data_payment; use ant_evm::{AttoTokens, ProofOfPayment}; use ant_networking::NetworkError; -use ant_protocol::storage::Transaction; +use ant_protocol::storage::LinkedList; use ant_protocol::{ storage::{ - try_deserialize_record, try_serialize_record, Chunk, RecordHeader, RecordKind, RecordType, - Scratchpad, TransactionAddress, + try_deserialize_record, try_serialize_record, Chunk, LinkedListAddress, Pointer, + RecordHeader, RecordKind, RecordType, Scratchpad, }, NetworkAddress, PrettyPrintRecordKey, }; @@ -163,19 +163,19 @@ impl Node { self.validate_and_store_scratchpad_record(scratchpad, key, false) .await } - RecordKind::Transaction => { + RecordKind::LinkedList => { // Transactions should always be paid for error!("Transaction should not be validated at this point"); Err(Error::InvalidPutWithoutPayment( PrettyPrintRecordKey::from(&record.key).into_owned(), )) } - RecordKind::TransactionWithPayment => { + RecordKind::LinkedListWithPayment => { let (payment, transaction) = - try_deserialize_record::<(ProofOfPayment, Transaction)>(&record)?; + try_deserialize_record::<(ProofOfPayment, LinkedList)>(&record)?; // check if the deserialized value's TransactionAddress matches the record's key - let net_addr = NetworkAddress::from_transaction_address(transaction.address()); + let net_addr = NetworkAddress::from_linked_list_address(transaction.address()); let key = net_addr.to_record_key(); let pretty_key = PrettyPrintRecordKey::from(&key); if record.key != key { @@ -311,6 +311,62 @@ impl Node { } res } + RecordKind::Pointer => { + // Pointers should always be paid for + error!("Pointer should not be validated at this point"); + Err(Error::InvalidPutWithoutPayment( + PrettyPrintRecordKey::from(&record.key).into_owned(), + )) + } + RecordKind::PointerWithPayment => { + let (payment, pointer) = + try_deserialize_record::<(ProofOfPayment, Pointer)>(&record)?; + + // check if the deserialized value's PointerAddress matches the record's key + let net_addr = NetworkAddress::from_pointer_address(pointer.network_address()); + let key = net_addr.to_record_key(); + let pretty_key = PrettyPrintRecordKey::from(&key); + if record.key != key { + warn!( + "Record's key {pretty_key:?} does not match with the value's PointerAddress, ignoring PUT." + ); + return Err(Error::RecordKeyMismatch); + } + + let already_exists = self.validate_key_and_existence(&net_addr, &key).await?; + + // The pointer may already exist during the replication. + // The payment shall get deposit to self even if the pointer already exists. + if let Err(err) = self + .payment_for_us_exists_and_is_still_valid(&net_addr, payment) + .await + { + if already_exists { + debug!("Payment of the incoming exists pointer {pretty_key:?} having error {err:?}"); + } else { + error!("Payment of the incoming non-exist pointer {pretty_key:?} having error {err:?}"); + return Err(err); + } + } + + let res = self.validate_and_store_pointer_record(pointer, key).await; + if res.is_ok() { + let content_hash = XorName::from_content(&record.value); + Marker::ValidPointerPutFromClient(&PrettyPrintRecordKey::from(&record.key)) + .log(); + self.replicate_valid_fresh_record( + record.key.clone(), + RecordType::NonChunk(content_hash), + ); + + // Notify replication_fetcher to mark the attempt as completed. + self.network().notify_fetch_completed( + record.key.clone(), + RecordType::NonChunk(content_hash), + ); + } + res + } } } @@ -321,9 +377,10 @@ impl Node { match record_header.kind { // A separate flow handles payment for chunks and registers RecordKind::ChunkWithPayment - | RecordKind::TransactionWithPayment + | RecordKind::LinkedListWithPayment | RecordKind::RegisterWithPayment - | RecordKind::ScratchpadWithPayment => { + | RecordKind::ScratchpadWithPayment + | RecordKind::PointerWithPayment => { warn!("Prepaid record came with Payment, which should be handled in another flow"); Err(Error::UnexpectedRecordWithPayment( PrettyPrintRecordKey::from(&record.key).into_owned(), @@ -352,9 +409,9 @@ impl Node { self.validate_and_store_scratchpad_record(scratchpad, key, false) .await } - RecordKind::Transaction => { + RecordKind::LinkedList => { let record_key = record.key.clone(); - let transactions = try_deserialize_record::>(&record)?; + let transactions = try_deserialize_record::>(&record)?; self.validate_merge_and_store_transactions(transactions, &record_key) .await } @@ -372,6 +429,11 @@ impl Node { } self.validate_and_store_register(register, false).await } + RecordKind::Pointer => { + let pointer = try_deserialize_record::(&record)?; + let key = record.key.clone(); + self.validate_and_store_pointer_record(pointer, key).await + } } } @@ -559,19 +621,19 @@ impl Node { /// If we already have a transaction at this address, the Vec is extended and stored. pub(crate) async fn validate_merge_and_store_transactions( &self, - transactions: Vec, + transactions: Vec, record_key: &RecordKey, ) -> Result<()> { let pretty_key = PrettyPrintRecordKey::from(record_key); debug!("Validating transactions before storage at {pretty_key:?}"); // only keep transactions that match the record key - let transactions_for_key: Vec = transactions + let transactions_for_key: Vec = transactions .into_iter() .filter(|s| { // get the record key for the transaction let transaction_address = s.address(); - let network_address = NetworkAddress::from_transaction_address(transaction_address); + let network_address = NetworkAddress::from_linked_list_address(transaction_address); let transaction_record_key = network_address.to_record_key(); let transaction_pretty = PrettyPrintRecordKey::from(&transaction_record_key); if &transaction_record_key != record_key { @@ -591,7 +653,7 @@ impl Node { } // verify the transactions - let mut validated_transactions: BTreeSet = transactions_for_key + let mut validated_transactions: BTreeSet = transactions_for_key .into_iter() .filter(|t| t.verify()) .collect(); @@ -608,12 +670,12 @@ impl Node { // add local transactions to the validated transactions, turn to Vec let local_txs = self.get_local_transactions(addr).await?; validated_transactions.extend(local_txs.into_iter()); - let validated_transactions: Vec = validated_transactions.into_iter().collect(); + let validated_transactions: Vec = validated_transactions.into_iter().collect(); // store the record into the local storage let record = Record { key: record_key.clone(), - value: try_serialize_record(&validated_transactions, RecordKind::Transaction)?.to_vec(), + value: try_serialize_record(&validated_transactions, RecordKind::LinkedList)?.to_vec(), publisher: None, expires: None, }; @@ -764,9 +826,9 @@ impl Node { /// Get the local transactions for the provided `TransactionAddress` /// This only fetches the transactions from the local store and does not perform any network operations. - async fn get_local_transactions(&self, addr: TransactionAddress) -> Result> { + async fn get_local_transactions(&self, addr: LinkedListAddress) -> Result> { // get the local transactions - let record_key = NetworkAddress::from_transaction_address(addr).to_record_key(); + let record_key = NetworkAddress::from_linked_list_address(addr).to_record_key(); debug!("Checking for local transactions with key: {record_key:?}"); let local_record = match self.network().get_local_record(&record_key).await? { Some(r) => r, @@ -779,11 +841,45 @@ impl Node { // deserialize the record and get the transactions let local_header = RecordHeader::from_record(&local_record)?; let record_kind = local_header.kind; - if !matches!(record_kind, RecordKind::Transaction) { + if !matches!(record_kind, RecordKind::LinkedList) { error!("Found a {record_kind} when expecting to find Spend at {addr:?}"); - return Err(NetworkError::RecordKindMismatch(RecordKind::Transaction).into()); + return Err(NetworkError::RecordKindMismatch(RecordKind::LinkedList).into()); } - let local_transactions: Vec = try_deserialize_record(&local_record)?; + let local_transactions: Vec = try_deserialize_record(&local_record)?; Ok(local_transactions) } + + /// Validate and store a pointer record + pub(crate) async fn validate_and_store_pointer_record( + &self, + pointer: Pointer, + key: RecordKey, + ) -> Result<()> { + // Verify the pointer's signature + if !pointer.verify() { + warn!("Pointer signature verification failed"); + return Err(Error::InvalidSignature); + } + + // Check if the pointer's address matches the record key + let net_addr = NetworkAddress::from_pointer_address(pointer.network_address()); + if key != net_addr.to_record_key() { + warn!("Pointer address does not match record key"); + return Err(Error::RecordKeyMismatch); + } + + // Store the pointer + let record = Record { + key: key.clone(), + value: try_serialize_record(&pointer, RecordKind::Pointer)?.to_vec(), + publisher: None, + expires: None, + }; + self.network().put_local_record(record); + + let content_hash = XorName::from_content(&pointer.network_address().to_bytes()); + self.replicate_valid_fresh_record(key, RecordType::NonChunk(content_hash)); + + Ok(()) + } } diff --git a/ant-node/src/quote.rs b/ant-node/src/quote.rs index f7c61b2af8..59d9eda832 100644 --- a/ant-node/src/quote.rs +++ b/ant-node/src/quote.rs @@ -12,6 +12,7 @@ use ant_networking::Network; use ant_protocol::{error::Error as ProtocolError, storage::ChunkAddress, NetworkAddress}; use libp2p::PeerId; use std::time::Duration; +use xor_name::XorName; impl Node { pub(crate) fn create_quote_for_storecost( @@ -20,7 +21,14 @@ impl Node { quoting_metrics: &QuotingMetrics, payment_address: &RewardsAddress, ) -> Result { - let content = address.as_xorname().unwrap_or_default(); + let content = match address { + NetworkAddress::ChunkAddress(addr) => *addr.xorname(), + NetworkAddress::LinkedListAddress(addr) => *addr.xorname(), + NetworkAddress::RegisterAddress(addr) => addr.xorname(), + NetworkAddress::ScratchpadAddress(addr) => addr.xorname(), + NetworkAddress::PointerAddress(addr) => *addr.xorname(), + NetworkAddress::PeerId(_) | NetworkAddress::RecordKey(_) => XorName::default(), + }; let timestamp = std::time::SystemTime::now(); let bytes = PaymentQuote::bytes_for_signing(content, timestamp, quoting_metrics, payment_address); @@ -51,7 +59,15 @@ pub(crate) fn verify_quote_for_storecost( debug!("Verifying payment quote for {address:?}: {quote:?}"); // check address - if address.as_xorname().unwrap_or_default() != quote.content { + let content = match address { + NetworkAddress::ChunkAddress(addr) => *addr.xorname(), + NetworkAddress::LinkedListAddress(addr) => *addr.xorname(), + NetworkAddress::RegisterAddress(addr) => addr.xorname(), + NetworkAddress::ScratchpadAddress(addr) => addr.xorname(), + NetworkAddress::PointerAddress(addr) => *addr.xorname(), + NetworkAddress::PeerId(_) | NetworkAddress::RecordKey(_) => XorName::default(), + }; + if content != quote.content { return Err(Error::InvalidQuoteContent); } diff --git a/ant-protocol/Cargo.toml b/ant-protocol/Cargo.toml index a6f54065ad..22a0571f1c 100644 --- a/ant-protocol/Cargo.toml +++ b/ant-protocol/Cargo.toml @@ -14,10 +14,10 @@ default = [] rpc = ["tonic", "prost"] [dependencies] +bls = { package = "blsttc", version = "8.0.2" } ant-build-info = { path = "../ant-build-info", version = "0.1.21" } ant-evm = { path = "../ant-evm", version = "0.1.6" } ant-registers = { path = "../ant-registers", version = "0.4.5" } -bls = { package = "blsttc", version = "8.0.1" } bytes = { version = "1.0.1", features = ["serde"] } color-eyre = "0.6.3" crdts = { version = "7.3", default-features = false, features = ["merkle"] } @@ -27,10 +27,8 @@ exponential-backoff = "2.0.0" hex = "~0.4.3" lazy_static = "1.4.0" libp2p = { version = "0.54.1", features = ["identify", "kad"] } -# # watch out updating this, protoc compiler needs to be installed on all build systems -# # arm builds + musl are very problematic -# prost and tonic are needed for the RPC server messages, not the underlying protocol prost = { version = "0.9", optional = true } +rand = "0.8" rmp-serde = "1.1.1" serde = { version = "1.0.133", features = ["derive", "rc"] } serde_json = "1.0" @@ -38,13 +36,18 @@ sha2 = "0.10.7" thiserror = "1.0.23" tiny-keccak = { version = "~2.0.2", features = ["sha3"] } tracing = { version = "~0.1.26" } -tonic = { version = "0.6.2", optional = true, default-features = false, features = ["prost", "tls", "codegen"] } +tonic = { version = "0.6.2", optional = true, default-features = false, features = [ + "prost", + "tls", + "codegen", +] } xor_name = "5.0.0" [build-dependencies] -# watch out updating this, protoc compiler needs to be installed on all build systems -# arm builds + musl are very problematic tonic-build = { version = "~0.6.2" } [lints] workspace = true + +[dev-dependencies] +rand = "0.8" diff --git a/ant-protocol/src/error.rs b/ant-protocol/src/error.rs index bc784860e1..4644e76c3a 100644 --- a/ant-protocol/src/error.rs +++ b/ant-protocol/src/error.rs @@ -6,7 +6,9 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::{storage::RegisterAddress, NetworkAddress, PrettyPrintRecordKey}; +use crate::{NetworkAddress, PrettyPrintRecordKey}; +use ant_registers::RegisterAddress; +use libp2p::kad::store; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -14,7 +16,7 @@ use thiserror::Error; pub type Result = std::result::Result; /// Main error types for the SAFE protocol. -#[derive(Error, Clone, PartialEq, Eq, Serialize, Deserialize, custom_debug::Debug)] +#[derive(Error, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[non_exhaustive] pub enum Error { // ---------- Misc errors @@ -82,3 +84,15 @@ pub enum Error { #[error("The record already exists, so do not charge for it: {0:?}")] RecordExists(PrettyPrintRecordKey<'static>), } + +impl From for store::Error { + fn from(_err: Error) -> Self { + store::Error::ValueTooLarge + } +} + +impl From for Error { + fn from(_err: store::Error) -> Self { + Error::RecordParsingFailed + } +} diff --git a/ant-protocol/src/lib.rs b/ant-protocol/src/lib.rs index 936d474246..08d8f621b7 100644 --- a/ant-protocol/src/lib.rs +++ b/ant-protocol/src/lib.rs @@ -29,14 +29,16 @@ pub mod antnode_proto { tonic::include_proto!("antnode_proto"); } pub use error::Error; +pub use error::Error as NetworkError; use storage::ScratchpadAddress; -use self::storage::{ChunkAddress, RegisterAddress, TransactionAddress}; +use self::storage::{ChunkAddress, LinkedListAddress, PointerAddress, RegisterAddress}; /// Re-export of Bytes used throughout the protocol pub use bytes::Bytes; use ant_evm::U256; +use hex; use libp2p::{ kad::{KBucketDistance as Distance, KBucketKey as Key, RecordKey}, multiaddr::Protocol, @@ -48,7 +50,6 @@ use std::{ borrow::Cow, fmt::{self, Debug, Display, Formatter, Write}, }; -use xor_name::XorName; /// The maximum number of peers to return in a `GetClosestPeers` response. /// This is the group size used in safe network protocol to be responsible for @@ -95,13 +96,15 @@ pub enum NetworkAddress { /// The NetworkAddress is representing a ChunkAddress. ChunkAddress(ChunkAddress), /// The NetworkAddress is representing a TransactionAddress. - TransactionAddress(TransactionAddress), - /// The NetworkAddress is representing a ChunkAddress. + LinkedListAddress(LinkedListAddress), + /// The NetworkAddress is representing a RegisterAddress. RegisterAddress(RegisterAddress), - /// The NetworkAddress is representing a RecordKey. - RecordKey(Bytes), /// The NetworkAddress is representing a ScratchpadAddress. ScratchpadAddress(ScratchpadAddress), + /// The NetworkAddress is representing a PointerAddress. + PointerAddress(PointerAddress), + /// The NetworkAddress is representing a RecordKey. + RecordKey(Bytes), } impl NetworkAddress { @@ -111,9 +114,10 @@ impl NetworkAddress { } /// Return a `NetworkAddress` representation of the `TransactionAddress`. - pub fn from_transaction_address(transaction_address: TransactionAddress) -> Self { - NetworkAddress::TransactionAddress(transaction_address) + pub fn from_linked_list_address(transaction_address: LinkedListAddress) -> Self { + NetworkAddress::LinkedListAddress(transaction_address) } + /// Return a `NetworkAddress` representation of the `TransactionAddress`. pub fn from_scratchpad_address(address: ScratchpadAddress) -> Self { NetworkAddress::ScratchpadAddress(address) @@ -134,18 +138,24 @@ impl NetworkAddress { NetworkAddress::RecordKey(Bytes::copy_from_slice(record_key.as_ref())) } + /// Return a `NetworkAddress` representation of the `PointerAddress`. + pub fn from_pointer_address(pointer_address: PointerAddress) -> Self { + NetworkAddress::PointerAddress(pointer_address) + } + /// Return the encapsulated bytes of this `NetworkAddress`. pub fn as_bytes(&self) -> Vec { match self { NetworkAddress::PeerId(bytes) | NetworkAddress::RecordKey(bytes) => bytes.to_vec(), NetworkAddress::ChunkAddress(chunk_address) => chunk_address.xorname().0.to_vec(), - NetworkAddress::TransactionAddress(transaction_address) => { - transaction_address.xorname().0.to_vec() + NetworkAddress::LinkedListAddress(linked_list_address) => { + linked_list_address.xorname().0.to_vec() } NetworkAddress::ScratchpadAddress(addr) => addr.xorname().0.to_vec(), NetworkAddress::RegisterAddress(register_address) => { register_address.xorname().0.to_vec() } + NetworkAddress::PointerAddress(pointer_address) => pointer_address.0.to_vec(), } } @@ -156,23 +166,9 @@ impl NetworkAddress { return Some(peer_id); } } - None } - /// Try to return the represented `XorName`. - pub fn as_xorname(&self) -> Option { - match self { - NetworkAddress::TransactionAddress(transaction_address) => { - Some(*transaction_address.xorname()) - } - NetworkAddress::ChunkAddress(chunk_address) => Some(*chunk_address.xorname()), - NetworkAddress::RegisterAddress(register_address) => Some(register_address.xorname()), - NetworkAddress::ScratchpadAddress(address) => Some(address.xorname()), - _ => None, - } - } - /// Try to return the represented `RecordKey`. pub fn as_record_key(&self) -> Option { match self { @@ -189,8 +185,11 @@ impl NetworkAddress { NetworkAddress::RegisterAddress(register_address) => { RecordKey::new(®ister_address.xorname()) } - NetworkAddress::TransactionAddress(transaction_address) => { - RecordKey::new(transaction_address.xorname()) + NetworkAddress::LinkedListAddress(linked_list_address) => { + RecordKey::new(linked_list_address.xorname()) + } + NetworkAddress::PointerAddress(pointer_address) => { + RecordKey::new(pointer_address.xorname()) } NetworkAddress::ScratchpadAddress(addr) => RecordKey::new(&addr.xorname()), NetworkAddress::PeerId(bytes) => RecordKey::new(bytes), @@ -211,16 +210,6 @@ impl NetworkAddress { pub fn distance(&self, other: &NetworkAddress) -> Distance { self.as_kbucket_key().distance(&other.as_kbucket_key()) } - - // NB: Leaving this here as to demonstrate what we can do with this. - // /// Return the uniquely determined key with the given distance to `self`. - // /// - // /// This implements the following equivalence: - // /// - // /// `self xor other = distance <==> other = self xor distance` - // pub fn for_distance(&self, d: Distance) -> libp2p::kad::kbucket::KeyBytes { - // self.as_kbucket_key().for_distance(d) - // } } impl Debug for NetworkAddress { @@ -239,7 +228,7 @@ impl Debug for NetworkAddress { &chunk_address.to_hex()[0..6] ) } - NetworkAddress::TransactionAddress(transaction_address) => { + NetworkAddress::LinkedListAddress(transaction_address) => { format!( "NetworkAddress::TransactionAddress({} - ", &transaction_address.to_hex()[0..6] @@ -251,20 +240,24 @@ impl Debug for NetworkAddress { &scratchpad_address.to_hex()[0..6] ) } - NetworkAddress::RegisterAddress(register_address) => format!( - "NetworkAddress::RegisterAddress({} - ", - ®ister_address.to_hex()[0..6] - ), - NetworkAddress::RecordKey(bytes) => format!( - "NetworkAddress::RecordKey({} - ", - &PrettyPrintRecordKey::from(&RecordKey::new(bytes)).no_kbucket_log()[0..6] - ), + NetworkAddress::RegisterAddress(register_address) => { + format!( + "NetworkAddress::RegisterAddress({} - ", + ®ister_address.to_hex()[0..6] + ) + } + NetworkAddress::PointerAddress(pointer_address) => { + format!( + "NetworkAddress::PointerAddress({} - ", + &pointer_address.to_hex()[0..6] + ) + } + NetworkAddress::RecordKey(bytes) => { + format!("NetworkAddress::RecordKey({:?} - ", bytes) + } }; - write!( - f, - "{name_str}{:?})", - PrettyPrintKBucketKey(self.as_kbucket_key()), - ) + + write!(f, "{name_str}{:?})", self.as_kbucket_key()) } } @@ -277,7 +270,7 @@ impl Display for NetworkAddress { NetworkAddress::ChunkAddress(addr) => { write!(f, "NetworkAddress::ChunkAddress({addr:?})") } - NetworkAddress::TransactionAddress(addr) => { + NetworkAddress::LinkedListAddress(addr) => { write!(f, "NetworkAddress::TransactionAddress({addr:?})") } NetworkAddress::ScratchpadAddress(addr) => { @@ -289,6 +282,9 @@ impl Display for NetworkAddress { NetworkAddress::RecordKey(key) => { write!(f, "NetworkAddress::RecordKey({})", hex::encode(key)) } + NetworkAddress::PointerAddress(addr) => { + write!(f, "NetworkAddress::PointerAddress({addr:?})") + } } } } @@ -413,15 +409,15 @@ impl std::fmt::Debug for PrettyPrintRecordKey<'_> { #[cfg(test)] mod tests { - use crate::storage::TransactionAddress; + use crate::storage::LinkedListAddress; use crate::NetworkAddress; use bls::rand::thread_rng; #[test] fn verify_transaction_addr_is_actionable() { let xorname = xor_name::XorName::random(&mut thread_rng()); - let transaction_addr = TransactionAddress::new(xorname); - let net_addr = NetworkAddress::from_transaction_address(transaction_addr); + let transaction_addr = LinkedListAddress::new(xorname); + let net_addr = NetworkAddress::from_linked_list_address(transaction_addr); let transaction_addr_hex = &transaction_addr.to_hex()[0..6]; // we only log the first 6 chars let net_addr_fmt = format!("{net_addr}"); diff --git a/ant-protocol/src/storage/address.rs b/ant-protocol/src/storage/address.rs deleted file mode 100644 index 57c7a18aeb..0000000000 --- a/ant-protocol/src/storage/address.rs +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -mod chunk; -mod scratchpad; -mod transaction; - -pub use self::chunk::ChunkAddress; -pub use self::scratchpad::ScratchpadAddress; -pub use self::transaction::TransactionAddress; -pub use ant_registers::RegisterAddress; diff --git a/ant-protocol/src/storage/address/transaction.rs b/ant-protocol/src/storage/address/linked_list.rs similarity index 91% rename from ant-protocol/src/storage/address/transaction.rs rename to ant-protocol/src/storage/address/linked_list.rs index 399a7a6397..4e290f9d37 100644 --- a/ant-protocol/src/storage/address/transaction.rs +++ b/ant-protocol/src/storage/address/linked_list.rs @@ -12,9 +12,9 @@ use xor_name::XorName; /// Address of a transaction, is derived from the owner's public key #[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -pub struct TransactionAddress(pub XorName); +pub struct LinkedListAddress(pub XorName); -impl TransactionAddress { +impl LinkedListAddress { pub fn from_owner(owner: PublicKey) -> Self { Self(XorName::from_content(&owner.to_bytes())) } @@ -32,7 +32,7 @@ impl TransactionAddress { } } -impl std::fmt::Debug for TransactionAddress { +impl std::fmt::Debug for LinkedListAddress { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "TransactionAddress({})", &self.to_hex()[0..6]) } diff --git a/ant-protocol/src/storage/address/mod.rs b/ant-protocol/src/storage/address/mod.rs new file mode 100644 index 0000000000..92bdd045e4 --- /dev/null +++ b/ant-protocol/src/storage/address/mod.rs @@ -0,0 +1,9 @@ +pub mod chunk; +pub mod linked_list; +pub mod pointer_address; +pub mod scratchpad; + +pub use chunk::ChunkAddress; +pub use linked_list::LinkedListAddress; +pub use pointer_address::PointerAddress; +pub use scratchpad::ScratchpadAddress; diff --git a/ant-protocol/src/storage/address/pointer_address.rs b/ant-protocol/src/storage/address/pointer_address.rs new file mode 100644 index 0000000000..5a1a2db943 --- /dev/null +++ b/ant-protocol/src/storage/address/pointer_address.rs @@ -0,0 +1,39 @@ +use bls::PublicKey; +use serde::{Deserialize, Serialize}; +use xor_name::XorName; + +/// Address of a pointer, is derived from the owner's public key +#[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub struct PointerAddress(pub XorName); + +impl PointerAddress { + pub fn from_owner(owner: PublicKey) -> Self { + Self(XorName::from_content(&owner.to_bytes())) + } + + pub fn new(xor_name: XorName) -> Self { + Self(xor_name) + } + + pub fn xorname(&self) -> &XorName { + &self.0 + } + + pub fn to_hex(&self) -> String { + hex::encode(self.0) + } + + pub fn to_bytes(&self) -> Vec { + rmp_serde::to_vec(self).expect("Failed to serialize PointerAddress") + } + + pub fn from_bytes(bytes: &[u8]) -> Result { + rmp_serde::from_slice(bytes) + } +} + +impl std::fmt::Debug for PointerAddress { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "PointerAddress({})", &self.to_hex()[0..6]) + } +} diff --git a/ant-protocol/src/storage/header.rs b/ant-protocol/src/storage/header.rs index 7cfd2ffedf..c62b0e8685 100644 --- a/ant-protocol/src/storage/header.rs +++ b/ant-protocol/src/storage/header.rs @@ -22,6 +22,8 @@ use xor_name::XorName; pub enum RecordType { Chunk, Scratchpad, + Pointer, + LinkedList, NonChunk(XorName), } @@ -34,12 +36,14 @@ pub struct RecordHeader { pub enum RecordKind { Chunk, ChunkWithPayment, - Transaction, - TransactionWithPayment, + LinkedList, + LinkedListWithPayment, Register, RegisterWithPayment, Scratchpad, ScratchpadWithPayment, + Pointer, + PointerWithPayment, } impl Serialize for RecordKind { @@ -50,12 +54,14 @@ impl Serialize for RecordKind { match *self { Self::ChunkWithPayment => serializer.serialize_u32(0), Self::Chunk => serializer.serialize_u32(1), - Self::Transaction => serializer.serialize_u32(2), + Self::LinkedList => serializer.serialize_u32(2), Self::Register => serializer.serialize_u32(3), Self::RegisterWithPayment => serializer.serialize_u32(4), Self::Scratchpad => serializer.serialize_u32(5), Self::ScratchpadWithPayment => serializer.serialize_u32(6), - Self::TransactionWithPayment => serializer.serialize_u32(7), + Self::LinkedListWithPayment => serializer.serialize_u32(7), + Self::Pointer => serializer.serialize_u32(8), + Self::PointerWithPayment => serializer.serialize_u32(9), } } } @@ -69,12 +75,14 @@ impl<'de> Deserialize<'de> for RecordKind { match num { 0 => Ok(Self::ChunkWithPayment), 1 => Ok(Self::Chunk), - 2 => Ok(Self::Transaction), + 2 => Ok(Self::LinkedList), 3 => Ok(Self::Register), 4 => Ok(Self::RegisterWithPayment), 5 => Ok(Self::Scratchpad), 6 => Ok(Self::ScratchpadWithPayment), - 7 => Ok(Self::TransactionWithPayment), + 7 => Ok(Self::LinkedListWithPayment), + 8 => Ok(Self::Pointer), + 9 => Ok(Self::PointerWithPayment), _ => Err(serde::de::Error::custom( "Unexpected integer for RecordKind variant", )), @@ -184,7 +192,7 @@ mod tests { assert_eq!(chunk.len(), RecordHeader::SIZE); let transaction = RecordHeader { - kind: RecordKind::Transaction, + kind: RecordKind::LinkedList, } .try_serialize()?; assert_eq!(transaction.len(), RecordHeader::SIZE); @@ -207,6 +215,45 @@ mod tests { .try_serialize()?; assert_eq!(scratchpad_with_payment.len(), RecordHeader::SIZE); + let pointer = RecordHeader { + kind: RecordKind::Pointer, + } + .try_serialize()?; + assert_eq!(pointer.len(), RecordHeader::SIZE); + + let pointer_with_payment = RecordHeader { + kind: RecordKind::PointerWithPayment, + } + .try_serialize()?; + assert_eq!(pointer_with_payment.len(), RecordHeader::SIZE); + + Ok(()) + } + + #[test] + fn test_record_kind_serialization() -> Result<()> { + let kinds = vec![ + RecordKind::Chunk, + RecordKind::ChunkWithPayment, + RecordKind::LinkedList, + RecordKind::LinkedListWithPayment, + RecordKind::Register, + RecordKind::RegisterWithPayment, + RecordKind::Scratchpad, + RecordKind::ScratchpadWithPayment, + RecordKind::Pointer, + RecordKind::PointerWithPayment, + ]; + + for kind in kinds { + let header = RecordHeader { kind }; + let header2 = RecordHeader { kind }; + + let serialized = header.try_serialize()?; + let deserialized = RecordHeader::try_deserialize(&serialized)?; + assert_eq!(header2.kind, deserialized.kind); + } + Ok(()) } } diff --git a/ant-protocol/src/storage/transaction.rs b/ant-protocol/src/storage/linked_list.rs similarity index 76% rename from ant-protocol/src/storage/transaction.rs rename to ant-protocol/src/storage/linked_list.rs index 6f7a7a9b11..e93a64c699 100644 --- a/ant-protocol/src/storage/transaction.rs +++ b/ant-protocol/src/storage/linked_list.rs @@ -6,7 +6,7 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use super::address::TransactionAddress; +use super::address::LinkedListAddress; use bls::SecretKey; use serde::{Deserialize, Serialize}; @@ -14,26 +14,26 @@ use serde::{Deserialize, Serialize}; pub use bls::{PublicKey, Signature}; /// Content of a transaction, limited to 32 bytes -pub type TransactionContent = [u8; 32]; +pub type LinkedListContent = [u8; 32]; /// A generic Transaction on the Network #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash, Ord, PartialOrd)] -pub struct Transaction { +pub struct LinkedList { pub owner: PublicKey, pub parents: Vec, - pub content: TransactionContent, - pub outputs: Vec<(PublicKey, TransactionContent)>, + pub content: LinkedListContent, + pub outputs: Option>, /// signs the above 4 fields with the owners key pub signature: Signature, } -impl Transaction { +impl LinkedList { /// Create a new transaction, signing it with the provided secret key. pub fn new( owner: PublicKey, parents: Vec, - content: TransactionContent, - outputs: Vec<(PublicKey, TransactionContent)>, + content: LinkedListContent, + outputs: Option>, signing_key: &SecretKey, ) -> Self { let signature = signing_key.sign(Self::bytes_to_sign(&owner, &parents, &content, &outputs)); @@ -50,8 +50,8 @@ impl Transaction { pub fn new_with_signature( owner: PublicKey, parents: Vec, - content: TransactionContent, - outputs: Vec<(PublicKey, TransactionContent)>, + content: LinkedListContent, + outputs: Option>, signature: Signature, ) -> Self { Self { @@ -68,7 +68,7 @@ impl Transaction { owner: &PublicKey, parents: &[PublicKey], content: &[u8], - outputs: &[(PublicKey, TransactionContent)], + outputs: &Option>, ) -> Vec { let mut bytes = Vec::new(); bytes.extend_from_slice(&owner.to_bytes()); @@ -83,17 +83,19 @@ impl Transaction { bytes.extend_from_slice("content".as_bytes()); bytes.extend_from_slice(content); bytes.extend_from_slice("outputs".as_bytes()); - bytes.extend_from_slice( - &outputs - .iter() - .flat_map(|(p, c)| [&p.to_bytes(), c.as_slice()].concat()) - .collect::>(), - ); + if let Some(outputs) = outputs { + bytes.extend_from_slice( + &outputs + .iter() + .flat_map(|(p, c)| [&p.to_bytes(), c.as_slice()].concat()) + .collect::>(), + ); + } bytes } - pub fn address(&self) -> TransactionAddress { - TransactionAddress::from_owner(self.owner) + pub fn address(&self) -> LinkedListAddress { + LinkedListAddress::from_owner(self.owner) } /// Get the bytes that the signature is calculated from. diff --git a/ant-protocol/src/storage.rs b/ant-protocol/src/storage/mod.rs similarity index 93% rename from ant-protocol/src/storage.rs rename to ant-protocol/src/storage/mod.rs index 9d3e675039..cb0cca01c5 100644 --- a/ant-protocol/src/storage.rs +++ b/ant-protocol/src/storage/mod.rs @@ -9,21 +9,25 @@ mod address; mod chunks; mod header; +mod linked_list; +pub mod pointer; +pub use pointer::{Pointer, PointerTarget}; mod scratchpad; -mod transaction; use core::fmt; use exponential_backoff::Backoff; use std::{num::NonZeroUsize, time::Duration}; pub use self::{ - address::{ChunkAddress, RegisterAddress, ScratchpadAddress, TransactionAddress}, + address::{ChunkAddress, LinkedListAddress, PointerAddress, ScratchpadAddress}, chunks::Chunk, header::{try_deserialize_record, try_serialize_record, RecordHeader, RecordKind, RecordType}, + linked_list::LinkedList, scratchpad::Scratchpad, - transaction::Transaction, }; +pub use ant_registers::RegisterAddress; + /// A strategy that translates into a configuration for exponential backoff. /// The first retry is done after 2 seconds, after which the backoff is roughly doubled each time. /// The interval does not go beyond 32 seconds. So the intervals increase from 2 to 4, to 8, to 16, to 32 seconds and diff --git a/ant-protocol/src/storage/pointer.rs b/ant-protocol/src/storage/pointer.rs new file mode 100644 index 0000000000..38d42347f1 --- /dev/null +++ b/ant-protocol/src/storage/pointer.rs @@ -0,0 +1,208 @@ +use crate::storage::{ChunkAddress, LinkedListAddress, PointerAddress, ScratchpadAddress}; +use bls::{Error as BlsError, PublicKey, SecretKey, Signature}; +use hex::FromHexError; +use rand::thread_rng; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use xor_name::XorName; + +#[derive(Error, Debug)] +pub enum PointerError { + #[error("Failed to decode hex string: {0}")] + HexDecoding(#[from] FromHexError), + #[error("Failed to create public key: {0}")] + BlsError(#[from] BlsError), + #[error("Invalid public key bytes length")] + InvalidPublicKeyLength, + #[error("Invalid signature")] + InvalidSignature, + #[error("Serialization error: {0}")] + SerializationError(String), +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct Pointer { + owner: PublicKey, + counter: u32, + target: PointerTarget, + signature: Signature, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub enum PointerTarget { + ChunkAddress(ChunkAddress), + LinkedListAddress(LinkedListAddress), + PointerAddress(PointerAddress), + ScratchpadAddress(ScratchpadAddress), +} + +impl PointerTarget { + pub fn xorname(&self) -> XorName { + match self { + PointerTarget::ChunkAddress(addr) => *addr.xorname(), + PointerTarget::LinkedListAddress(addr) => *addr.xorname(), + PointerTarget::PointerAddress(ptr) => *ptr.xorname(), + PointerTarget::ScratchpadAddress(addr) => addr.xorname(), + } + } +} + +impl Pointer { + /// Create a new pointer, signing it with the provided secret key. + pub fn new( + owner: PublicKey, + counter: u32, + target: PointerTarget, + signing_key: &SecretKey, + ) -> Self { + let bytes_to_sign = Self::bytes_to_sign(&owner, counter, &target); + let signature = signing_key.sign(&bytes_to_sign); + + Self { + owner, + counter, + target, + signature, + } + } + + /// Create a new pointer with an existing signature + pub fn new_with_signature( + owner: PublicKey, + counter: u32, + target: PointerTarget, + signature: Signature, + ) -> Self { + Self { + owner, + counter, + target, + signature, + } + } + + /// Get the bytes that the signature is calculated from + fn bytes_to_sign(owner: &PublicKey, counter: u32, target: &PointerTarget) -> Vec { + let mut bytes = Vec::new(); + // Add owner public key bytes + bytes.extend_from_slice(&owner.to_bytes()); + // Add counter + bytes.extend_from_slice(&counter.to_le_bytes()); + // Add target bytes using MessagePack serialization + if let Ok(target_bytes) = rmp_serde::to_vec(target) { + bytes.extend_from_slice(&target_bytes); + } + bytes + } + + /// Get the bytes that were signed for this pointer + pub fn bytes_for_signature(&self) -> Vec { + Self::bytes_to_sign(&self.owner, self.counter, &self.target) + } + + pub fn xorname(&self) -> XorName { + self.target.xorname() + } + + pub fn count(&self) -> u32 { + self.counter + } + + /// Get the network address for this pointer + pub fn network_address(&self) -> PointerAddress { + PointerAddress::from_owner(self.owner) + } + + /// Verifies if the pointer has a valid signature + pub fn verify(&self) -> bool { + let bytes = self.bytes_for_signature(); + self.owner.verify(&self.signature, &bytes) + } + + pub fn encode_hex(&self) -> String { + hex::encode(self.owner.to_bytes()) + } + + pub fn decode_hex(hex_str: &str) -> Result { + let bytes = hex::decode(hex_str)?; + if bytes.len() != 48 { + return Err(PointerError::InvalidPublicKeyLength); + } + let mut bytes_array = [0u8; 48]; + bytes_array.copy_from_slice(&bytes); + + let owner = PublicKey::from_bytes(bytes_array).map_err(PointerError::BlsError)?; + + let mut rng = thread_rng(); + let target = PointerTarget::ChunkAddress(ChunkAddress::new(XorName::random(&mut rng))); + + // Create a temporary secret key just for hex decoding test purposes + let sk = SecretKey::random(); + Ok(Self::new(owner, 0, target, &sk)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_pointer_creation_and_validation() { + let owner_sk = SecretKey::random(); + let owner_pk = owner_sk.public_key(); + let counter = 1; + let mut rng = thread_rng(); + let target = + PointerTarget::LinkedListAddress(LinkedListAddress::new(XorName::random(&mut rng))); + + // Create and sign pointer + let pointer = Pointer::new(owner_pk, counter, target.clone(), &owner_sk); + assert!(pointer.verify()); // Should be valid with correct signature + + // Create pointer with wrong signature + let wrong_sk = SecretKey::random(); + let wrong_pointer = Pointer::new(owner_pk, counter, target.clone(), &wrong_sk); + assert!(!wrong_pointer.verify()); // Should be invalid with wrong signature + } + + #[test] + fn test_pointer_xorname() { + let owner_sk = SecretKey::random(); + let owner_pk = owner_sk.public_key(); + let counter = 1; + let mut rng = thread_rng(); + let target = + PointerTarget::LinkedListAddress(LinkedListAddress::new(XorName::random(&mut rng))); + + let pointer = Pointer::new(owner_pk, counter, target.clone(), &owner_sk); + let xorname = pointer.xorname(); + assert_eq!(xorname, target.xorname()); + } + + #[test] + fn test_pointer_hex_encoding() { + let owner_sk = SecretKey::random(); + let owner_pk = owner_sk.public_key(); + let counter = 1; + let mut rng = thread_rng(); + let target = + PointerTarget::LinkedListAddress(LinkedListAddress::new(XorName::random(&mut rng))); + + let pointer = Pointer::new(owner_pk, counter, target, &owner_sk); + let hex = pointer.encode_hex(); + let expected_hex = hex::encode(owner_pk.to_bytes()); + assert_eq!(hex, expected_hex); + } + + #[test] + fn test_pointer_hex_decoding() { + let owner_sk = SecretKey::random(); + let owner_pk = owner_sk.public_key(); + let hex = hex::encode(owner_pk.to_bytes()); + + let result = Pointer::decode_hex(&hex); + assert!(result.is_ok()); + let pointer = result.unwrap(); + assert_eq!(pointer.owner, owner_pk); + } +} diff --git a/ant-protocol/src/storage/scratchpad.rs b/ant-protocol/src/storage/scratchpad.rs index 97f0d2dda4..348ad3a0bf 100644 --- a/ant-protocol/src/storage/scratchpad.rs +++ b/ant-protocol/src/storage/scratchpad.rs @@ -135,7 +135,7 @@ impl Scratchpad { pub fn to_xor_name_vec(&self) -> Vec { [self.network_address()] .iter() - .filter_map(|f| f.as_xorname()) + .filter_map(|f| Some(XorName::from_content(f.as_bytes().as_ref()))) .collect::>() } diff --git a/autonomi/README_PYTHON.md b/autonomi/README_PYTHON.md index 6772ce14a1..84810159a9 100644 --- a/autonomi/README_PYTHON.md +++ b/autonomi/README_PYTHON.md @@ -1,14 +1,17 @@ -## Python Bindings +# Autonomi Python Bindings The Autonomi client library provides Python bindings for easy integration with Python applications. -### Installation +## Installation + +We recommend using `uv` for Python environment management: ```bash -pip install autonomi-client +uv venv +uv pip install autonomi-client ``` -### Quick Start +## Quick Start ```python from autonomi_client import Client, Wallet, PaymentOption @@ -34,155 +37,180 @@ retrieved = client.data_get_public(addr) print(f"Retrieved: {retrieved.decode()}") ``` -### Available Modules +## API Reference -#### Core Components +### Client -- `Client`: Main interface to the Autonomi network - - `connect(peers: List[str])`: Connect to network nodes - - `data_put_public(data: bytes, payment: PaymentOption)`: Upload data - - `data_get_public(addr: str)`: Download data - - `data_put(data: bytes, payment: PaymentOption)`: Store private data - - `data_get(access: DataMapChunk)`: Retrieve private data - - `register_generate_key()`: Generate register key +The main interface to interact with the Autonomi network. -- `Wallet`: Ethereum wallet management - - `new(private_key: str)`: Create wallet from private key - - `address()`: Get wallet address - - `balance()`: Get current balance +#### Connection Methods -- `PaymentOption`: Payment configuration - - `wallet(wallet: Wallet)`: Create payment option from wallet +- `connect(peers: List[str]) -> Client` + - Connect to network nodes + - `peers`: List of multiaddresses for initial network nodes -#### Private Data +#### Data Operations -- `DataMapChunk`: Handle private data storage - - `from_hex(hex: str)`: Create from hex string - - `to_hex()`: Convert to hex string - - `address()`: Get short reference address +- `data_put_public(data: bytes, payment: PaymentOption) -> str` + - Upload public data to the network + - Returns address where data is stored -```python -# Private data example -access = client.data_put(secret_data, payment) -print(f"Private data stored at: {access.to_hex()}") -retrieved = client.data_get(access) -``` +- `data_get_public(addr: str) -> bytes` + - Download public data from the network + - `addr`: Address returned from `data_put_public` -#### Registers +- `data_put(data: bytes, payment: PaymentOption) -> DataMapChunk` + - Store private (encrypted) data + - Returns access information for later retrieval -- Register operations for mutable data - - `register_create(value: bytes, name: str, key: RegisterSecretKey, wallet: Wallet)` - - `register_get(address: str)` - - `register_update(register: Register, value: bytes, key: RegisterSecretKey)` +- `data_get(access: DataMapChunk) -> bytes` + - Retrieve private data + - `access`: DataMapChunk from previous `data_put` -```python -# Register example -key = client.register_generate_key() -register = client.register_create(b"Initial value", "my_register", key, wallet) -client.register_update(register, b"New value", key) -``` +#### Pointer Operations -#### Vaults +- `pointer_get(address: str) -> Pointer` + - Retrieve pointer from network + - `address`: Hex-encoded pointer address -- `VaultSecretKey`: Manage vault access - - `new()`: Generate new key - - `from_hex(hex: str)`: Create from hex string - - `to_hex()`: Convert to hex string +- `pointer_put(pointer: Pointer, wallet: Wallet)` + - Store pointer on network + - Requires payment via wallet -- `UserData`: User data management - - `new()`: Create new user data - - `add_file_archive(archive: str)`: Add file archive - - `add_private_file_archive(archive: str)`: Add private archive - - `file_archives()`: List archives - - `private_file_archives()`: List private archives +- `pointer_cost(key: VaultSecretKey) -> str` + - Calculate pointer storage cost + - Returns cost in atto tokens -```python -# Vault example -vault_key = VaultSecretKey.new() -cost = client.vault_cost(vault_key) -client.write_bytes_to_vault(data, payment, vault_key, content_type=1) -data, content_type = client.fetch_and_decrypt_vault(vault_key) -``` +#### Vault Operations -#### Utility Functions +- `vault_cost(key: VaultSecretKey) -> str` + - Calculate vault storage cost -- `encrypt(data: bytes)`: Self-encrypt data -- `hash_to_short_string(input: str)`: Generate short reference +- `write_bytes_to_vault(data: bytes, payment: PaymentOption, key: VaultSecretKey, content_type: int) -> str` + - Write data to vault + - Returns vault address -### Complete Examples +- `fetch_and_decrypt_vault(key: VaultSecretKey) -> Tuple[bytes, int]` + - Retrieve vault data + - Returns (data, content_type) -#### Data Management +- `get_user_data_from_vault(key: VaultSecretKey) -> UserData` + - Get user data from vault -```python -def handle_data_operations(client, payment): - # Upload text - text_data = b"Hello, Safe Network!" - text_addr = client.data_put_public(text_data, payment) - - # Upload binary data - with open("image.jpg", "rb") as f: - image_data = f.read() - image_addr = client.data_put_public(image_data, payment) - - # Download and verify - downloaded = client.data_get_public(text_addr) - assert downloaded == text_data -``` +- `put_user_data_to_vault(key: VaultSecretKey, payment: PaymentOption, user_data: UserData) -> str` + - Store user data in vault + - Returns vault address -#### Private Data and Encryption +### Wallet -```python -def handle_private_data(client, payment): - # Create and encrypt private data - secret = {"api_key": "secret_key"} - data = json.dumps(secret).encode() - - # Store privately - access = client.data_put(data, payment) - print(f"Access token: {access.to_hex()}") - - # Retrieve - retrieved = client.data_get(access) - secret = json.loads(retrieved.decode()) -``` +Ethereum wallet management for payments. -#### Vault Management +- `new(private_key: str) -> Wallet` + - Create wallet from private key + - `private_key`: 64-char hex string without '0x' prefix -```python -def handle_vault(client, payment): - # Create vault - vault_key = VaultSecretKey.new() - - # Store user data - user_data = UserData() - user_data.add_file_archive("archive_address") - - # Save to vault - cost = client.put_user_data_to_vault(vault_key, payment, user_data) - - # Retrieve - retrieved = client.get_user_data_from_vault(vault_key) - archives = retrieved.file_archives() -``` +- `address() -> str` + - Get wallet's Ethereum address -### Error Handling +- `balance() -> str` + - Get wallet's token balance -All operations can raise exceptions. It's recommended to use try-except blocks: +- `balance_of_gas() -> str` + - Get wallet's gas balance -```python -try: - client = Client.connect(peers) - # ... operations ... -except Exception as e: - print(f"Error: {e}") -``` +### PaymentOption + +Configure payment methods. + +- `wallet(wallet: Wallet) -> PaymentOption` + - Create payment option from wallet + +### Pointer + +Handle network pointers for referencing data. + +- `new(target: str) -> Pointer` + - Create new pointer + - `target`: Hex-encoded target address + +- `address() -> str` + - Get pointer's network address + +- `target() -> str` + - Get pointer's target address + +### VaultSecretKey + +Manage vault access keys. + +- `new() -> VaultSecretKey` + - Generate new key + +- `from_hex(hex: str) -> VaultSecretKey` + - Create from hex string + +- `to_hex() -> str` + - Convert to hex string + +### UserData + +Manage user data in vaults. + +- `new() -> UserData` + - Create new user data + +- `add_file_archive(archive: str) -> Optional[str]` + - Add file archive + - Returns archive ID if successful + +- `add_private_file_archive(archive: str) -> Optional[str]` + - Add private archive + - Returns archive ID if successful + +- `file_archives() -> List[Tuple[str, str]]` + - List archives as (id, address) pairs + +- `private_file_archives() -> List[Tuple[str, str]]` + - List private archives as (id, address) pairs + +### DataMapChunk + +Handle private data storage references. + +- `from_hex(hex: str) -> DataMapChunk` + - Create from hex string + +- `to_hex() -> str` + - Convert to hex string + +- `address() -> str` + - Get short reference address + +### Utility Functions + +- `encrypt(data: bytes) -> Tuple[bytes, List[bytes]]` + - Self-encrypt data + - Returns (data_map, chunks) + +## Examples + +See the `examples/` directory for complete examples: +- `autonomi_example.py`: Basic data operations +- `autonomi_pointers.py`: Working with pointers +- `autonomi_vault.py`: Vault operations +- `autonomi_private_data.py`: Private data handling +- `autonomi_data_registers.py`: Using data registers +- `autonomi_private_encryption.py`: Data encryption +- `autonomi_advanced.py`: Advanced usage scenarios -### Best Practices +## Best Practices -1. Always keep private keys secure -2. Use error handling for all network operations -3. Clean up resources when done +1. Always handle wallet private keys securely +2. Check operation costs before executing +3. Use appropriate error handling 4. Monitor wallet balance for payments 5. Use appropriate content types for vault storage +6. Consider using pointers for updatable references +7. Properly manage and backup vault keys -For more examples, see the `examples/` directory in the repository. +For more examples and detailed usage, see the examples in the repository. diff --git a/autonomi/pyproject.toml b/autonomi/pyproject.toml index 0a17202968..b3c9a2d080 100644 --- a/autonomi/pyproject.toml +++ b/autonomi/pyproject.toml @@ -2,33 +2,50 @@ requires = ["maturin>=1.0,<2.0"] build-backend = "maturin" -[tool.maturin] -features = ["extension-module"] -python-source = "python" -module-name = "autonomi_client.autonomi_client" -bindings = "pyo3" -target-dir = "target/wheels" - [project] name = "autonomi-client" dynamic = ["version"] description = "Autonomi client API" -readme = "README.md" +authors = [{ name = "MaidSafe Developers", email = "dev@maidsafe.net" }] +dependencies = ["maturin>=1.7.4", "pip>=24.0"] +readme = "README_PYTHON.md" requires-python = ">=3.8" license = { text = "GPL-3.0" } -keywords = ["safe", "network", "autonomi"] -authors = [{ name = "MaidSafe Developers", email = "dev@maidsafe.net" }] classifiers = [ - "Programming Language :: Python", - "Programming Language :: Python :: Implementation :: CPython", + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Rust", - "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Programming Language :: Python :: Implementation :: CPython", ] -dependencies = [ - "pip>=24.3.1", + +[project.urls] +Homepage = "https://maidsafe.net" +Repository = "https://github.com/maidsafe/autonomi" + +[tool.maturin] +features = ["extension-module"] +module-name = "autonomi_client" +python-source = "python" +bindings = "pyo3" +include = ["README_PYTHON.md", "src/*", "python/*", "pyproject.toml"] +manifest-path = "Cargo.toml" +sdist-include = [ + "README_PYTHON.md", + "src/**/*", + "python/**/*", + "pyproject.toml", + "Cargo.toml", ] +workspace = false + +[tool.pytest.ini_options] +testpaths = ["tests/python"] +python_files = ["test_*.py"] +addopts = "-v -s" diff --git a/autonomi/python/autonomi_client/__init__.py b/autonomi/python/autonomi_client/__init__.py index b1e437b894..b149985473 100644 --- a/autonomi/python/autonomi_client/__init__.py +++ b/autonomi/python/autonomi_client/__init__.py @@ -1,4 +1,18 @@ -from .autonomi_client import Client, Wallet, PaymentOption, VaultSecretKey, UserData, DataMapChunk, encrypt +from .autonomi_client import ( + Client, + Wallet, + PaymentOption, + VaultSecretKey, + UserData, + DataMapChunk, + encrypt, + ChunkAddress, + PointerTarget, + Pointer, + PointerAddress, + SecretKey, + PublicKey, +) __all__ = [ "Client", @@ -7,5 +21,11 @@ "VaultSecretKey", "UserData", "DataMapChunk", - "encrypt" + "encrypt", + "ChunkAddress", + "PointerTarget", + "Pointer", + "PointerAddress", + "SecretKey", + "PublicKey", ] diff --git a/autonomi/python/examples/autonomi_pointers.py b/autonomi/python/examples/autonomi_pointers.py new file mode 100644 index 0000000000..d5380915e0 --- /dev/null +++ b/autonomi/python/examples/autonomi_pointers.py @@ -0,0 +1,54 @@ +""" +Example demonstrating the use of pointers in the Autonomi network. +Pointers allow for creating references to data that can be updated. +""" + +from autonomi_client import Client, Wallet, PaymentOption, PublicKey, SecretKey, PointerTarget, ChunkAddress + +def main(): + # Initialize a wallet with a private key + # This should be a valid Ethereum private key (64 hex chars without '0x' prefix) + private_key = "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" + wallet = Wallet(private_key) + print(f"Wallet address: {wallet.address()}") + print(f"Wallet balance: {wallet.balance()}") + + # Connect to the network + peers = [ + "/ip4/127.0.0.1/tcp/12000", + "/ip4/127.0.0.1/tcp/12001" + ] + client = Client.connect(peers) + + # First, let's upload some data that we want to point to + target_data = b"Hello, I'm the target data!" + target_addr = client.data_put_public(target_data, PaymentOption.wallet(wallet)) + print(f"Target data uploaded to: {target_addr}") + + # Create a pointer target from the address + chunk_addr = ChunkAddress.from_hex(target_addr) + target = PointerTarget.from_chunk_address(chunk_addr) + + # Create owner key pair + owner_key = SecretKey.new() + owner_pub = PublicKey.from_secret_key(owner_key) + + # Create and store the pointer + counter = 0 # Start with counter 0 + client.pointer_put(owner_pub, counter, target, owner_key, wallet) + print(f"Pointer stored successfully") + + # Calculate the pointer address + pointer_addr = client.pointer_address(owner_pub, counter) + print(f"Pointer address: {pointer_addr}") + + # Later, we can retrieve the pointer + pointer = client.pointer_get(pointer_addr) + print(f"Retrieved pointer target: {pointer.target().hex()}") + + # We can then use the target address to get the original data + retrieved_data = client.data_get_public(pointer.target().hex()) + print(f"Retrieved target data: {retrieved_data.decode()}") + +if __name__ == "__main__": + main() diff --git a/autonomi/setup.py b/autonomi/setup.py new file mode 100644 index 0000000000..f7d5530dd0 --- /dev/null +++ b/autonomi/setup.py @@ -0,0 +1,35 @@ +from setuptools import setup +from setuptools_rust import RustExtension + +setup( + name="autonomi-client", + version="0.3.0", + description="Autonomi client API", + long_description=open("README_PYTHON.md").read(), + long_description_content_type="text/markdown", + author="MaidSafe Developers", + author_email="dev@maidsafe.net", + url="https://github.com/maidsafe/autonomi", + rust_extensions=[ + RustExtension( + "autonomi_client.autonomi_client", + "Cargo.toml", + features=["extension-module"], + py_limited_api=True, + debug=False, + ) + ], + packages=["autonomi_client"], + package_dir={"": "python"}, + zip_safe=False, + python_requires=">=3.8", + classifiers=[ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + ], +) \ No newline at end of file diff --git a/autonomi/src/client/transactions.rs b/autonomi/src/client/linked_list.rs similarity index 91% rename from autonomi/src/client/transactions.rs rename to autonomi/src/client/linked_list.rs index 1585709960..a3a3a359c4 100644 --- a/autonomi/src/client/transactions.rs +++ b/autonomi/src/client/linked_list.rs @@ -13,8 +13,8 @@ use crate::client::UploadSummary; use ant_evm::Amount; use ant_evm::AttoTokens; -pub use ant_protocol::storage::Transaction; -use ant_protocol::storage::TransactionAddress; +pub use ant_protocol::storage::LinkedList; +use ant_protocol::storage::LinkedListAddress; pub use bls::SecretKey; use ant_evm::{EvmWallet, EvmWalletError}; @@ -44,23 +44,23 @@ pub enum TransactionError { #[error("Received invalid quote from node, this node is possibly malfunctioning, try another node by trying another transaction name")] InvalidQuote, #[error("Transaction already exists at this address: {0:?}")] - TransactionAlreadyExists(TransactionAddress), + TransactionAlreadyExists(LinkedListAddress), } impl Client { /// Fetches a Transaction from the network. pub async fn transaction_get( &self, - address: TransactionAddress, - ) -> Result, TransactionError> { - let transactions = self.network.get_transactions(address).await?; + address: LinkedListAddress, + ) -> Result, TransactionError> { + let transactions = self.network.get_linked_list(address).await?; Ok(transactions) } pub async fn transaction_put( &self, - transaction: Transaction, + transaction: LinkedList, wallet: &EvmWallet, ) -> Result<(), TransactionError> { let address = transaction.address(); @@ -88,8 +88,8 @@ impl Client { // prepare the record for network storage let payees = proof.payees(); let record = Record { - key: NetworkAddress::from_transaction_address(address).to_record_key(), - value: try_serialize_record(&(proof, &transaction), RecordKind::TransactionWithPayment) + key: NetworkAddress::from_linked_list_address(address).to_record_key(), + value: try_serialize_record(&(proof, &transaction), RecordKind::LinkedListWithPayment) .map_err(|_| TransactionError::Serialization)? .to_vec(), publisher: None, @@ -137,7 +137,7 @@ impl Client { let pk = key.public_key(); trace!("Getting cost for transaction of {pk:?}"); - let address = TransactionAddress::from_owner(pk); + let address = LinkedListAddress::from_owner(pk); let xor = *address.xorname(); let store_quote = self.get_store_quotes(std::iter::once(xor)).await?; let total_cost = AttoTokens::from_atto( diff --git a/autonomi/src/client/mod.rs b/autonomi/src/client/mod.rs index 699a98703f..73e1add961 100644 --- a/autonomi/src/client/mod.rs +++ b/autonomi/src/client/mod.rs @@ -15,7 +15,8 @@ pub mod quote; pub mod data; pub mod files; -pub mod transactions; +pub mod linked_list; +pub mod pointer; #[cfg(feature = "external-signer")] #[cfg_attr(docsrs, doc(cfg(feature = "external-signer")))] diff --git a/autonomi/src/client/pointer.rs b/autonomi/src/client/pointer.rs new file mode 100644 index 0000000000..ce2c3f4462 --- /dev/null +++ b/autonomi/src/client/pointer.rs @@ -0,0 +1,138 @@ +use crate::client::Client; +use crate::client::data::PayError; +use tracing::{debug, error, trace}; + +use ant_evm::{Amount, AttoTokens, EvmWallet, EvmWalletError}; +use ant_networking::{GetRecordCfg, NetworkError, PutRecordCfg, VerificationKind}; +use ant_protocol::{ + storage::{Pointer, PointerAddress, RecordKind, RetryStrategy, try_serialize_record}, + NetworkAddress, +}; +use bls::SecretKey; +use libp2p::kad::{Quorum, Record}; + +use super::data::CostError; + +#[derive(Debug, thiserror::Error)] +pub enum PointerError { + #[error("Cost error: {0}")] + Cost(#[from] CostError), + #[error("Network error")] + Network(#[from] NetworkError), + #[error("Serialization error")] + Serialization, + #[error("Pointer could not be verified (corrupt)")] + Corrupt, + #[error("Payment failure occurred during pointer creation.")] + Pay(#[from] PayError), + #[error("Failed to retrieve wallet payment")] + Wallet(#[from] EvmWalletError), + #[error("Received invalid quote from node, this node is possibly malfunctioning, try another node by trying another pointer name")] + InvalidQuote, + #[error("Pointer already exists at this address: {0:?}")] + PointerAlreadyExists(PointerAddress), +} + +impl Client { + /// Get a pointer from the network + pub async fn pointer_get( + &self, + address: PointerAddress, + ) -> Result { + let key = NetworkAddress::from_pointer_address(address).to_record_key(); + let record = self.network.get_local_record(&key).await?; + + match record { + Some(record) => { + let (_, pointer): (Vec, Pointer) = rmp_serde::from_slice(&record.value) + .map_err(|_| PointerError::Serialization)?; + Ok(pointer) + } + None => Err(PointerError::Corrupt), + } + } + + /// Store a pointer on the network + pub async fn pointer_put( + &self, + pointer: Pointer, + wallet: &EvmWallet, + ) -> Result<(), PointerError> { + let address = pointer.network_address(); + + // pay for the pointer storage + let xor_name = *address.xorname(); + debug!("Paying for pointer at address: {address:?}"); + let payment_proofs = self + .pay(std::iter::once(xor_name), wallet) + .await + .inspect_err(|err| { + error!("Failed to pay for pointer at address: {address:?} : {err}") + })?; + + // verify payment was successful + let (proof, _price) = match payment_proofs.get(&xor_name) { + Some((proof, price)) => (proof, price), + None => { + error!("Pointer at address: {address:?} was already paid for"); + return Err(PointerError::PointerAlreadyExists(address)); + } + }; + + let payees = proof.payees(); + + let record = Record { + key: NetworkAddress::from_pointer_address(address).to_record_key(), + value: try_serialize_record(&(proof, &pointer), RecordKind::PointerWithPayment) + .map_err(|_| PointerError::Serialization)? + .to_vec(), + publisher: None, + expires: None, + }; + + let get_cfg = GetRecordCfg { + get_quorum: Quorum::Majority, + retry_strategy: Some(RetryStrategy::default()), + target_record: None, + expected_holders: Default::default(), + is_register: false, + }; + + let put_cfg = PutRecordCfg { + put_quorum: Quorum::All, + retry_strategy: None, + verification: Some((VerificationKind::Crdt, get_cfg)), + use_put_record_to: Some(payees), + }; + + // store the pointer on the network + debug!("Storing pointer at address {address:?} to the network"); + self.network + .put_record(record, &put_cfg) + .await + .inspect_err(|err| { + error!("Failed to put record - pointer {address:?} to the network: {err}") + })?; + + Ok(()) + } + + /// Calculate the cost of storing a pointer + pub async fn pointer_cost(&self, key: SecretKey) -> Result { + let pk = key.public_key(); + trace!("Getting cost for pointer of {pk:?}"); + + let address = PointerAddress::from_owner(pk); + let xor = *address.xorname(); + let store_quote = self.get_store_quotes(std::iter::once(xor)).await?; + let total_cost = AttoTokens::from_atto( + store_quote + .0 + .values() + .map(|quote| quote.price()) + .sum::(), + ); + debug!("Calculated the cost to create pointer of {pk:?} is {total_cost}"); + Ok(total_cost) + } +} diff --git a/autonomi/src/client/vault.rs b/autonomi/src/client/vault.rs index dd69f8f9d7..f53875010f 100644 --- a/autonomi/src/client/vault.rs +++ b/autonomi/src/client/vault.rs @@ -149,7 +149,7 @@ impl Client { let client_pk = owner.public_key(); let content_type = Default::default(); let scratch = Scratchpad::new(client_pk, content_type); - let vault_xor = scratch.network_address().as_xorname().unwrap_or_default(); + let vault_xor = scratch.address().xorname(); // NB TODO: vault should be priced differently from other data let store_quote = self.get_store_quotes(std::iter::once(vault_xor)).await?; diff --git a/autonomi/src/python.rs b/autonomi/src/python.rs index 1f1c4d443b..f2dd5e1056 100644 --- a/autonomi/src/python.rs +++ b/autonomi/src/python.rs @@ -5,21 +5,27 @@ use crate::client::{ data::DataMapChunk, files::{archive::PrivateArchiveAccess, archive_public::ArchiveAddr}, payment::PaymentOption as RustPaymentOption, - vault::{UserData, VaultSecretKey}, + vault::{UserData, VaultSecretKey as RustVaultSecretKey}, Client as RustClient, }; use crate::{Bytes, Network, Wallet as RustWallet}; +use ant_protocol::storage::{ + ChunkAddress, Pointer as RustPointer, PointerAddress as RustPointerAddress, + PointerTarget as RustPointerTarget, +}; +use bls::{PublicKey as RustPublicKey, SecretKey as RustSecretKey}; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; +use rand::thread_rng; use xor_name::XorName; #[pyclass(name = "Client")] -pub(crate) struct PyClient { +pub(crate) struct Client { inner: RustClient, } #[pymethods] -impl PyClient { +impl Client { #[staticmethod] fn connect(peers: Vec) -> PyResult { let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); @@ -40,7 +46,7 @@ impl PyClient { Ok(Self { inner: client }) } - fn data_put(&self, data: Vec, payment: &PyPaymentOption) -> PyResult { + fn data_put(&self, data: Vec, payment: &PaymentOption) -> PyResult { let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); let access = rt .block_on( @@ -48,7 +54,7 @@ impl PyClient { .data_put(Bytes::from(data), payment.inner.clone()), ) .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to put private data: {e}")) + pyo3::exceptions::PyValueError::new_err(format!("Failed to put data: {e}")) })?; Ok(PyDataMapChunk { inner: access }) @@ -59,12 +65,12 @@ impl PyClient { let data = rt .block_on(self.inner.data_get(access.inner.clone())) .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to get private data: {e}")) + pyo3::exceptions::PyValueError::new_err(format!("Failed to get data: {e}")) })?; Ok(data.to_vec()) } - fn data_put_public(&self, data: Vec, payment: &PyPaymentOption) -> PyResult { + fn data_put_public(&self, data: Vec, payment: &PaymentOption) -> PyResult { let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); let addr = rt .block_on( @@ -104,7 +110,7 @@ impl PyClient { fn write_bytes_to_vault( &self, data: Vec, - payment: &PyPaymentOption, + payment: &PaymentOption, key: &PyVaultSecretKey, content_type: u64, ) -> PyResult { @@ -137,38 +143,275 @@ impl PyClient { let user_data = rt .block_on(self.inner.get_user_data_from_vault(&key.inner)) .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to get user data: {e}")) + pyo3::exceptions::PyValueError::new_err(format!( + "Failed to get user data from vault: {e}" + )) })?; + Ok(PyUserData { inner: user_data }) } fn put_user_data_to_vault( &self, key: &PyVaultSecretKey, - payment: &PyPaymentOption, + payment: &PaymentOption, user_data: &PyUserData, - ) -> PyResult { + ) -> PyResult<()> { + let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); + rt.block_on(self.inner.put_user_data_to_vault( + &key.inner, + payment.inner.clone(), + user_data.inner.clone(), + )) + .map_err(|e| { + pyo3::exceptions::PyValueError::new_err(format!("Failed to put user data: {e}")) + })?; + Ok(()) + } + + fn pointer_get(&self, address: &str) -> PyResult { + let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); + let xorname = XorName::from_content(&hex::decode(address).map_err(|e| { + pyo3::exceptions::PyValueError::new_err(format!("Invalid pointer address: {e}")) + })?); + let address = RustPointerAddress::new(xorname); + + let pointer = rt.block_on(self.inner.pointer_get(address)).map_err(|e| { + pyo3::exceptions::PyValueError::new_err(format!("Failed to get pointer: {e}")) + })?; + + Ok(PyPointer { inner: pointer }) + } + + fn pointer_put( + &self, + owner: &PyPublicKey, + counter: u32, + target: &PyPointerTarget, + key: &PySecretKey, + wallet: &Wallet, + ) -> PyResult<()> { + let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); + let pointer = RustPointer::new( + owner.inner.clone(), + counter, + target.inner.clone(), + &key.inner, + ); + rt.block_on(self.inner.pointer_put(pointer, &wallet.inner)) + .map_err(|e| PyValueError::new_err(format!("Failed to put pointer: {}", e))) + } + + fn pointer_cost(&self, key: &PySecretKey) -> PyResult { let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime"); let cost = rt - .block_on(self.inner.put_user_data_to_vault( - &key.inner, - payment.inner.clone(), - user_data.inner.clone(), - )) + .block_on(self.inner.pointer_cost(key.inner.clone())) .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to put user data: {e}")) + pyo3::exceptions::PyValueError::new_err(format!("Failed to get pointer cost: {e}")) })?; Ok(cost.to_string()) } + + fn pointer_address(&self, owner: &PyPublicKey, counter: u32) -> PyResult { + let mut rng = thread_rng(); + let pointer = RustPointer::new( + owner.inner.clone(), + counter, + RustPointerTarget::ChunkAddress(ChunkAddress::new(XorName::random(&mut rng))), + &RustSecretKey::random(), + ); + let address = pointer.network_address(); + let bytes: [u8; 32] = address.xorname().0; + Ok(hex::encode(bytes)) + } +} + +#[pyclass(name = "PointerAddress")] +#[derive(Debug, Clone)] +pub struct PyPointerAddress { + inner: RustPointerAddress, +} + +#[pymethods] +impl PyPointerAddress { + #[new] + pub fn new(hex_str: String) -> PyResult { + let bytes = hex::decode(&hex_str) + .map_err(|e| PyValueError::new_err(format!("Invalid hex string: {}", e)))?; + let xorname = XorName::from_content(&bytes); + Ok(Self { + inner: RustPointerAddress::new(xorname), + }) + } + + #[getter] + pub fn hex(&self) -> String { + let bytes: [u8; 32] = self.inner.xorname().0; + hex::encode(bytes) + } +} + +#[pyclass(name = "Pointer")] +#[derive(Debug, Clone)] +pub struct PyPointer { + inner: RustPointer, +} + +#[pymethods] +impl PyPointer { + #[new] + pub fn new( + owner: &PyPublicKey, + counter: u32, + target: &PyPointerTarget, + key: &PySecretKey, + ) -> PyResult { + Ok(Self { + inner: RustPointer::new( + owner.inner.clone(), + counter, + target.inner.clone(), + &key.inner, + ), + }) + } + + pub fn network_address(&self) -> PyPointerAddress { + PyPointerAddress { + inner: self.inner.network_address(), + } + } + + #[getter] + fn hex(&self) -> String { + let bytes: [u8; 32] = self.inner.xorname().0; + hex::encode(bytes) + } + + #[getter] + fn target(&self) -> PyPointerTarget { + PyPointerTarget { + inner: RustPointerTarget::ChunkAddress(ChunkAddress::new(self.inner.xorname().clone())), + } + } +} + +#[pyclass(name = "PointerTarget")] +#[derive(Debug, Clone)] +pub struct PyPointerTarget { + inner: RustPointerTarget, +} + +#[pymethods] +impl PyPointerTarget { + #[new] + fn new(xorname: &[u8]) -> PyResult { + Ok(Self { + inner: RustPointerTarget::ChunkAddress(ChunkAddress::new(XorName::from_content( + xorname, + ))), + }) + } + + #[getter] + fn hex(&self) -> String { + let bytes: [u8; 32] = self.inner.xorname().0; + hex::encode(bytes) + } + + #[getter] + fn target(&self) -> PyPointerTarget { + PyPointerTarget { + inner: RustPointerTarget::ChunkAddress(ChunkAddress::new(self.inner.xorname().clone())), + } + } + + #[staticmethod] + fn from_xorname(xorname: &[u8]) -> PyResult { + Ok(Self { + inner: RustPointerTarget::ChunkAddress(ChunkAddress::new(XorName::from_content( + xorname, + ))), + }) + } + + #[staticmethod] + fn from_chunk_address(addr: &PyChunkAddress) -> Self { + Self { + inner: RustPointerTarget::ChunkAddress(addr.inner.clone()), + } + } +} + +#[pyclass(name = "ChunkAddress")] +#[derive(Debug, Clone)] +pub struct PyChunkAddress { + inner: ChunkAddress, +} + +impl From for PyChunkAddress { + fn from(addr: ChunkAddress) -> Self { + Self { inner: addr } + } +} + +impl From for ChunkAddress { + fn from(addr: PyChunkAddress) -> Self { + addr.inner + } +} + +#[pymethods] +impl PyChunkAddress { + #[new] + fn new(xorname: &[u8]) -> PyResult { + Ok(Self { + inner: ChunkAddress::new(XorName::from_content(xorname)), + }) + } + + #[getter] + fn hex(&self) -> String { + let bytes: [u8; 32] = self.inner.xorname().0; + hex::encode(bytes) + } + + #[staticmethod] + fn from_chunk_address(addr: &str) -> PyResult { + let bytes = hex::decode(addr).map_err(|e| { + pyo3::exceptions::PyValueError::new_err(format!("Invalid chunk address: {e}")) + })?; + + if bytes.len() != 32 { + return Err(pyo3::exceptions::PyValueError::new_err( + "Invalid chunk address length: must be 32 bytes", + )); + } + + let mut xorname = [0u8; 32]; + xorname.copy_from_slice(&bytes); + + Ok(Self { + inner: ChunkAddress::new(XorName(xorname)), + }) + } + + fn __str__(&self) -> PyResult { + Ok(self.hex()) + } + + fn __repr__(&self) -> PyResult { + Ok(format!("ChunkAddress({})", self.hex())) + } } #[pyclass(name = "Wallet")] -pub(crate) struct PyWallet { - inner: RustWallet, +pub struct Wallet { + pub(crate) inner: RustWallet, } #[pymethods] -impl PyWallet { +impl Wallet { #[new] fn new(private_key: String) -> PyResult { let wallet = RustWallet::new_from_private_key( @@ -210,23 +453,79 @@ impl PyWallet { } #[pyclass(name = "PaymentOption")] -pub(crate) struct PyPaymentOption { - inner: RustPaymentOption, +pub struct PaymentOption { + pub(crate) inner: RustPaymentOption, } #[pymethods] -impl PyPaymentOption { +impl PaymentOption { #[staticmethod] - fn wallet(wallet: &PyWallet) -> Self { + fn wallet(wallet: &Wallet) -> Self { Self { inner: RustPaymentOption::Wallet(wallet.inner.clone()), } } } +#[pyclass(name = "SecretKey")] +#[derive(Debug, Clone)] +pub struct PySecretKey { + inner: RustSecretKey, +} + +#[pymethods] +impl PySecretKey { + #[new] + fn new() -> PyResult { + Ok(Self { + inner: RustSecretKey::random(), + }) + } + + #[staticmethod] + fn from_hex(hex_str: &str) -> PyResult { + RustSecretKey::from_hex(hex_str) + .map(|key| Self { inner: key }) + .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Invalid hex key: {e}"))) + } + + fn to_hex(&self) -> String { + self.inner.to_hex() + } +} + +#[pyclass(name = "PublicKey")] +#[derive(Debug, Clone)] +pub struct PyPublicKey { + inner: RustPublicKey, +} + +#[pymethods] +impl PyPublicKey { + #[new] + fn new() -> PyResult { + let secret = RustSecretKey::random(); + Ok(Self { + inner: secret.public_key(), + }) + } + + #[staticmethod] + fn from_hex(hex_str: &str) -> PyResult { + RustPublicKey::from_hex(hex_str) + .map(|key| Self { inner: key }) + .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Invalid hex key: {e}"))) + } + + fn to_hex(&self) -> String { + self.inner.to_hex() + } +} + #[pyclass(name = "VaultSecretKey")] -pub(crate) struct PyVaultSecretKey { - inner: VaultSecretKey, +#[derive(Debug, Clone)] +pub struct PyVaultSecretKey { + inner: RustVaultSecretKey, } #[pymethods] @@ -234,13 +533,13 @@ impl PyVaultSecretKey { #[new] fn new() -> PyResult { Ok(Self { - inner: VaultSecretKey::random(), + inner: RustVaultSecretKey::random(), }) } #[staticmethod] fn from_hex(hex_str: &str) -> PyResult { - VaultSecretKey::from_hex(hex_str) + RustVaultSecretKey::from_hex(hex_str) .map(|key| Self { inner: key }) .map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Invalid hex key: {e}"))) } @@ -251,7 +550,8 @@ impl PyVaultSecretKey { } #[pyclass(name = "UserData")] -pub(crate) struct PyUserData { +#[derive(Debug, Clone)] +pub struct PyUserData { inner: UserData, } @@ -297,8 +597,8 @@ impl PyUserData { } #[pyclass(name = "DataMapChunk")] -#[derive(Clone)] -pub(crate) struct PyDataMapChunk { +#[derive(Debug, Clone)] +pub struct PyDataMapChunk { inner: DataMapChunk, } @@ -339,12 +639,18 @@ fn encrypt(data: Vec) -> PyResult<(Vec, Vec>)> { #[pymodule] #[pyo3(name = "autonomi_client")] fn autonomi_client_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_function(wrap_pyfunction!(encrypt, m)?)?; Ok(()) } diff --git a/autonomi/tests/python/conftest.py b/autonomi/tests/python/conftest.py new file mode 100644 index 0000000000..cbbd1c6f2b --- /dev/null +++ b/autonomi/tests/python/conftest.py @@ -0,0 +1,5 @@ +import os +import sys + +# Add the project root to Python path +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) \ No newline at end of file diff --git a/autonomi/tests/python/test_bindings.py b/autonomi/tests/python/test_bindings.py new file mode 100644 index 0000000000..ce1d37cd10 --- /dev/null +++ b/autonomi/tests/python/test_bindings.py @@ -0,0 +1,92 @@ +import pytest +from autonomi_client import ( + ChunkAddress, + PointerTarget, + Pointer, + PointerAddress, + SecretKey, + PublicKey, + Wallet +) + +def test_chunk_address_creation(): + # Test creating a ChunkAddress from bytes + test_data = b"test data for chunk address" + chunk_addr = ChunkAddress(test_data) + + # Test hex representation + hex_str = chunk_addr.hex + assert isinstance(hex_str, str) + assert len(hex_str) == 64 # 32 bytes = 64 hex chars + + # Test string representation + str_repr = str(chunk_addr) + assert str_repr == hex_str + + # Test repr + repr_str = repr(chunk_addr) + assert repr_str == f"ChunkAddress({hex_str})" + +def test_chunk_address_from_hex(): + # Create a chunk address + original = ChunkAddress(b"test data") + hex_str = original.hex + + # Create new chunk address from hex + recreated = ChunkAddress.from_chunk_address(hex_str) + assert recreated.hex == hex_str + +def test_pointer_target_with_chunk_address(): + # Create a chunk address + chunk_addr = ChunkAddress(b"test data for pointer target") + + # Create pointer target from chunk address + target = PointerTarget.from_chunk_address(chunk_addr) + + # Verify the hex matches + assert isinstance(target.hex, str) + assert len(target.hex) == 64 + +def test_pointer_creation(): + # Create necessary components + owner = PublicKey() + counter = 42 + chunk_addr = ChunkAddress(b"test data for pointer") + target = PointerTarget.from_chunk_address(chunk_addr) + key = SecretKey() + + # Create pointer + pointer = Pointer(owner, counter, target, key) + + # Verify pointer properties + assert isinstance(pointer.hex, str) + assert len(pointer.hex) == 64 + + # Test network address + addr = pointer.network_address() + assert isinstance(addr, PointerAddress) + assert isinstance(addr.hex, str) + assert len(addr.hex) == 64 + +def test_pointer_target_creation(): + # Test direct creation + test_data = b"test data for pointer target" + target = PointerTarget(test_data) + + # Verify hex + assert isinstance(target.hex, str) + assert len(target.hex) == 64 + + # Test from_xorname + target2 = PointerTarget.from_xorname(test_data) + assert isinstance(target2.hex, str) + assert len(target2.hex) == 64 + +def test_invalid_hex(): + # Test invalid hex string for chunk address + with pytest.raises(ValueError): + ChunkAddress.from_chunk_address("invalid hex") + + # Test invalid hex string for pointer address + with pytest.raises(ValueError): + PointerAddress("invalid hex") \ No newline at end of file diff --git a/autonomi/tests/transaction.rs b/autonomi/tests/transaction.rs index b0523618b3..af25785126 100644 --- a/autonomi/tests/transaction.rs +++ b/autonomi/tests/transaction.rs @@ -7,8 +7,8 @@ // permissions and limitations relating to use of the SAFE Network Software. use ant_logging::LogBuilder; -use ant_protocol::storage::Transaction; -use autonomi::{client::transactions::TransactionError, Client}; +use ant_protocol::storage::LinkedList; +use autonomi::{client::linked_list::TransactionError, Client}; use eyre::Result; use test_utils::evm::get_funded_wallet; @@ -21,7 +21,7 @@ async fn transaction_put() -> Result<()> { let key = bls::SecretKey::random(); let content = [0u8; 32]; - let transaction = Transaction::new(key.public_key(), vec![], content, vec![], &key); + let transaction = LinkedList::new(key.public_key(), vec![], content, vec![].into(), &key); // estimate the cost of the transaction let cost = client.transaction_cost(key.clone()).await?; @@ -41,7 +41,7 @@ async fn transaction_put() -> Result<()> { // try put another transaction with the same address let content2 = [1u8; 32]; - let transaction2 = Transaction::new(key.public_key(), vec![], content2, vec![], &key); + let transaction2 = LinkedList::new(key.public_key(), vec![], content2, vec![].into(), &key); let res = client.transaction_put(transaction2.clone(), &wallet).await; assert!(matches!( diff --git a/docs/pointer_design_doc.md b/docs/pointer_design_doc.md new file mode 100644 index 0000000000..390887d1e4 --- /dev/null +++ b/docs/pointer_design_doc.md @@ -0,0 +1,75 @@ +# Pointer Data Type Design Document + +## Overview + +The `Pointer` data type is designed to represent a reference to a `LinkedList` in the system. It will include metadata such as the owner, a counter, and a signature to ensure data integrity and authenticity. + +## Structure + +```rust +struct Pointer { + owner: PubKey, // This is the address of this data type + counter: U32, + target: PointerTarget, // Can be PointerAddress, LinkedListAddress, ChunksAddress, or ScratchpadAddress + signature: Sig, // Signature of counter and pointer (and target) +} +``` + +## Pointer Target + +The `PointerTarget` enum will define the possible target types for a `Pointer`: + +```rust +enum PointerTarget { + PointerAddress(PointerAddress), + LinkedListAddress(LinkedListAddress), + ChunkAddress(ChunkAddress), + ScratchpadAddress(ScratchpadAddress), +} +``` + +## Detailed Implementation and Testing Strategy + +1. **Define the `Pointer` Struct**: + - Implement the `Pointer` struct in a new Rust file alongside `linked_list.rs`. + - **Testing**: Write unit tests to ensure the struct is correctly defined and can be instantiated. + +2. **Address Handling**: + - Implement address handling similar to `LinkedListAddress`. + - **Testing**: Verify address conversion and serialization through unit tests. + +3. **Integration with `record_store.rs`**: + - Ensure that the `Pointer` type is properly integrated into the `record_store.rs` to handle storage and retrieval operations. + - **Testing**: Use integration tests to confirm that `Pointer` records can be stored and retrieved correctly. + +4. **Signature Verification**: + - Implement methods to sign and verify the `Pointer` data using the owner's private key. + - **Testing**: Write tests to validate the signature creation and verification process. + +5. **Output Handling**: + - The `Pointer` will point to a `LinkedList`, and the `LinkedList` output will be used as the value. If there is more than one output, the return will be a vector of possible values. + - **Testing**: Test the output handling logic to ensure it returns the correct values. + +6. **Integration with ant-networking**: + - Implement methods to serialize and deserialize `Pointer` records, similar to how `LinkedList` records are handled. + - Ensure that the `Pointer` type is supported in the `NodeRecordStore` for storage and retrieval operations. + - **Testing**: Conduct end-to-end tests to verify the integration with `ant-networking`. + +7. **Payment Handling**: + - Introduce `RecordKind::PointerWithPayment` to handle `Pointer` records with payments. + - Implement logic to process `Pointer` records with payments, similar to `LinkedListWithPayment`. + - **Testing**: Test the payment processing logic to ensure it handles payments correctly. + +8. **Documentation and Review**: + - Update documentation to reflect the new `Pointer` type and its usage. + - Conduct code reviews to ensure quality and adherence to best practices. + +## Next Steps + +- Develop a detailed implementation plan for each component. +- Identify any additional dependencies or libraries required. +- Plan for testing and validation of the `Pointer` data type. + +## Conclusion + +The `Pointer` data type will enhance the system's ability to reference and manage `LinkedList` structures efficiently. Further details will be added as the implementation progresses.