Time out error when running get_quote_and_pay for storing values on the testnet

[emanuel-skai]

Hi,

I'm updating my Python client script so it connects and uses the testnet for an RAG application.

My script currently looks as follows

#!/usr/bin/env python

import asyncio
import logging
import os
import shutil
import sys
import tempfile
import uuid

import psutil
import py_nillion_client as nillion
import pytest
from dotenv import load_dotenv

sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))

from cosmpy.aerial.client import LedgerClient
from cosmpy.aerial.wallet import LocalWallet
from cosmpy.crypto.keypairs import PrivateKey
from dotenv import load_dotenv, find_dotenv
from nillion_python_helpers import (
    create_nillion_client,
    create_payments_config,
    get_quote_and_pay,
    get_quote, 
    pay_with_quote
)
from py_nillion_client import NodeKey, UserKey

from helpers.nillion_client_helper import create_nillion_client
from helpers.nillion_keypath_helper import getNodeKeyFromFile, getUserKeyFromFile
from helpers.nillion_secrets_inputs import merge_dictionaries

home = os.getenv("HOME")
#load_dotenv(f"{home}/.config/nillion/nillion-devnet.env")
load_dotenv()

project_directory = os.path.dirname(os.path.abspath(__file__))
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

def clear_env_variables():
    """
    Clear the existing environment variables loaded by the previous .env file.
    """
    for key in list(os.environ.keys()):
        if key.startswith('NILLION_'):
            del os.environ[key]

def log_environment_variables(env_path):
    """
    Log environment variables loaded from the specified .env file.
    """
    logger.info(f"Logging environment variables from {env_path}...")
    for key, value in os.environ.items():
        if key.startswith("NILLION_"):
            logger.info(f"{key}={value}")

def load_new_env():
    """
    Clear old environment variables and load the new .env file from the root of the app.
    """
    clear_env_variables()
    
    # Find and load the new .env file from the root of the app
    dotenv_path = find_dotenv()
    if dotenv_path:
        logger.info(f"Loading .env file from: {dotenv_path}")
        load_dotenv(dotenv_path)
        log_environment_variables(dotenv_path)  # Log the path and variables
    else:
        logger.info("No .env file found at the root of the app")

async def get_quote_and_pay_with_timeout(client, operation, payments_wallet, payments_client, cluster_id, timeout=300):
    try:
        return await asyncio.wait_for(
            get_quote_and_pay(client, operation, payments_wallet, payments_client, cluster_id),
            timeout=timeout
        )
    except asyncio.TimeoutError as e:
        logger.error(f"Operation timed out after {timeout} seconds")
        raise Exception(f"Operation timed out after {timeout} seconds") from e

async def run_euclidean_distance(nillion_user_key, *secret_inputs):
    logger.info("Running euclidean_distance Client...")
    load_new_env()  # Load new environment variables
    try:
        # Ensure old environment variables are cleared (if necessary)
        cluster_id = os.getenv("NILLION_CLUSTER_ID")
        grpc_endpoint = os.getenv("NILLION_NILCHAIN_GRPC")
        chain_id = os.getenv("NILLION_NILCHAIN_CHAIN_ID")

        logger.info(f"** Cluster ID: {cluster_id}")
        logger.info(f"** GRPC Endpoint: {grpc_endpoint}")
        logger.info(f"** Chain ID: {chain_id}")

        userkey_path = os.getenv("NILLION_USERKEY_PATH_PARTY_1")
        nodekey_path = os.getenv("NILLION_NODEKEY_PATH_PARTY_1")

        logger.info(f"Userkey path: {userkey_path}")
        logger.info(f"Nodekey path: {nodekey_path}")
        logger.info(f"Cluster ID: {cluster_id}")

        # 1.2 pick a seed and generate user and node keys
        seed = "42"
        userkey = UserKey.from_seed(nillion_user_key)
        nodekey = NodeKey.from_seed(f"seed-{str(uuid.uuid4())}")

        logger.info(f"Userkey: {userkey}")
        logger.info(f"Nodekey: {nodekey}")
    except Exception as e:
        logger.error("Error getting environment variables from .env file")
        logger.error(f"Error is {e}")
        raise e

    try:
        # 1. Initialize NillionClient against nillion-devnet
        client = create_nillion_client(userkey, nodekey)
        logger.info("Created Nillion Client")

        # 2. Get the user id and party id from NillionClient
        party_id = client.party_id
        user_id = client.user_id
        logger.info(f"Got user_id: {user_id}")
        logger.info(f"Got party_id: {party_id}")

        # 3. Store a compiled Nada program in the network
        program_name = "euclidean_distance" + str(uuid.uuid4())
        script_dir = os.path.dirname(os.path.abspath(__file__))
        program_mir_rel_path = "../nada_programs/target/euclidean_distance.nada.bin"
        program_mir_abs_path = os.path.abspath(
            os.path.join(script_dir, program_mir_rel_path)
        )
        logger.info(f"Program MIR Absolute Path: {program_mir_abs_path}")
    except Exception as e:
        logger.error("Error initializing NillionClient and defining program")
        logger.error(f"Error is {e}")
        raise e

    try:
        # 4. Create payments config, client and wallet
        payments_config = create_payments_config(chain_id, grpc_endpoint)
        payments_client = LedgerClient(payments_config)
        payments_wallet = LocalWallet(
            PrivateKey(bytes.fromhex(os.getenv("NILLION_NILCHAIN_PRIVATE_KEY"))),
            prefix="nillion",
        )
    except Exception as e:
        logger.error("Error creating payments config, client and wallet")
        logger.error(f"Error is: {e}")
        raise e

    try:
        # Pay to store the program and obtain a receipt of the payment
        receipt_store_program = await get_quote_and_pay_with_timeout(
            client,
            nillion.Operation.store_program(program_mir_abs_path),
            payments_wallet,
            payments_client,
            cluster_id,
            timeout=300
        )
    except Exception as e:
        logger.error("Error paying to store the program and obtaining a receipt")
        logger.error(f"Error is: {e}")
        raise e

    try:
        # Store the program
        action_id = await client.store_program(
            cluster_id, program_name, program_mir_abs_path, receipt_store_program
        )
        # Create a variable for the program_id, which is the {user_id}/{program_name}. We will need this later
        program_id = f"{user_id}/{program_name}"
        logger.info(f"Stored program. action_id: {action_id}")
        logger.info(f"Stored program_id: {program_id}")
    except Exception as e:
        logger.error("Error storing program on the network")
        logger.error(f"Error is {e}")
        raise e

    try:
        # 5. Create the 1st secret with bindings to the program
        nillion_compute_secrets, nillion_compute_secrets_raw = merge_dictionaries(
            *secret_inputs
        )

        party_name = "Party1"
        # Set permissions for the client to compute on the program
        permissions = nillion.Permissions.default_for_user(client.user_id)
        permissions.add_compute_permissions({client.user_id: {program_id}})
        logger.info(f"Permissions: {permissions}")
        # Pay for and store the secret in the network and print the returned store_id
        logger.info("<<<Storing secret>>>...")
        # logger.info(f"Secrets: {nillion_compute_secrets_raw}")
        logger.info(f'client: {client}')
        logger.info(f'operation: {nillion.Operation.store_values(nillion_compute_secrets, ttl_days=5)}')    
        logger.info(f'payments_wallet: {payments_wallet}')
        logger.info(f'payments_client: {payments_client}')
        logger.info(f'cluster_id: {cluster_id}')
    except Exception as e:
        logger.error("Error creating the 1st secret with bindings to the program")
        logger.error(f"Error is {e}")
        raise e

    try:
        timeout = 300
        
        receipt_store  = await get_quote_and_pay( client, 
                                    nillion.Operation.store_values(nillion_compute_secrets, ttl_days=5),
                                    payments_wallet, 
                                    payments_client, 
                                    cluster_id
                                    )
        '''
        secrets_quote =  await get_quote(client, 
                                         nillion.Operation.store_values(nillion_compute_secrets, ttl_days=5), 
                                        cluster_id=cluster_id)
                    
        logger.info(f"Secrets Quote: {secrets_quote}") 
        receipt_store = await pay_with_quote(client, secrets_quote, payments_wallet, payments_client)    
        '''
        logger.info(f"Stored secret. receipt_store: {receipt_store}")
    except asyncio.TimeoutError as e:
        logger.error("Timeout occurred while storing the secret.")
        raise e
    except Exception as e:
        logger.error("Error storing secret on the network")
        logger.error(f"Error is {e}")
        raise e
    try :
        # Store a secret
        store_id = await client.store_values(
            cluster_id, nillion_compute_secrets, permissions, receipt_store
        )
        print(f"Computing using program {program_id}")
        print(f"Use secret store_id: {store_id}")
    except Exception as e:
        logger.error("Error storing secret on the network")
        logger.error(f"Error is {e}")
        raise e

    try:
        # 6. Pay for and action the computation
        compute_bindings = nillion.ProgramBindings(program_id)
        compute_bindings.add_input_party(party_name, party_id)
        compute_bindings.add_output_party(party_name, party_id)

        # Add my_int2, the 2nd secret at computation time
        computation_time_secrets = nillion.NadaValues({})

        # Pay for the compute
        receipt_compute = await get_quote_and_pay(
            client,
            nillion.Operation.compute(program_id, computation_time_secrets),
            payments_wallet,
            payments_client,
            cluster_id,
        )

        # Compute on the secret
        compute_id = await client.compute(
            cluster_id,
            compute_bindings,
            [store_id],
            computation_time_secrets,
            receipt_compute,
        )
    except Exception as e:
        logger.error("Error paying for and actioning the computation")
        logger.error(f"error is {e}")
        raise e
    try:
        # 7. Return the result of the computation
        print(f"The computation was sent to the network. compute_id: {compute_id}")
        while True:
            compute_event = await client.next_compute_event()
            if isinstance(compute_event, nillion.ComputeFinishedEvent):
                print(f"✅  Compute complete for compute_id {compute_event.uuid}")
                print(f"🖥️  The result is {compute_event.result.value}")
                return compute_event.result.value

    except Exception as e:
        logger.error("Error running euclidean_distance client")
        logger.error(e)
        return
    finally:
        # log_open_files_and_connections("final")
        pass


if __name__ == "__main__":
    asyncio.run(run_euclidean_distance())


@pytest.mark.asyncio
async def test_main():
    result = await run_euclidean_distance()
    assert result == {
        "closest_document_index": 0,
        "distance0": 6245790988,
        "distance1": 15476544776,
        "distance2": 18906393104,
    }

I've configured my .env file to wi9th the testnet network configuration described here: https://docs.nillion.com/network-configuration

However, I'm facing some time out issues when running.

        receipt_store  = await get_quote_and_pay( client, 
                                    nillion.Operation.store_values(nillion_compute_secrets, ttl_days=5),
                                    payments_wallet, 
                                    payments_client, 
                                    cluster_id
                                    )

The logs produced by my application are the following.

INFO:web_app.clients.euclidean_distance:Running euclidean_distance Client...
INFO:web_app.clients.euclidean_distance:Loading .env file from: /Users/emanuelhc/Desktop/code/nillion-mvp/mvp/services/nillion/.env
INFO:web_app.clients.euclidean_distance:Logging environment variables from /Users/emanuelhc/Desktop/code/nillion-mvp/mvp/services/nillion/.env...
INFO:web_app.clients.euclidean_distance:NILLION_CLUSTER_ID=b13880d3-dde8-4a75-a171-8a1a9d985e6c
INFO:web_app.clients.euclidean_distance:NILLION_BOOTNODE_MULTIADDRESS=/dns/node-1.testnet-photon.nillion-network.nilogy.xyz/tcp/14111/p2p/12D3KooWCfFYAb77NCjEk711e9BVe2E6mrasPZTtAjJAPtVAdbye
INFO:web_app.clients.euclidean_distance:NILLION_BOOTNODE_WEBSOCKET=/dns/node-1.testnet-photon.nillion-network.nilogy.xyz/tcp/14211/wss/p2p/12D3KooWCfFYAb77NCjEk711e9BVe2E6mrasPZTtAjJAPtVAdbye
INFO:web_app.clients.euclidean_distance:NILLION_NILCHAIN_CHAIN_ID=nillion-chain-testnet-1
INFO:web_app.clients.euclidean_distance:NILLION_NILCHAIN_JSON_RPC=http://65.109.222.111:26657
INFO:web_app.clients.euclidean_distance:NILLION_NILCHAIN_REST_API=http://65.109.222.111:26657
INFO:web_app.clients.euclidean_distance:NILLION_NILCHAIN_GRPC=65.109.228.73:9090

INFO:web_app.clients.euclidean_distance:Cluster ID: b13880d3-dde8-4a75-a171-8a1a9d985e6c
INFO:web_app.clients.euclidean_distance:Userkey: <builtins.UserKey object at 0x34f2df110>
INFO:web_app.clients.euclidean_distance:Nodekey: <builtins.NodeKey object at 0x1044acb30>
INFO:web_app.clients.euclidean_distance:Created Nillion Client
INFO:web_app.clients.euclidean_distance:Got user_id: 4h9pG29W8rkqgskhDxJtbmFS1aZUZRHNj6nqEtdTC5EvWrKu8WdH4X4yC2mG3tqEVicTvrWL6FUxRQxGQPCnry3C
INFO:web_app.clients.euclidean_distance:Got party_id: 12D3KooWPMQZ2SPSSdFt63uxCSXL7e6Ai4roJfnmPugcA3VbVAaL
INFO:web_app.clients.euclidean_distance:Program MIR Absolute Path: /Users/emanuelhc/Desktop/code/nillion-mvp/mvp/services/nillion/web_app/nada_programs/target/euclidean_distance.nada.bin
INFO:web_app.clients.euclidean_distance:Stored program. action_id: 4h9pG29W8rkqgskhDxJtbmFS1aZUZRHNj6nqEtdTC5EvWrKu8WdH4X4yC2mG3tqEVicTvrWL6FUxRQxGQPCnry3C/euclidean_distance52ed642a-bb02-45ae-826d-3214c2ef31f4
INFO:web_app.clients.euclidean_distance:Stored program_id: 4h9pG29W8rkqgskhDxJtbmFS1aZUZRHNj6nqEtdTC5EvWrKu8WdH4X4yC2mG3tqEVicTvrWL6FUxRQxGQPCnry3C/euclidean_distance52ed642a-bb02-45ae-826d-3214c2ef31f4
INFO:web_app.clients.euclidean_distance:Permissions: <builtins.Permissions object at 0x39cc44120>
INFO:web_app.clients.euclidean_distance:<<<Storing secret>>>...
INFO:web_app.clients.euclidean_distance:client: <builtins.NillionClient object at 0x104415710>
INFO:web_app.clients.euclidean_distance:operation: <builtins.Operation object at 0x39cc5dd80>
INFO:web_app.clients.euclidean_distance:payments_wallet: nillion1zqfj3cujymt2t2ykyvsac9f8lc6a2sh2mqzjye
INFO:web_app.clients.euclidean_distance:payments_client: <cosmpy.aerial.client.LedgerClient object at 0x351d629f0>
INFO:web_app.clients.euclidean_distance:cluster_id: b13880d3-dde8-4a75-a171-8a1a9d985e6c
ERROR:web_app.clients.euclidean_distance:Error storing secret on the network
ERROR:web_app.clients.euclidean_distance:Error is requesting price quote: price quote query failed: price quote query failed: timed out
ERROR:web_app.main:Error running euclidean_distance client: requesting price quote: price quote query failed: price quote query failed: timed out

Notice how I'm able to store properly the Nada program, however it times out when trying to obtain the quote for storing the compute values.

I wonder if there's any way I can increase the timeout setting for this operation.

[Davetbutler]

Hey. Interesting. A couple of things to check first:

• Are you using the latest public version of the SDK? nilup use latest will do it.
• Can you try to run a simple example using the testnet configs you mention. I am surprised that you are able to store a program, but not store secrets. How many secrets are you trying to store here? Maybe you are trying to store too many for the public testnet.

I just ran some simple examples on the testnet configs you pointed to, and everything seemed fine to me.

[emanuel-skai]

Thanks for the quick response, @Davetbutler.

I am indeed using the latest version of the SDK as shown below:

(.venv) (base) emanuelhc@Emanuels-MBP nada_programs % nilup use latest     
SDK version latest set as default

Originally, I was trying to store 4 vectors, each with 384 entries, totaling 1536 secret values. However, following your suggestion, I decreased the number of entries to 3 per vector, resulting in a total of 12 secret values. Unfortunately, I continue to see the same error as shown below.

INFO:web_app.clients.euclidean_distance:Running euclidean_distance Client...
INFO:web_app.clients.euclidean_distance:Loading .env file from: /Users/emanuelhc/Desktop/code/nillion-mvp/mvp/services/nillion/.env
INFO:web_app.clients.euclidean_distance:Logging environment variables from /Users/emanuelhc/Desktop/code/nillion-mvp/mvp/services/nillion/.env...
INFO:web_app.clients.euclidean_distance:NILLION_CLUSTER_ID=b13880d3-dde8-4a75-a171-8a1a9d985e6c
INFO:web_app.clients.euclidean_distance:NILLION_BOOTNODE_MULTIADDRESS=/dns/node-1.testnet-photon.nillion-network.nilogy.xyz/tcp/14111/p2p/12D3KooWCfFYAb77NCjEk711e9BVe2E6mrasPZTtAjJAPtVAdbye
INFO:web_app.clients.euclidean_distance:NILLION_BOOTNODE_WEBSOCKET=/dns/node-1.testnet-photon.nillion-network.nilogy.xyz/tcp/14211/wss/p2p/12D3KooWCfFYAb77NCjEk711e9BVe2E6mrasPZTtAjJAPtVAdbye
INFO:web_app.clients.euclidean_distance:NILLION_NILCHAIN_CHAIN_ID=nillion-chain-testnet-1
INFO:web_app.clients.euclidean_distance:NILLION_NILCHAIN_JSON_RPC=http://65.109.222.111:26657
INFO:web_app.clients.euclidean_distance:NILLION_NILCHAIN_REST_API=http://65.109.222.111:26657
INFO:web_app.clients.euclidean_distance:NILLION_NILCHAIN_GRPC=65.109.228.73:9090

INFO:web_app.clients.euclidean_distance:Cluster ID: b13880d3-dde8-4a75-a171-8a1a9d985e6c
INFO:web_app.clients.euclidean_distance:Userkey: <builtins.UserKey object at 0x346a0a830>
INFO:web_app.clients.euclidean_distance:Nodekey: <builtins.NodeKey object at 0x105644770>
INFO:web_app.clients.euclidean_distance:Created Nillion Client
INFO:web_app.clients.euclidean_distance:Got user_id: 4h9pG29W8rkqgskhDxJtbmFS1aZUZRHNj6nqEtdTC5EvWrKu8WdH4X4yC2mG3tqEVicTvrWL6FUxRQxGQPCnry3C
INFO:web_app.clients.euclidean_distance:Got party_id: 12D3KooWBzciVPcJAw4XDsHV2jEKSFcHan7MgzaHaazzV1vs1zeT
INFO:web_app.clients.euclidean_distance:Program MIR Absolute Path: /Users/emanuelhc/Desktop/code/nillion-mvp/mvp/services/nillion/web_app/nada_programs/target/euclidean_distance.nada.bin
INFO:web_app.clients.euclidean_distance:Stored program. action_id: 4h9pG29W8rkqgskhDxJtbmFS1aZUZRHNj6nqEtdTC5EvWrKu8WdH4X4yC2mG3tqEVicTvrWL6FUxRQxGQPCnry3C/euclidean_distancea12baced-1302-4ad5-bebb-41a7ab3d93a5
INFO:web_app.clients.euclidean_distance:Stored program_id: 4h9pG29W8rkqgskhDxJtbmFS1aZUZRHNj6nqEtdTC5EvWrKu8WdH4X4yC2mG3tqEVicTvrWL6FUxRQxGQPCnry3C/euclidean_distancea12baced-1302-4ad5-bebb-41a7ab3d93a5
INFO:helpers.nillion_secrets_inputs:Reduced dictionaries: [{'doc0_0': <builtins.SecretInteger object at 0x34bbd9030>, 'doc0_1': <builtins.SecretInteger object at 0x1057d6370>, 'doc0_2': <builtins.SecretInteger object at 0x1057c1bf0>}, {'doc1_0': <builtins.SecretInteger object at 0x3997293b0>, 'doc1_1': <builtins.SecretInteger object at 0x399447730>, 'doc1_2': <builtins.SecretInteger object at 0x399593f30>}, {'doc2_0': <builtins.SecretInteger object at 0x34884c330>, 'doc2_1': <builtins.SecretInteger object at 0x34bbdec30>, 'doc2_2': <builtins.SecretInteger object at 0x399446cf0>}, {'qv_0': <builtins.SecretInteger object at 0x3995ca6f0>, 'qv_1': <builtins.SecretInteger object at 0x39956d3b0>, 'qv_2': <builtins.SecretInteger object at 0x39955f170>}]
INFO:web_app.clients.euclidean_distance:Permissions: <builtins.Permissions object at 0x399761020>
INFO:web_app.clients.euclidean_distance:<<<Storing secret>>>...
INFO:web_app.clients.euclidean_distance:Secrets: {'doc0_0': <builtins.SecretInteger object at 0x34bbd9030>, 'doc0_1': <builtins.SecretInteger object at 0x1057d6370>, 'doc0_2': <builtins.SecretInteger object at 0x1057c1bf0>, 'doc1_0': <builtins.SecretInteger object at 0x3997293b0>, 'doc1_1': <builtins.SecretInteger object at 0x399447730>, 'doc1_2': <builtins.SecretInteger object at 0x399593f30>, 'doc2_0': <builtins.SecretInteger object at 0x34884c330>, 'doc2_1': <builtins.SecretInteger object at 0x34bbdec30>, 'doc2_2': <builtins.SecretInteger object at 0x399446cf0>, 'qv_0': <builtins.SecretInteger object at 0x3995ca6f0>, 'qv_1': <builtins.SecretInteger object at 0x39956d3b0>, 'qv_2': <builtins.SecretInteger object at 0x39955f170>}
INFO:web_app.clients.euclidean_distance:client: <builtins.NillionClient object at 0x3995cdad0>
INFO:web_app.clients.euclidean_distance:operation: <builtins.Operation object at 0x39977fe50>
INFO:web_app.clients.euclidean_distance:payments_wallet: nillion1zqfj3cujymt2t2ykyvsac9f8lc6a2sh2mqzjye
INFO:web_app.clients.euclidean_distance:payments_client: <cosmpy.aerial.client.LedgerClient object at 0x34e675e50>
INFO:web_app.clients.euclidean_distance:cluster_id: b13880d3-dde8-4a75-a171-8a1a9d985e6c
ERROR:web_app.clients.euclidean_distance:Error storing secret on the network
ERROR:web_app.clients.euclidean_distance:Error is requesting price quote: price quote query failed: price quote query failed: timed out
ERROR:web_app.main:Error running euclidean_distance client: requesting price quote: price quote query failed: price quote query failed: timed out

Later today, I'll try using the simple examples provided in the GitHub repository to see if I can identify the bug.