From 863a9be6de4b9319f3dc3f1da533f6f850468711 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 17 Apr 2024 16:55:12 +0000 Subject: [PATCH] fam --- commune/subspace/network.py | 345 +++++++++++++++++++++++++++++++++++ commune/subspace/subspace.py | 254 ++------------------------ commune/subspace/wallet.py | 169 ++++++++++++++++- 3 files changed, 526 insertions(+), 242 deletions(-) create mode 100644 commune/subspace/network.py diff --git a/commune/subspace/network.py b/commune/subspace/network.py new file mode 100644 index 00000000..1b456873 --- /dev/null +++ b/commune/subspace/network.py @@ -0,0 +1,345 @@ + +from retry import retry +from typing import * +import json +import os +import commune as c +import requests +from substrateinterface import SubstrateInterface + +U32_MAX = 2**32 - 1 +U16_MAX = 2**16 - 1 + +class Network(c.Module): + """ + Handles interactions with the subspace chain. + """ + + def __init__( + self, + **kwargs, + ): + self.set_config(kwargs=kwargs) + + def resolve_url(self, url:str = None, network:str = network, mode=None , **kwargs): + mode = mode or self.config.connection_mode + network = 'network' or self.config.network + if url == None: + + url_search_terms = [x.strip() for x in self.config.url_search.split(',')] + is_match = lambda x: any([url in x for url in url_search_terms]) + urls = [] + for provider, mode2url in self.config.urls.items(): + if is_match(provider): + chain = c.module('subspace.chain') + if provider == 'commune': + url = chain.resolve_node_url(url=url, chain=network, mode=mode) + elif provider == 'local': + url = chain.resolve_node_url(url=url, chain='local', mode=mode) + else: + url = mode2url[mode] + + if isinstance(url, list): + urls += url + else: + urls += [url] + + url = c.choice(urls) + + url = url.replace(c.ip(), '0.0.0.0') + + + return url + + url2substrate = {} + def get_substrate(self, + network:str = 'main', + url : str = None, + websocket:str=None, + ss58_format:int=42, + type_registry:dict=None, + type_registry_preset='substrate-node-template', + cache_region=None, + runtime_config=None, + ws_options=None, + auto_discover=True, + auto_reconnect=True, + trials:int = 10, + cache:bool = True, + mode = 'http',): + + network = network or self.config.network + + + + ''' + A specialized class in interfacing with a Substrate node. + + Parameters + A specialized class in interfacing with a Substrate node. + + Parameters + url : the URL to the substrate node, either in format or wss://127.0.0.1:9944 + + ss58_format : The address type which account IDs will be SS58-encoded to Substrate addresses. Defaults to 42, for Kusama the address type is 2 + + type_registry : A dict containing the custom type registry in format: {'types': {'customType': 'u32'},..} + + type_registry_preset : The name of the predefined type registry shipped with the SCALE-codec, e.g. kusama + + cache_region : a Dogpile cache region as a central store for the metadata cache + + use_remote_preset : When True preset is downloaded from Github master, otherwise use files from local installed scalecodec package + + ws_options : dict of options to pass to the websocket-client create_connection function + : dict of options to pass to the websocket-client create_connection function + + ''' + if cache: + if url in self.url2substrate: + return self.url2substrate[url] + + + while trials > 0: + try: + url = self.resolve_url(url, mode=mode, network=network) + + substrate= SubstrateInterface(url=url, + websocket=websocket, + ss58_format=ss58_format, + type_registry=type_registry, + type_registry_preset=type_registry_preset, + cache_region=cache_region, + runtime_config=runtime_config, + ws_options=ws_options, + auto_discover=auto_discover, + auto_reconnect=auto_reconnect) + break + except Exception as e: + trials = trials - 1 + if trials > 0: + raise e + + if cache: + self.url2substrate[url] = substrate + + self.network = network + self.url = url + + return substrate + + + def set_network(self, + network:str = 'main', + mode = 'http', + trials = 10, + url : str = None, **kwargs): + + self.substrate = self.get_substrate(network=network, url=url, mode=mode, trials=trials , **kwargs) + response = {'network': self.network, 'url': self.url} + c.print(response) + + return response + + + + def query(self, + name:str, + params = None, + module:str='SubspaceModule', + block=None, + netuid = None, + network: str = network, + save= True, + max_age=1000, + mode = 'http', + update=False): + + """ + query a subspace storage function with params and block. + """ + + network = self.resolve_network(network) + path = f'query/{network}/{module}.{name}' + + params = params or [] + if not isinstance(params, list): + params = [params] + if netuid != None and netuid != 'all': + params = [netuid] + params + + # we want to cache based on the params if there are any + if len(params) > 0 : + path = path + f'::params::' + '-'.join([str(p) for p in params]) + + value = self.get(path, None, max_age=max_age, update=update) + if value != None: + return value + substrate = self.get_substrate(network=network, mode=mode) + response = substrate.query( + module=module, + storage_function = name, + block_hash = None if block == None else substrate.get_block_hash(block), + params = params + ) + value = response.value + + # if the value is a tuple then we want to convert it to a list + if save: + self.put(path, value) + + return value + + def query_constant( self, + constant_name: str, + module_name: str = 'SubspaceModule', + block: Optional[int] = None , + network: str = None) -> Optional[object]: + """ + Gets a constant from subspace with + module_name, constant_name, and block. + """ + + network = self.resolve_network(network) + substrate = self.get_substrate(network=network) + + value = substrate.query( + module=module_name, + storage_function=constant_name, + block_hash = None if block == None else substrate.get_block_hash(block) + ) + + return value + + + + def query_map(self, name: str = 'StakeFrom', + params: list = None, + block: Optional[int] = None, + network:str = 'main', + netuid = None, + page_size=1000, + max_results=100000, + module='SubspaceModule', + update: bool = True, + max_age : str = 1000, # max age in seconds + mode = 'http', + **kwargs + ) -> Optional[object]: + """ Queries subspace map storage with params and block. """ + # if all lowercase then we want to capitalize the first letter + if name[0].islower(): + _splits = name.split('_') + name = _splits[0].capitalize() + ''.join([s[0].capitalize() + s[1:] for s in _splits[1:]]) + if name == 'Account': + module = 'System' + network = self.resolve_network(network, new_connection=False, mode=mode) + + path = f'query/{network}/{module}.{name}' + # resolving the params + params = params or [] + + is_single_subnet = bool(netuid != 'all' and netuid != None) + if is_single_subnet: + params = [netuid] + params + if not isinstance(params, list): + params = [params] + if len(params) > 0 : + path = path + f'::params::' + '-'.join([str(p) for p in params]) + path = path+"::block::" + paths = self.glob(path + '*') + update = update or len(paths) == 0 or block != None + if not update: + last_path = sorted(paths, reverse=True)[0] + value = self.get(last_path, None , max_age=max_age) + else: + value = None + + if value == None: + # block = block or self.block + path = path + f'{block}' + network = self.resolve_network(network) + # if the value is a tuple then we want to convert it to a list + + substrate = self.get_substrate(network=network, mode=mode) + qmap = substrate.query_map( + module=module, + storage_function = name, + params = params, + page_size = page_size, + max_results = max_results, + block_hash =substrate.get_block_hash(block) + ) + + new_qmap = {} + progress_bar = c.progress(qmap, desc=f'Querying {name} ma') + for (k,v) in qmap: + progress_bar.update(1) + if not isinstance(k, tuple): + k = [k] + if type(k) in [tuple,list]: + # this is a double map + k = [_k.value for _k in k] + if hasattr(v, 'value'): + v = v.value + c.dict_put(new_qmap, k, v) + + self.put(path, new_qmap) + + else: + new_qmap = value + + def convert_dict_k_digit_to_int(d): + is_int_bool = False + for k,v in c.copy(d).items(): + if c.is_int(k): + is_int_bool = True + d[int(k)] = d.pop(k) + if isinstance(v, dict): + d[int(k)] = convert_dict_k_digit_to_int(v) + if is_int_bool: + # sort the dictionary by key + d = dict(sorted(d.items())) + + return d + + + new_map = convert_dict_k_digit_to_int(new_qmap) + + return new_map + + + def query_multi(self, params_batch , substrate=None, module='SubspaceModule', feature='SubnetNames', network='main'): + substrate = substrate or self.get_substrate(network=network) + + # check if the params_batch is a list of lists + for i,p in enumerate(params_batch): + if isinstance(p, dict): + p = [p.get('module', module), p.get('feature', feature), p.get('netuid', 0)] + if len(p) == 1: + p = [module, feature, p] + assert len(p) == 3, f"[module, feature, netuid] should be of length 4. Got {p}" + params_batch[i] = p + + assert isinstance(params_batch, list), f"params_batch should be a list of lists" + multi_query = [substrate.create_storage_key(*p) for p in params_batch] + results = substrate.query_multi(multi_query) + return results + + + def storage_functions(self, network=network, block_hash = None): + self.resolve_network(network) + return self.substrate.get_metadata_storage_functions( block_hash=block_hash) + + + def storage_names(self, search=None, network=network, block_hash = None): + self.resolve_network(network) + storage_names = [f['storage_name'] for f in self.substrate.get_metadata_storage_functions( block_hash=block_hash)] + if search != None: + storage_names = [s for s in storage_names if search in s.lower()] + return storage_names + + + def check_storage(self, block_hash = None, network=network): + self.resolve_network(network) + return self.substrate.get_metadata_storage_functions( block_hash=block_hash) diff --git a/commune/subspace/subspace.py b/commune/subspace/subspace.py index e60b75ee..ade180ab 100644 --- a/commune/subspace/subspace.py +++ b/commune/subspace/subspace.py @@ -1994,34 +1994,7 @@ def archive_history(cls, return archive_history - - - - - def key_usage_path(self, key:str): - key_ss58 = self.resolve_key_ss58(key) - return f'key_usage/{key_ss58}' - def key_used(self, key:str): - return self.exists(self.key_usage_path(key)) - - def use_key(self, key:str): - return self.put(self.key_usage_path(key), c.time()) - - def unuse_key(self, key:str): - return self.rm(self.key_usage_path(key)) - - def test_key_usage(self): - key_path = 'test_key_usage' - c.add_key(key_path) - self.use_key(key_path) - assert self.key_used(key_path) - self.unuse_key(key_path) - assert not self.key_used(key_path) - c.rm_key('test_key_usage') - assert not c.key_exists(key_path) - return {'success': True, 'msg': f'Tested key usage for {key_path}'} - def get_nonce(self, key:str=None, network=None, **kwargs): key_ss58 = self.resolve_key_ss58(key) @@ -2127,6 +2100,9 @@ def status(cls): def storage_functions(self, network=network, block_hash = None): self.resolve_network(network) return self.substrate.get_metadata_storage_functions( block_hash=block_hash) + + + storage_fns = storage_functions @@ -2235,8 +2211,6 @@ def sand(cls): c.print(k, state['balances'].get(addy, 0)) - - def test_balance(self, network:str = network, n:int = 10, timeout:int = 10, verbose:bool = False, min_amount = 10, key=None): key = c.get_key(key) @@ -2279,23 +2253,6 @@ def total_balance(self, **kwargs): WALLET VIBES """ - - - """ - ######################################### - CHAIN LAND - ######################################### - - """ - - def chain(self, *args, **kwargs): - return c.module('subspace.chain')(*args, **kwargs) - - def chain_config(self, *args, **kwargs): - return self.chain(*args, **kwargs).config - - def chains(self, *args, **kwargs): - return self.chain(*args, **kwargs).chains() """ ######################################### @@ -2341,9 +2298,6 @@ def register( serve_info = c.serve(module, name=name, **kwargs) address = serve_info['address'] - - - network =self.resolve_network(network) key = self.resolve_key(key) address = address or c.namespace(network='local').get(name, '0.0.0.0:8888') @@ -2627,104 +2581,6 @@ def propose_subnet_update( return response - - - ################# - #### Serving #### - ################# - def vote_proposal( - self, - proposal_id: int = None, - key: str = None, - network = 'main', - nonce = None, - netuid = 0, - **params, - - ) -> bool: - - self.resolve_network(network) - # remove the params that are the same as the module info - params = { - 'proposal_id': proposal_id, - 'netuid': netuid, - } - - response = self.compose_call(fn='add_subnet_proposal', - params=params, - key=key, - nonce=nonce) - - return response - - ################# - #### Serving #### - ################# - def update_global( - self, - key: str = None, - network : str = 'main', - sudo: bool = True, - **params, - ) -> bool: - - key = self.resolve_key(key) - network = self.resolve_network(network) - global_params = self.global_params(fmt='nanos') - global_params.update(params) - params = global_params - for k,v in params.items(): - if isinstance(v, str): - params[k] = v.encode('utf-8') - # this is a sudo call - return self.compose_call(fn='update_global', - params=params, - key=key, - sudo=sudo) - - - - - - ################# - #### set_code #### - ################# - def set_code( - self, - wasm_file_path = None, - key: str = None, - network = network, - ) -> bool: - - if wasm_file_path == None: - wasm_file_path = self.wasm_file_path() - - assert os.path.exists(wasm_file_path), f'Wasm file not found at {wasm_file_path}' - - self.resolve_network(network) - key = self.resolve_key(key) - - # Replace with the path to your compiled WASM file - with open(wasm_file_path, 'rb') as file: - wasm_binary = file.read() - wasm_hex = wasm_binary.hex() - - code = '0x' + wasm_hex - - # Construct the extrinsic - response = self.compose_call( - module='System', - fn='set_code', - params={ - 'code': code.encode('utf-8') - }, - unchecked_weight=True, - sudo = True, - key=key - ) - - return response - def resolve_module_key(self, x, netuid=0, max_age=10): if not c.valid_ss58_address(x): name2key = self.name2key(netuid=netuid, max_age=max_age) @@ -3171,10 +3027,11 @@ def staked(self, - def my_keys(self, *args, search=None, netuid=0, **kwargs): + def my_keys(self, search=None, netuid=0, max_age=1000, update=False, **kwargs): netuid = self.resolve_netuid(netuid) - keys = self.keys(*args,netuid=netuid, **kwargs) - key2address = c.key2address() + + keys = self.keys(netuid=netuid, max_age=max_age, update=update, **kwargs) + key2address = c.key2address(search=search, max_age=max_age, update=update) if search != None: key2address = {k: v for k,v in key2address.items() if search in k} addresses = list(key2address.values()) @@ -3389,20 +3246,20 @@ def batch_fn(batch_keys): return {k.params[0]: v['data']['free'].value for k, v in results} key2balance = {} progress = c.progress(num_batches) + + for batch_keys in batched_keys: fails = 0 while fails < max_trials: + if fails > max_trials: + raise Exception(f'Error getting balances {fails}/{max_trials}') try: result = batch_fn(batch_keys) progress.update(1) - break + break # if successful, break except Exception as e: fails += 1 - if fails == max_trials: - raise e c.print(f'Error getting balances {fails}/{max_trials} {e}') - - if c.is_error(result): c.print(result, color='red') else: @@ -3415,34 +3272,6 @@ def batch_fn(batch_keys): key2balance = {address2key[k]: v for k,v in key2balance.items()} return key2balance - - def clean_keys(self, - network='main', - min_value=1, - update = True): - """ - description: - Removes keys with a value less than min_value - params: - network: str = 'main', # network to remove keys from - min_value: int = 1, # min value of the key - update: bool = True, # update the key2value cache - max_age: int = 0 # max age of the key2value cache - """ - key2value= self.key2value(netuid='all', update=update, network=network, fmt='j', min_value=0) - address2key = c.address2key() - rm_keys = [] - for k,v in key2value.items(): - if k in address2key and v < min_value: - c.print(f'Removing key {k} with value {v}') - c.rm_key(address2key[k]) - rm_keys += [k] - return rm_keys - - - - - def registered_servers(self, netuid = 0, network = network, **kwargs): netuid = self.resolve_netuid(netuid) network = self.resolve_network(network) @@ -3456,10 +3285,9 @@ def registered_servers(self, netuid = 0, network = network, **kwargs): registered_keys += [s] return registered_keys + reged = registered_servers - reged = reged_servers = registered_servers - - unreged = unreged_servers = unregistered_servers + unreged = unregistered_servers def key2balance(self, search=None, batch_size = 64, @@ -3482,17 +3310,7 @@ def key2balance(self, search=None, for k,v in key2balance.items(): key2balance[k] = self.format_amount(v, fmt=fmt) return key2balance - - - - key2balances = key2balance - - # key2balances = {key:balances[address] for address,key in address2key.items() if address in balances} - # if min_value > 0: - # key2balances = {k:v for k,v in key2balances.items() if v > min_value} - # return key2balances - def my_value( self, *args, **kwargs ): @@ -3500,57 +3318,13 @@ def my_value( my_supply = my_value - def subnet2stake(self, network=None, update=False) -> dict: - subnet2stake = {} - for subnet_name in self.subnet_names(network=network): - c.print(f'Getting stake for subnet {subnet_name}') - subnet2stake[subnet_name] = self.my_total_stake(network=network, netuid=subnet_name , update=update) - return subnet2stake - def my_total_stake(self, netuid='all', network = 'main', fmt='j', update=False): my_stake_to = self.my_stake_to(netuid=netuid, network=network, fmt=fmt, update=update) return sum([sum(list(v.values())) for k,v in my_stake_to.items()]) - - - - - - def staker2stake(self, update=False, network='main', fmt='j', local=False): - staker2netuid2stake = self.staker2netuid2stake(update=update, network=network, fmt=fmt, local=local) - staker2stake = {} - for staker, netuid2stake in staker2netuid2stake.items(): - if staker not in staker2stake: - staker2stake[staker] = 0 - - return staker2stake - - - def staker2netuid2stake(self, update=False, network='main', fmt='j', local=False, **kwargs): - stake_to = self.query_map("StakeTo", update=update, network=network, **kwargs) - staker2netuid2stake = {} - for netuid , stake_to_subnet in stake_to.items(): - for staker, stake_tuples in stake_to_subnet.items(): - staker2netuid2stake[staker] = staker2netuid2stake.get(staker, {}) - staker2netuid2stake[staker][netuid] = staker2netuid2stake[staker].get(netuid, []) - staker2netuid2stake[staker][netuid] = sum(list(map(lambda x: x[-1], stake_tuples ))) - staker2netuid2stake[staker][netuid] += self.format_amount(staker2netuid2stake[staker][netuid],fmt=fmt) - - if local: - address2key = c.address2key() - staker2netuid2stake = {address:staker2netuid2stake.get(address,{}) for address in address2key.keys()} - - return staker2netuid2stake - - - - def my_total_balance(self, network = None, fmt='j', update=False): - return sum(self.key2balance(network=network, fmt=fmt, update=update ).values()) - def check_valis(self, **kwargs): return self.check_servers(search='vali', **kwargs) - def check_servers(self, search='vali',update:bool=False, netuid=0, min_lag=100, remote=False, **kwargs): if remote: kwargs = c.locals2kwargs(locals()) diff --git a/commune/subspace/wallet.py b/commune/subspace/wallet.py index ab6a4a17..b6e98d50 100644 --- a/commune/subspace/wallet.py +++ b/commune/subspace/wallet.py @@ -1,8 +1,7 @@ import commune as c - -class Wallet(c.Module): +class Wallet(c.m('subspace')): def __init__(self, network='main', **kwargs): @@ -11,3 +10,169 @@ def __init__(self, network='main', **kwargs): def call(self, *text, **kwargs): return self.talk(*text, **kwargs) + + + def key_usage_path(self, key:str): + key_ss58 = self.resolve_key_ss58(key) + return f'key_usage/{key_ss58}' + + def key_used(self, key:str): + return self.exists(self.key_usage_path(key)) + + def use_key(self, key:str): + return self.put(self.key_usage_path(key), c.time()) + + def unuse_key(self, key:str): + return self.rm(self.key_usage_path(key)) + + def test_key_usage(self): + key_path = 'test_key_usage' + c.add_key(key_path) + self.use_key(key_path) + assert self.key_used(key_path) + self.unuse_key(key_path) + assert not self.key_used(key_path) + c.rm_key('test_key_usage') + assert not c.key_exists(key_path) + return {'success': True, 'msg': f'Tested key usage for {key_path}'} + + + + ################# + #### Serving #### + ################# + def update_global( + self, + key: str = None, + network : str = 'main', + sudo: bool = True, + **params, + ) -> bool: + + key = self.resolve_key(key) + network = self.resolve_network(network) + global_params = self.global_params(fmt='nanos') + global_params.update(params) + params = global_params + for k,v in params.items(): + if isinstance(v, str): + params[k] = v.encode('utf-8') + # this is a sudo call + return self.compose_call(fn='update_global', + params=params, + key=key, + sudo=sudo) + + + + + ################# + #### Serving #### + ################# + def vote_proposal( + self, + proposal_id: int = None, + key: str = None, + network = 'main', + nonce = None, + netuid = 0, + **params, + + ) -> bool: + + self.resolve_network(network) + # remove the params that are the same as the module info + params = { + 'proposal_id': proposal_id, + 'netuid': netuid, + } + + response = self.compose_call(fn='add_subnet_proposal', + params=params, + key=key, + nonce=nonce) + + return response + + + + + ################# + #### set_code #### + ################# + def set_code( + self, + wasm_file_path = None, + key: str = None, + network = network, + ) -> bool: + + if wasm_file_path == None: + wasm_file_path = self.wasm_file_path() + + assert os.path.exists(wasm_file_path), f'Wasm file not found at {wasm_file_path}' + + self.resolve_network(network) + key = self.resolve_key(key) + + # Replace with the path to your compiled WASM file + with open(wasm_file_path, 'rb') as file: + wasm_binary = file.read() + wasm_hex = wasm_binary.hex() + + code = '0x' + wasm_hex + + # Construct the extrinsic + response = self.compose_call( + module='System', + fn='set_code', + params={ + 'code': code.encode('utf-8') + }, + unchecked_weight=True, + sudo = True, + key=key + ) + + return response + + + + + def unregistered_servers(self, search=None, netuid = 0, network = network, timeout=42, key=None, max_age=None, update=False, transfer_multiple=True,**kwargs): + netuid = self.resolve_netuid(netuid) + network = self.resolve_network(network) + servers = c.servers(search=search) + key2address = c.key2address(update=1) + keys = self.keys(netuid=netuid, max_age=max_age, update=update) + uniregistered_keys = [] + unregister_servers = [] + for s in servers: + if key2address[s] not in keys: + unregister_servers += [s] + return unregister_servers + + + + def clean_keys(self, + network='main', + min_value=1, + update = True): + """ + description: + Removes keys with a value less than min_value + params: + network: str = 'main', # network to remove keys from + min_value: int = 1, # min value of the key + update: bool = True, # update the key2value cache + max_age: int = 0 # max age of the key2value cache + """ + key2value= self.key2value(netuid='all', update=update, network=network, fmt='j', min_value=0) + address2key = c.address2key() + rm_keys = [] + for k,v in key2value.items(): + if k in address2key and v < min_value: + c.print(f'Removing key {k} with value {v}') + c.rm_key(address2key[k]) + rm_keys += [k] + return rm_keys