From 0c74dc76fa47591d92d4d9af0e81a4cea037c0dd Mon Sep 17 00:00:00 2001 From: sergeyradchenkop2p Date: Sun, 14 Jan 2024 18:07:30 +0300 Subject: [PATCH 01/10] astar exporter fix --- exporters/acala/config.yaml | 2 +- exporters/acala/exporter.py | 55 +++++++---------------- exporters/acala/functions.py | 79 +++++++++++++++++++++++++++++++++ exporters/astar/config.yaml | 9 ++-- exporters/astar/exporter.py | 66 ++++++++++----------------- exporters/astar/functions.py | 79 +++++++++++++++++++++++++++++++++ exporters/moonbeam/config.yaml | 2 +- exporters/moonbeam/exporter.py | 56 +++++++---------------- exporters/moonbeam/functions.py | 79 +++++++++++++++++++++++++++++++++ 9 files changed, 297 insertions(+), 130 deletions(-) create mode 100644 exporters/acala/functions.py create mode 100644 exporters/astar/functions.py create mode 100644 exporters/moonbeam/functions.py diff --git a/exporters/acala/config.yaml b/exporters/acala/config.yaml index 5e5c3e8..4cc6321 100644 --- a/exporters/acala/config.yaml +++ b/exporters/acala/config.yaml @@ -1,5 +1,5 @@ --- -api_substrate: https://acala-api.polka.p2p.world/api +ws_endpoint: ws://scaleway-acala-node1:9934 chain: acala exporter: listen: 0.0.0.0 diff --git a/exporters/acala/exporter.py b/exporters/acala/exporter.py index 78680a0..c47f118 100644 --- a/exporters/acala/exporter.py +++ b/exporters/acala/exporter.py @@ -6,8 +6,15 @@ import time import json import logging +import operator +import traceback from collections import deque from flask import Flask, request, make_response +from functions import SUBSTRATE_INTERFACE, get_config, get_era_points, get_chain_info +from _thread import interrupt_main +from numpy import median, average, percentile +from decimal import * + logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %I:%M:%S') app = Flask(__name__) @@ -51,40 +58,6 @@ def metrics(): return response -def api_request(endpoint = "api_substrate",method = None,args = None): - if endpoint == 'api_substrate': - url = get_config('api_substrate') - - if isinstance(args, list): - for i in range(len(args)): - if isinstance(args[i], str): - args[i] = '"' + args[i] + '"' - elif isinstance(args, str): - args = '"' + args + '"' - elif not args: - args = "" - - data = {'method': method, - 'args': args} - - elif endpoint == 'api_registry': - url = get_config('api_registry') - - data = {'method': method} - - try: - r = requests.post(url, json=data) - except (ConnectionRefusedError,requests.exceptions.ConnectionError): - logging.critical('Coulnd not get data from ' + endpoint) - - return None - - if r.status_code == 200: - return r.json()['result'] - else: - logging.critical('Request to ' + endpoint + ' finished with code ' + str(r.status_code)) - return None - def get_config(part): with open('./config.yaml') as config_file: data = yaml.load(config_file, Loader=yaml.FullLoader) @@ -97,18 +70,17 @@ def main(): while True: try: - last_block = int(api_request(method = 'api.query.system.number'),16) - + last_block = substrate_interface.request('System','Number').value if last_block != block: - validators = api_request(method = 'api.query.session.validators') - current_session = int(api_request(method = 'api.query.session.currentIndex'),16) - disabled_validators = api_request(method = 'api.query.session.disabledValidators') + validators = substrate_interface.request('Session','Validators').value + current_session = substrate_interface.request('Session','CurrentIndex').value + disabled_validators = substrate_interface.request('Session','DisabledValidators').value result = {'validators':{},'common':{}} result['common'] = {} result['common']['active_validators_count'] = len(validators) result['common']['current_session'] = current_session for addr in validators: - points = int(api_request(method = 'api.query.collatorSelection.sessionPoints', args = addr),16) + points = substrate_interface.request('CollatorSelection','SessionPoints', [addr]).value validator_points = {k:points for k in validators if k == addr} result['validators'].update(validator_points) @@ -127,6 +99,9 @@ def main(): if __name__ == '__main__': endpoint_listen = get_config('exporter')['listen'] endpoint_port = get_config('exporter')['port'] + ws_endpoint = get_config('ws_endpoint') + chain = get_config('chain') + substrate_interface = SUBSTRATE_INTERFACE(ws_endpoint,chain) q_metrics = deque([]) diff --git a/exporters/acala/functions.py b/exporters/acala/functions.py new file mode 100644 index 0000000..e9034e7 --- /dev/null +++ b/exporters/acala/functions.py @@ -0,0 +1,79 @@ +import yaml +from substrateinterface import SubstrateInterface, Keypair +from substrateinterface.exceptions import SubstrateRequestException +from websocket._exceptions import WebSocketConnectionClosedException + +class SUBSTRATE_INTERFACE: + def __init__(self, ws_endpoint, chain): + self.substrate = SubstrateInterface( + url=ws_endpoint, + ss58_format=42, + type_registry_preset=chain) + + + def request(self, module: str, function: str, params: str = None): + try: + r = self.substrate.query( + module=module, + storage_function=function, + params=params) + + return r + except (WebSocketConnectionClosedException,ConnectionRefusedError,SubstrateRequestException) as e: + self.substrate.connect_websocket() + logging.critical('The substrate api call failed with error ' + str(e)) + r = None + +def get_config(part): + with open('./config.yaml') as config_file: + data = yaml.load(config_file, Loader=yaml.FullLoader) + + return data[part] + +def get_era_points(data): + result = {} + + for i in data['individual']: + result[i[0]] = i[1] + + return {'result':result,'total':data['total']} + +def get_chain_info(chain,substrate_interface): + constants = {'polkadot':{'session_length':1200,'era_length':7200}} + + session_length = constants[chain]['session_length'] + era_length = constants[chain]['era_length'] + + current_era = substrate_interface.request('Staking','ActiveEra').value['index'] + current_session = substrate_interface.request('Session','CurrentIndex').value + + eras_start_session_index = substrate_interface.request('Staking','ErasStartSessionIndex',[current_era]).value + + genesis_slot = substrate_interface.request('Babe','GenesisSlot').value + current_slot = substrate_interface.request('Babe','CurrentSlot').value + + session_start_slot = int(current_session) * int(session_length) + int(genesis_slot) + session_progress = int(current_slot) - int(session_start_slot) + + era_session_index = int(current_session) - int(eras_start_session_index) + era_progress = int(era_session_index) * int(session_length) + int(session_progress) + + return {'current_era': current_era, 'eras_start_session_index': eras_start_session_index, 'current_session': current_session, 'era_progress': era_progress / era_length * 100, 'session_progress': session_progress / session_length * 100} + +def ss58_convert(data): + r = [] + + for key in data: + pubkey = Keypair(ss58_address=key).public_key.hex() + r.append('0x' + pubkey) + + return r + +def get_keys(validators,keys): + result = {i:None for i in validators} + + for i in keys: + if str(i[0]) in result.keys(): + result[str(i[0])] = str(i[1]['grandpa']) + + return result diff --git a/exporters/astar/config.yaml b/exporters/astar/config.yaml index 6418622..7c1a8ca 100644 --- a/exporters/astar/config.yaml +++ b/exporters/astar/config.yaml @@ -1,10 +1,7 @@ --- -api_substrate: https://shiden-api.polka.p2p.world/api -api_registry: https://registry-para-api.polka.p2p.world/api -chain: shiden +chain: astar +ws_endpoint: wss://rpc.astar.network exporter: listen: 0.0.0.0 port: 9618 -validators: - scaleway-shiden-collator1: - account: 'ZPsVRDiDTsWr9wENxE5iyBFCd2ySckF9mn8PtGSevnKjMwU' + diff --git a/exporters/astar/exporter.py b/exporters/astar/exporter.py index b0ec44b..e6903cc 100644 --- a/exporters/astar/exporter.py +++ b/exporters/astar/exporter.py @@ -6,10 +6,15 @@ import time import json import logging -from numpy import median, average, percentile -from decimal import * +import operator +import traceback from collections import deque from flask import Flask, request, make_response +from functions import SUBSTRATE_INTERFACE, get_config, get_era_points, get_chain_info +from _thread import interrupt_main +from numpy import median, average, percentile +from decimal import * + logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %I:%M:%S') app = Flask(__name__) @@ -103,32 +108,6 @@ def metrics(): return response -def api_request(method = None,args = None): - url = get_config('api_substrate') - - if isinstance(args, list): - for i in range(len(args)): - if isinstance(args[i], str): - args[i] = '"' + args[i] + '"' - elif isinstance(args, str): - args = '"' + args + '"' - elif not args: - args = "" - - data = {'method': method, - 'args': args} - - try: - r = requests.post(url, json=data) - except (ConnectionRefusedError,requests.exceptions.ConnectionError) as e: - logging.critical(e) - return None - - if r.status_code == 200: - return r.json()['result'] - else: - logging.critical('Request to ' + url + ' finished with code ' + str(r.status_code)) - return None def get_config(part): with open('./config.yaml') as config_file: @@ -138,8 +117,8 @@ def get_config(part): def get_block_count(addr): try: - authored_block = int(api_request(method = 'api.query.collatorSelection.lastAuthoredBlock', args = addr),16) - + authored_block = substrate_interface.request('CollatorSelection','LastAuthoredBlock', ['addr']).value + for i in q_collators: idx = q_collators.index(i) if addr in i.keys(): @@ -155,9 +134,9 @@ def main(): session = 0 while True: - try: - current_era = int(api_request(method = 'api.query.dappsStaking.currentEra')[2:],16) - current_session = int(api_request(method = 'api.query.session.currentIndex')[2:],16) + try: + current_era = substrate_interface.request('DappsStaking','CurrentEra').value + current_session = substrate_interface.request('Session','CurrentIndex').value if era != current_era: logging.info('New era ' + str(current_era) + ' has just begun') @@ -165,9 +144,9 @@ def main(): if session != current_session: processed = [] q_collators.clear() - active_collators = api_request(method = 'api.query.session.validators') - disabled_collators = api_request(method = 'api.query.session.disabledValidators') - + active_collators = substrate_interface.request('Session','Validators').value + disabled_collators = substrate_interface.request('Session','DisabledValidators').value + for addr in active_collators: if addr not in collators.keys() and addr not in processed: q_collators.append({addr:[]}) @@ -189,7 +168,7 @@ def main(): result['common']['session_blocks'] = 0 logging.info('New session ' + str(current_session) + ' has just begun') - last_block = int(api_request(method = 'api.query.system.number'),16) + last_block = substrate_interface.request('System','Number').value if last_block != block: logging.info('Processing block ' + str(last_block)) @@ -199,9 +178,8 @@ def main(): for addr,params in result['collators'].items(): if addr in active_collators: - authored_block = api_request(method = 'api.query.collatorSelection.lastAuthoredBlock', args = addr) - authored_block = int(authored_block, 16) - + authored_block = substrate_interface.request('CollatorSelection','LastAuthoredBlock', ['addr']).value + if 'last_authored_block' not in params: params['last_authored_block'] = authored_block @@ -247,11 +225,13 @@ def main(): if __name__ == '__main__': endpoint_listen = get_config('exporter')['listen'] endpoint_port = get_config('exporter')['port'] - + ws_endpoint = get_config('ws_endpoint') + chain = get_config('chain') + substrate_interface = SUBSTRATE_INTERFACE(ws_endpoint,chain) + collators = {} - for k,v in get_config('validators').items(): - collators[v['account']] = {'node':k} + q_metrics = deque([]) q_collators = deque([]) diff --git a/exporters/astar/functions.py b/exporters/astar/functions.py new file mode 100644 index 0000000..e9034e7 --- /dev/null +++ b/exporters/astar/functions.py @@ -0,0 +1,79 @@ +import yaml +from substrateinterface import SubstrateInterface, Keypair +from substrateinterface.exceptions import SubstrateRequestException +from websocket._exceptions import WebSocketConnectionClosedException + +class SUBSTRATE_INTERFACE: + def __init__(self, ws_endpoint, chain): + self.substrate = SubstrateInterface( + url=ws_endpoint, + ss58_format=42, + type_registry_preset=chain) + + + def request(self, module: str, function: str, params: str = None): + try: + r = self.substrate.query( + module=module, + storage_function=function, + params=params) + + return r + except (WebSocketConnectionClosedException,ConnectionRefusedError,SubstrateRequestException) as e: + self.substrate.connect_websocket() + logging.critical('The substrate api call failed with error ' + str(e)) + r = None + +def get_config(part): + with open('./config.yaml') as config_file: + data = yaml.load(config_file, Loader=yaml.FullLoader) + + return data[part] + +def get_era_points(data): + result = {} + + for i in data['individual']: + result[i[0]] = i[1] + + return {'result':result,'total':data['total']} + +def get_chain_info(chain,substrate_interface): + constants = {'polkadot':{'session_length':1200,'era_length':7200}} + + session_length = constants[chain]['session_length'] + era_length = constants[chain]['era_length'] + + current_era = substrate_interface.request('Staking','ActiveEra').value['index'] + current_session = substrate_interface.request('Session','CurrentIndex').value + + eras_start_session_index = substrate_interface.request('Staking','ErasStartSessionIndex',[current_era]).value + + genesis_slot = substrate_interface.request('Babe','GenesisSlot').value + current_slot = substrate_interface.request('Babe','CurrentSlot').value + + session_start_slot = int(current_session) * int(session_length) + int(genesis_slot) + session_progress = int(current_slot) - int(session_start_slot) + + era_session_index = int(current_session) - int(eras_start_session_index) + era_progress = int(era_session_index) * int(session_length) + int(session_progress) + + return {'current_era': current_era, 'eras_start_session_index': eras_start_session_index, 'current_session': current_session, 'era_progress': era_progress / era_length * 100, 'session_progress': session_progress / session_length * 100} + +def ss58_convert(data): + r = [] + + for key in data: + pubkey = Keypair(ss58_address=key).public_key.hex() + r.append('0x' + pubkey) + + return r + +def get_keys(validators,keys): + result = {i:None for i in validators} + + for i in keys: + if str(i[0]) in result.keys(): + result[str(i[0])] = str(i[1]['grandpa']) + + return result diff --git a/exporters/moonbeam/config.yaml b/exporters/moonbeam/config.yaml index 03f1d54..f624900 100644 --- a/exporters/moonbeam/config.yaml +++ b/exporters/moonbeam/config.yaml @@ -1,5 +1,5 @@ --- -api_substrate: https://moonbeam-api.polka.p2p.world/api +ws_endpoint: ws://scaleway-moonbeam-node1:9934 chain: moonbeam exporter: listen: 0.0.0.0 diff --git a/exporters/moonbeam/exporter.py b/exporters/moonbeam/exporter.py index 73be592..0478168 100755 --- a/exporters/moonbeam/exporter.py +++ b/exporters/moonbeam/exporter.py @@ -6,10 +6,15 @@ import time import json import logging -from numpy import median, average, percentile -from decimal import * +import operator +import traceback from collections import deque from flask import Flask, request, make_response +from functions import SUBSTRATE_INTERFACE, get_config, get_era_points, get_chain_info +from _thread import interrupt_main +from numpy import median, average, percentile +from decimal import * + logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %I:%M:%S') app = Flask(__name__) @@ -91,32 +96,6 @@ def metrics(): return response -def api_request(method = None,args = None): - url = get_config('api_substrate') - - if isinstance(args, list): - for i in range(len(args)): - if isinstance(args[i], str): - args[i] = '"' + args[i] + '"' - elif isinstance(args, str): - args = '"' + args + '"' - elif not args: - args = "" - - data = {'method': method, - 'args': args} - - try: - r = requests.post(url, json=data) - except (ConnectionRefusedError,requests.exceptions.ConnectionError) as e: - logging.critical(e) - return None - - if r.status_code == 200: - return r.json()['result'] - else: - logging.critical('Request to ' + url + ' finished with code ' + str(r.status_code)) - return None def get_round_progress(chain): constants = { @@ -125,8 +104,8 @@ def get_round_progress(chain): } round_length = constants[chain]['round_length'] - round_first_block = api_request(method = 'api.query.parachainStaking.round')['first'] - current_block = int(api_request(method = 'api.query.system.number'),16) + round_first_block = substrate_interface.request('ParachainStaking','Round', ['first']).value + current_block = substrate_interface.request('System','Number').value round_progress = (int(current_block) - int(round_first_block)) / round_length * 100 return round_progress @@ -143,11 +122,10 @@ def main(): while True: try: - current_rnd = api_request(method = 'api.query.parachainStaking.round')['current'] + current_rnd = substrate_interface.request('ParachainStaking','Round').value.current if current_rnd != rnd: rnd_blocks_count = 0 - - active_collators = api_request(method = 'api.query.parachainStaking.selectedCandidates') + active_collators = substrate_interface.request('ParachainStaking','SelectedCandidates').value common = {} common['active_collators'] = len(active_collators) common['current_round'] = current_rnd @@ -157,13 +135,12 @@ def main(): result = {'collators':all_collators, 'common':common} logging.info('New round ' + str(current_rnd) + ' has just begun') - last_block = int(api_request(method = 'api.query.system.number'),16) + last_block = substrate_interface.request('System','Number').value if last_block != block: logging.info('Processing block ' + str(last_block)) - - block_author = api_request(method = 'api.query.authorInherent.author') - + block_author = substrate_interface.request('AuthorInherent','Author').value + for addr,params in result['collators'].items(): if block_author == addr: @@ -195,11 +172,12 @@ def main(): if __name__ == '__main__': endpoint_listen = get_config('exporter')['listen'] endpoint_port = get_config('exporter')['port'] + ws_endpoint = get_config('ws_endpoint') + chain = get_config('chain') + substrate_interface = SUBSTRATE_INTERFACE(ws_endpoint,chain) collators = {} - # for k,v in get_config('validators').items(): - # collators[v['account']] = {'node':k} q_metrics = deque([]) diff --git a/exporters/moonbeam/functions.py b/exporters/moonbeam/functions.py new file mode 100644 index 0000000..e9034e7 --- /dev/null +++ b/exporters/moonbeam/functions.py @@ -0,0 +1,79 @@ +import yaml +from substrateinterface import SubstrateInterface, Keypair +from substrateinterface.exceptions import SubstrateRequestException +from websocket._exceptions import WebSocketConnectionClosedException + +class SUBSTRATE_INTERFACE: + def __init__(self, ws_endpoint, chain): + self.substrate = SubstrateInterface( + url=ws_endpoint, + ss58_format=42, + type_registry_preset=chain) + + + def request(self, module: str, function: str, params: str = None): + try: + r = self.substrate.query( + module=module, + storage_function=function, + params=params) + + return r + except (WebSocketConnectionClosedException,ConnectionRefusedError,SubstrateRequestException) as e: + self.substrate.connect_websocket() + logging.critical('The substrate api call failed with error ' + str(e)) + r = None + +def get_config(part): + with open('./config.yaml') as config_file: + data = yaml.load(config_file, Loader=yaml.FullLoader) + + return data[part] + +def get_era_points(data): + result = {} + + for i in data['individual']: + result[i[0]] = i[1] + + return {'result':result,'total':data['total']} + +def get_chain_info(chain,substrate_interface): + constants = {'polkadot':{'session_length':1200,'era_length':7200}} + + session_length = constants[chain]['session_length'] + era_length = constants[chain]['era_length'] + + current_era = substrate_interface.request('Staking','ActiveEra').value['index'] + current_session = substrate_interface.request('Session','CurrentIndex').value + + eras_start_session_index = substrate_interface.request('Staking','ErasStartSessionIndex',[current_era]).value + + genesis_slot = substrate_interface.request('Babe','GenesisSlot').value + current_slot = substrate_interface.request('Babe','CurrentSlot').value + + session_start_slot = int(current_session) * int(session_length) + int(genesis_slot) + session_progress = int(current_slot) - int(session_start_slot) + + era_session_index = int(current_session) - int(eras_start_session_index) + era_progress = int(era_session_index) * int(session_length) + int(session_progress) + + return {'current_era': current_era, 'eras_start_session_index': eras_start_session_index, 'current_session': current_session, 'era_progress': era_progress / era_length * 100, 'session_progress': session_progress / session_length * 100} + +def ss58_convert(data): + r = [] + + for key in data: + pubkey = Keypair(ss58_address=key).public_key.hex() + r.append('0x' + pubkey) + + return r + +def get_keys(validators,keys): + result = {i:None for i in validators} + + for i in keys: + if str(i[0]) in result.keys(): + result[str(i[0])] = str(i[1]['grandpa']) + + return result From 83e4d25e9bec0be42444bff6c7283513aff92cab Mon Sep 17 00:00:00 2001 From: sergeyradchenkop2p Date: Mon, 15 Jan 2024 15:38:45 +0300 Subject: [PATCH 02/10] astar fix --- exporters/astar/config.yaml | 10 -- exporters/astar/exporter.py | 263 ------------------------------------ 2 files changed, 273 deletions(-) delete mode 100644 exporters/astar/config.yaml delete mode 100644 exporters/astar/exporter.py diff --git a/exporters/astar/config.yaml b/exporters/astar/config.yaml deleted file mode 100644 index 6418622..0000000 --- a/exporters/astar/config.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -api_substrate: https://shiden-api.polka.p2p.world/api -api_registry: https://registry-para-api.polka.p2p.world/api -chain: shiden -exporter: - listen: 0.0.0.0 - port: 9618 -validators: - scaleway-shiden-collator1: - account: 'ZPsVRDiDTsWr9wENxE5iyBFCd2ySckF9mn8PtGSevnKjMwU' diff --git a/exporters/astar/exporter.py b/exporters/astar/exporter.py deleted file mode 100644 index b0ec44b..0000000 --- a/exporters/astar/exporter.py +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env python3 - -import threading -import requests -import yaml -import time -import json -import logging -from numpy import median, average, percentile -from decimal import * -from collections import deque -from flask import Flask, request, make_response - -logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %I:%M:%S') -app = Flask(__name__) - -@app.route("/metrics") -def metrics(): - chain = get_config('chain') - metrics = q_metrics[0].copy() - - out = "" - - try: - out += '# HELP astar_currentEra Current era\n' - out += '# TYPE astar_currentEra Current era counter\n' - - out += 'astar_currentEra{chain="%s"} %s\n' % (chain,metrics['common']['current_era']) - except KeyError: - pass - - try: - out += '# HELP astar_currentSession Current session\n' - out += '# TYPE astar_currentSession Current session counter\n' - - out += 'astar_currentSession{chain="%s"} %s\n' % (chain,metrics['common']['current_session']) - except KeyError: - pass - - try: - out += '# HELP astar_activeCollatorsCount Active collators\n' - out += '# TYPE astar_activeCollatorsCount Active collators counter\n' - - out += 'astar_activeCollatorsCount{chain="%s"} %s\n' % (chain,metrics['common']['active_collators_count']) - except KeyError: - pass - - try: - out += '# HELP astar_sessionBlocks Session blocks\n' - out += '# TYPE astar_sessionBlocks Session blocks counter\n' - - out += 'astar_sessionBlocks{chain="%s"} %s\n' % (chain,metrics['common']['session_blocks']) - except KeyError: - pass - - try: - out += '# HELP astar_blocksAvg Blocks avarage\n' - out += '# TYPE astar_blocksAvg Blocks avarage counter\n' - - out += 'astar_blocksAvg{chain="%s"} %s\n' % (chain,metrics['common']['average']) - except KeyError: - pass - - try: - out += '# HELP astar_blocksMedian Blocks median\n' - out += '# TYPE astar_blocksMedian Blocks median counter\n' - - out += 'astar_blocksMedian{chain="%s"} %s\n' % (chain,metrics['common']['median']) - except KeyError: - pass - - try: - out += '# HELP astar_blocksP95 Blocks p95\n' - out += '# TYPE astar_blocksP95 Blocks p95 counter\n' - - out += 'astar_blocksP95{chain="%s"} %s\n' % (chain,metrics['common']['p95']) - except KeyError: - pass - - try: - out += '# HELP astar_activeCollators Active collators\n' - out += '# TYPE astar_activeCollators Active collators counter\n' - - for k,v in metrics['collators'].items(): - - out += 'astar_activeCollators{node="%s",chain="%s",account="%s"} %s\n' % (v['node'],chain,k,v['is_active']) - - except KeyError: - pass - - try: - out += '# HELP astar_blockAuthorship Blocks authored\n' - out += '# TYPE astar_blockAuthorship Blocks authored counter\n' - - for k,v in metrics['collators'].items(): - out += 'astar_blockAuthorship{node="%s",chain="%s",account="%s"} %s\n' % (v['node'],chain,k,v['authored_blocks_count']) - except KeyError: - pass - - - response = make_response(out, 200) - response.mimetype = "text/plain" - - return response - -def api_request(method = None,args = None): - url = get_config('api_substrate') - - if isinstance(args, list): - for i in range(len(args)): - if isinstance(args[i], str): - args[i] = '"' + args[i] + '"' - elif isinstance(args, str): - args = '"' + args + '"' - elif not args: - args = "" - - data = {'method': method, - 'args': args} - - try: - r = requests.post(url, json=data) - except (ConnectionRefusedError,requests.exceptions.ConnectionError) as e: - logging.critical(e) - return None - - if r.status_code == 200: - return r.json()['result'] - else: - logging.critical('Request to ' + url + ' finished with code ' + str(r.status_code)) - return None - -def get_config(part): - with open('./config.yaml') as config_file: - data = yaml.load(config_file, Loader=yaml.FullLoader) - - return data[part] - -def get_block_count(addr): - try: - authored_block = int(api_request(method = 'api.query.collatorSelection.lastAuthoredBlock', args = addr),16) - - for i in q_collators: - idx = q_collators.index(i) - if addr in i.keys(): - if authored_block not in q_collators[idx][addr]: - q_collators[idx][addr].append(authored_block) - - except Exception as e: - logging.critical('The collators thread been stucked with error "' + str(e) + '"') - -def main(): - block = 0 - era = 0 - session = 0 - - while True: - try: - current_era = int(api_request(method = 'api.query.dappsStaking.currentEra')[2:],16) - current_session = int(api_request(method = 'api.query.session.currentIndex')[2:],16) - - if era != current_era: - logging.info('New era ' + str(current_era) + ' has just begun') - - if session != current_session: - processed = [] - q_collators.clear() - active_collators = api_request(method = 'api.query.session.validators') - disabled_collators = api_request(method = 'api.query.session.disabledValidators') - - for addr in active_collators: - if addr not in collators.keys() and addr not in processed: - q_collators.append({addr:[]}) - processed.append(addr) - - for k,v in collators.items(): - if v in active_collators: - collator_idx = active_collators.index(k) - v['authored_blocks_count'] = 0 - v['is_active'] = 1 - result = {'collators':collators, 'common':{}} - else: - result = {'collators':{}, 'common':{}} - v['is_active'] = 0 - break - - result['common']['current_era'] = current_era - result['common']['current_session'] = current_session - result['common']['session_blocks'] = 0 - logging.info('New session ' + str(current_session) + ' has just begun') - - last_block = int(api_request(method = 'api.query.system.number'),16) - - if last_block != block: - logging.info('Processing block ' + str(last_block)) - - blocks_per_collator = [] - if result['collators'].items(): - for addr,params in result['collators'].items(): - - if addr in active_collators: - authored_block = api_request(method = 'api.query.collatorSelection.lastAuthoredBlock', args = addr) - authored_block = int(authored_block, 16) - - if 'last_authored_block' not in params: - params['last_authored_block'] = authored_block - - if params['last_authored_block'] != authored_block: - params['authored_blocks_count'] += 1 - logging.info('Collator ' + str(addr) + ' has just constructed block ' + str(authored_block)) - - params['last_authored_block'] = authored_block - - threads = [] - - for addr in active_collators: - if addr not in collators.keys(): - th = threading.Thread(target=get_block_count,args=(addr,)) - threads.append(th) - - for t in threads: - t.start() - t.join() - - for c in q_collators: - blocks_per_collator.append(len(list(c.values())[0])) - - result['common']['session_blocks'] += 1 - result['common']['median'] = int(Decimal(median(blocks_per_collator))) - result['common']['average'] = int(Decimal(average(blocks_per_collator))) - result['common']['p95'] = int(Decimal(percentile(blocks_per_collator,95))) - - q_metrics.clear() - q_metrics.append(result) - - era = current_era - session = current_session - block = last_block - - except Exception as e: - logging.critical('The main thread been stucked with error "' + str(e) + '"') - time.sleep(10) - continue - - time.sleep(3) - -if __name__ == '__main__': - endpoint_listen = get_config('exporter')['listen'] - endpoint_port = get_config('exporter')['port'] - - collators = {} - - for k,v in get_config('validators').items(): - collators[v['account']] = {'node':k} - - q_metrics = deque([]) - q_collators = deque([]) - - worker = threading.Thread(target=main) - worker.daemon = True - worker.start() - - app.run(host="0.0.0.0", port=int(endpoint_port)) From d7b8add3d2bb6ab4223c9941536ce73ae2b4352a Mon Sep 17 00:00:00 2001 From: sergeyradchenkop2p Date: Mon, 15 Jan 2024 16:50:55 +0300 Subject: [PATCH 03/10] moonbeam exporter fix --- exporters/moonbeam/exporter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exporters/moonbeam/exporter.py b/exporters/moonbeam/exporter.py index 0478168..5794862 100755 --- a/exporters/moonbeam/exporter.py +++ b/exporters/moonbeam/exporter.py @@ -104,7 +104,7 @@ def get_round_progress(chain): } round_length = constants[chain]['round_length'] - round_first_block = substrate_interface.request('ParachainStaking','Round', ['first']).value + round_first_block = substrate_interface.request('ParachainStaking','Round').value['first'] current_block = substrate_interface.request('System','Number').value round_progress = (int(current_block) - int(round_first_block)) / round_length * 100 @@ -122,7 +122,7 @@ def main(): while True: try: - current_rnd = substrate_interface.request('ParachainStaking','Round').value.current + current_rnd = substrate_interface.request('ParachainStaking','Round').value['current'] if current_rnd != rnd: rnd_blocks_count = 0 active_collators = substrate_interface.request('ParachainStaking','SelectedCandidates').value From e71ddc7c9fe8176f9e0b15bcc4fcf56d6d3435cd Mon Sep 17 00:00:00 2001 From: sergeyradchenkop2p Date: Fri, 19 Jan 2024 19:05:42 +0300 Subject: [PATCH 04/10] rm astar --- exporters/astar/Dockerfile | 10 -- exporters/astar/config.yaml | 6 - exporters/astar/exporter.py | 243 ------------------------------- exporters/astar/functions.py | 79 ---------- exporters/astar/requirements.txt | 4 - 5 files changed, 342 deletions(-) delete mode 100644 exporters/astar/Dockerfile delete mode 100644 exporters/astar/config.yaml delete mode 100644 exporters/astar/exporter.py delete mode 100644 exporters/astar/functions.py delete mode 100644 exporters/astar/requirements.txt diff --git a/exporters/astar/Dockerfile b/exporters/astar/Dockerfile deleted file mode 100644 index 9d01651..0000000 --- a/exporters/astar/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM --platform=linux/amd64 python:3.9.10-slim-buster - -WORKDIR / - -COPY requirements.txt requirements.txt -RUN pip3 install -r requirements.txt - -COPY exporter.py app.py - -CMD ["python3", "app.py"] diff --git a/exporters/astar/config.yaml b/exporters/astar/config.yaml deleted file mode 100644 index 7249b24..0000000 --- a/exporters/astar/config.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -chain: astar -ws_endpoint: wss://rpc.astar.network -exporter: - listen: 0.0.0.0 - port: 9618 diff --git a/exporters/astar/exporter.py b/exporters/astar/exporter.py deleted file mode 100644 index e6903cc..0000000 --- a/exporters/astar/exporter.py +++ /dev/null @@ -1,243 +0,0 @@ -#!/usr/bin/env python3 - -import threading -import requests -import yaml -import time -import json -import logging -import operator -import traceback -from collections import deque -from flask import Flask, request, make_response -from functions import SUBSTRATE_INTERFACE, get_config, get_era_points, get_chain_info -from _thread import interrupt_main -from numpy import median, average, percentile -from decimal import * - - -logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %I:%M:%S') -app = Flask(__name__) - -@app.route("/metrics") -def metrics(): - chain = get_config('chain') - metrics = q_metrics[0].copy() - - out = "" - - try: - out += '# HELP astar_currentEra Current era\n' - out += '# TYPE astar_currentEra Current era counter\n' - - out += 'astar_currentEra{chain="%s"} %s\n' % (chain,metrics['common']['current_era']) - except KeyError: - pass - - try: - out += '# HELP astar_currentSession Current session\n' - out += '# TYPE astar_currentSession Current session counter\n' - - out += 'astar_currentSession{chain="%s"} %s\n' % (chain,metrics['common']['current_session']) - except KeyError: - pass - - try: - out += '# HELP astar_activeCollatorsCount Active collators\n' - out += '# TYPE astar_activeCollatorsCount Active collators counter\n' - - out += 'astar_activeCollatorsCount{chain="%s"} %s\n' % (chain,metrics['common']['active_collators_count']) - except KeyError: - pass - - try: - out += '# HELP astar_sessionBlocks Session blocks\n' - out += '# TYPE astar_sessionBlocks Session blocks counter\n' - - out += 'astar_sessionBlocks{chain="%s"} %s\n' % (chain,metrics['common']['session_blocks']) - except KeyError: - pass - - try: - out += '# HELP astar_blocksAvg Blocks avarage\n' - out += '# TYPE astar_blocksAvg Blocks avarage counter\n' - - out += 'astar_blocksAvg{chain="%s"} %s\n' % (chain,metrics['common']['average']) - except KeyError: - pass - - try: - out += '# HELP astar_blocksMedian Blocks median\n' - out += '# TYPE astar_blocksMedian Blocks median counter\n' - - out += 'astar_blocksMedian{chain="%s"} %s\n' % (chain,metrics['common']['median']) - except KeyError: - pass - - try: - out += '# HELP astar_blocksP95 Blocks p95\n' - out += '# TYPE astar_blocksP95 Blocks p95 counter\n' - - out += 'astar_blocksP95{chain="%s"} %s\n' % (chain,metrics['common']['p95']) - except KeyError: - pass - - try: - out += '# HELP astar_activeCollators Active collators\n' - out += '# TYPE astar_activeCollators Active collators counter\n' - - for k,v in metrics['collators'].items(): - - out += 'astar_activeCollators{node="%s",chain="%s",account="%s"} %s\n' % (v['node'],chain,k,v['is_active']) - - except KeyError: - pass - - try: - out += '# HELP astar_blockAuthorship Blocks authored\n' - out += '# TYPE astar_blockAuthorship Blocks authored counter\n' - - for k,v in metrics['collators'].items(): - out += 'astar_blockAuthorship{node="%s",chain="%s",account="%s"} %s\n' % (v['node'],chain,k,v['authored_blocks_count']) - except KeyError: - pass - - - response = make_response(out, 200) - response.mimetype = "text/plain" - - return response - - -def get_config(part): - with open('./config.yaml') as config_file: - data = yaml.load(config_file, Loader=yaml.FullLoader) - - return data[part] - -def get_block_count(addr): - try: - authored_block = substrate_interface.request('CollatorSelection','LastAuthoredBlock', ['addr']).value - - for i in q_collators: - idx = q_collators.index(i) - if addr in i.keys(): - if authored_block not in q_collators[idx][addr]: - q_collators[idx][addr].append(authored_block) - - except Exception as e: - logging.critical('The collators thread been stucked with error "' + str(e) + '"') - -def main(): - block = 0 - era = 0 - session = 0 - - while True: - try: - current_era = substrate_interface.request('DappsStaking','CurrentEra').value - current_session = substrate_interface.request('Session','CurrentIndex').value - - if era != current_era: - logging.info('New era ' + str(current_era) + ' has just begun') - - if session != current_session: - processed = [] - q_collators.clear() - active_collators = substrate_interface.request('Session','Validators').value - disabled_collators = substrate_interface.request('Session','DisabledValidators').value - - for addr in active_collators: - if addr not in collators.keys() and addr not in processed: - q_collators.append({addr:[]}) - processed.append(addr) - - for k,v in collators.items(): - if v in active_collators: - collator_idx = active_collators.index(k) - v['authored_blocks_count'] = 0 - v['is_active'] = 1 - result = {'collators':collators, 'common':{}} - else: - result = {'collators':{}, 'common':{}} - v['is_active'] = 0 - break - - result['common']['current_era'] = current_era - result['common']['current_session'] = current_session - result['common']['session_blocks'] = 0 - logging.info('New session ' + str(current_session) + ' has just begun') - - last_block = substrate_interface.request('System','Number').value - - if last_block != block: - logging.info('Processing block ' + str(last_block)) - - blocks_per_collator = [] - if result['collators'].items(): - for addr,params in result['collators'].items(): - - if addr in active_collators: - authored_block = substrate_interface.request('CollatorSelection','LastAuthoredBlock', ['addr']).value - - if 'last_authored_block' not in params: - params['last_authored_block'] = authored_block - - if params['last_authored_block'] != authored_block: - params['authored_blocks_count'] += 1 - logging.info('Collator ' + str(addr) + ' has just constructed block ' + str(authored_block)) - - params['last_authored_block'] = authored_block - - threads = [] - - for addr in active_collators: - if addr not in collators.keys(): - th = threading.Thread(target=get_block_count,args=(addr,)) - threads.append(th) - - for t in threads: - t.start() - t.join() - - for c in q_collators: - blocks_per_collator.append(len(list(c.values())[0])) - - result['common']['session_blocks'] += 1 - result['common']['median'] = int(Decimal(median(blocks_per_collator))) - result['common']['average'] = int(Decimal(average(blocks_per_collator))) - result['common']['p95'] = int(Decimal(percentile(blocks_per_collator,95))) - - q_metrics.clear() - q_metrics.append(result) - - era = current_era - session = current_session - block = last_block - - except Exception as e: - logging.critical('The main thread been stucked with error "' + str(e) + '"') - time.sleep(10) - continue - - time.sleep(3) - -if __name__ == '__main__': - endpoint_listen = get_config('exporter')['listen'] - endpoint_port = get_config('exporter')['port'] - ws_endpoint = get_config('ws_endpoint') - chain = get_config('chain') - substrate_interface = SUBSTRATE_INTERFACE(ws_endpoint,chain) - - collators = {} - - - - q_metrics = deque([]) - q_collators = deque([]) - - worker = threading.Thread(target=main) - worker.daemon = True - worker.start() - - app.run(host="0.0.0.0", port=int(endpoint_port)) diff --git a/exporters/astar/functions.py b/exporters/astar/functions.py deleted file mode 100644 index e9034e7..0000000 --- a/exporters/astar/functions.py +++ /dev/null @@ -1,79 +0,0 @@ -import yaml -from substrateinterface import SubstrateInterface, Keypair -from substrateinterface.exceptions import SubstrateRequestException -from websocket._exceptions import WebSocketConnectionClosedException - -class SUBSTRATE_INTERFACE: - def __init__(self, ws_endpoint, chain): - self.substrate = SubstrateInterface( - url=ws_endpoint, - ss58_format=42, - type_registry_preset=chain) - - - def request(self, module: str, function: str, params: str = None): - try: - r = self.substrate.query( - module=module, - storage_function=function, - params=params) - - return r - except (WebSocketConnectionClosedException,ConnectionRefusedError,SubstrateRequestException) as e: - self.substrate.connect_websocket() - logging.critical('The substrate api call failed with error ' + str(e)) - r = None - -def get_config(part): - with open('./config.yaml') as config_file: - data = yaml.load(config_file, Loader=yaml.FullLoader) - - return data[part] - -def get_era_points(data): - result = {} - - for i in data['individual']: - result[i[0]] = i[1] - - return {'result':result,'total':data['total']} - -def get_chain_info(chain,substrate_interface): - constants = {'polkadot':{'session_length':1200,'era_length':7200}} - - session_length = constants[chain]['session_length'] - era_length = constants[chain]['era_length'] - - current_era = substrate_interface.request('Staking','ActiveEra').value['index'] - current_session = substrate_interface.request('Session','CurrentIndex').value - - eras_start_session_index = substrate_interface.request('Staking','ErasStartSessionIndex',[current_era]).value - - genesis_slot = substrate_interface.request('Babe','GenesisSlot').value - current_slot = substrate_interface.request('Babe','CurrentSlot').value - - session_start_slot = int(current_session) * int(session_length) + int(genesis_slot) - session_progress = int(current_slot) - int(session_start_slot) - - era_session_index = int(current_session) - int(eras_start_session_index) - era_progress = int(era_session_index) * int(session_length) + int(session_progress) - - return {'current_era': current_era, 'eras_start_session_index': eras_start_session_index, 'current_session': current_session, 'era_progress': era_progress / era_length * 100, 'session_progress': session_progress / session_length * 100} - -def ss58_convert(data): - r = [] - - for key in data: - pubkey = Keypair(ss58_address=key).public_key.hex() - r.append('0x' + pubkey) - - return r - -def get_keys(validators,keys): - result = {i:None for i in validators} - - for i in keys: - if str(i[0]) in result.keys(): - result[str(i[0])] = str(i[1]['grandpa']) - - return result diff --git a/exporters/astar/requirements.txt b/exporters/astar/requirements.txt deleted file mode 100644 index f363289..0000000 --- a/exporters/astar/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -numpy -flask -requests -pyyaml From 0378097decd71534e15202c61d2bdbee5e17d991 Mon Sep 17 00:00:00 2001 From: sergeyradchenkop2p Date: Wed, 24 Jan 2024 18:25:16 +0300 Subject: [PATCH 05/10] dockerfile fix for parachains --- exporters/acala/Dockerfile | 9 +++++++-- exporters/acala/requirements.txt | 7 ++++--- exporters/moonbeam/Dockerfile | 8 +------- exporters/moonbeam/requirements.txt | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/exporters/acala/Dockerfile b/exporters/acala/Dockerfile index 9d01651..e53d535 100644 --- a/exporters/acala/Dockerfile +++ b/exporters/acala/Dockerfile @@ -1,10 +1,15 @@ -FROM --platform=linux/amd64 python:3.9.10-slim-buster +FROM --platform=linux/amd64 python:3.11-slim-buster + +ARG exporter WORKDIR / +RUN apt-get update && apt-get install -y gcc g++ COPY requirements.txt requirements.txt -RUN pip3 install -r requirements.txt +RUN pip3 install -r requirements.txt --no-cache-dir +RUN groupadd -r exporter && useradd -r -g exporter exporter COPY exporter.py app.py +USER exporter CMD ["python3", "app.py"] diff --git a/exporters/acala/requirements.txt b/exporters/acala/requirements.txt index 09a313b..6188367 100644 --- a/exporters/acala/requirements.txt +++ b/exporters/acala/requirements.txt @@ -1,3 +1,4 @@ -flask -requests -pyyaml +pyaml==23.5.9 +substrate-interface==1.7.1 +flask==2.3.2 +numpy==1.24.3 diff --git a/exporters/moonbeam/Dockerfile b/exporters/moonbeam/Dockerfile index b3cfdb9..e53d535 100644 --- a/exporters/moonbeam/Dockerfile +++ b/exporters/moonbeam/Dockerfile @@ -1,9 +1,3 @@ -FROM alpine/flake8:latest as linter -WORKDIR /apps/ -COPY . /apps/ -## ingore E501 line too long (XX > 79 characters) -RUN flake8 --ignore="E501" *.py - FROM --platform=linux/amd64 python:3.11-slim-buster ARG exporter @@ -15,7 +9,7 @@ COPY requirements.txt requirements.txt RUN pip3 install -r requirements.txt --no-cache-dir RUN groupadd -r exporter && useradd -r -g exporter exporter -COPY --from=linter /apps/${exporter}.py app.py +COPY exporter.py app.py USER exporter CMD ["python3", "app.py"] diff --git a/exporters/moonbeam/requirements.txt b/exporters/moonbeam/requirements.txt index f363289..6188367 100644 --- a/exporters/moonbeam/requirements.txt +++ b/exporters/moonbeam/requirements.txt @@ -1,4 +1,4 @@ -numpy -flask -requests -pyyaml +pyaml==23.5.9 +substrate-interface==1.7.1 +flask==2.3.2 +numpy==1.24.3 From 31725ba1a93e909f25008915e735bae05ce8b77f Mon Sep 17 00:00:00 2001 From: sergeyradchenkop2p Date: Wed, 24 Jan 2024 19:00:05 +0300 Subject: [PATCH 06/10] dockerfile fix for parachains --- exporters/acala/Dockerfile | 1 + exporters/moonbeam/Dockerfile | 1 + 2 files changed, 2 insertions(+) diff --git a/exporters/acala/Dockerfile b/exporters/acala/Dockerfile index e53d535..550d77c 100644 --- a/exporters/acala/Dockerfile +++ b/exporters/acala/Dockerfile @@ -10,6 +10,7 @@ RUN pip3 install -r requirements.txt --no-cache-dir RUN groupadd -r exporter && useradd -r -g exporter exporter COPY exporter.py app.py +COPY functions.py functions.py USER exporter CMD ["python3", "app.py"] diff --git a/exporters/moonbeam/Dockerfile b/exporters/moonbeam/Dockerfile index e53d535..550d77c 100644 --- a/exporters/moonbeam/Dockerfile +++ b/exporters/moonbeam/Dockerfile @@ -10,6 +10,7 @@ RUN pip3 install -r requirements.txt --no-cache-dir RUN groupadd -r exporter && useradd -r -g exporter exporter COPY exporter.py app.py +COPY functions.py functions.py USER exporter CMD ["python3", "app.py"] From d770267d2d64c3eead7bbfe41ba1f56c339b86eb Mon Sep 17 00:00:00 2001 From: sergeyradchenkop2p Date: Thu, 25 Jan 2024 12:44:12 +0300 Subject: [PATCH 07/10] parachains fix --- acala.env | 2 ++ acala.yml | 17 +++++++++++++++++ exporters/acala/exporter.py | 16 +++++++--------- exporters/moonbeam/exporter.py | 16 ++++++---------- karura.env | 2 ++ karura.yml | 17 +++++++++++++++++ 6 files changed, 51 insertions(+), 19 deletions(-) create mode 100644 acala.env create mode 100644 acala.yml create mode 100644 karura.env create mode 100644 karura.yml diff --git a/acala.env b/acala.env new file mode 100644 index 0000000..a64aba9 --- /dev/null +++ b/acala.env @@ -0,0 +1,2 @@ +WS_ENDPOINT="wss://acala-rpc.dwellir.com" +WS_ENDPOINTS="wss://acala-rpc.dwellir.com" diff --git a/acala.yml b/acala.yml new file mode 100644 index 0000000..a6b5c3b --- /dev/null +++ b/acala.yml @@ -0,0 +1,17 @@ +version: '3.4' + +services: + acala_exporter: + build: + context: ./exporters/acala + args: + exporter: "acala_exporter" + environment: + - "LISTEN=0.0.0.0" + - "PORT=9150" + - "CHAIN=acala" + env_file: + - ./acala.env + networks: + - exporters + diff --git a/exporters/acala/exporter.py b/exporters/acala/exporter.py index c47f118..b85e15d 100644 --- a/exporters/acala/exporter.py +++ b/exporters/acala/exporter.py @@ -58,11 +58,6 @@ def metrics(): return response -def get_config(part): - with open('./config.yaml') as config_file: - data = yaml.load(config_file, Loader=yaml.FullLoader) - - return data[part] def main(): block = 0 @@ -97,10 +92,13 @@ def main(): time.sleep(3) if __name__ == '__main__': - endpoint_listen = get_config('exporter')['listen'] - endpoint_port = get_config('exporter')['port'] - ws_endpoint = get_config('ws_endpoint') - chain = get_config('chain') + + endpoint_listen = os.environ['LISTEN'] + endpoint_port = os.environ['PORT'] + ws_endpoint = os.environ['WS_ENDPOINT'] + chain = os.environ['CHAIN'] + + substrate_interface = SUBSTRATE_INTERFACE(ws_endpoint,chain) q_metrics = deque([]) diff --git a/exporters/moonbeam/exporter.py b/exporters/moonbeam/exporter.py index 5794862..ab2dec5 100755 --- a/exporters/moonbeam/exporter.py +++ b/exporters/moonbeam/exporter.py @@ -110,12 +110,6 @@ def get_round_progress(chain): return round_progress -def get_config(part): - with open('./config.yaml') as config_file: - data = yaml.load(config_file, Loader=yaml.FullLoader) - - return data[part] - def main(): block = 0 rnd = 0 @@ -170,10 +164,12 @@ def main(): time.sleep(3) if __name__ == '__main__': - endpoint_listen = get_config('exporter')['listen'] - endpoint_port = get_config('exporter')['port'] - ws_endpoint = get_config('ws_endpoint') - chain = get_config('chain') + + endpoint_listen = os.environ['LISTEN'] + endpoint_port = os.environ['PORT'] + ws_endpoint = os.environ['WS_ENDPOINT'] + chain = os.environ['CHAIN'] + substrate_interface = SUBSTRATE_INTERFACE(ws_endpoint,chain) collators = {} diff --git a/karura.env b/karura.env new file mode 100644 index 0000000..2726043 --- /dev/null +++ b/karura.env @@ -0,0 +1,2 @@ +WS_ENDPOINT="wss://karura-rpc.dwellir.com" +WS_ENDPOINTS="wss://karura-rpc.dwellir.com" diff --git a/karura.yml b/karura.yml new file mode 100644 index 0000000..5f21bba --- /dev/null +++ b/karura.yml @@ -0,0 +1,17 @@ +version: '3.4' + +services: + acala_exporter: + build: + context: ./exporters/acala + args: + exporter: "acala_exporter" + environment: + - "LISTEN=0.0.0.0" + - "PORT=9150" + - "CHAIN=karura" + env_file: + - ./karura.env + networks: + - exporters + From fa289b6c08cbc2fa92087c3c5486bee505bcb51f Mon Sep 17 00:00:00 2001 From: sergeyradchenkop2p Date: Thu, 25 Jan 2024 14:18:20 +0300 Subject: [PATCH 08/10] fix exporters for parachains --- exporters/acala/exporter.py | 1 + exporters/moonbeam/exporter.py | 1 + 2 files changed, 2 insertions(+) diff --git a/exporters/acala/exporter.py b/exporters/acala/exporter.py index b85e15d..ab18e70 100644 --- a/exporters/acala/exporter.py +++ b/exporters/acala/exporter.py @@ -8,6 +8,7 @@ import logging import operator import traceback +import os from collections import deque from flask import Flask, request, make_response from functions import SUBSTRATE_INTERFACE, get_config, get_era_points, get_chain_info diff --git a/exporters/moonbeam/exporter.py b/exporters/moonbeam/exporter.py index ab2dec5..6b67d1d 100755 --- a/exporters/moonbeam/exporter.py +++ b/exporters/moonbeam/exporter.py @@ -8,6 +8,7 @@ import logging import operator import traceback +import os from collections import deque from flask import Flask, request, make_response from functions import SUBSTRATE_INTERFACE, get_config, get_era_points, get_chain_info From c32614ae7765d72d0962d3f07e2be7f76401584c Mon Sep 17 00:00:00 2001 From: sergeyradchenkop2p Date: Thu, 25 Jan 2024 14:50:56 +0300 Subject: [PATCH 09/10] another small fix to parachains maas --- exporters/acala/exporter.py | 2 +- exporters/acala/functions.py | 5 ----- exporters/moonbeam/exporter.py | 2 +- exporters/moonbeam/functions.py | 5 ----- 4 files changed, 2 insertions(+), 12 deletions(-) diff --git a/exporters/acala/exporter.py b/exporters/acala/exporter.py index ab18e70..d1f1e8d 100644 --- a/exporters/acala/exporter.py +++ b/exporters/acala/exporter.py @@ -22,7 +22,7 @@ @app.route("/metrics") def metrics(): - chain = get_config('chain') + chain = os.environ['CHAIN'] metrics = q_metrics[0].copy() out = "" diff --git a/exporters/acala/functions.py b/exporters/acala/functions.py index e9034e7..e8ab882 100644 --- a/exporters/acala/functions.py +++ b/exporters/acala/functions.py @@ -24,11 +24,6 @@ def request(self, module: str, function: str, params: str = None): logging.critical('The substrate api call failed with error ' + str(e)) r = None -def get_config(part): - with open('./config.yaml') as config_file: - data = yaml.load(config_file, Loader=yaml.FullLoader) - - return data[part] def get_era_points(data): result = {} diff --git a/exporters/moonbeam/exporter.py b/exporters/moonbeam/exporter.py index 6b67d1d..84b2f87 100755 --- a/exporters/moonbeam/exporter.py +++ b/exporters/moonbeam/exporter.py @@ -22,7 +22,7 @@ @app.route("/metrics") def metrics(): - chain = get_config('chain') + chain = os.environ['CHAIN'] metrics = q_metrics[0].copy() out = "" diff --git a/exporters/moonbeam/functions.py b/exporters/moonbeam/functions.py index e9034e7..e8ab882 100644 --- a/exporters/moonbeam/functions.py +++ b/exporters/moonbeam/functions.py @@ -24,11 +24,6 @@ def request(self, module: str, function: str, params: str = None): logging.critical('The substrate api call failed with error ' + str(e)) r = None -def get_config(part): - with open('./config.yaml') as config_file: - data = yaml.load(config_file, Loader=yaml.FullLoader) - - return data[part] def get_era_points(data): result = {} From fed8c70599e28c0fe084c1f187f0cdb18304682a Mon Sep 17 00:00:00 2001 From: master Date: Fri, 26 Jan 2024 12:33:18 +0300 Subject: [PATCH 10/10] fix exporters for maas --- exporters/acala/exporter.py | 2 +- exporters/moonbeam/exporter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/exporters/acala/exporter.py b/exporters/acala/exporter.py index d1f1e8d..3ae350b 100644 --- a/exporters/acala/exporter.py +++ b/exporters/acala/exporter.py @@ -11,7 +11,7 @@ import os from collections import deque from flask import Flask, request, make_response -from functions import SUBSTRATE_INTERFACE, get_config, get_era_points, get_chain_info +from functions import SUBSTRATE_INTERFACE, get_era_points, get_chain_info from _thread import interrupt_main from numpy import median, average, percentile from decimal import * diff --git a/exporters/moonbeam/exporter.py b/exporters/moonbeam/exporter.py index 84b2f87..69e6de6 100755 --- a/exporters/moonbeam/exporter.py +++ b/exporters/moonbeam/exporter.py @@ -11,7 +11,7 @@ import os from collections import deque from flask import Flask, request, make_response -from functions import SUBSTRATE_INTERFACE, get_config, get_era_points, get_chain_info +from functions import SUBSTRATE_INTERFACE, get_era_points, get_chain_info from _thread import interrupt_main from numpy import median, average, percentile from decimal import *