Rework of labels discovery

bot/app/__main__.py

@@ -30,6 +30,7 @@
     prometheus_alert_path = os.environ.get('prometheus_alert_path', '../prometheus/rules/')
     prometheus_alert_tmpl = os.environ.get('prometheus_alert_tmpl', './prom_alerts/alerts_tmpl.yml')
     prometheus_alert_api = os.environ.get('prometheus_alert_api', 'http://prometheus:9090/api/v1/rules')
+    prometheus_metric_api = os.environ.get('prometheus_metric_api', 'http://prometheus:9090/api/v1/series')
     prometheus_config_reload = os.environ.get('prometheus_config_reload', 'http://prometheus:9090/-/reload')
 
     cache = CACHE(redis_host, redis_port)

bot/app/callback_query_handlers/subscribtions.py

@@ -124,6 +124,7 @@ async def sub_view(query: CallbackQuery):
 
     await query.answer()
 
+
 @router.callback_query(CbData.filter(F.dst == 'sub_edit'))
 async def sub_edit(query: CallbackQuery, state: FSMContext):
     chat_id = query.message.chat.id
@@ -140,6 +141,7 @@ async def sub_edit(query: CallbackQuery, state: FSMContext):
         await state.set_state(Form.subscribtions)
 
         check_list = {'id': uniqueid, 'check_list':{i:{'data': 'undefined', 'emoji': '❌ '} for i in mandatory_filters if i != 'chat_id'}}
+
     elif isinstance(d,dict) and len(d.keys()) > 0:
         check_list = d
     else:
@@ -150,6 +152,7 @@ async def sub_edit(query: CallbackQuery, state: FSMContext):
             validators = db.get_records('validators', 'id', chat_id).split(' ')
         except AttributeError:
             await query.answer('Please define accounts. Subsbtions menu -> My accounts')
+
             validators = 'undefined'
 
         if len(validators) > 0 and validators != 'undefined':
@@ -164,23 +167,26 @@ async def sub_edit(query: CallbackQuery, state: FSMContext):
         if k == 'accounts' and v['data'] != 'undefined':
             check_list_text += v['emoji'] + k.capitalize() + ': ' + str(len(v['data'])) + ' accounts in portfolio.'
         else:
-            check_list_text += v['emoji'] + k.capitalize() + ': ' + v['data']
+            check_list_text += v['emoji'] + k.capitalize() + ': ' + str(v['data'])
 
         check_list_text += "\n      "
 
     text = '🔹 ' + template['alert'] + '\n\n' + '🔻 ' + template['annotations']['bot_description'] + "\n\n🔸 Mandatory filters:\n" + check_list_text + "\n\n"
 
     menu = MenuBuilder()
 
+    if 'project' in check_list['check_list'].keys():
+        menu.button(text='Select projects',callback_data=CbData(dst="sub_set_labels", data=".project", id=uniqueid).pack()) + "size=1"
+
+    if 'chain' in check_list['check_list'].keys():
+        menu.button(text='Select chains', callback_data=CbData(dst="sub_set_labels", data=".chain", id=uniqueid).pack()) + "size=1"
+
     if 'interval' in check_list['check_list'].keys():
         menu.button(text='Set time interval',callback_data=CbData(dst="sub_set_interval", data="", id=uniqueid).pack()) + "size=1"
 
     if 'threshold' in check_list['check_list'].keys():
-        menu.button(text="Select threshold", callback_data=CbData(dst="sub_set_threshold", data="reset", id=uniqueid).pack()) + "size=1"
+        menu.button(text="Set threshold", callback_data=CbData(dst="sub_set_threshold", data="reset", id=uniqueid).pack()) + "size=1"
 
-    if 'chain' in check_list['check_list'].keys():
-        menu.button(text='Set chain', callback_data=CbData(dst="sub_set_chain", data="", id=uniqueid).pack()) + "size=1"
-
     if isinstance(d,dict) and len(d.keys()) > 0:
         statuses = [v['data'] for k,v in d['check_list'].items()]
 
@@ -204,12 +210,68 @@ async def sub_edit(query: CallbackQuery, state: FSMContext):
 
     await query.answer()
 
+
+@router.callback_query(CbData.filter(F.dst == 'sub_set_labels'))
+async def sub_set_labels(query: CallbackQuery, state: FSMContext):
+    chat_id = query.message.chat.id
+    uniqueid = int(query.data.split(':')[3])
+
+    d = await state.get_data()
+
+    if query.data.split(':')[2].startswith('.'):
+        label_name = query.data.split(':')[2].replace('.', '').split(',')[0]
+
+        try:
+            label_value = query.data.split(':')[2].replace('.', '').split(',')[1]
+        except IndexError:
+            label_value = None
+
+    try:
+        if label_value and label_value == "reset":
+            d['check_list'][label_name]['data'] = ""
+            d['check_list'][label_name]['emoji'] = '❌ '
+
+        else:
+            try:
+                if label_value:
+                    d['check_list'][label_name]['data'].append(label_value)
+            except AttributeError:
+                d['check_list'][label_name]['data'] = [label_value]
+                d['check_list'][label_name]['emoji'] = '✅ '
+
+    except (TypeError, AttributeError):
+        d['check_list'][label_name]['data'] = ""
+        d['check_list'][label_name]['emoji'] = '❌ '
+
+    alerts = Alerts(chat_id)
+    template = alerts.get_template(uniqueid)
+
+    label_values = alerts.get_labels(template, label_name)
+
+    menu = MenuBuilder()
+
+    if isinstance(label_values, list) and len(label_values) > 0:
+        for label in label_values:
+            if label in d['check_list'][label_name]['data']:
+                menu.button(text='🟢 ' + label, callback_data=CbData(dst="sub_set_labels", data="reset", id=uniqueid).pack()) + "size=1"
+            else:
+                menu.button(text=label, callback_data=CbData(dst="sub_set_labels", data="."+label_name + ',' + label, id=uniqueid).pack()) + "size=1"
+
+    menu.button(text="⬅️  Back", callback_data=CbData(dst="sub_edit", data="", id=uniqueid).pack()) + "size=2"
+    menu.button(text="Reset", callback_data=CbData(dst="sub_set_labels", data="."+label_name + ',' + "reset", id=uniqueid).pack()) + "size=2"
+    menu.build()
+
+    await state.set_data(d)
+    await query.message.edit_text(text="Please select labels from list bellow.\n\n", reply_markup=menu.as_markup())
+
+
 @router.callback_query(CbData.filter(F.dst == 'sub_set_interval'))
 async def sub_set_interval(query: CallbackQuery, state: FSMContext):
     uniqueid = int(query.data.split(':')[3])
     interval = query.data.split(':')[2]
 
     d = await state.get_data()
+
     intervals = ['30s','1m','2m','5m','10m','30m']
 
     if interval:
@@ -224,38 +286,12 @@ async def sub_set_interval(query: CallbackQuery, state: FSMContext):
         for interval in intervals:
             menu.button(text=interval, callback_data=CbData(dst="sub_set_interval", data=interval, id=uniqueid).pack()) + "size=3"
 
-        menu.button(text="Back", callback_data=CbData(dst="sub_edit", data="", id=uniqueid).pack()) + "size=1"
+        menu.button(text="⬅️   Back", callback_data=CbData(dst="sub_edit", data="", id=uniqueid).pack()) + "size=1"
         menu.build()
 
         await state.set_data(d)
         await query.message.edit_text(text="Please select time interval.\n\n", reply_markup=menu.as_markup())
 
-@router.callback_query(CbData.filter(F.dst == 'sub_set_chain'))
-async def sub_set_interval(query: CallbackQuery, state: FSMContext):
-    uniqueid = int(query.data.split(':')[3])
-    chain = query.data.split(':')[2]
-
-    d = await state.get_data()
-    chains = ['polkadot','kusama']
-
-    if chain:
-        d['check_list']['chain']['data'] = query.data.split(':')[2]
-        d['check_list']['chain']['emoji'] = '✅ '
-
-        await state.set_data(d)
-        await sub_edit(query,state)
-
-    else:
-        menu = MenuBuilder()
-
-        for chain in chains:
-            menu.button(text=chain, callback_data=CbData(dst="sub_set_chain", data=chain, id=uniqueid).pack()) + "size=1"
-
-        menu.button(text="Back", callback_data=CbData(dst="sub_edit", data="", id=uniqueid).pack()) + "size=1"
-        menu.build()
-
-        await state.set_data(d)
-        await query.message.edit_text(text="Please select the chain.\n\nCurrently only Polkadot and Kusama possible.\n\n", reply_markup=menu.as_markup())
 
 @router.callback_query(CbData.filter(F.dst == 'sub_set_threshold'))
 async def sub_set_threshold(query: CallbackQuery, state: FSMContext):
@@ -303,7 +339,7 @@ async def sub_set_threshold(query: CallbackQuery, state: FSMContext):
     menu.build()
 
     try:
-        await query.message.edit_text(text="Value: " + d['check_list']['threshold']['data'], reply_markup=menu.as_markup())
+        await query.message.edit_text(text="Here you can type necessary threshold value.\n\nValue: " + d['check_list']['threshold']['data'], reply_markup=menu.as_markup())
     except TelegramBadRequest:
         pass
 

bot/app/utils/alerts.py

@@ -1,4 +1,4 @@
-from __main__ import prometheus_alert_path, prometheus_alert_tmpl, prometheus_alert_api, prometheus_config_reload
+from __main__ import prometheus_alert_path, prometheus_alert_tmpl, prometheus_alert_api, prometheus_metric_api, prometheus_config_reload
 import yaml
 import json
 import requests
@@ -19,6 +19,28 @@ def _save_yml(self,path,data):
         with open(path, 'w') as file:
             yaml.dump(data, file)
 
+    def get_labels(self, template: Union[dict, str] = None, key: str = None):
+        metric = template['expr'].split('{')[0].replace("(", "").split(" ")[-1]
+        self.content = requests.get(prometheus_metric_api + '?match[]=' + metric).json()
+
+        try:
+            if isinstance(self.content['data'], list):
+                labels = []
+
+                for i in self.content['data']:
+                    if key in i:
+                        if i[key] not in labels:
+                            labels.append(i[key])
+
+                return labels
+
+            else:
+                return []
+
+        except KeyError:
+            return []
+
+
     def list_templates(self):
         rules = self._load_yml(self.prometheus_alert_tmpl)['rules']
 
@@ -95,8 +117,12 @@ def add_rule(self, uniqueid: int = None, check_list: Union[dict, str] = None, te
 
         check_list = {k:v['data'] for k,v in check_list.items()}
         check_list['chat_id'] = self.chat_id
+
+
+        for k,v in check_list.items():
+            if isinstance(v, list):
+                check_list[k] = '(' + '|'.join(check_list[k]) + ')'
 
-        check_list['accounts'] = '(' + '|'.join(check_list['accounts']) + ')'
         template['labels'].update(check_list) 
 
         for label,line in template.items():

bot/prom_alerts/alerts_tmpl.yml

@@ -10,25 +10,23 @@ rules:
     description: "Test alert for user with id {{ labels.chat_id }}"
     chain: "[[chain]]"
     account: "[[accounts]]"
-    bot_description: "Here is a test alert, which will shout according [[interval]] value.\n\n❗ This text renders from annotations['bot_description'] key.\nFrom template if you are adding or from exist rule if view.\n\n❗ You must have accounts in portfolio and also set all possible filters here.\n\n☝️ [[interval]] [[chain]] [[threshold]] [[accounts]] - represent placeholders if just adding or rendered data if you already activated subscribtion."
+    bot_description: "Here is a test alert, which will shout according [[interval]] value.\n\n❗ This text renders from annotations['bot_description'] key.\nFrom template if you are adding or from exist rule if view.\n\n❗ You must have accounts in portfolio and also set all possible filters here.\n\n☝️ [[interval]] [[threshold]] [[accounts]] [[chains]] [[names]] - represent placeholders if just adding or rendered data if you already activated subscribtion."
 
-- alert: "[Relay]Validator kicked from active set"
-  expr: | 
-    polkadot_session_disabledValidators{account=~"[[accounts]]"} == 1
+- alert: "Latest releases on github"
+  expr: |
+    github_latest_release_seconds{project=~"[[project]]"} < [[threshold]]
   for: "[[interval]]"
   labels:
     uniqueid: 2
     chat_id: "[[chat_id]]"
   annotations:
-    summary: "{{ $labels.chain }}: Validator has been kicked from the active set."
-    description: "Validator {{ $labels.account }} has been kicked from the active set."
-    chain: "{{ $labels.chain }}"
-    account: "{{ $labels.account }}"
-    bot_description: "Will shout if some of selected validators has been kicked from the set."
+    summary: "{{ $labels.name }}: Has been released recently."
+    description: "New version for {{ $labels.name }} has been released recently."
+    bot_description: "Will shout if some of monitored project been release LESS than threshold value in seconds ago."
 
 - alert: "[Relay]Finality GRANDPA precommits ratio"
   expr: | 
-    (polkadot_finality_precommits{account=~"[[accounts]]"} / on (chain) group_left() polkadot_finality_roundsProcessed * 100) < [[threshold]] and on(account) polkadot_session_validators == 1 and on (chain) polkadot_session_sessionProgress >= 3
+    (polkadot_finality_precommits{chain=~"[[chain]]", account=~"[[accounts]]"} / on (chain) group_left() polkadot_finality_roundsProcessed * 100) < [[threshold]] and on(account) polkadot_session_validators == 1 and on (chain) polkadot_session_sessionProgress >= 3
   for: "[[interval]]"
   labels:
     uniqueid: 3
@@ -42,7 +40,7 @@ rules:
 
 - alert: "[Relay]Finality GRANDPA prevotes ratio"
   expr: | 
-    (polkadot_finality_prevotes{account=~"[[accounts]]"} / on (chain) group_left() polkadot_finality_roundsProcessed * 100) < [[threshold]] and on(account) polkadot_session_validators == 1 and on (chain) polkadot_session_sessionProgress >= 3
+    (polkadot_finality_prevotes{chain=~"[[chain]]", account=~"[[accounts]]"} / on (chain) group_left() polkadot_finality_roundsProcessed * 100) < [[threshold]] and on(account) polkadot_session_validators == 1 and on (chain) polkadot_session_sessionProgress >= 3
   for: "[[interval]]"
   labels:
     uniqueid: 4
@@ -56,7 +54,7 @@ rules:
 
 - alert: "[Relay]Epoch ParaValidator rewards ratio"
   expr: | 
-    (polkadot_pv_eraPoints{account=~"[[accounts]]"} / on (chain) group_left() polkadot_pv_pointsP95 * 100) < [[threshold]] and on(account) polkadot_session_paraValidators == 1 and on (chain) polkadot_session_sessionProgress >= 25
+    (polkadot_pv_eraPoints{chain=~"[[chain]]", account=~"[[accounts]]"} / on (chain) group_left() polkadot_pv_pointsP95 * 100) < [[threshold]] and on(account) polkadot_session_paraValidators == 1 and on (chain) polkadot_session_sessionProgress >= 25
   for: "[[interval]]"
   labels:
     uniqueid: 5
@@ -70,7 +68,7 @@ rules:
 
 - alert: "[Relay]Epoch ParaValidator rewards raw"
   expr: | 
-    (polkadot_pv_eraPoints{account=~"[[accounts]]"} < [[threshold]] and on(account) polkadot_session_paraValidators == 1 and on (chain) polkadot_session_sessionProgress >= 85
+    (polkadot_pv_eraPoints{chain=~"[[chain]]", account=~"[[accounts]]"} < [[threshold]] and on(account) polkadot_session_paraValidators == 1 and on (chain) polkadot_session_sessionProgress >= 85
   for: "[[interval]]"
   labels:
     uniqueid: 6
@@ -84,7 +82,7 @@ rules:
 
 - alert: "[Relay]Unapplied slashes for validators"
   expr: |
-    polkadot_staking_unappliedSlashes{account=~"[[accounts]]"} == 1
+    polkadot_staking_unappliedSlashes{chain=~"[[chain]]", account=~"[[accounts]]"} == 1
   for: "[[interval]]"
   labels:
     uniqueid: 7