Merge pull request #24 from p2p-org/subscribtions_improvement

validators_cache + middlewares

bot.env

@@ -5,4 +5,7 @@ prometheus_alert_groups="maas-rules"
 db_name="maas"
 db_user="adm"
 db_pass="adm"
-db_port="5432"
+db_host="postgress"
+db_port="5432"
+redis_host="redis"
+redis_port="6379"

bot/app/__main__.py

@@ -2,15 +2,16 @@
 import sys
 import os
 import asyncio
+from aiohttp import web
 from aiogram import Bot,Dispatcher,Router
 from aiogram.fsm.storage.memory import MemoryStorage
 from message_handlers.setup import setup_message_handler
 from web_apps.setup import setup_web_app
 from forms.setup import setup_message_form
+from utils import subscriptions
 from callback_data.main import CbData
-from aiohttp import web
 from utils.db import DB
-from utils import subscriptions
+from utils.cache import CACHE
 
 logging.basicConfig(level=logging.INFO, stream=sys.stdout)
 
@@ -23,22 +24,28 @@
     db_pass = os.environ['db_pass']
     db_host = os.environ['db_host']
     db_port = os.environ['db_port']
+
+    redis_host = os.environ['redis_host']
+    redis_port = os.environ['redis_port']
 
     grafana_url = os.environ.get('grafana_url', 'http://127.0.0.1:3000/d/fDrj0_EGz/p2p-org-polkadot-kusama-dashboard?orgId=1')
     prometheus_rules_url = os.environ.get('prometheus_rules_url', 'http://localhost:9090/api/v1/rules')
     prometheus_alert_groups = os.environ.get('prometheus_alert_groups', [])
+
     if isinstance(prometheus_alert_groups, str):
         prometheus_alert_groups = prometheus_alert_groups.split(',')
 
     web_app = web.Application()
     db = DB(db_name,db_user,db_pass,db_host,db_port)
     subs = subscriptions.Subscriptions(db, prometheus_rules_url, prometheus_alert_groups)
     bot = Bot(token=tg_token, parse_mode="HTML")
+    validators_cache = CACHE(redis_host,redis_port)
 
     storage = MemoryStorage()
     dp = Dispatcher(storage=storage)
 
     router = Router()
+
     dp.include_router(router)
 
     cb = CbData
@@ -52,6 +59,7 @@
     setup_web_app('prom_alert')
 
     from callback_query_handlers import promalert,main_menu,support,subscriptions
+    from middlewares import acl
 
     web_runner = web.AppRunner(web_app)
     loop = asyncio.get_event_loop()

bot/app/callback_query_handlers/support.py

@@ -100,13 +100,14 @@ async def handle_support_off(query: CallbackQuery, callback_data: CbData):
 @router.callback_query(CbData.filter(F.dst == 'toggle_ban'))
 async def handle_toggle_ban(query: CallbackQuery, callback_data: CbData):
     chat_id = int(callback_data.data)
+
     if db.get_records('account_status', 'id', chat_id) == 'off':
         db.update_record(chat_id, 'account_status', 'on')
-        await bot.send_message(chat_id, 'Your account has been activated🤷\nHave a good day.')
+        await bot.send_message(chat_id, 'Your account has been activated🎉\n')
         await query.answer('Unbanned')
     else:
         db.update_record(chat_id, 'account_status', 'off')
         db.update_record(chat_id, 'support_status', 'off')
-        await bot.send_message(chat_id, 'Your account has been disabled 🤷\nSorry and have a good day.')
+        await bot.send_message(chat_id, 'Your account has been deactivated by our team 🤷\n')
         await query.answer('Banned')
-
+

bot/app/message_handlers/start.py

@@ -5,27 +5,18 @@
 
 @router.message(F.text == '/start')
 async def command_start(message: Message) -> None:
-    if str(message.chat.id).startswith('-'):
-        await message.answer("🧑🤝🧑 Group chats are not allowed.\nSorry and have a good day.")
-        return
-
     username = message.chat.username
     chat_id = message.from_user.id
     account_status = db.get_records('account_status','id',chat_id)
 
     menu = MenuBuilder()
     menu = menu.build(preset='main_menu')
 
-    if account_status and account_status == 'off':
-        await message.answer(chat_id,"Your account has been disabled 🤷\nSorry and have a good day.")
-        return
-
     if not account_status:
         keyboard = MenuBuilder()
         keyboard.add(preset='toggle_ban',data=str(message.from_user.id))
         await message.bot.send_message(admin_chat, text="Username: @{} ID: {}\nHas just PRE-registered.".format(username,chat_id), reply_markup=keyboard.build().as_markup())
         db.add_account(chat_id,username)
 
-        await message.answer("Hi there 👋\n\n\nWelcome to a validator monitoring bot by P2P.org\n\n\n\n",reply_markup=menu.as_markup())
-    else:
-        await message.answer("Here is main menu",reply_markup=menu.as_markup())
+
+    await message.answer("Welcome to a validator monitoring bot by P2P.org\n\n\n\n",reply_markup=menu.as_markup())

bot/app/middlewares/acl.py

@@ -0,0 +1,31 @@
+from __main__ import db,dp, bot
+from aiogram import BaseMiddleware
+from aiogram.types import Message,CallbackQuery
+from aiogram.dispatcher.event.bases import CancelHandler
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional,Awaitable
+from aiogram.types.update import Update
+import asyncio
+
+@dp.update.outer_middleware()
+async def check_if_blocked(handler: Callable[[Update, Dict[str, Any]], Awaitable[Any]],event: Update,data: Dict[str, Any]) -> Any:
+    if event.event_type == 'message':
+        chat_id = event.message.chat.id
+    elif event.event_type == 'callback_query':
+        chat_id = event.callback_query.from_user.id
+
+    if str(chat_id).startswith('-'):
+        await event.message.answer("🧑🤝🧑 Group chats are not allowed.\nSorry and have a good day.")
+        return
+
+    account_status = db.get_records('account_status','id',chat_id)
+
+    if account_status and account_status == 'off':
+        if event.event_type == 'message':
+            await event.message.answer("Your account has been disabled 🤷\nSorry and have a good day.")
+            CancelHandler()
+        elif event.event_type == 'callback_query':
+            await event.callback_query.answer('Your account has been disabled.')
+            CancelHandler()
+    else:
+        return await handler(event, data)
+

bot/app/utils/cache.py

@@ -0,0 +1,26 @@
+import logging
+import redis
+
+class CACHE():
+    def __init__(self, redis_host, redis_port):
+        self.redis_host = redis_host
+        self.redis_port = redis_port
+
+    def connect(self):
+        self.r = redis.Redis(host=self.redis_host, port=self.redis_port, password=self.redis_pass, db=0)
+
+    def list(self,expr):
+        self.connect()
+        result = []
+
+        try:
+            for key in self.r.scan_iter(expr + "*"):
+                result.append(key.decode())
+
+        except redis.exceptions.ConnectionError:
+            logging.error("Unable connect to redis.")
+            return []
+
+        finally: self.r.close()
+
+        return result

bot/app/utils/db.py

@@ -8,7 +8,6 @@
 
 
 class DB():
-
     table_bot             = Identifier('maas_bot_v1')
     table_subscription    = Identifier('maas_subscription')
     col_id                = Identifier('id')

bot/app/utils/subscriptions.py

@@ -1,6 +1,6 @@
-
 from utils.db import DB
-import asyncio, aiohttp
+import asyncio
+import aiohttp
 
 class Alert():
     def __init__(self, prom_json: dict) -> None:

bot/requirements.txt

@@ -3,3 +3,4 @@ psycopg
 aiogram
 aiohttp
 pyyaml
+redis

docker-compose.yml

@@ -36,12 +36,11 @@ services:
 
   alertmanager:
     image: prom/alertmanager:latest
-    restart: unless-stopped
     ports:
-      - 9093:9093
+    - 9093:9093
     volumes:
-      - ./alertmanager:/config
-      - alertmanager-data:/data
+    - ./alertmanager:/config
+    - alertmanager-data:/data
     command: --config.file=/config/alertmanager.yml 
     networks:
     - monitoring
@@ -50,11 +49,6 @@ services:
   bot:
     build:
       context: ./bot
-    environment:
-    - "db_host=postgres"
-    - "db_port=5432"
-    - "prometheus_rules_url=http://prometheus:9090/api/v1/rules"
-    - "prometheus_alert_groups=maas-rules"
     env_file:
     - ./bot.env
     ports:
@@ -65,7 +59,6 @@ services:
     - monitoring
 
   postgres:
-    container_name: postgres
     image: postgres:13
     environment:
       POSTGRES_USER: adm
@@ -79,21 +72,36 @@ services:
     networks:
     - postgres
 
+  redis:
+    image: redis:latest
+    command: ["redis-server"]
+    ports:
+    - "6379:6379"
+    networks:
+    - bot
+    - monitoring
+
   pgadmin:
-    container_name: pgadmin
     image: dpage/pgadmin4
     environment:
       PGADMIN_DEFAULT_EMAIL: [email protected]
       PGADMIN_DEFAULT_PASSWORD: adm
       PGADMIN_CONFIG_SERVER_MODE: 'False'
     volumes:
        - pgadmin:/var/lib/pgadmin
-
     ports:
     - "5050:80"
     networks:
     - postgres
 
+  validators_cache:
+    build:
+      context: ./validators_cache
+    networks:
+    - redis
+    - monitoring
+    restart: unless-stopped
+
 volumes:
     postgres: {}
     prometheus_data: {}
@@ -106,3 +114,4 @@ networks:
   exporters: 
   monitoring:
   postgres:
+  redis:

validators_cache/Dockerfile

@@ -0,0 +1,12 @@
+FROM --platform=linux/amd64 ubuntu:22.04
+
+WORKDIR /
+
+COPY requirements.txt requirements.txt
+COPY update_cache.py update_cache.py
+COPY validators.txt validators.txt
+
+RUN apt update && apt install -y gcc g++ libpq-dev git python3 python3-pip
+RUN pip3 install -r requirements.txt
+
+CMD ["python3", "update_cache.py"]

validators_cache/requirements.txt

@@ -0,0 +1,2 @@
+requests
+redis

validators_cache/update_cache.py

@@ -0,0 +1,49 @@
+import requests
+import redis
+import json
+import time
+import logging
+
+def write_to_cache(redis_host,redis_port,data):
+    r = redis.Redis(host=redis_host, port=redis_port, db=0)
+
+    for key in r.keys():
+        r.delete(key)
+
+    for i in data:
+        r.set(i,"")
+
+    amount = len(r.keys())
+
+    return amount
+
+def get_from_file(path):
+    with open(path) as file:
+        lines = file.readlines()
+        lines = [line.rstrip() for line in lines]
+
+    return lines
+
+def get_from_prom(url):
+    r = requests.get(url)
+
+    return r.json()['data']
+
+if __name__ == '__main__':
+    logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.DEBUG, datefmt='%Y-%m-%d %I:%M:%S')
+
+    validators_from_prom = get_from_prom('http://prometheus:9090/api/v1/label/account/values')
+    validators_from_file = get_from_file('./validators.txt')
+
+    data = validators_from_prom
+
+    for i in validators_from_file:
+        if i not in data:
+            data.append(i)
+
+    r = write_to_cache('redis',6379,data)
+
+    if isinstance(r, int):
+        logging.info("Written validators to cache " + str(r))
+
+    time.sleep(300)