New admin interface. Internal API Key registration/removal request page.

dev/local/Makefile

@@ -105,6 +105,7 @@ web:
 	@# Run the web server
 	@# MODULE_NAME specifies the location of the `app` variable, the actual WSGI application object to run.
 	@# see https://github.com/tiangolo/meinheld-gunicorn-docker#module_name
+	@# To receive Slack notification, please add SLACK_WEBHOOK_URL with proper webhook link to the list of env variables.
 	@touch $(ENV_FILE)
 	@docker run --rm -p 127.0.0.1:10080:80 \
 		$(M1) \
@@ -116,7 +117,8 @@ web:
 		--env "REDIS_PASSWORD=1234" \
 		--env "API_KEY_ADMIN_PASSWORD=test_admin_password" \
 		--env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" \
-		$(rate_limit_settings) \
+		--env "RECAPTCHA_SITE_KEY=6Le3wcMnAAAAAL0phFNTaWIHWDDb78I8aSzA34f2" \
+		--env "RECAPTCHA_SECRET_KEY=6Le3wcMnAAAAANlmnp_oApGewLwOmiNFjxlTe2Zo" \
 		--network delphi-net --name delphi_web_epidata \
 		delphi_web_epidata >$(LOG_WEB) 2>&1 &
 

requirements.api.txt

@@ -1,6 +1,8 @@
 delphi_utils==0.3.15
 epiweeks==2.1.2
 Flask==2.2.5
+Flask-Admin==1.6.1
+Flask-Login==0.6.2
 Flask-Limiter==3.3.0
 jinja2==3.1.3
 more_itertools==8.4.0

src/ddl/api_user.sql

@@ -29,3 +29,19 @@ CREATE TABLE IF NOT EXISTS `user_role_link` (
   `role_id` int(11) UNSIGNED NOT NULL,
   PRIMARY KEY (`user_id`, `role_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+
+-- `registration_responses` table
+
+CREATE TABLE IF NOT EXISTS `registration_responses` (
+    `email` varchar(320) UNIQUE NOT NULL,
+    `organization` varchar(120),
+    `purpose` varchar(320)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+-- `removal_requests` table
+
+CREATE TABLE IF NOT EXISTS `removal_requests` (
+    `api_key` varchar(50) UNIQUE NOT NULL,
+    `comment` varchar(320)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;

src/maintenance/remove_outdated_keys.py

@@ -4,19 +4,10 @@
 import delphi.operations.secrets as secrets
 import mysql.connector
 from delphi.epidata.server._config import API_KEY_REGISTRATION_FORM_LINK_LOCAL
+from delphi.epidata.server._common import send_email
 
 ApiUserRecord = namedtuple("APIUserRecord", ("api_key", "email", "date_diff"))
 
-SMTP_HOST = "relay.andrew.cmu.edu"
-SMTP_PORT = 25
-
-EMAIL_SUBJECT = "Your API Key was deleted."
-EMAIL_FROM = "[email protected]"
-ALERT_EMAIL_MESSAGE = f"""Hi! \n Your API Key is going to be removed due to inactivity.
-To renew it, pelase use it within one month from now."""
-DELETED_EMAIL_MESSAGE = f"""Hi! \n Your API Key was removed due to inactivity.
-To get new one, please use registration form ({API_KEY_REGISTRATION_FORM_LINK_LOCAL}) or contact us."""
-
 
 def get_old_keys(cur):
     cur.execute(
@@ -41,25 +32,25 @@ def remove_outdated_key(cur, api_key):
     )
 
 
-def send_notification(to_addr, alert=True):
-    message = ALERT_EMAIL_MESSAGE if alert else DELETED_EMAIL_MESSAGE
-    BODY = "\r\n".join((f"FROM: {EMAIL_FROM}", f"TO: {to_addr}", f"Subject: {EMAIL_SUBJECT}", "", message))
-    smtp_server = SMTP(host=SMTP_HOST, port=SMTP_PORT)
-    smtp_server.starttls()
-    smtp_server.sendmail(EMAIL_FROM, to_addr, BODY)
-
-
 def main():
     u, p = secrets.db.epi
     cnx = mysql.connector.connect(database="epidata", user=u, password=p, host=secrets.db.host)
     cur = cnx.cursor()
     outdated_keys_list = get_old_keys(cur)
     for item in outdated_keys_list:
         if item.date_diff == 5:
-            send_notification(item.email)
+            send_email(
+                to_addr=item.email,
+                subject="API Key will be expired soon",
+                message=f"Hi! \n Your API Key: {item.api_key}, is going to be expired due to inactivity.",
+            )
         else:
             remove_outdated_key(cur, item.api_key)
-            send_notification(item.email, alert=False)
+            send_email(
+                to_addr=item.email,
+                subject="Your API Key was expired",
+                message=f"Hi! \n Your API Key: {item.api_key}, was removed due to inactivity. To get new one, please use registration form ({API_KEY_REGISTRATION_FORM_LINK_LOCAL}) or contact us.""
+            )
     cur.close()
     cnx.commit()
     cnx.close()

src/server/_common.py

@@ -1,20 +1,24 @@
 from typing import cast
 import time
+from datetime import datetime
+import requests
+import json
 
 from flask import Flask, g, request
 from sqlalchemy import event
 from sqlalchemy.engine import Connection, Engine
 from werkzeug.exceptions import Unauthorized
 from werkzeug.local import LocalProxy
+from smtplib import SMTP
 
 from delphi.epidata.common.logger import get_structured_logger
-from ._config import SECRET, REVERSE_PROXY_DEPTH
+from ._config import SECRET, REVERSE_PROXY_DEPTH, SMTP_HOST, SMTP_PORT, EMAIL_FROM, SLACK_WEBHOOK_URL, RECAPTCHA_SECRET_KEY
 from ._db import engine
 from ._exceptions import DatabaseErrorException, EpiDataException
 from ._security import current_user, _is_public_route, resolve_auth_token, update_key_last_time_used, ERROR_MSG_INVALID_KEY
 
 
-app = Flask("EpiData", static_url_path="")
+app = Flask("EpiData", static_url_path="", template_folder="/app/delphi/epidata/server/templates")
 app.config["SECRET"] = SECRET
 
 
@@ -210,3 +214,33 @@ def set_compatibility_mode():
     sets the compatibility mode for this request
     """
     g.compatibility = True
+
+
+def send_email(to_addr: str, subject: str, message: str):
+    """Send email messages
+    Args:
+        to_addr (str): Reciever email address
+        subject (str): Email subject
+        message (str): Email message
+    """
+    smtp_server = SMTP(host=SMTP_HOST, port=SMTP_PORT)
+    smtp_server.starttls()
+    body = "\r\n".join((f"FROM: {EMAIL_FROM}", f"TO: {to_addr}", f"Subject: {subject}", "", message))
+    smtp_server.sendmail(EMAIL_FROM, to_addr, body)
+
+
+def send_slack_notification(payload: dict):
+    headers = {"Accept": "application/json", "Content-Type": "application/json"}
+    response = requests.post(
+        SLACK_WEBHOOK_URL,
+        headers=headers,
+        data=json.dumps(payload)
+    )
+    return response
+
+
+def verify_recaptcha_response(response_token: str):
+    verification_url = "https://www.google.com/recaptcha/api/siteverify"
+    payload = {"secret": RECAPTCHA_SECRET_KEY, "response": response_token}
+    response = requests.post(verification_url, data=payload)
+    return response.json()["success"]

src/server/_config.py

@@ -119,3 +119,13 @@
 # ^ shortcut to "https://docs.google.com/forms/d/e/1FAIpQLSff30tsq4xwPCoUbvaIygLSMs_Mt8eDhHA0rifBoIrjo8J5lw/viewform"
 API_KEY_REMOVAL_REQUEST_LINK_LOCAL = "https://api.delphi.cmu.edu/epidata/admin/removal_request"
 # ^ redirects to API_KEY_REMOVAL_REQUEST_LINK
+
+# STMP credentials
+SMTP_HOST = "relay.andrew.cmu.edu"
+SMTP_PORT = 25
+EMAIL_FROM = "[email protected]"
+
+RECAPTCHA_SITE_KEY = os.environ.get("RECAPTCHA_SITE_KEY")
+RECAPTCHA_SECRET_KEY = os.environ.get("RECAPTCHA_SECRET_KEY")
+
+SLACK_WEBHOOK_URL = os.environ.get("SLACK_WEBHOOK_URL")

src/server/_security.py

@@ -54,10 +54,12 @@ def _get_current_user():
 
 
 def _is_public_route() -> bool:
-    public_routes_list = ["lib", "admin", "version"]
+    public_routes_list = ["lib", "version", "diagnostics"]
     for route in public_routes_list:
         if request.path.startswith(f"{URL_PREFIX}/{route}"):
             return True
+        if "admin" in request.path:
+            return True
     return False
 
 

src/server/admin/models.py

@@ -6,8 +6,8 @@
 from .._db import Session, WriteSession, default_session
 from delphi.epidata.common.logger import get_structured_logger
 
-from typing import Set, Optional, List
-from datetime import datetime as dtime
+from typing import Set, Optional
+from datetime import date
 
 
 Base = declarative_base()
@@ -20,7 +20,7 @@
 )
 
 def _default_date_now():
-    return dtime.strftime(dtime.now(), "%Y-%m-%d")
+    return date.today()
 
 class User(Base):
     __tablename__ = "api_user"
@@ -35,16 +35,6 @@ def __init__(self, api_key: str, email: str = None) -> None:
         self.api_key = api_key
         self.email = email
 
-    @property
-    def as_dict(self):
-        return {
-            "id": self.id,
-            "api_key": self.api_key,
-            "email": self.email,
-            "roles": set(role.name for role in self.roles),
-            "created": self.created,
-            "last_time_used": self.last_time_used
-        }
 
     def has_role(self, required_role: str) -> bool:
         return required_role in set(role.name for role in self.roles)
@@ -121,6 +111,9 @@ class UserRole(Base):
     id = Column(Integer, primary_key=True, autoincrement=True)
     name = Column(String(50), unique=True)
 
+    def __repr__(self):
+        return self.name
+
     @staticmethod
     @default_session(WriteSession)
     def create_role(name: str, session) -> None:
@@ -142,3 +135,41 @@ def create_role(name: str, session) -> None:
     def list_all_roles(session):
         roles = session.query(UserRole).all()
         return [role.name for role in roles]
+
+
+class RegistrationResponse(Base):
+    __tablename__ = "registration_responses"
+
+    email = Column(String(320), unique=True, nullable=False, primary_key=True)
+    organization = Column(String(120), unique=False, nullable=True)
+    purpose = Column(String(320), unique=False, nullable=True)
+
+    def __init__(self, email: str, organization: str = None, purpose: str = None) -> None:
+        self.email = email
+        self.organization = organization
+        self.purpose = purpose
+
+    @staticmethod
+    @default_session(WriteSession)
+    def add_response(email: str, organization: str, purpose: str, session):
+        new_response = RegistrationResponse(email, organization, purpose)
+        session.add(new_response)
+        session.commit()
+
+
+class RemovalRequest(Base):
+    __tablename__ = "removal_requests"
+
+    api_key = Column(String(50), unique=True, nullable=False, primary_key=True)
+    comment = Column(String(320), unique=False, nullable=True)
+
+    def __init__(self, api_key: str, comment: str = None) -> None:
+        self.api_key = api_key
+        self.comment = comment
+
+    @staticmethod
+    @default_session(WriteSession)
+    def add_request(api_key: str, comment: str, session):
+        new_request = RemovalRequest(api_key, comment)
+        session.add(new_request)
+        session.commit()

src/server/endpoints/__init__.py

@@ -30,6 +30,8 @@
     wiki,
     signal_dashboard_status,
     signal_dashboard_coverage,
+    registration,
+    api_key_removal_request
 )
 
 endpoints = [
@@ -64,6 +66,8 @@
     wiki,
     signal_dashboard_status,
     signal_dashboard_coverage,
+    registration,
+    api_key_removal_request
 ]
 
 __all__ = ["endpoints"]