Content Blocker Bot

.gitignore

@@ -168,3 +168,6 @@ cython_debug/
 # Custom gitignore
 *.db
 # End of custom ignore
+
+# 
+/**/cache/*

3rdparty/py/requirements-all.txt

@@ -1,3 +1,5 @@
+aiohttp==3.9.3
+backoff==2.2.1
 boto3==1.34.98
 celery==5.4.0
 dj-rest-auth[with_social]==5.1.0
@@ -13,6 +15,8 @@ google-auth-oauthlib==1.2.0
 greenlet==3.0.3
 gunicorn[gevent, setproctitle]==22.0.0
 html2text==2024.2.26
+pyairtable==2.3.3
+python-dotenv==1.0.1
 redis==5.0.4
 requests==2.31.0
 sentry-sdk==2.1.0

docker-compose.yml

@@ -55,9 +55,18 @@ services:
       - pesacheck-data:/app/database
     env_file:
       - ./pesacheck_meedan_bridge/.env
+
+  mediadata_ai_blocklist:
+    image: codeforafrica/mediadata_ai_blocklist:latest
+    command: ["tail", "-f", "/dev/null"]
+    volumes:
+      - mediadata-data:/app/database
+    env_file:
+      - ./mediadata_ai_blocklist/.env
 
 volumes:
   app-media:
   app-staticfiles:
   db-data:
   pesacheck-data:
+  mediadata-data:

mediadata_ai_blocklist/.env.example

@@ -0,0 +1,5 @@
+AIRTABLE_BASE_ID=
+AIRTABLE_API_KEY=
+AIRTABLE_ORGANISATION_TABLE=
+AIRTABLE_CONTENT_TABLE=
+DB_FILE=mediadata_ai_blocklist.db

mediadata_ai_blocklist/docker/BUILD

@@ -0,0 +1,32 @@
+docker_image(
+    name="mediadata-deps",
+    image_tags=["deps"],
+    build_platform=["linux/amd64"],
+    registries=["mediadata_ai_blocklist"],
+    repository="app",
+    skip_push=True,
+    source="Dockerfile.deps",
+)
+
+file(name="app.json", source="app.json")
+
+docker_image(
+    name="mediadata-srcs",
+    image_tags=["srcs"],
+    build_platform=["linux/amd64"],
+    registries=["mediadata_ai_blocklist"],
+    repository="app",
+    skip_push=True,
+    source="Dockerfile.srcs",
+)
+
+docker_image(
+    name="mediadata_ai_blocklist",
+    build_platform=["linux/amd64"],
+    dependencies=[":mediadata-srcs", ":mediadata-deps", ":app.json"],
+    image_tags=[
+        "{build_args.VERSION}",
+        "latest",
+    ],
+    source="Dockerfile",
+)

mediadata_ai_blocklist/docker/Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.11-slim-bullseye AS python-base
+FROM mediadata_ai_blocklist/app:deps AS app-deps
+FROM mediadata_ai_blocklist/app:srcs AS app-srcs
+FROM python-base AS python-app
+
+WORKDIR /app
+COPY  mediadata_ai_blocklist/docker/app.json ./
+COPY --from=app-deps /app ./
+COPY --from=app-srcs /app ./
+
+CMD ["tail", "-f", "/dev/null"]

mediadata_ai_blocklist/docker/Dockerfile.deps

@@ -0,0 +1,4 @@
+FROM python:3.11-slim-bookworm
+
+COPY mediadata_ai_blocklist.py/mediadata-deps@environment=linux.pex /mediadata-deps.pex
+RUN PEX_TOOLS=1 python /mediadata-deps.pex venv --scope=deps --compile /app

mediadata_ai_blocklist/docker/Dockerfile.srcs

@@ -0,0 +1,4 @@
+FROM python:3.11-slim-bookworm
+
+COPY mediadata_ai_blocklist.py/mediadata-srcs@environment=linux.pex /mediadata-srcs.pex
+RUN PEX_TOOLS=1 python /mediadata-srcs.pex venv --scope=srcs --compile /app

mediadata_ai_blocklist/docker/app.json

@@ -0,0 +1,9 @@
+{
+    "name": "mediadata_ai_blocklist",
+    "cron": [
+        {
+            "command": "./pex",
+            "schedule": "@daily"
+        }
+    ]
+}

mediadata_ai_blocklist/py/BUILD

@@ -0,0 +1,42 @@
+python_sources(
+    name="lib",
+    dependencies=[
+        "3rdparty/py:requirements-all#aiohttp",
+        "3rdparty/py:requirements-all#backoff",
+        "3rdparty/py:requirements-all#pyairtable",
+        "3rdparty/py:requirements-all#python-dotenv",
+    ],
+)
+
+pex_binary(
+    name="mediadata-deps",
+    environment=parametrize("__local__", "linux"),
+    dependencies=[
+        ":lib",
+    ],
+    entry_point="main.py",
+    include_sources=False,
+    include_tools=True,
+    layout="packed",
+)
+
+pex_binary(
+    name="mediadata-srcs",
+    environment=parametrize("__local__", "linux"),
+    dependencies=[
+        ":lib",
+    ],
+    entry_point="main.py",
+    include_requirements=False,
+    include_tools=True,
+    layout="packed",
+)
+
+
+pex_binary(
+    name="mediadata",
+    dependencies=[
+        ":lib",
+    ],
+    entry_point="main.py",
+)

mediadata_ai_blocklist/py/VERSION

@@ -0,0 +1 @@
+0.0.1

mediadata_ai_blocklist/py/airtable.py

@@ -0,0 +1,75 @@
+from pyairtable import Api
+from dotenv import load_dotenv
+from utils import validate_url, clean_url
+import os
+import logging
+import re
+
+
+logging.basicConfig(level=logging.INFO,
+                    format='%(asctime)s - %(levelname)s - %(message)s')
+dotenv_path = os.path.join(os.path.dirname(__file__), '..', '.env')
+load_dotenv(dotenv_path)
+
+api_key = os.getenv('AIRTABLE_API_KEY')
+base_id = os.getenv('AIRTABLE_BASE_ID')
+organisations_table = os.getenv('AIRTABLE_ORGANISATION_TABLE')
+content_table = os.getenv('AIRTABLE_CONTENT_TABLE')
+
+if not api_key or not base_id or not organisations_table or not content_table:
+    raise ValueError('API key, base ID and Organisation table are required')
+
+at = Api(api_key)
+
+
+def get_table_data(table_name, formula=None, fields=None):
+    table = at.table(base_id, table_name)
+    return table.all(formula=formula, fields=fields)
+
+
+def get_formula(allowed_countries=None):
+    base_formula = 'AND(NOT({Organisation Name} = ""), NOT({Website} = ""), NOT({HQ Country} = ""))'
+    if allowed_countries:
+        countries_formula = ', '.join(
+            [f'({{HQ Country}} = "{country}")' for country in allowed_countries])
+        formula = f'AND({base_formula}, OR({countries_formula}))'
+    else:
+        formula = base_formula
+    return formula
+
+
+def process_records(data):
+    organizations = []
+    for record in data:
+        website = validate_url(record['fields'].get('Website', None))
+        name = record['fields'].get('Organisation Name', None)
+        country = record['fields'].get('HQ Country', None)
+        id: str = record['id']
+        if website:
+            org = {}
+            org['id'] = id
+            org['name'] = re.sub(
+                r'[\\/*?:"<>|]', '-', name) if name else None
+            org['url'] = clean_url(website)
+            org['country'] = country
+
+            organizations.append(org)
+    return organizations
+
+
+def get_organizations(allowed_countries=None):
+    logging.info('Fetching organizations from Airtable')
+    formula = get_formula(allowed_countries)
+    fields = ['Organisation Name', 'Website', 'HQ Country']
+    data = get_table_data(organisations_table, formula, fields)
+    organizations = process_records(data)
+    return organizations
+
+
+async def batch_upsert_organizations(data):
+    logging.info('Upserting organizations in Airtable')
+    try:
+        table = at.table(base_id, content_table)
+        table.batch_upsert(records=data, key_fields=['URL',])
+    except Exception as e:
+        logging.error(f'Error upserting organization: {e}')