Merge pull request #53 from Bot-detector/bf/add-data-validation-and-t…

…ransformation

hopefully this should work

api/cogs/requests.py

@@ -4,6 +4,7 @@
 import aiohttp
 
 import api.config as config
+from api.models import ScraperDataV3, HighscoreData
 
 logger = logging.getLogger(__name__)
 
@@ -114,7 +115,17 @@ async def get_hiscore_data(label_id: int, limit: int = 5000):
     # Continue making requests until all data is retrieved
     while True:
         data = await retry_request(url=url, params=params)
-        hiscores.extend(data)
+        for d in data:
+            scraper_data = ScraperDataV3(**d)
+            skills = {r.skill_name: r.skill_value for r in scraper_data.skills}
+            activities = {
+                r.activity_name: r.activity_value for r in scraper_data.activities
+            }
+            hiscores.append(
+                HighscoreData(
+                    **skills, **activities, Player_id=scraper_data.player_id
+                ).model_dump()
+            )
 
         logger.info(f"received: {len(data)}, in total {len(hiscores)}")
 

api/models.py

@@ -0,0 +1,121 @@
+from pydantic import BaseModel
+
+from datetime import date, datetime
+from typing import Optional
+
+
+class Skill(BaseModel):
+    skill_name: str
+    skill_value: int
+
+
+class Activity(BaseModel):
+    activity_name: str
+    activity_value: int
+
+
+class ScraperDataV3(BaseModel):
+    created_at: datetime
+    record_date: date
+    scraper_id: int
+    player_id: int
+    skills: list[Skill]
+    activities: list[Activity]
+
+
+class HighscoreData(BaseModel):
+    Player_id: int
+    attack: Optional[int] = 0
+    defence: Optional[int] = 0
+    strength: Optional[int] = 0
+    hitpoints: Optional[int] = 0
+    ranged: Optional[int] = 0
+    prayer: Optional[int] = 0
+    magic: Optional[int] = 0
+    cooking: Optional[int] = 0
+    woodcutting: Optional[int] = 0
+    fletching: Optional[int] = 0
+    fishing: Optional[int] = 0
+    firemaking: Optional[int] = 0
+    crafting: Optional[int] = 0
+    smithing: Optional[int] = 0
+    mining: Optional[int] = 0
+    herblore: Optional[int] = 0
+    agility: Optional[int] = 0
+    thieving: Optional[int] = 0
+    slayer: Optional[int] = 0
+    farming: Optional[int] = 0
+    runecraft: Optional[int] = 0
+    hunter: Optional[int] = 0
+    construction: Optional[int] = 0
+    league: Optional[int] = 0
+    bounty_hunter_hunter: Optional[int] = 0
+    bounty_hunter_rogue: Optional[int] = 0
+    cs_all: Optional[int] = 0
+    cs_beginner: Optional[int] = 0
+    cs_easy: Optional[int] = 0
+    cs_medium: Optional[int] = 0
+    cs_hard: Optional[int] = 0
+    cs_elite: Optional[int] = 0
+    cs_master: Optional[int] = 0
+    lms_rank: Optional[int] = 0
+    soul_wars_zeal: Optional[int] = 0
+    abyssal_sire: Optional[int] = 0
+    alchemical_hydra: Optional[int] = 0
+    barrows_chests: Optional[int] = 0
+    bryophyta: Optional[int] = 0
+    callisto: Optional[int] = 0
+    cerberus: Optional[int] = 0
+    chambers_of_xeric: Optional[int] = 0
+    chambers_of_xeric_challenge_mode: Optional[int] = 0
+    chaos_elemental: Optional[int] = 0
+    chaos_fanatic: Optional[int] = 0
+    commander_zilyana: Optional[int] = 0
+    corporeal_beast: Optional[int] = 0
+    crazy_archaeologist: Optional[int] = 0
+    dagannoth_prime: Optional[int] = 0
+    dagannoth_rex: Optional[int] = 0
+    dagannoth_supreme: Optional[int] = 0
+    deranged_archaeologist: Optional[int] = 0
+    general_graardor: Optional[int] = 0
+    giant_mole: Optional[int] = 0
+    grotesque_guardians: Optional[int] = 0
+    hespori: Optional[int] = 0
+    kalphite_queen: Optional[int] = 0
+    king_black_dragon: Optional[int] = 0
+    kraken: Optional[int] = 0
+    kreearra: Optional[int] = 0
+    kril_tsutsaroth: Optional[int] = 0
+    mimic: Optional[int] = 0
+    nightmare: Optional[int] = 0
+    nex: Optional[int] = 0
+    phosanis_nightmare: Optional[int] = 0
+    obor: Optional[int] = 0
+    phantom_muspah: Optional[int] = 0
+    sarachnis: Optional[int] = 0
+    scorpia: Optional[int] = 0
+    skotizo: Optional[int] = 0
+    tempoross: Optional[int] = 0
+    the_gauntlet: Optional[int] = 0
+    the_corrupted_gauntlet: Optional[int] = 0
+    theatre_of_blood: Optional[int] = 0
+    theatre_of_blood_hard: Optional[int] = 0
+    thermonuclear_smoke_devil: Optional[int] = 0
+    tombs_of_amascut: Optional[int] = 0
+    tombs_of_amascut_expert: Optional[int] = 0
+    tzkal_zuk: Optional[int] = 0
+    tztok_jad: Optional[int] = 0
+    venenatis: Optional[int] = 0
+    vetion: Optional[int] = 0
+    vorkath: Optional[int] = 0
+    wintertodt: Optional[int] = 0
+    zalcano: Optional[int] = 0
+    zulrah: Optional[int] = 0
+    rifts_closed: Optional[int] = 0
+    artio: Optional[int] = 0
+    calvarion: Optional[int] = 0
+    duke_sucellus: Optional[int] = 0
+    spindel: Optional[int] = 0
+    the_leviathan: Optional[int] = 0
+    the_whisperer: Optional[int] = 0
+    vardorvis: Optional[int] = 0

docker-compose-env.yml

@@ -1,21 +1,20 @@
-version: '3'
 services:
-  machine-learning:
+  machine_learning:
+    container_name: bd-ml
     build:
       context: .
       dockerfile: Dockerfile
       target: base
       args:
-        root_path: '/'
+        root_path: /
         api_port: 8000
-    container_name: bd-ml
     command: uvicorn api.app:app --host 0.0.0.0 --reload --reload-include api/*
     env_file:
       - .env
     volumes:
-      - ../bot-detector-ML/api:/project/api
+      - ./api:/project/api
     ports:
-      - 8000:8000
+      - 5003:8000
     networks:
       - botdetector-network
 

requirements.txt

@@ -1,50 +1,72 @@
-aiohttp==3.8.3
+aiohappyeyeballs==2.4.0
+aiohttp==3.10.5
+aiokafka==0.8.1
 aiosignal==1.3.1
-anyio==3.6.2
-asgiref==3.5.2
-async-timeout==4.0.2
-attrs==22.1.0
-black==22.10.0
-certifi==2023.7.22
-chardet==5.1.0
-charset-normalizer==2.1.1
-click==8.1.3
+annotated-types==0.7.0
+anyio==4.4.0
+APScheduler==3.10.1
+asgiref==3.8.1
+async-timeout==4.0.3
+attrs==24.2.0
+black==24.8.0
+certifi==2024.8.30
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
 colorama==0.4.6
-fastapi==0.88.0
-frozenlist==1.3.3
+discord-webhook==1.1.0
+discord.py==2.2.3
+exceptiongroup==1.2.2
+fastapi==0.112.2
+frozenlist==1.4.1
+greenlet==2.0.2
 h11==0.14.0
 h2==4.1.0
 hpack==4.0.0
-hypercorn==0.14.3
+Hypercorn==0.17.3
 hyperframe==6.0.1
-idna==3.4
-iniconfig==1.1.1
-joblib==1.2.0
-multidict==6.0.3
-mypy-extensions==0.4.3
-numpy==1.23.5
-packaging==21.3
-pandas==1.5.2
-pathspec==0.10.1
-platformdirs==2.5.2
-pluggy==1.0.0
+idna==3.8
+iniconfig==2.0.0
+joblib==1.4.2
+kafka-python==2.0.2
+multidict==6.0.5
+mypy-extensions==1.0.0
+mysql-connector-python==8.0.33
+numpy==2.1.0
+oauthlib==3.2.2
+packaging==24.1
+pandas==2.2.2
+pathspec==0.12.1
+platformdirs==4.2.2
+pluggy==1.5.0
 priority==2.0.0
-pydantic==1.10.2
-pyparsing==3.0.9
-pytest==7.2.0
-python-dateutil==2.8.2
-python-dotenv==0.21.0
-pytz==2022.6
-requests==2.28.1
-scikit-learn==1.2.0
-scipy==1.9.3
+protobuf==3.20.3
+pydantic==2.8.2
+pydantic_core==2.20.1
+PyMySQL==1.0.3
+pyparsing==3.1.4
+pytest==8.3.2
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+pytz==2024.1
+requests==2.32.3
+requests-oauthlib==1.3.1
+ruff==0.6.3
+scikit-learn==1.5.1
+scipy==1.14.1
 six==1.16.0
-sniffio==1.3.0
-starlette==0.22.0
-threadpoolctl==3.1.0
+sniffio==1.3.1
+SQLAlchemy==2.0.16
+starlette==0.38.4
+taskgroup==0.0.0a4
+threadpoolctl==3.5.0
 toml==0.10.2
-typing_extensions==4.4.0
-urllib3==1.26.13
-uvicorn==0.20.0
+tomli==2.0.1
+tweepy==4.14.0
+typing_extensions==4.12.2
+tzdata==2023.3
+tzlocal==5.0.1
+urllib3==2.2.2
+uvicorn==0.30.6
 wsproto==1.2.0
-yarl==1.8.2
+yarl==1.9.7