From f8c01c5ac2bd8493000632b3dcac1e08f7b5d87c Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Sun, 1 Sep 2024 22:51:50 +0200 Subject: [PATCH] hopefully this should work --- api/cogs/requests.py | 13 ++++- api/models.py | 121 +++++++++++++++++++++++++++++++++++++++++ docker-compose-env.yml | 11 ++-- requirements.txt | 102 ++++++++++++++++++++-------------- 4 files changed, 200 insertions(+), 47 deletions(-) create mode 100644 api/models.py diff --git a/api/cogs/requests.py b/api/cogs/requests.py index 11e2951..2cfa235 100644 --- a/api/cogs/requests.py +++ b/api/cogs/requests.py @@ -4,6 +4,7 @@ import aiohttp import api.config as config +from api.models import ScraperDataV3, HighscoreData logger = logging.getLogger(__name__) @@ -114,7 +115,17 @@ async def get_hiscore_data(label_id: int, limit: int = 5000): # Continue making requests until all data is retrieved while True: data = await retry_request(url=url, params=params) - hiscores.extend(data) + for d in data: + scraper_data = ScraperDataV3(**d) + skills = {r.skill_name: r.skill_value for r in scraper_data.skills} + activities = { + r.activity_name: r.activity_value for r in scraper_data.activities + } + hiscores.append( + HighscoreData( + **skills, **activities, Player_id=scraper_data.player_id + ).model_dump() + ) logger.info(f"received: {len(data)}, in total {len(hiscores)}") diff --git a/api/models.py b/api/models.py new file mode 100644 index 0000000..d766ad6 --- /dev/null +++ b/api/models.py @@ -0,0 +1,121 @@ +from pydantic import BaseModel + +from datetime import date, datetime +from typing import Optional + + +class Skill(BaseModel): + skill_name: str + skill_value: int + + +class Activity(BaseModel): + activity_name: str + activity_value: int + + +class ScraperDataV3(BaseModel): + created_at: datetime + record_date: date + scraper_id: int + player_id: int + skills: list[Skill] + activities: list[Activity] + + +class HighscoreData(BaseModel): + Player_id: int + attack: Optional[int] = 0 + defence: Optional[int] = 0 + strength: Optional[int] = 0 + hitpoints: Optional[int] = 0 + ranged: Optional[int] = 0 + prayer: Optional[int] = 0 + magic: Optional[int] = 0 + cooking: Optional[int] = 0 + woodcutting: Optional[int] = 0 + fletching: Optional[int] = 0 + fishing: Optional[int] = 0 + firemaking: Optional[int] = 0 + crafting: Optional[int] = 0 + smithing: Optional[int] = 0 + mining: Optional[int] = 0 + herblore: Optional[int] = 0 + agility: Optional[int] = 0 + thieving: Optional[int] = 0 + slayer: Optional[int] = 0 + farming: Optional[int] = 0 + runecraft: Optional[int] = 0 + hunter: Optional[int] = 0 + construction: Optional[int] = 0 + league: Optional[int] = 0 + bounty_hunter_hunter: Optional[int] = 0 + bounty_hunter_rogue: Optional[int] = 0 + cs_all: Optional[int] = 0 + cs_beginner: Optional[int] = 0 + cs_easy: Optional[int] = 0 + cs_medium: Optional[int] = 0 + cs_hard: Optional[int] = 0 + cs_elite: Optional[int] = 0 + cs_master: Optional[int] = 0 + lms_rank: Optional[int] = 0 + soul_wars_zeal: Optional[int] = 0 + abyssal_sire: Optional[int] = 0 + alchemical_hydra: Optional[int] = 0 + barrows_chests: Optional[int] = 0 + bryophyta: Optional[int] = 0 + callisto: Optional[int] = 0 + cerberus: Optional[int] = 0 + chambers_of_xeric: Optional[int] = 0 + chambers_of_xeric_challenge_mode: Optional[int] = 0 + chaos_elemental: Optional[int] = 0 + chaos_fanatic: Optional[int] = 0 + commander_zilyana: Optional[int] = 0 + corporeal_beast: Optional[int] = 0 + crazy_archaeologist: Optional[int] = 0 + dagannoth_prime: Optional[int] = 0 + dagannoth_rex: Optional[int] = 0 + dagannoth_supreme: Optional[int] = 0 + deranged_archaeologist: Optional[int] = 0 + general_graardor: Optional[int] = 0 + giant_mole: Optional[int] = 0 + grotesque_guardians: Optional[int] = 0 + hespori: Optional[int] = 0 + kalphite_queen: Optional[int] = 0 + king_black_dragon: Optional[int] = 0 + kraken: Optional[int] = 0 + kreearra: Optional[int] = 0 + kril_tsutsaroth: Optional[int] = 0 + mimic: Optional[int] = 0 + nightmare: Optional[int] = 0 + nex: Optional[int] = 0 + phosanis_nightmare: Optional[int] = 0 + obor: Optional[int] = 0 + phantom_muspah: Optional[int] = 0 + sarachnis: Optional[int] = 0 + scorpia: Optional[int] = 0 + skotizo: Optional[int] = 0 + tempoross: Optional[int] = 0 + the_gauntlet: Optional[int] = 0 + the_corrupted_gauntlet: Optional[int] = 0 + theatre_of_blood: Optional[int] = 0 + theatre_of_blood_hard: Optional[int] = 0 + thermonuclear_smoke_devil: Optional[int] = 0 + tombs_of_amascut: Optional[int] = 0 + tombs_of_amascut_expert: Optional[int] = 0 + tzkal_zuk: Optional[int] = 0 + tztok_jad: Optional[int] = 0 + venenatis: Optional[int] = 0 + vetion: Optional[int] = 0 + vorkath: Optional[int] = 0 + wintertodt: Optional[int] = 0 + zalcano: Optional[int] = 0 + zulrah: Optional[int] = 0 + rifts_closed: Optional[int] = 0 + artio: Optional[int] = 0 + calvarion: Optional[int] = 0 + duke_sucellus: Optional[int] = 0 + spindel: Optional[int] = 0 + the_leviathan: Optional[int] = 0 + the_whisperer: Optional[int] = 0 + vardorvis: Optional[int] = 0 diff --git a/docker-compose-env.yml b/docker-compose-env.yml index 88cb120..460b199 100644 --- a/docker-compose-env.yml +++ b/docker-compose-env.yml @@ -1,21 +1,20 @@ -version: '3' services: - machine-learning: + machine_learning: + container_name: bd-ml build: context: . dockerfile: Dockerfile target: base args: - root_path: '/' + root_path: / api_port: 8000 - container_name: bd-ml command: uvicorn api.app:app --host 0.0.0.0 --reload --reload-include api/* env_file: - .env volumes: - - ../bot-detector-ML/api:/project/api + - ./api:/project/api ports: - - 8000:8000 + - 5003:8000 networks: - botdetector-network diff --git a/requirements.txt b/requirements.txt index d0ac310..2cbfea6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,50 +1,72 @@ -aiohttp==3.8.3 +aiohappyeyeballs==2.4.0 +aiohttp==3.10.5 +aiokafka==0.8.1 aiosignal==1.3.1 -anyio==3.6.2 -asgiref==3.5.2 -async-timeout==4.0.2 -attrs==22.1.0 -black==22.10.0 -certifi==2023.7.22 -chardet==5.1.0 -charset-normalizer==2.1.1 -click==8.1.3 +annotated-types==0.7.0 +anyio==4.4.0 +APScheduler==3.10.1 +asgiref==3.8.1 +async-timeout==4.0.3 +attrs==24.2.0 +black==24.8.0 +certifi==2024.8.30 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 colorama==0.4.6 -fastapi==0.88.0 -frozenlist==1.3.3 +discord-webhook==1.1.0 +discord.py==2.2.3 +exceptiongroup==1.2.2 +fastapi==0.112.2 +frozenlist==1.4.1 +greenlet==2.0.2 h11==0.14.0 h2==4.1.0 hpack==4.0.0 -hypercorn==0.14.3 +Hypercorn==0.17.3 hyperframe==6.0.1 -idna==3.4 -iniconfig==1.1.1 -joblib==1.2.0 -multidict==6.0.3 -mypy-extensions==0.4.3 -numpy==1.23.5 -packaging==21.3 -pandas==1.5.2 -pathspec==0.10.1 -platformdirs==2.5.2 -pluggy==1.0.0 +idna==3.8 +iniconfig==2.0.0 +joblib==1.4.2 +kafka-python==2.0.2 +multidict==6.0.5 +mypy-extensions==1.0.0 +mysql-connector-python==8.0.33 +numpy==2.1.0 +oauthlib==3.2.2 +packaging==24.1 +pandas==2.2.2 +pathspec==0.12.1 +platformdirs==4.2.2 +pluggy==1.5.0 priority==2.0.0 -pydantic==1.10.2 -pyparsing==3.0.9 -pytest==7.2.0 -python-dateutil==2.8.2 -python-dotenv==0.21.0 -pytz==2022.6 -requests==2.28.1 -scikit-learn==1.2.0 -scipy==1.9.3 +protobuf==3.20.3 +pydantic==2.8.2 +pydantic_core==2.20.1 +PyMySQL==1.0.3 +pyparsing==3.1.4 +pytest==8.3.2 +python-dateutil==2.9.0.post0 +python-dotenv==1.0.1 +pytz==2024.1 +requests==2.32.3 +requests-oauthlib==1.3.1 +ruff==0.6.3 +scikit-learn==1.5.1 +scipy==1.14.1 six==1.16.0 -sniffio==1.3.0 -starlette==0.22.0 -threadpoolctl==3.1.0 +sniffio==1.3.1 +SQLAlchemy==2.0.16 +starlette==0.38.4 +taskgroup==0.0.0a4 +threadpoolctl==3.5.0 toml==0.10.2 -typing_extensions==4.4.0 -urllib3==1.26.13 -uvicorn==0.20.0 +tomli==2.0.1 +tweepy==4.14.0 +typing_extensions==4.12.2 +tzdata==2023.3 +tzlocal==5.0.1 +urllib3==2.2.2 +uvicorn==0.30.6 wsproto==1.2.0 -yarl==1.8.2 +yarl==1.9.7