Migrate to beatmaps-service (#15)

* Migrate to beatmaps-service

* refactor some stuff

* more warning logs

* fix double logs

* rewrite background image fetching to hit beatmaps service

* fix type err

* fix type errsa

* Update url

* take over prod

* Delete unused file

* Delete unused module reference

* r

app/adapters/__init__.py

@@ -1,5 +1,3 @@
 from . import aws_s3
-from . import beatmaps
 from . import database
-from . import osu_api_v1
 from . import webdriver

app/common/settings.py

@@ -16,6 +16,7 @@ def read_bool(value: str) -> bool:
 APP_PERFORMANCE_URL = os.environ["APP_PERFORMANCE_URL"]
 APP_API_URL = os.environ["APP_API_URL"]
 APP_SCORE_SERVICE_URL = os.environ["APP_SCORE_SERVICE_URL"]
+APP_BEATMAPS_SERVICE_URL = os.environ["APP_BEATMAPS_SERVICE_URL"]
 
 DISCORD_TOKEN = os.environ["DISCORD_TOKEN"]
 

app/osu_beatmaps.py

@@ -1,133 +1,85 @@
-import io
 import logging
 import typing
-import zipfile
-from typing import TypedDict
 
-from PIL import Image
+import httpx
 
-from app import state
-from app.adapters import aws_s3
-from app.adapters import osu_api_v1
-from app.adapters.beatmaps.mirrors import BeatmapMirror
-from app.adapters.beatmaps.mirrors import CatboyBestMirror
-from app.adapters.beatmaps.mirrors import OsuDirectMirror
+from app.common import settings
 
 
-class Beatmap(TypedDict):
+beatmaps_service_http_client = httpx.AsyncClient(
+    base_url=settings.APP_BEATMAPS_SERVICE_URL,
+)
+
+
+class BeatmapMetadata(typing.TypedDict):
     artist: str
     title: str
     creator: str
     version: str
 
 
-BEATMAP_MIRRORS: list[BeatmapMirror] = [
-    OsuDirectMirror(),
-    CatboyBestMirror(),
-]
-
-
-async def get_beatmap(beatmap_id: int) -> bytes | None:
-    osu_file_contents = await aws_s3.get_object_data(f"/beatmaps/{beatmap_id}.osu")
-    if not osu_file_contents:
-        osu_file_contents = await osu_api_v1.get_osu_file_contents(beatmap_id)
-
-    if not osu_file_contents:
+async def get_osu_file_contents(beatmap_id: int) -> bytes | None:
+    """Fetch the .osu file content for a beatmap."""
+    try:
+        response = await beatmaps_service_http_client.get(
+            f"/api/osu-api/v1/osu-files/{beatmap_id}",
+        )
+        if response.status_code == 404:
+            return None
+        response.raise_for_status()
+        return response.read()
+    except Exception:
+        logging.warning(
+            "Failed to fetch .osu file contents from beatmaps-service",
+            extra={"beatmap_id": beatmap_id},
+            exc_info=True,
+        )
         return None
 
-    # NOTE: intentionally not saving to s3 here, because we don't want to
-    # disrupt other online systems with potentially newer data.
-    return osu_file_contents
-
 
-async def get_beatmap_background_image(
-    beatmap_id: int,
-    beatmapset_id: int,
-) -> Image.Image | None:
-    """Gets a beatmap's background image by any means."""
-    background_image = await _get_beatmap_background_image_online(
-        beatmap_id,
-        beatmapset_id,
-    )
-
-    if background_image is None:
-        background_image = await _get_beatmap_background_image_io(
-            beatmap_id,
-            beatmapset_id,
+async def get_osz2_file_contents(beatmapset_id: int) -> bytes | None:
+    """Fetch the .osz2 file content for a beatmapset."""
+    try:
+        response = await beatmaps_service_http_client.get(
+            f"/public/api/d/{beatmapset_id}",
+        )
+        if response.status_code == 404:
+            return None
+        response.raise_for_status()
+        return response.read()
+    except Exception:
+        logging.warning(
+            "Failed to fetch .osz2 file contents from beatmaps-service",
+            extra={"beatmapset_id": beatmapset_id},
+            exc_info=True,
         )
-
-    if background_image is None:
         return None
 
-    return background_image
-
-
-async def _get_beatmap_background_image_online(
-    beatmap_id: int,
-    _: int,
-) -> Image.Image | None:
-    osu_background_url = f"https://api.osu.direct/media/background/{beatmap_id}"
-    response = await state.http_client.get(
-        osu_background_url,
-    )
-    response.raise_for_status()
 
-    background_data = response.read()
-    if b"beatmap not found!" in background_data:
+async def get_beatmap_background_image_contents(beatmap_id: int) -> bytes | None:
+    try:
+        response = await beatmaps_service_http_client.get(
+            f"/api/osu-assets/backgrounds/{beatmap_id}",
+        )
+        if response.status_code == 404:
+            logging.warning(
+                "Failed to retrieve beatmap background image from beatmaps-service",
+                extra={"beatmap_id": beatmap_id},
+            )
+            return None
+        response.raise_for_status()
+        return response.read()
+    except Exception:
         logging.warning(
-            "Failed to find beatmap background image on osu.direct",
+            "Failed to retrieve beatmap background image from beatmaps-service",
             extra={"beatmap_id": beatmap_id},
         )
         return None
 
-    with io.BytesIO(background_data) as image_file:
-        image = Image.open(image_file)
-        image.load()
-
-    return image
-
-
-async def _get_beatmap_background_image_io(
-    beatmap_id: int,
-    beatmapset_id: int,
-) -> Image.Image | None:
-    """Gets a beatmap's background image by any means."""
-    beatmap = await get_beatmap(beatmap_id)
-    if beatmap is None:
-        return None
-
-    background_filename = find_beatmap_background_filename(beatmap)
-    if background_filename is None:
-        return None
-
-    for mirror in BEATMAP_MIRRORS:
-        data = await mirror.fetch_beatmap_zip_data(beatmapset_id)
-        if data is None:  # try next mirror
-            continue
 
-        with io.BytesIO(data) as zip_file:
-            with zipfile.ZipFile(zip_file) as zip_ref:
-                for file_name in zip_ref.namelist():
-                    if file_name == background_filename:
-                        break
-                else:  # try next mirror
-                    continue
-
-                with zip_ref.open(file_name) as image_file:
-                    image = Image.open(image_file)
-                    image.load()
-                    return image
-
-    logging.warning(
-        "Could not find a beatmap by set id on any of our mirrors",
-        extra={"beatmapset_id": beatmapset_id},
-    )
-    return None
-
-
-def parse_beatmap_metadata(osu_file_bytes: bytes) -> Beatmap:
+def parse_beatmap_metadata(osu_file_bytes: bytes) -> BeatmapMetadata:
     lines = osu_file_bytes.decode().splitlines()
-    beatmap = {}
+    beatmap: dict[str, str] = {}
     for line in lines[1:]:
         if line.startswith("Artist:"):
             beatmap["artist"] = line.split(":")[1].strip()
@@ -138,15 +90,4 @@ def parse_beatmap_metadata(osu_file_bytes: bytes) -> Beatmap:
         elif line.startswith("Version:"):
             beatmap["version"] = line.split(":")[1].strip()
 
-    return typing.cast(Beatmap, beatmap)
-
-
-def find_beatmap_background_filename(osu_file_bytes: bytes) -> str | None:
-    lines = osu_file_bytes.decode("utf-8-sig").splitlines()
-
-    for line in lines:
-        if line.startswith("0,0,"):
-            background_path = line.split(",")[2].strip().strip('"')
-            return background_path
-
-    return None
+    return typing.cast(BeatmapMetadata, beatmap)

app/osu_replays.py

@@ -2,22 +2,32 @@
 import logging
 
 import aiosu
+import httpx
 from aiosu.models.files import ReplayFile
 
-from app import state
 from app.common import settings
 
 
+score_service_http_client = httpx.AsyncClient(
+    base_url=settings.APP_SCORE_SERVICE_URL,
+)
+
+
 class Replay(ReplayFile):
     raw_replay_data: bytes
 
 
 async def get_replay(score_id: int) -> Replay | None:
-    response = await state.http_client.get(
-        f"{settings.APP_SCORE_SERVICE_URL}/replays/{score_id}",
-    )
-    response.raise_for_status()
-    osu_replay_data = response.read()
+    try:
+        response = await score_service_http_client.get(f"/replays/{score_id}")
+        response.raise_for_status()
+        osu_replay_data = response.read()
+    except Exception:
+        logging.exception(
+            "An error occurred while fetching osu! replay file data",
+            extra={"score_id": score_id},
+        )
+        return None
 
     if not osu_replay_data or osu_replay_data == b"Score not found!":
         logging.warning("Failed to find osu! replay file data")

app/usecases/scorewatch.py

@@ -1,4 +1,5 @@
 import datetime
+import io
 import os
 import tempfile
 import typing
@@ -7,6 +8,7 @@
 import discord
 from aiosu.models.mods import Mod
 from discord.ext import commands
+from PIL import Image
 
 from app import osu
 from app import osu_beatmaps
@@ -111,25 +113,26 @@ async def generate_score_upload_resources(
     mods = aiosu.models.mods.Mods(score_data["mods"])
 
     beatmap_id = score_data["beatmap"]["beatmap_id"]
-    beatmapset_id = score_data["beatmap"]["beatmapset_id"]
 
-    beatmap_bytes = await osu_beatmaps.get_beatmap(beatmap_id)
+    beatmap_bytes = await osu_beatmaps.get_osu_file_contents(beatmap_id)
 
     if not beatmap_bytes:
         return "Couldn't find beatmap associated with this score!"
 
     beatmap = osu_beatmaps.parse_beatmap_metadata(beatmap_bytes)
-    beatmap_background_image = await osu_beatmaps.get_beatmap_background_image(
-        beatmap_id,
-        beatmapset_id,
+    background_image_contents = (
+        await osu_beatmaps.get_beatmap_background_image_contents(
+            beatmap_id,
+        )
     )
-
-    if not beatmap_background_image:
+    if not background_image_contents:
         return "Couldn't find beatmap associated with this score!"
 
-    beatmap_background_image = postprocessing.apply_effects_normal_template(
-        beatmap_background_image,
-    )
+    with io.BytesIO(background_image_contents) as image_file:
+        background_image = Image.open(image_file)
+        background_image.load()
+
+    background_image = postprocessing.apply_effects_normal_template(background_image)
 
     if not artist:
         artist = beatmap["artist"]
@@ -160,7 +163,7 @@ async def generate_score_upload_resources(
         template = f.read()
 
     with tempfile.NamedTemporaryFile(suffix=".png") as background_file:
-        beatmap_background_image.save(background_file.name, format="PNG")
+        background_image.save(background_file.name, format="PNG")
         template = template.replace(
             r"<% beatmap.background_url %>",
             background_file.name,