From d41c2ff33cc1e88325da6c8f9e10c24199eeb291 Mon Sep 17 00:00:00 2001 From: iPromKnight <156901906+iPromKnight@users.noreply.github.com> Date: Mon, 4 Nov 2024 21:19:49 +0000 Subject: [PATCH] feat: requests second pass (#848) * feat: add support for HTTP method and response type enums - Introduced `HttpMethod` and `ResponseType` enums to standardize HTTP method usage and response handling. - Updated `BaseRequestHandler` and `ResponseObject` to utilize the new enums for improved code clarity and flexibility. - Modified `RealDebridRequestHandler` and `AllDebridRequestHandler` to use `HttpMethod` and `ResponseType` enums. - Enhanced request handling in downloader classes to support the new enums, ensuring consistent API interactions. * refactor: implement request handler classes for API communication - Introduced new request handler classes for various APIs to streamline HTTP request handling. - Replaced direct requests with handler executions across multiple modules, including TraktAPI, OverseerrAPI, and others. - Enhanced error handling by defining custom exceptions for each API. - Updated session creation and rate limiting configurations for improved performance and reliability. - Improved code readability and maintainability by encapsulating request logic within dedicated classes. * fix: handle missing 'streams' attribute in Torrentio response - Added a check for the presence of the 'streams' attribute in the Torrentio scraper response to prevent errors when the attribute is missing. - Updated the MdblistRequestHandler to include the API key in the request parameters by default, simplifying API calls. - Improved error handling in the ListrrAPI to break on specific HTTP errors. - Enhanced logging for better debugging and error tracking across various modules. * fix: jackett exception handling ordering * feat: add support for overriding response type in request handlers - Introduced an optional `overriden_response_type` parameter in the `_request` method of `BaseRequestHandler` and `execute` method of `PlexRequestHandler`. - Updated `get_items_from_rss` in `PlexAPI` to use `ResponseType.DICT` for RSS feed requests. - Enhanced flexibility in handling different response formats dynamically. * fix: remove retry_if_failed parameter from request handler calls in Orionoid scraper * fix: correct parameter name in Plex RSS feed request handler call --- src/program/apis/listrr_api.py | 24 ++- src/program/apis/mdblist_api.py | 33 ++-- src/program/apis/overseerr_api.py | 55 ++---- src/program/apis/plex_api.py | 31 ++- src/program/apis/trakt_api.py | 37 ++-- src/program/services/content/trakt.py | 15 +- src/program/services/downloaders/alldebrid.py | 22 +-- .../services/downloaders/realdebrid.py | 27 +-- src/program/services/scrapers/comet.py | 16 +- src/program/services/scrapers/jackett.py | 15 +- .../services/scrapers/knightcrawler.py | 12 +- src/program/services/scrapers/mediafusion.py | 13 +- src/program/services/scrapers/orionoid.py | 14 +- src/program/services/scrapers/prowlarr.py | 15 +- src/program/services/scrapers/shared.py | 11 +- src/program/services/scrapers/torbox.py | 10 +- src/program/services/scrapers/torrentio.py | 16 +- src/program/services/scrapers/zilean.py | 10 +- src/program/services/updaters/emby.py | 20 +- src/program/services/updaters/jellyfin.py | 20 +- src/program/utils/request.py | 186 +++++++----------- 21 files changed, 320 insertions(+), 282 deletions(-) diff --git a/src/program/apis/listrr_api.py b/src/program/apis/listrr_api.py index 67fc8d2f..d6c8590c 100644 --- a/src/program/apis/listrr_api.py +++ b/src/program/apis/listrr_api.py @@ -3,8 +3,17 @@ from program.apis.trakt_api import TraktAPI from program.media.item import MediaItem -from program.utils.request import get, ping, create_service_session +from program.utils.request import create_service_session, BaseRequestHandler, Session, ResponseType, ResponseObject, HttpMethod +class ListrrAPIError(Exception): + """Base exception for ListrrAPI related errors""" + +class ListrrRequestHandler(BaseRequestHandler): + def __init__(self, session: Session, base_url: str, request_logging: bool = False): + super().__init__(session, base_url=base_url, response_type=ResponseType.SIMPLE_NAMESPACE, custom_exception=ListrrAPIError, request_logging=request_logging) + + def execute(self, method: HttpMethod, endpoint: str, **kwargs) -> ResponseObject: + return super()._request(method, endpoint, **kwargs) class ListrrAPI: """Handles Listrr API communication""" @@ -13,12 +22,13 @@ def __init__(self, api_key: str): self.BASE_URL = "https://listrr.pro" self.api_key = api_key self.headers = {"X-Api-Key": self.api_key} - self.session = create_service_session() - self.session.headers.update(self.headers) + session = create_service_session() + session.headers.update(self.headers) + self.request_handler = ListrrRequestHandler(session, base_url=self.BASE_URL) self.trakt_api = TraktAPI(rate_limit=False) def validate(self): - return ping(session=self.session, url=self.BASE_URL) + return self.request_handler.execute(HttpMethod.GET, self.BASE_URL) def get_items_from_Listrr(self, content_type, content_lists) -> list[MediaItem] | list[str]: # noqa: C901, PLR0912 """Fetch unique IMDb IDs from Listrr for a given type and list of content.""" @@ -33,9 +43,9 @@ def get_items_from_Listrr(self, content_type, content_lists) -> list[MediaItem] page, total_pages = 1, 1 while page <= total_pages: try: - url = f"{self.BASE_URL}/api/List/{content_type}/{list_id}/ReleaseDate/Descending/{page}" - response = get(session=self.session, url=url).response - data = response.json() + url = f"api/List/{content_type}/{list_id}/ReleaseDate/Descending/{page}" + response = self.request_handler.execute(HttpMethod.GET, url) + data = response.data total_pages = data.get("pages", 1) for item in data.get("items", []): imdb_id = item.get("imDbId") diff --git a/src/program/apis/mdblist_api.py b/src/program/apis/mdblist_api.py index 57aace72..19541ebd 100644 --- a/src/program/apis/mdblist_api.py +++ b/src/program/apis/mdblist_api.py @@ -1,4 +1,16 @@ -from program.utils.request import get_rate_limit_params, create_service_session, get, ping +from program.utils.request import get_rate_limit_params, create_service_session, BaseRequestHandler, Session, ResponseType, ResponseObject, HttpMethod + + +class MdblistAPIError(Exception): + """Base exception for MdblistAPI related errors""" + +class MdblistRequestHandler(BaseRequestHandler): + def __init__(self, session: Session, base_url: str, api_key: str, request_logging: bool = False): + self.api_key = api_key + super().__init__(session, base_url=base_url, response_type=ResponseType.SIMPLE_NAMESPACE, custom_exception=MdblistAPIError, request_logging=request_logging) + + def execute(self, method: HttpMethod, endpoint: str, ignore_base_url: bool = False, **kwargs) -> ResponseObject: + return super()._request(method, endpoint, ignore_base_url=ignore_base_url, params={"apikey": self.api_key}, **kwargs) class MdblistAPI: @@ -6,32 +18,25 @@ class MdblistAPI: BASE_URL = "https://mdblist.com" def __init__(self, api_key: str): - self.api_key = api_key - rate_limit_params = get_rate_limit_params(per_minute=60) - - self.session = create_service_session( - rate_limit_params=rate_limit_params, - use_cache=False - ) + session = create_service_session(rate_limit_params=rate_limit_params) + self.request_handler = MdblistRequestHandler(session, base_url=self.BASE_URL, api_key=api_key) def validate(self): - return ping(session=self.session, url=f"{self.BASE_URL}/api/user?apikey={self.api_key}") + return self.request_handler.execute(HttpMethod.GET, f"api/user") def my_limits(self): """Wrapper for mdblist api method 'My limits'""" - response = get(session=self.session, url=f"{self.BASE_URL}/api/user?apikey={self.api_key}") + response = self.request_handler.execute(HttpMethod.GET,f"api/user") return response.data def list_items_by_id(self, list_id: int): """Wrapper for mdblist api method 'List items'""" - response = get(session=self.session, - url=f"{self.BASE_URL}/api/lists/{str(list_id)}/items?apikey={self.api_key}" - ) + response = self.request_handler.execute(HttpMethod.GET,f"api/lists/{str(list_id)}/items") return response.data def list_items_by_url(self, url: str): url = url if url.endswith("/") else f"{url}/" url = url if url.endswith("json/") else f"{url}json/" - response = get(session=self.session, url=url, params={"apikey": self.api_key}) + response = self.request_handler.execute(HttpMethod.GET, url, ignore_base_url=True) return response.data \ No newline at end of file diff --git a/src/program/apis/overseerr_api.py b/src/program/apis/overseerr_api.py index 68811d48..99031f91 100644 --- a/src/program/apis/overseerr_api.py +++ b/src/program/apis/overseerr_api.py @@ -7,7 +7,17 @@ from program.apis.trakt_api import TraktAPI from program.media.item import MediaItem from program.settings.manager import settings_manager -from program.utils.request import delete, get, ping, post, get_rate_limit_params, create_service_session +from program.utils.request import BaseRequestHandler, Session, ResponseType, HttpMethod, ResponseObject, get_rate_limit_params, create_service_session + +class OverseerrAPIError(Exception): + """Base exception for OverseerrAPI related errors""" + +class OverseerrRequestHandler(BaseRequestHandler): + def __init__(self, session: Session, base_url: str, request_logging: bool = False): + super().__init__(session, base_url=base_url, response_type=ResponseType.SIMPLE_NAMESPACE, custom_exception=OverseerrAPIError, request_logging=request_logging) + + def execute(self, method: HttpMethod, endpoint: str, **kwargs) -> ResponseObject: + return super()._request(method, endpoint, **kwargs) class OverseerrAPI: @@ -15,24 +25,20 @@ class OverseerrAPI: def __init__(self, api_key: str, base_url: str): self.api_key = api_key - self.BASE_URL = base_url rate_limit_params = get_rate_limit_params(max_calls=1000, period=300) - self.session = create_service_session(rate_limit_params=rate_limit_params, use_cache=False) + session = create_service_session(rate_limit_params=rate_limit_params) self.trakt_api = TraktAPI(rate_limit=False) self.headers = {"X-Api-Key": self.api_key} - self.session.headers.update(self.headers) + session.headers.update(self.headers) + self.request_handler = OverseerrRequestHandler(session, base_url=base_url) def validate(self): - return ping( - session=self.session, - url=self.BASE_URL + "/api/v1/auth/me", - timeout=30, - ) + return self.request_handler.execute(HttpMethod.GET, "api/v1/auth/me", timeout=30) def get_media_requests(self, service_key: str) -> list[MediaItem]: """Get media requests from `Overseerr`""" try: - response = get(session=self.session, url=self.BASE_URL + f"/api/v1/request?take={10000}&filter=approved&sort=added") + response = self.request_handler.execute(HttpMethod.GET, f"api/v1/request?take={10000}&filter=approved&sort=added") if not response.is_ok: logger.error(f"Failed to fetch requests from overseerr: {response.data}") return [] @@ -82,10 +88,7 @@ def get_imdb_id(self, data) -> str | None: external_id = data.tmdbId try: - response = get( - session=self.session, - url=self.BASE_URL + f"/api/v1/{data.mediaType}/{external_id}?language=en", - ) + response = self.request_handler.execute(HttpMethod.GET, f"api/v1/{data.mediaType}/{external_id}?language=en") except (ConnectionError, RetryError, MaxRetryError) as e: logger.error(f"Failed to fetch media details from overseerr: {str(e)}") return None @@ -122,11 +125,7 @@ def delete_request(self, mediaId: int) -> bool: settings = settings_manager.settings.content.overseerr headers = {"X-Api-Key": settings.api_key} try: - response = delete( - session=self.session, - url=self.BASE_URL + f"/api/v1/request/{mediaId}", - additional_headers=headers, - ) + response = self.request_handler.execute(HttpMethod.DELETE, f"api/v1/request/{mediaId}", additional_headers=headers) logger.debug(f"Deleted request {mediaId} from overseerr") return response.is_ok == True except Exception as e: @@ -136,11 +135,7 @@ def delete_request(self, mediaId: int) -> bool: def mark_processing(self, mediaId: int) -> bool: """Mark item as processing in overseerr""" try: - response = post( - session=self.session, - url=self.BASE_URL + f"/api/v1/media/{mediaId}/pending", - data={"is4k": False}, - ) + response = self.request_handler.execute(HttpMethod.POST, f"api/v1/media/{mediaId}/pending", data={"is4k": False}) logger.info(f"Marked media {mediaId} as processing in overseerr") return response.is_ok except Exception as e: @@ -150,11 +145,7 @@ def mark_processing(self, mediaId: int) -> bool: def mark_partially_available(self, mediaId: int) -> bool: """Mark item as partially available in overseerr""" try: - response = post( - session=self.session, - url=self.BASE_URL + f"/api/v1/media/{mediaId}/partial", - data={"is4k": False}, - ) + response = self.request_handler.execute(HttpMethod.POST, f"api/v1/media/{mediaId}/partial", data={"is4k": False}) logger.info(f"Marked media {mediaId} as partially available in overseerr") return response.is_ok except Exception as e: @@ -164,11 +155,7 @@ def mark_partially_available(self, mediaId: int) -> bool: def mark_completed(self, mediaId: int) -> bool: """Mark item as completed in overseerr""" try: - response = post( - session=self.session, - url=self.BASE_URL + f"/api/v1/media/{mediaId}/available", - data={"is4k": False}, - ) + response = self.request_handler.execute(HttpMethod.POST, f"api/v1/media/{mediaId}/available", data={"is4k": False}) logger.info(f"Marked media {mediaId} as completed in overseerr") return response.is_ok except Exception as e: diff --git a/src/program/apis/plex_api.py b/src/program/apis/plex_api.py index b72979c7..927a2e10 100644 --- a/src/program/apis/plex_api.py +++ b/src/program/apis/plex_api.py @@ -1,46 +1,57 @@ from typing import List, Optional, Dict, Union from loguru import logger +from requests import Session from plexapi.myplex import MyPlexAccount from plexapi.server import PlexServer from plexapi.library import LibrarySection - from program.media import Movie, Episode from program.settings.manager import settings_manager -from program.utils.request import get, ping, create_service_session +from program.utils.request import create_service_session, BaseRequestHandler, HttpMethod, ResponseType, ResponseObject + + +class PlexAPIError(Exception): + """Base exception for PlexApi related errors""" +class PlexRequestHandler(BaseRequestHandler): + def __init__(self, session: Session, request_logging: bool = False): + super().__init__(session, response_type=ResponseType.SIMPLE_NAMESPACE, custom_exception=PlexAPIError, request_logging=request_logging) + + def execute(self, method: HttpMethod, endpoint: str, overriden_response_type: ResponseType = None, **kwargs) -> ResponseObject: + return super()._request(method, endpoint, overriden_response_type=overriden_response_type, **kwargs) class PlexAPI: """Handles Plex API communication""" def __init__(self, token: str, base_url: str, rss_urls: Optional[List[str]]): - self.BASE_URL = base_url self.rss_urls = rss_urls self.token = token - self.session = create_service_session() + self.BASE_URL = base_url + session = create_service_session() + self.request_handler = PlexRequestHandler(session) self.account = None self.plex_server = None self.rss_enabled = False def validate_account(self): try: - self.account = MyPlexAccount(session=self.session, token=self.token) + self.account = MyPlexAccount(session=self.request_handler.session, token=self.token) except Exception as e: logger.error(f"Failed to authenticate Plex account: {e}") return False return True def validate_server(self): - self.plex_server = PlexServer(self.BASE_URL, token=self.token, session=self.session, timeout=60) + self.plex_server = PlexServer(self.BASE_URL, token=self.token, session=self.request_handler.session, timeout=60) def validate_rss(self, url: str): - return ping(session=self.session, url=url) + return self.request_handler.execute(HttpMethod.GET, url) def ratingkey_to_imdbid(self, ratingKey: str) -> str | None: """Convert Plex rating key to IMDb ID""" token = settings_manager.settings.updaters.plex.token filter_params = "includeGuids=1&includeFields=guid,title,year&includeElements=Guid" url = f"https://metadata.provider.plex.tv/library/metadata/{ratingKey}?X-Plex-Token={token}&{filter_params}" - response = get(session=self.session, url=url) + response = self.request_handler.execute(HttpMethod.GET, url) if response.is_ok and hasattr(response.data, "MediaContainer"): metadata = response.data.MediaContainer.Metadata[0] return next((guid.id.split("//")[-1] for guid in metadata.Guid if "imdb://" in guid.id), None) @@ -52,8 +63,8 @@ def get_items_from_rss(self) -> list[str]: rss_items: list[str] = [] for rss_url in self.rss_urls: try: - response = self.session.get(rss_url + "?format=json", timeout=60) - for _item in response.json().get("items", []): + response = self.request_handler.execute(HttpMethod.GET, rss_url + "?format=json", overriden_response_type=ResponseType.DICT, timeout=60) + for _item in response.data.get("items", []): imdb_id = self.extract_imdb_ids(_item.get("guids", [])) if imdb_id and imdb_id.startswith("tt"): rss_items.append(imdb_id) diff --git a/src/program/apis/trakt_api.py b/src/program/apis/trakt_api.py index 72f85e60..17d57822 100644 --- a/src/program/apis/trakt_api.py +++ b/src/program/apis/trakt_api.py @@ -2,12 +2,22 @@ from datetime import datetime from types import SimpleNamespace from typing import Union, List, Optional -from requests import RequestException - +from requests import RequestException, Session from program import MediaItem from program.media import Movie, Show, Season, Episode -from program.utils.request import get_rate_limit_params, create_service_session, get, logger +from program.utils.request import get_rate_limit_params, create_service_session, logger, BaseRequestHandler, \ + ResponseType, HttpMethod, ResponseObject + + +class TraktAPIError(Exception): + """Base exception for TraktApi related errors""" + +class TraktRequestHandler(BaseRequestHandler): + def __init__(self, session: Session, request_logging: bool = False): + super().__init__(session, response_type=ResponseType.SIMPLE_NAMESPACE, custom_exception=TraktAPIError, request_logging=request_logging) + def execute(self, method: HttpMethod, endpoint: str, **kwargs) -> ResponseObject: + return super()._request(method, endpoint, **kwargs) class TraktAPI: """Handles Trakt API communication""" @@ -22,7 +32,7 @@ class TraktAPI: def __init__(self, api_key: Optional[str] = None, rate_limit: bool = True): self.api_key = api_key rate_limit_params = get_rate_limit_params(max_calls=1000, period=300) if rate_limit else None - self.session = create_service_session( + session = create_service_session( rate_limit_params=rate_limit_params, use_cache=False ) @@ -31,10 +41,11 @@ def __init__(self, api_key: Optional[str] = None, rate_limit: bool = True): "trakt-api-key": self.api_key or self.CLIENT_ID, "trakt-api-version": "2" } - self.session.headers.update(self.headers) + session.headers.update(self.headers) + self.request_handler = TraktRequestHandler(session) def validate(self): - return get(session=self.session, url=f"{self.BASE_URL}/lists/2") + return self.request_handler.execute(HttpMethod.GET, f"{self.BASE_URL}/lists/2") def _fetch_data(self, url, params): """Fetch paginated data from Trakt API with rate limiting.""" @@ -43,7 +54,7 @@ def _fetch_data(self, url, params): while True: try: - response = get(session=self.session, url=url, params={**params, "page": page}) + response = self.request_handler.execute(HttpMethod.GET, url, params={**params, "page": page}) if response.is_ok: data = response.data if not data: @@ -138,7 +149,7 @@ def get_show(self, imdb_id: str) -> dict: if not imdb_id: return {} url = f"https://api.trakt.tv/shows/{imdb_id}/seasons?extended=episodes,full" - response = get(self.session, url, timeout=30) + response = self.request_handler.execute(HttpMethod.GET, url, timeout=30) return response.data if response.is_ok and response.data else {} def get_show_aliases(self, imdb_id: str, item_type: str) -> List[dict]: @@ -147,7 +158,7 @@ def get_show_aliases(self, imdb_id: str, item_type: str) -> List[dict]: return [] url = f"https://api.trakt.tv/{item_type}/{imdb_id}/aliases" try: - response = get(self.session, url, timeout=30) + response = self.request_handler.execute(HttpMethod.GET, url, timeout=30) if response.is_ok and response.data: aliases = {} for ns in response.data: @@ -168,7 +179,7 @@ def get_show_aliases(self, imdb_id: str, item_type: str) -> List[dict]: def create_item_from_imdb_id(self, imdb_id: str, type: str = None) -> Optional[MediaItem]: """Wrapper for trakt.tv API search method.""" url = f"https://api.trakt.tv/search/imdb/{imdb_id}?extended=full" - response = get(self.session, url, timeout=30) + response = self.request_handler.execute(HttpMethod.GET, url, timeout=30) if not response.is_ok or not response.data: logger.error( f"Failed to create item using imdb id: {imdb_id}") # This returns an empty list for response.data @@ -184,7 +195,7 @@ def create_item_from_imdb_id(self, imdb_id: str, type: str = None) -> Optional[M def get_imdbid_from_tmdb(self, tmdb_id: str, type: str = "movie") -> Optional[str]: """Wrapper for trakt.tv API search method.""" url = f"https://api.trakt.tv/search/tmdb/{tmdb_id}" # ?extended=full - response = get(self.session, url, timeout=30) + response = self.request_handler.execute(HttpMethod.GET, url, timeout=30) if not response.is_ok or not response.data: return None imdb_id = self._get_imdb_id_from_list(response.data, id_type="tmdb", _id=tmdb_id, type=type) @@ -196,7 +207,7 @@ def get_imdbid_from_tmdb(self, tmdb_id: str, type: str = "movie") -> Optional[st def get_imdbid_from_tvdb(self, tvdb_id: str, type: str = "show") -> Optional[str]: """Wrapper for trakt.tv API search method.""" url = f"https://api.trakt.tv/search/tvdb/{tvdb_id}" - response = get(self.session, url, timeout=30) + response = self.request_handler.execute(HttpMethod.GET, url, timeout=30) if not response.is_ok or not response.data: return None imdb_id = self._get_imdb_id_from_list(response.data, id_type="tvdb", _id=tvdb_id, type=type) @@ -208,7 +219,7 @@ def get_imdbid_from_tvdb(self, tvdb_id: str, type: str = "show") -> Optional[str def resolve_short_url(self, short_url) -> Union[str, None]: """Resolve short URL to full URL""" try: - response = get(session=self.session, url=short_url, additional_headers={"Content-Type": "application/json", "Accept": "text/html"}) + response = self.request_handler.execute(HttpMethod.GET, url=short_url, additional_headers={"Content-Type": "application/json", "Accept": "text/html"}) if response.is_ok: return response.response.url else: diff --git a/src/program/services/content/trakt.py b/src/program/services/content/trakt.py index aaf31458..b8e831f5 100644 --- a/src/program/services/content/trakt.py +++ b/src/program/services/content/trakt.py @@ -1,6 +1,7 @@ """Trakt content module""" from datetime import datetime, timedelta +from typing import Type, Optional from urllib.parse import urlencode from loguru import logger @@ -9,7 +10,15 @@ from program.apis.trakt_api import TraktAPI from program.media.item import MediaItem from program.settings.manager import settings_manager -from program.utils.request import post +from program.utils.request import create_service_session, BaseRequestHandler, Session, ResponseType, ResponseObject, HttpMethod + + +class TraktOAuthRequestHandler(BaseRequestHandler): + def __init__(self, session: Session, response_type=ResponseType.SIMPLE_NAMESPACE, custom_exception: Optional[Type[Exception]] = None, request_logging: bool = False): + super().__init__(session, response_type=response_type, custom_exception=custom_exception, request_logging=request_logging) + + def execute(self, method: HttpMethod, endpoint: str, **kwargs) -> ResponseObject: + return super()._request(method, endpoint, **kwargs) class TraktContent: """Content class for Trakt""" @@ -18,6 +27,8 @@ def __init__(self): self.key = "trakt" self.settings = settings_manager.settings.content.trakt self.api = TraktAPI(self.settings.api_key) + session = create_service_session() + self.oauth_request_handler = TraktOAuthRequestHandler(session) self.initialized = self.validate() if not self.initialized: return @@ -195,7 +206,7 @@ def handle_oauth_callback(self, code: str) -> bool: "redirect_uri": self.settings.oauth_redirect_uri, "grant_type": "authorization_code", } - response = post(session=self.api.session, url=token_url, data=payload, additional_headers=self.api.headers) + response = self.oauth_request_handler.execute(HttpMethod.POST, token_url, data=payload, additional_headers=self.api.headers) if response.is_ok: token_data = response.data self.settings.access_token = token_data.get("access_token") diff --git a/src/program/services/downloaders/alldebrid.py b/src/program/services/downloaders/alldebrid.py index 5eb05f5a..cf2aff5a 100644 --- a/src/program/services/downloaders/alldebrid.py +++ b/src/program/services/downloaders/alldebrid.py @@ -5,7 +5,7 @@ from loguru import logger from requests.exceptions import ConnectTimeout from program.utils.request import get_rate_limit_params, create_service_session, BaseRequestHandler, \ - BaseRequestParameters + BaseRequestParameters, HttpMethod, ResponseType from program.settings.manager import settings_manager @@ -21,13 +21,13 @@ class AllDebridBaseRequestParameters(BaseRequestParameters): class AllDebridRequestHandler(BaseRequestHandler): def __init__(self, session: Session, base_url: str, base_params: AllDebridBaseRequestParameters, request_logging: bool = False): - super().__init__(session, base_url, base_params, custom_exception=AllDebridError, request_logging=request_logging) + super().__init__(session, response_type=ResponseType.DICT, base_url=base_url, base_params=base_params, custom_exception=AllDebridError, request_logging=request_logging) - def execute(self, method: str, endpoint: str, **kwargs) -> dict: - data, status_code = super()._request(method, endpoint, **kwargs) - if not data or "data" not in data: + def execute(self, method: HttpMethod, endpoint: str, **kwargs) -> dict: + response = super()._request(method, endpoint, **kwargs) + if not response.is_ok or not response.data or "data" not in response.data: raise AllDebridError("Invalid response from AllDebrid") - return data["data"] + return response.data["data"] class AllDebridAPI: """Handles AllDebrid API communication""" @@ -93,7 +93,7 @@ def _validate_settings(self) -> bool: def _validate_premium(self) -> bool: """Validate premium status""" try: - user_info = self.api.request_handler.execute("GET", "user") + user_info = self.api.request_handler.execute(HttpMethod.GET, "user") user = user_info.get("user", {}) if not user.get("isPremium", False): @@ -121,7 +121,7 @@ def get_instant_availability(self, infohashes: List[str]) -> Dict[str, list]: try: params = {f"magnets[{i}]": infohash for i, infohash in enumerate(infohashes)} - response = self.api.request_handler.execute("GET", "magnet/instant", **params) + response = self.api.request_handler.execute(HttpMethod.GET, "magnet/instant", **params) magnets = response.get("magnets", []) availability = {} @@ -175,7 +175,7 @@ def add_torrent(self, infohash: str) -> str: try: response = self.api.request_handler.execute( - "GET", + HttpMethod.GET, "magnet/upload", **{"magnets[]": infohash} ) @@ -216,7 +216,7 @@ def get_torrent_info(self, torrent_id: str) -> dict: raise AllDebridError("Downloader not properly initialized") try: - response = self.api.request_handler.execute("GET", "magnet/status", id=torrent_id) + response = self.api.request_handler.execute(HttpMethod.GET, "magnet/status", id=torrent_id) info = response.get("magnets", {}) if "filename" not in info: raise AllDebridError("Invalid torrent info response") @@ -234,7 +234,7 @@ def delete_torrent(self, torrent_id: str): raise AllDebridError("Downloader not properly initialized") try: - self.api.request_handler.execute("GET", "magnet/delete", id=torrent_id) + self.api.request_handler.execute(HttpMethod.GET, "magnet/delete", id=torrent_id) except Exception as e: logger.error(f"Failed to delete torrent {torrent_id}: {e}") raise \ No newline at end of file diff --git a/src/program/services/downloaders/realdebrid.py b/src/program/services/downloaders/realdebrid.py index 7387b6bb..d46275d9 100644 --- a/src/program/services/downloaders/realdebrid.py +++ b/src/program/services/downloaders/realdebrid.py @@ -10,7 +10,8 @@ from program.settings.manager import settings_manager from .shared import VIDEO_EXTENSIONS, DownloaderBase, FileFinder, premium_days_left -from program.utils.request import get_rate_limit_params, create_service_session, BaseRequestHandler +from program.utils.request import get_rate_limit_params, create_service_session, BaseRequestHandler, HttpMethod, \ + ResponseType class RDTorrentStatus(str, Enum): @@ -44,15 +45,15 @@ class RealDebridError(Exception): class RealDebridRequestHandler(BaseRequestHandler): def __init__(self, session: Session, base_url: str, request_logging: bool = False): - super().__init__(session, base_url, custom_exception=RealDebridError, request_logging=request_logging) + super().__init__(session, response_type=ResponseType.DICT, base_url=base_url, custom_exception=RealDebridError, request_logging=request_logging) - def execute(self, method: str, endpoint: str, **kwargs) -> Union[dict, list]: - data, status_code = super()._request(method, endpoint, **kwargs) - if status_code == 204: + def execute(self, method: HttpMethod, endpoint: str, **kwargs) -> Union[dict, list]: + response = super()._request(method, endpoint, **kwargs) + if response.status_code == 204: return {} - if not data: + if not response.data and not response.is_ok: raise RealDebridError("Invalid JSON response from RealDebrid") - return data + return response.data class RealDebridAPI: """Handles Real-Debrid API communication""" @@ -110,7 +111,7 @@ def _validate_settings(self) -> bool: def _validate_premium(self) -> bool: """Validate premium status""" try: - user_info = self.api.request_handler.execute("GET", "user") + user_info = self.api.request_handler.execute(HttpMethod.GET, "user") if not user_info.get("premium"): logger.error("Premium membership required") return False @@ -136,7 +137,7 @@ def get_instant_availability(self, infohashes: List[str]) -> Dict[str, list]: for attempt in range(self.MAX_RETRIES): try: response = self.api.request_handler.execute( - "GET", + HttpMethod.GET, f"torrents/instantAvailability/{'/'.join(infohashes)}" ) @@ -194,7 +195,7 @@ def add_torrent(self, infohash: str) -> str: try: magnet = f"magnet:?xt=urn:btih:{infohash}" response = self.api.request_handler.execute( - "POST", + HttpMethod.POST, "torrents/addMagnet", data={"magnet": magnet.lower()} ) @@ -213,7 +214,7 @@ def select_files(self, torrent_id: str, files: List[str]): try: self.api.request_handler.execute( - "POST", + HttpMethod.POST, f"torrents/selectFiles/{torrent_id}", data={"files": ",".join(files)} ) @@ -230,7 +231,7 @@ def get_torrent_info(self, torrent_id: str) -> dict: raise RealDebridError("Downloader not properly initialized") try: - return self.api.request_handler.execute("GET", f"torrents/info/{torrent_id}") + return self.api.request_handler.execute(HttpMethod.GET, f"torrents/info/{torrent_id}") except Exception as e: logger.error(f"Failed to get torrent info for {torrent_id}: {e}") raise @@ -245,7 +246,7 @@ def delete_torrent(self, torrent_id: str): raise RealDebridError("Downloader not properly initialized") try: - self.api.request_handler.execute("DELETE", f"torrents/delete/{torrent_id}") + self.api.request_handler.execute(HttpMethod.DELETE, f"torrents/delete/{torrent_id}") except Exception as e: logger.error(f"Failed to delete torrent {torrent_id}: {e}") raise \ No newline at end of file diff --git a/src/program/services/scrapers/comet.py b/src/program/services/scrapers/comet.py index 301a0266..d7df747d 100644 --- a/src/program/services/scrapers/comet.py +++ b/src/program/services/scrapers/comet.py @@ -9,9 +9,10 @@ from requests.exceptions import RequestException from program.media.item import MediaItem, Show -from program.services.scrapers.shared import _get_stremio_identifier +from program.services.scrapers.shared import _get_stremio_identifier, ScraperRequestHandler from program.settings.manager import settings_manager -from program.utils.request import get, ping, create_service_session, get_rate_limit_params, RateLimitExceeded +from program.utils.request import create_service_session, get_rate_limit_params, RateLimitExceeded, \ + HttpMethod class Comet: @@ -31,11 +32,8 @@ def __init__(self): "debridStreamProxyPassword": "" }).encode("utf-8")).decode("utf-8") rate_limit_params = get_rate_limit_params(per_hour=300) if self.settings.ratelimit else None - - self.session = create_service_session( - rate_limit_params=rate_limit_params, - use_cache=False - ) + session = create_service_session(rate_limit_params=rate_limit_params) + self.request_handler = ScraperRequestHandler(session) self.initialized = self.validate() if not self.initialized: return @@ -56,7 +54,7 @@ def validate(self) -> bool: return False try: url = f"{self.settings.url}/manifest.json" - response = ping(session=self.session, url=url, timeout=self.timeout) + response = self.request_handler.execute(HttpMethod.GET, url, timeout=self.timeout) if response.is_ok: return True except Exception as e: @@ -92,7 +90,7 @@ def scrape(self, item: MediaItem) -> tuple[Dict[str, str], int]: url = f"{self.settings.url}/{self.encoded_string}/stream/{scrape_type}/{imdb_id}{identifier or ''}.json" - response = get(self.session, url=url, timeout=self.timeout) + response = self.request_handler.execute(HttpMethod.GET, url, timeout=self.timeout) if not response.is_ok or not getattr(response.data, "streams", None): return {} diff --git a/src/program/services/scrapers/jackett.py b/src/program/services/scrapers/jackett.py index 5628e768..b9bd947c 100644 --- a/src/program/services/scrapers/jackett.py +++ b/src/program/services/scrapers/jackett.py @@ -12,8 +12,9 @@ from requests import HTTPError, ReadTimeout, RequestException, Timeout from program.media.item import Episode, MediaItem, Movie, Season, Show +from program.services.scrapers.shared import ScraperRequestHandler from program.settings.manager import settings_manager -from program.utils.request import get, create_service_session, get_rate_limit_params, RateLimitExceeded +from program.utils.request import create_service_session, get_rate_limit_params, RateLimitExceeded, HttpMethod class JackettIndexer(BaseModel): @@ -35,7 +36,7 @@ def __init__(self): self.api_key = None self.indexers = None self.settings = settings_manager.settings.scraping.jackett - self.session = None + self.request_handler = None self.initialized = self.validate() if not self.initialized and not self.api_key: return @@ -61,7 +62,8 @@ def validate(self) -> bool: self.indexers = indexers rate_limit_params = get_rate_limit_params(max_calls=len(self.indexers), period=2) if self.settings.ratelimit else None - self.session = create_service_session(rate_limit_params=rate_limit_params, use_cache=False) + session = create_service_session(rate_limit_params=rate_limit_params) + self.request_handler = ScraperRequestHandler(session) self._log_indexers() return True except ReadTimeout: @@ -242,8 +244,11 @@ def _get_indexer_from_xml(self, xml_content: str) -> list[JackettIndexer]: def _fetch_results(self, url: str, params: Dict[str, str], indexer_title: str, search_type: str) -> List[Tuple[str, str]]: """Fetch results from the given indexer""" try: - response = get(session=self.session, url=url, params=params, timeout=self.settings.timeout) - return self._parse_xml(response.response.text) + response = self.request_handler.execute(HttpMethod.GET, url, params=params, timeout=self.settings.timeout) + return self._parse_xml(response.data) + except RateLimitExceeded: + logger.warning(f"Rate limit exceeded while fetching results for {search_type}: {indexer_title}") + return [] except (HTTPError, ConnectionError, Timeout): logger.debug(f"Indexer failed to fetch results for {search_type}: {indexer_title}") except Exception as e: diff --git a/src/program/services/scrapers/knightcrawler.py b/src/program/services/scrapers/knightcrawler.py index 90bcb290..310d2ee4 100644 --- a/src/program/services/scrapers/knightcrawler.py +++ b/src/program/services/scrapers/knightcrawler.py @@ -6,9 +6,10 @@ from requests.exceptions import RequestException from program.media.item import MediaItem -from program.services.scrapers.shared import _get_stremio_identifier +from program.services.scrapers.shared import _get_stremio_identifier, ScraperRequestHandler from program.settings.manager import settings_manager -from program.utils.request import get, ping, create_service_session, get_rate_limit_params, RateLimitExceeded +from program.utils.request import create_service_session, get_rate_limit_params, RateLimitExceeded, HttpMethod + class Knightcrawler: """Scraper for `Knightcrawler`""" @@ -18,7 +19,8 @@ def __init__(self): self.settings = settings_manager.settings.scraping.knightcrawler self.timeout = self.settings.timeout rate_limit_params = get_rate_limit_params(max_calls=1, period=5) if self.settings.ratelimit else None - self.session = create_service_session(rate_limit_params=rate_limit_params,use_cache=False) + session = create_service_session(rate_limit_params=rate_limit_params) + self.request_handler = ScraperRequestHandler(session) self.initialized = self.validate() if not self.initialized: return @@ -39,7 +41,7 @@ def validate(self) -> bool: return False try: url = f"{self.settings.url}/{self.settings.filter}/manifest.json" - response = ping(session=self.session, url=url, timeout=self.timeout) + response = self.request_handler.execute(HttpMethod.GET, url, timeout=self.timeout) if response.is_ok: return True except Exception as e: @@ -78,7 +80,7 @@ def scrape(self, item: MediaItem) -> Dict[str, str]: if identifier: url += identifier - response = get(session=self.session, url=f"{url}.json", timeout=self.timeout) + response = self.request_handler.execute(HttpMethod.GET, f"{url}.json", timeout=self.timeout) if not response.is_ok or len(response.data.streams) <= 0: return {} diff --git a/src/program/services/scrapers/mediafusion.py b/src/program/services/scrapers/mediafusion.py index 2dfb827f..fff743e9 100644 --- a/src/program/services/scrapers/mediafusion.py +++ b/src/program/services/scrapers/mediafusion.py @@ -7,10 +7,10 @@ from requests.exceptions import RequestException from program.media.item import MediaItem -from program.services.scrapers.shared import _get_stremio_identifier +from program.services.scrapers.shared import _get_stremio_identifier, ScraperRequestHandler from program.settings.manager import settings_manager from program.settings.models import AppModel -from program.utils.request import get, ping, create_service_session, get_rate_limit_params, RateLimitExceeded, post +from program.utils.request import create_service_session, get_rate_limit_params, RateLimitExceeded, HttpMethod class Mediafusion: @@ -25,7 +25,8 @@ def __init__(self): self.timeout = self.settings.timeout self.encrypted_string = None rate_limit_params = get_rate_limit_params(max_calls=1, period=2) if self.settings.ratelimit else None - self.session = create_service_session(rate_limit_params=rate_limit_params,use_cache=False) + session = create_service_session(rate_limit_params=rate_limit_params) + self.request_handler = ScraperRequestHandler(session) self.initialized = self.validate() if not self.initialized: return @@ -78,7 +79,7 @@ def validate(self) -> bool: headers = {"Content-Type": "application/json"} try: - response = post(session=self.session, url=url, json=payload, additional_headers=headers) + response = self.request_handler.execute(HttpMethod.POST, url, json=payload, additional_headers=headers) self.encrypted_string = json.loads(response.data)["encrypted_str"] except Exception as e: logger.error(f"Failed to encrypt user data: {e}") @@ -86,7 +87,7 @@ def validate(self) -> bool: try: url = f"{self.settings.url}/manifest.json" - response = ping(session=self.session, url=url, timeout=self.timeout) + response = self.request_handler.execute(HttpMethod.GET, url, timeout=self.timeout) return response.is_ok except Exception as e: logger.error(f"Mediafusion failed to initialize: {e}") @@ -120,7 +121,7 @@ def scrape(self, item: MediaItem) -> tuple[Dict[str, str], int]: if identifier: url += identifier - response = get(session=self.session, url=f"{url}.json", timeout=self.timeout) + response = self.request_handler.execute(HttpMethod.GET, f"{url}.json", timeout=self.timeout) if not response.is_ok or len(response.data.streams) <= 0: return {} diff --git a/src/program/services/scrapers/orionoid.py b/src/program/services/scrapers/orionoid.py index b4c46def..f0c8c82b 100644 --- a/src/program/services/scrapers/orionoid.py +++ b/src/program/services/scrapers/orionoid.py @@ -4,8 +4,9 @@ from loguru import logger from program.media.item import MediaItem +from program.services.scrapers.shared import ScraperRequestHandler from program.settings.manager import settings_manager -from program.utils.request import get, create_service_session, get_rate_limit_params, RateLimitExceeded +from program.utils.request import create_service_session, get_rate_limit_params, RateLimitExceeded, HttpMethod KEY_APP = "D3CH6HMX9KD9EMD68RXRCDUNBDJV5HRR" @@ -22,7 +23,8 @@ def __init__(self): self.is_unlimited = False self.initialized = False rate_limit_params = get_rate_limit_params(max_calls=1, period=5) if self.settings.ratelimit else None - self.session = create_service_session(rate_limit_params=rate_limit_params,use_cache=False) + session = create_service_session(rate_limit_params=rate_limit_params) + self.request_handler = ScraperRequestHandler(session) if self.validate(): self.is_premium = self.check_premium() self.initialized = True @@ -42,7 +44,7 @@ def validate(self) -> bool: return False try: url = f"{self.base_url}?keyapp={KEY_APP}&keyuser={self.settings.api_key}&mode=user&action=retrieve" - response = get(session=self.session, url=url, retry_if_failed=True, timeout=self.timeout) + response = self.request_handler.execute(HttpMethod.GET, url, timeout=self.timeout) if response.is_ok and hasattr(response.data, "result"): if response.data.result.status != "success": logger.error( @@ -64,7 +66,7 @@ def validate(self) -> bool: def check_premium(self) -> bool: """Check if the user is active, has a premium account, and has RealDebrid service enabled.""" url = f"{self.base_url}?keyapp={KEY_APP}&keyuser={self.settings.api_key}&mode=user&action=retrieve" - response = get(self.session, url, retry_if_failed=False) + response = self.request_handler.execute(HttpMethod.GET, url) if response.is_ok and hasattr(response.data, "data"): active = response.data.data.status == "active" premium = response.data.data.subscription.package.premium @@ -77,7 +79,7 @@ def check_limit(self) -> bool: """Check if the user has exceeded the rate limit for the Orionoid API.""" url = f"{self.base_url}?keyapp={KEY_APP}&keyuser={self.settings.api_key}&mode=user&action=retrieve" try: - response = get(self.session, url) + response = self.request_handler.execute(HttpMethod.GET, url) if response.is_ok and hasattr(response.data, "data"): remaining = response.data.data.requests.streams.daily.remaining if remaining is None: @@ -144,7 +146,7 @@ def _build_query_params(self, item: MediaItem) -> dict: def scrape(self, item: MediaItem) -> Dict[str, str]: """Wrapper for `Orionoid` scrape method""" params = self._build_query_params(item) - response = get(self.session, self.base_url, params=params, timeout=self.timeout) + response = self.request_handler.execute(HttpMethod.GET, self.base_url, params=params, timeout=self.timeout) if not response.is_ok or not hasattr(response.data, "data"): return {} diff --git a/src/program/services/scrapers/prowlarr.py b/src/program/services/scrapers/prowlarr.py index 4dc5292f..709b16a5 100644 --- a/src/program/services/scrapers/prowlarr.py +++ b/src/program/services/scrapers/prowlarr.py @@ -13,8 +13,9 @@ from requests import HTTPError, ReadTimeout, RequestException, Timeout from program.media.item import Episode, MediaItem, Movie, Season, Show +from program.services.scrapers.shared import ScraperRequestHandler from program.settings.manager import settings_manager -from program.utils.request import get, create_service_session, get_rate_limit_params, RateLimitExceeded +from program.utils.request import create_service_session, get_rate_limit_params, RateLimitExceeded, HttpMethod class ProwlarrIndexer(BaseModel): @@ -37,7 +38,7 @@ def __init__(self): self.indexers = None self.settings = settings_manager.settings.scraping.prowlarr self.timeout = self.settings.timeout - self.session = None + self.request_handler = None self.initialized = self.validate() if not self.initialized and not self.api_key: return @@ -62,7 +63,8 @@ def validate(self) -> bool: return False self.indexers = indexers rate_limit_params = get_rate_limit_params(max_calls=len(self.indexers), period=self.settings.limiter_seconds) if self.settings.ratelimit else None - self.session = create_service_session(rate_limit_params=rate_limit_params, use_cache=False) + session = create_service_session(rate_limit_params=rate_limit_params) + self.request_handler = ScraperRequestHandler(session) self._log_indexers() return True except ReadTimeout: @@ -83,10 +85,7 @@ def run(self, item: MediaItem) -> Dict[str, str]: try: return self.scrape(item) except RateLimitExceeded: - if self.second_limiter: - self.second_limiter.limit_hit() - else: - logger.warning(f"Prowlarr ratelimit exceeded for item: {item.log_string}") + logger.warning(f"Prowlarr ratelimit exceeded for item: {item.log_string}") except RequestException as e: logger.error(f"Prowlarr request exception: {e}") except Exception as e: @@ -233,7 +232,7 @@ def _get_indexer_from_json(self, json_content: str) -> list[ProwlarrIndexer]: def _fetch_results(self, url: str, params: Dict[str, str], indexer_title: str, search_type: str) -> List[Tuple[str, str]]: """Fetch results from the given indexer""" try: - response = get(self.session, url, params=params, timeout=self.timeout) + response = self.request_handler.execute(HttpMethod.GET, url, params=params, timeout=self.timeout) return self._parse_xml(response.response.text, indexer_title) except (HTTPError, ConnectionError, Timeout): logger.debug(f"Indexer failed to fetch results for {search_type.title()} with indexer {indexer_title}") diff --git a/src/program/services/scrapers/shared.py b/src/program/services/scrapers/shared.py index bf27503a..ef79f5d1 100644 --- a/src/program/services/scrapers/shared.py +++ b/src/program/services/scrapers/shared.py @@ -1,5 +1,5 @@ """Shared functions for scrapers.""" -from typing import Dict, Set, Union +from typing import Dict, Set, Union, Type, Optional from loguru import logger from RTN import RTN, ParsedData, Torrent, sort_torrents @@ -10,6 +10,7 @@ from program.media.stream import Stream from program.settings.manager import settings_manager from program.settings.versions import models +from program.utils.request import BaseRequestHandler, Session, ResponseType, ResponseObject, HttpMethod enable_aliases = settings_manager.settings.scraping.enable_aliases settings_model = settings_manager.settings.ranking @@ -17,6 +18,14 @@ rtn = RTN(settings_model, ranking_model) +class ScraperRequestHandler(BaseRequestHandler): + def __init__(self, session: Session, response_type=ResponseType.SIMPLE_NAMESPACE, custom_exception: Optional[Type[Exception]] = None, request_logging: bool = False): + super().__init__(session, response_type=response_type, custom_exception=custom_exception, request_logging=request_logging) + + def execute(self, method: HttpMethod, endpoint: str, **kwargs) -> ResponseObject: + return super()._request(method, endpoint, **kwargs) + + def _get_stremio_identifier(item: MediaItem) -> tuple[str | None, str, str]: """Get the stremio identifier for a media item based on its type.""" if isinstance(item, Show): diff --git a/src/program/services/scrapers/torbox.py b/src/program/services/scrapers/torbox.py index e014a3a0..0008f191 100644 --- a/src/program/services/scrapers/torbox.py +++ b/src/program/services/scrapers/torbox.py @@ -5,8 +5,9 @@ from requests.exceptions import ConnectTimeout from program.media.item import MediaItem +from program.services.scrapers.shared import ScraperRequestHandler from program.settings.manager import settings_manager -from program.utils.request import get, create_service_session, RateLimitExceeded, ping +from program.utils.request import create_service_session, RateLimitExceeded, HttpMethod class TorBoxScraper: @@ -16,7 +17,8 @@ def __init__(self): self.base_url = "http://search-api.torbox.app" self.user_plan = None self.timeout = self.settings.timeout - self.session = create_service_session() + session = create_service_session() + self.request_handler = ScraperRequestHandler(session) self.initialized = self.validate() if not self.initialized: return @@ -30,7 +32,7 @@ def validate(self) -> bool: logger.error("TorBox timeout is not set or invalid.") return False try: - response = ping(self.session, f"{self.base_url}/torrents/imdb:tt0944947?metadata=false&season=1&episode=1", timeout=self.timeout) + response = self.request_handler.execute(HttpMethod.GET, f"{self.base_url}/torrents/imdb:tt0944947?metadata=false&season=1&episode=1", timeout=self.timeout) return response.is_ok except Exception as e: logger.exception(f"Error validating TorBox Scraper: {e}") @@ -69,7 +71,7 @@ def scrape(self, item: MediaItem) -> tuple[Dict[str, str], int]: query_params = self._build_query_params(item) url = f"{self.base_url}/torrents/{query_params}?metadata=false" - response = get(self.session, url, timeout=self.timeout) + response = self.request_handler.execute(HttpMethod.GET, url, timeout=self.timeout) if not response.is_ok or not response.data.data.torrents: return {} diff --git a/src/program/services/scrapers/torrentio.py b/src/program/services/scrapers/torrentio.py index 581e1fd9..ab352b58 100644 --- a/src/program/services/scrapers/torrentio.py +++ b/src/program/services/scrapers/torrentio.py @@ -4,10 +4,10 @@ from loguru import logger from program.media.item import MediaItem -from program.services.scrapers.shared import _get_stremio_identifier +from program.services.scrapers.shared import _get_stremio_identifier, ScraperRequestHandler from program.settings.manager import settings_manager from program.settings.models import TorrentioConfig -from program.utils.request import get, create_service_session, get_rate_limit_params, RateLimitExceeded, ping +from program.utils.request import create_service_session, get_rate_limit_params, RateLimitExceeded, HttpMethod class Torrentio: @@ -18,7 +18,8 @@ def __init__(self): self.settings: TorrentioConfig = settings_manager.settings.scraping.torrentio self.timeout: int = self.settings.timeout rate_limit_params = get_rate_limit_params(max_calls=1, period=5) if self.settings.ratelimit else None - self.session = create_service_session(rate_limit_params=rate_limit_params, use_cache=False) + session = create_service_session(rate_limit_params=rate_limit_params) + self.request_handler = ScraperRequestHandler(session) self.initialized: bool = self.validate() if not self.initialized: return @@ -36,7 +37,7 @@ def validate(self) -> bool: return False try: url = f"{self.settings.url}/{self.settings.filter}/manifest.json" - response = ping(self.session, url=url, timeout=10) + response = self.request_handler.execute(HttpMethod.GET, url, timeout=10) if response.is_ok: return True except Exception as e: @@ -64,8 +65,11 @@ def scrape(self, item: MediaItem) -> tuple[Dict[str, str], int]: if identifier: url += identifier - response = get(self.session, f"{url}.json", timeout=self.timeout) - if not response.is_ok or not response.data.streams: + response = self.request_handler.execute(HttpMethod.GET, f"{url}.json", timeout=self.timeout) + if not response.is_ok: + return {} + + if not hasattr(response.data, 'streams') or not response.data.streams: return {} torrents: Dict[str, str] = {} diff --git a/src/program/services/scrapers/zilean.py b/src/program/services/scrapers/zilean.py index 2ab4ed2e..3eb07ebe 100644 --- a/src/program/services/scrapers/zilean.py +++ b/src/program/services/scrapers/zilean.py @@ -3,8 +3,9 @@ from typing import Dict from loguru import logger from program.media.item import Episode, MediaItem, Season, Show +from program.services.scrapers.shared import ScraperRequestHandler from program.settings.manager import settings_manager -from program.utils.request import get, create_service_session, get_rate_limit_params, RateLimitExceeded, ping +from program.utils.request import create_service_session, get_rate_limit_params, RateLimitExceeded, HttpMethod class Zilean: @@ -15,7 +16,8 @@ def __init__(self): self.settings = settings_manager.settings.scraping.zilean self.timeout = self.settings.timeout rate_limit_params = get_rate_limit_params(max_calls=1, period=2) if self.settings.ratelimit else None - self.session = create_service_session(rate_limit_params=rate_limit_params, use_cache=False) + session = create_service_session(rate_limit_params=rate_limit_params) + self.request_handler = ScraperRequestHandler(session) self.initialized = self.validate() if not self.initialized: return @@ -33,7 +35,7 @@ def validate(self) -> bool: return False try: url = f"{self.settings.url}/healthchecks/ping" - response = ping(self.session, url=url, timeout=self.timeout) + response = self.request_handler.execute(HttpMethod.GET, url, timeout=self.timeout) return response.is_ok except Exception as e: logger.error(f"Zilean failed to initialize: {e}") @@ -66,7 +68,7 @@ def scrape(self, item: MediaItem) -> Dict[str, str]: url = f"{self.settings.url}/dmm/filtered" params = self._build_query_params(item) - response = get(self.session, url, params=params, timeout=self.timeout) + response = self.request_handler.execute(HttpMethod.GET, url, params=params, timeout=self.timeout) if not response.is_ok or not response.data: return {} diff --git a/src/program/services/updaters/emby.py b/src/program/services/updaters/emby.py index 219e5f8f..d76f4c46 100644 --- a/src/program/services/updaters/emby.py +++ b/src/program/services/updaters/emby.py @@ -1,19 +1,29 @@ """Emby Updater module""" from types import SimpleNamespace -from typing import Generator +from typing import Generator, Optional, Type from loguru import logger from program.media.item import MediaItem from program.settings.manager import settings_manager -from program.utils.request import get, post +from program.utils.request import BaseRequestHandler, ResponseType, ResponseObject, Session, HttpMethod, \ + create_service_session +class EmbyRequestHandler(BaseRequestHandler): + def __init__(self, session: Session, response_type=ResponseType.SIMPLE_NAMESPACE, custom_exception: Optional[Type[Exception]] = None, request_logging: bool = False): + super().__init__(session, response_type=response_type, custom_exception=custom_exception, request_logging=request_logging) + + def execute(self, method: HttpMethod, endpoint: str, **kwargs) -> ResponseObject: + return super()._request(method, endpoint, **kwargs) + class EmbyUpdater: def __init__(self): self.key = "emby" self.initialized = False self.settings = settings_manager.settings.updaters.emby + session = create_service_session() + self.request_handler = EmbyRequestHandler(session) self.initialized = self.validate() if not self.initialized: return @@ -30,7 +40,7 @@ def validate(self) -> bool: logger.error("Emby URL is not set!") return False try: - response = get(f"{self.settings.url}/Users?api_key={self.settings.api_key}") + response = self.request_handler.execute(HttpMethod.GET, f"{self.settings.url}/Users?api_key={self.settings.api_key}") if response.is_ok: self.initialized = True return True @@ -79,7 +89,7 @@ def update_item(self, item: MediaItem) -> bool: """Update the Emby item""" if item.symlinked and item.update_folder != "updated" and item.symlink_path: try: - response = post( + response = self.request_handler.execute(HttpMethod.POST, f"{self.settings.url}/Library/Media/Updated", json={"Updates": [{"Path": item.symlink_path, "UpdateType": "Created"}]}, params={"api_key": self.settings.api_key}, @@ -94,7 +104,7 @@ def update_item(self, item: MediaItem) -> bool: def get_libraries(self) -> list[SimpleNamespace]: """Get the libraries from Emby""" try: - response = get( + response = self.request_handler.execute(HttpMethod.GET, f"{self.settings.url}/Library/VirtualFolders", params={"api_key": self.settings.api_key}, ) diff --git a/src/program/services/updaters/jellyfin.py b/src/program/services/updaters/jellyfin.py index 56733e21..93335852 100644 --- a/src/program/services/updaters/jellyfin.py +++ b/src/program/services/updaters/jellyfin.py @@ -1,19 +1,29 @@ """Jellyfin Updater module""" from types import SimpleNamespace -from typing import Generator +from typing import Generator, Type, Optional from loguru import logger from program.media.item import MediaItem from program.settings.manager import settings_manager -from program.utils.request import get, post +from program.utils.request import create_service_session, BaseRequestHandler, ResponseType, ResponseObject, Session, \ + HttpMethod +class JellyfinRequestHandler(BaseRequestHandler): + def __init__(self, session: Session, response_type=ResponseType.SIMPLE_NAMESPACE, custom_exception: Optional[Type[Exception]] = None, request_logging: bool = False): + super().__init__(session, response_type=response_type, custom_exception=custom_exception, request_logging=request_logging) + + def execute(self, method: HttpMethod, endpoint: str, **kwargs) -> ResponseObject: + return super()._request(method, endpoint, **kwargs) + class JellyfinUpdater: def __init__(self): self.key = "jellyfin" self.initialized = False self.settings = settings_manager.settings.updaters.jellyfin + session = create_service_session() + self.request_handler = JellyfinRequestHandler(session) self.initialized = self.validate() if not self.initialized: return @@ -31,7 +41,7 @@ def validate(self) -> bool: return False try: - response = get(f"{self.settings.url}/Users?api_key={self.settings.api_key}") + response = self.request_handler.execute(HttpMethod.GET, f"{self.settings.url}/Users", params={"api_key": self.settings.api_key}) if response.is_ok: self.initialized = True return True @@ -80,7 +90,7 @@ def update_item(self, item: MediaItem) -> bool: """Update the Jellyfin item""" if item.symlinked and item.update_folder != "updated" and item.symlink_path: try: - response = post( + response = self.request_handler.execute(HttpMethod.POST, f"{self.settings.url}/Library/Media/Updated", json={"Updates": [{"Path": item.symlink_path, "UpdateType": "Created"}]}, params={"api_key": self.settings.api_key}, @@ -95,7 +105,7 @@ def update_item(self, item: MediaItem) -> bool: def get_libraries(self) -> list[SimpleNamespace]: """Get the libraries from Jellyfin""" try: - response = get( + response = self.request_handler.execute(HttpMethod.GET, f"{self.settings.url}/Library/VirtualFolders", params={"api_key": self.settings.api_key}, ) diff --git a/src/program/utils/request.py b/src/program/utils/request.py index a5b26bce..df772d57 100644 --- a/src/program/utils/request.py +++ b/src/program/utils/request.py @@ -1,82 +1,52 @@ import json +from enum import Enum from types import SimpleNamespace -from typing import Optional, Dict, Union, Type, Any, Tuple +from typing import Dict, Type, Optional, Any from requests import Session from lxml import etree -from requests.adapters import HTTPAdapter -from requests.exceptions import ConnectTimeout, RequestException, JSONDecodeError +from requests.exceptions import ConnectTimeout, RequestException from requests.models import Response from requests_cache import CacheMixin, CachedSession from requests_ratelimiter import LimiterMixin, LimiterSession -from urllib3.util.retry import Retry from xmltodict import parse as parse_xml from loguru import logger from program.utils import data_dir_path +from pyrate_limiter import RequestRate, Duration, Limiter, MemoryQueueBucket, MemoryListBucket +from requests_ratelimiter import SQLiteBucket -class BaseRequestParameters: - """Holds base parameters that may be included in every request.""" - def to_dict(self) -> Dict[str, Optional[str]]: - """Convert all non-None attributes to a dictionary for inclusion in requests.""" - return {key: value for key, value in self.__dict__.items() if value is not None} +class HttpMethod(Enum): + GET = "GET" + POST = "POST" + PUT = "PUT" + DELETE = "DELETE" + PATCH = "PATCH" -class BaseRequestHandler: - def __init__(self, session: Session, base_url: str, base_params: Optional[BaseRequestParameters] = None, - custom_exception: Optional[Type[Exception]] = None, request_logging: bool = False): - self.session = session - self.BASE_URL = base_url - self.BASE_REQUEST_PARAMS = base_params or BaseRequestParameters() - self.custom_exception = custom_exception or Exception - self.request_logging = request_logging - - def _request(self, method: str, endpoint: str, **kwargs) -> tuple[None, Any] | Any: - """Generic request handler with error handling, using kwargs for flexibility.""" - try: - url = f"{self.BASE_URL}/{endpoint}" +class ResponseType(Enum): + SIMPLE_NAMESPACE = "simple_namespace" + DICT = "dict" - request_params = self.BASE_REQUEST_PARAMS.to_dict() - if request_params: - kwargs.setdefault('params', {}).update(request_params) - elif 'params' in kwargs and not kwargs['params']: - del kwargs['params'] - if self.request_logging: - logger.debug(f"Making request to {url} with kwargs: {kwargs}") +class BaseRequestParameters: + """Holds base parameters that may be included in every request.""" - response = self.session.request(method, url, **kwargs) - response.raise_for_status() + def to_dict(self) -> Dict[str, Any]: + """Convert all non-None attributes to a dictionary for inclusion in requests.""" + return {key: value for key, value in self.__dict__.items() if value is not None} - if response.content: - try: - data = response.json() - if self.request_logging: - logger.debug(f"Response JSON from {endpoint}: {data}") - return data, response.status_code - except JSONDecodeError: - logger.error("Received non-JSON response") - raise self.custom_exception("Non-JSON response received from API") - else: - return None, response.status_code - except RequestException as e: - logger.error(f"Request failed: {e}") - raise self.custom_exception(f"Request failed: {e}") from e - -class RateLimitExceeded(Exception): - """Rate limit exceeded exception""" - def __init__(self, message, response=None): - super().__init__(message) - self.response = response class ResponseObject: """Response object to handle different response formats.""" - def __init__(self, response: Response): + def __init__(self, response: Response, response_type: ResponseType = ResponseType.SIMPLE_NAMESPACE): self.response = response self.is_ok = response.ok self.status_code = response.status_code - self.data = self.handle_response(response) + self.response_type = response_type + self.data = self.handle_response(response, response_type) - def handle_response(self, response: Response) -> dict: + + def handle_response(self, response: Response, response_type: ResponseType) -> dict | SimpleNamespace: """Parse the response content based on content type.""" timeout_statuses = [408, 460, 504, 520, 524, 522, 598, 599] rate_limit_statuses = [429] @@ -100,6 +70,8 @@ def handle_response(self, response: Response) -> dict: try: if "application/json" in content_type: + if response_type == ResponseType.DICT: + return response.json() return json.loads(response.content, object_hook=lambda item: SimpleNamespace(**item)) elif "application/xml" in content_type or "text/xml" in content_type: return xml_to_simplenamespace(response.content) @@ -111,6 +83,52 @@ def handle_response(self, response: Response) -> dict: logger.error(f"Failed to parse response content: {e}", exc_info=True) return {} +class BaseRequestHandler: + def __init__(self, session: Session, response_type: ResponseType = ResponseType.SIMPLE_NAMESPACE, base_url: Optional[str] = None, base_params: Optional[BaseRequestParameters] = None, + custom_exception: Optional[Type[Exception]] = None, request_logging: bool = False): + self.session = session + self.response_type = response_type + self.BASE_URL = base_url + self.BASE_REQUEST_PARAMS = base_params or BaseRequestParameters() + self.custom_exception = custom_exception or Exception + self.request_logging = request_logging + + def _request(self, method: HttpMethod, endpoint: str, ignore_base_url: Optional[bool] = None, overriden_response_type: ResponseType = None, **kwargs) -> ResponseObject: + """Generic request handler with error handling, using kwargs for flexibility.""" + try: + url = f"{self.BASE_URL}/{endpoint}" if not ignore_base_url and self.BASE_URL else endpoint + + # Add base parameters to kwargs if they exist + request_params = self.BASE_REQUEST_PARAMS.to_dict() + if request_params: + kwargs.setdefault('params', {}).update(request_params) + elif 'params' in kwargs and not kwargs['params']: + del kwargs['params'] + + if self.request_logging: + logger.debug(f"Making request to {url} with kwargs: {kwargs}") + + response = self.session.request(method.value, url, **kwargs) + response.raise_for_status() + + request_response_type = overriden_response_type or self.response_type + + response_obj = ResponseObject(response=response, response_type=request_response_type) + if self.request_logging: + logger.debug(f"ResponseObject: status_code={response_obj.status_code}, data={response_obj.data}") + return response_obj + + except RequestException as e: + logger.error(f"Request failed: {e}") + raise self.custom_exception(f"Request failed: {e}") from e + + +class RateLimitExceeded(Exception): + """Rate limit exceeded exception""" + def __init__(self, message, response=None): + super().__init__(message) + self.response = response + class CachedLimiterSession(CacheMixin, LimiterMixin, Session): """Session class with caching and rate-limiting behavior.""" pass @@ -142,53 +160,6 @@ def create_service_session( return Session() -def _handle_request_exception() -> ResponseObject: - """Handle exceptions during requests and return a default ResponseObject.""" - logger.error("Request failed", exc_info=True) - mock_response = SimpleNamespace(ok=False, status_code=500, content={}, headers={}) - return ResponseObject(mock_response) - -def _make_request( - session: Session, - method: str, - url: str, - data: dict = None, - params: dict = None, - timeout=5, - additional_headers=None, - retry_if_failed=True, - proxies=None, - json=None, -) -> ResponseObject: - if retry_if_failed: - retry_strategy = Retry(total=3, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504]) - adapter = HTTPAdapter(max_retries=retry_strategy) - session.mount("http://", adapter) - session.mount("https://", adapter) - - try: - response = session.request( - method, url, headers=additional_headers, data=data, params=params, timeout=timeout, proxies=proxies, - json=json - ) - except RequestException as e: - logger.error(f"Request failed: {e}", exc_info=True) - response = _handle_request_exception() - finally: - session.close() - - return ResponseObject(response) - -def ping(session: Session, url: str, timeout: int = 10, additional_headers=None, proxies=None, params=None) -> ResponseObject: - """Ping method to check connectivity to a URL by making a simple GET request.""" - return get(session=session, url=url, timeout=timeout, additional_headers=additional_headers, proxies=proxies, params=params) - - -from pyrate_limiter import RequestRate, Duration, Limiter, MemoryQueueBucket, MemoryListBucket -from requests_ratelimiter import SQLiteBucket -from typing import Optional - - def get_rate_limit_params( per_second: Optional[int] = None, per_minute: Optional[int] = None, @@ -257,16 +228,3 @@ def element_to_simplenamespace(element): attributes.update(children_as_ns) return SimpleNamespace(**attributes, text=element.text) return element_to_simplenamespace(root) - -# HTTP method wrappers -def get(session: Session, url: str, **kwargs) -> ResponseObject: - return _make_request(session, "GET", url, **kwargs) - -def post(session: Session, url: str, **kwargs) -> ResponseObject: - return _make_request(session, "POST", url, **kwargs) - -def put(session: Session, url: str, **kwargs) -> ResponseObject: - return _make_request(session, "PUT", url, **kwargs) - -def delete(session: Session, url: str, **kwargs) -> ResponseObject: - return _make_request(session, "DELETE", url, **kwargs)