diff --git a/backend/program/__init__.py b/backend/program/__init__.py index 9733cbf3..3de49ec7 100644 --- a/backend/program/__init__.py +++ b/backend/program/__init__.py @@ -56,6 +56,7 @@ class Program: """Program class""" def __init__(self): + logger.info("Iceberg initializing...") self.settings = settings_manager.get_all() self.media_items = MediaItemContainer(items=[]) self.data_path = get_data_path() @@ -67,7 +68,7 @@ def __init__(self): Symlinker(self.media_items), Scraping(self.media_items), ] - logger.info("Iceberg initialized") + logger.info("Iceberg initialized!") def start(self): for thread in self.threads: diff --git a/backend/program/debrid/realdebrid.py b/backend/program/debrid/realdebrid.py index a4d9453d..6129c753 100644 --- a/backend/program/debrid/realdebrid.py +++ b/backend/program/debrid/realdebrid.py @@ -4,6 +4,7 @@ import threading import time import requests +import PTN from requests import ConnectTimeout from utils.logger import logger from utils.request import get, post, ping @@ -40,7 +41,7 @@ def __init__(self, media_items: MediaItemContainer): if self._validate_settings(): self._torrents = {} break - logger.error("Realdebrid settings incorrect, retrying in 2...") + logger.error("Realdebrid settings incorrect or not premium, retrying in 2...") time.sleep(2) def _validate_settings(self): @@ -49,7 +50,9 @@ def _validate_settings(self): "https://api.real-debrid.com/rest/1.0/user", additional_headers=self.auth_headers, ) - return response.ok + if response.ok: + json = response.json() + return json["premium"] > 0 except ConnectTimeout: return False @@ -93,11 +96,21 @@ def download(self): def _download(self, item): """Download movie from real-debrid.com""" - self.check_stream_availability(item) + self._check_stream_availability(item) self._determine_best_stream(item) - self._download_item(item) - # item.change_state(MediaItemState.DOWNLOAD) - return 1 + if not self._is_downloaded(item): + self._download_item(item) + return 1 + return 0 + + def _is_downloaded(self, item): + if not item.get("active_stream", None): + return False + torrents = self.get_torrents() + if any(torrent.hash == item.active_stream.get("hash") for torrent in torrents): + logger.debug("Torrent already downloaded") + return True + return False def _download_item(self, item): if not item.get("active_stream", None): @@ -167,12 +180,12 @@ def _determine_best_stream(self, item) -> bool: item.streams = {} return False - def check_stream_availability(self, item: MediaItem): + def _check_stream_availability(self, item: MediaItem): if len(item.streams) == 0: return streams = "/".join( list(item.streams) - ) # THIS IT TO SLOW, LETS CHECK ONE STREAM AT A TIME + ) response = get( f"https://api.real-debrid.com/rest/1.0/torrents/instantAvailability/{streams}/", additional_headers=self.auth_headers, @@ -184,12 +197,25 @@ def check_stream_availability(self, item: MediaItem): continue for containers in provider_list.values(): for container in containers: - wanted_files = { - file_id: file - for file_id, file in container.items() - if os.path.splitext(file["filename"])[1] in WANTED_FORMATS - and file["filesize"] > 50000000 - } + wanted_files = None + if item.type in ["movie", "season"]: + wanted_files = { + file_id: file + for file_id, file in container.items() + if os.path.splitext(file["filename"])[1] in WANTED_FORMATS + and file["filesize"] > 50000000 + } + if item.type == "episode": + for file_id, file in container.items(): + parse = PTN.parse(file["filename"]) + episode = parse.get("episode") + if type(episode) == list: + if item.number in episode: + wanted_files = {file_id: file} + break + elif item.number == episode: + wanted_files = {file_id: file} + break if wanted_files: cached = False if item.type == "season": @@ -246,6 +272,20 @@ def add_magnet(self, item: MediaItem) -> str: return response.data.id return None + def get_torrents(self) -> str: + """Add magnet link to real-debrid.com""" + response = get( + "https://api.real-debrid.com/rest/1.0/torrents/", + data = { + "offset": 0, + "limit": 2500 + }, + additional_headers=self.auth_headers, + ) + if response.is_ok: + return response.data + return None + def select_files(self, request_id, item) -> bool: """Select files from real-debrid.com""" files = item.active_stream.get("files") diff --git a/backend/program/scrapers/torrentio.py b/backend/program/scrapers/torrentio.py index 72f4a661..a7cad2fc 100644 --- a/backend/program/scrapers/torrentio.py +++ b/backend/program/scrapers/torrentio.py @@ -21,7 +21,7 @@ def __init__(self, media_items: MediaItemContainer): self.last_scrape = 0 self.filters = self.class_settings["filter"] self.minute_limiter = RateLimiter( - max_calls=140, period=60 * 5, raise_on_limit=True + max_calls=60, period=60, raise_on_limit=True ) self.second_limiter = RateLimiter(max_calls=1, period=1) self.initialized = True @@ -37,11 +37,11 @@ def run(self): scraped_amount += self._scrape_items([item]) else: scraped_amount += self._scrape_show(item) - except RequestException as exception: - logger.error("%s, trying again next cycle", exception) + except RequestException: + self.minute_limiter.limit_hit() break - except RateLimitExceeded as exception: - logger.error("%s, trying again next cycle", exception) + except RateLimitExceeded: + self.minute_limiter.limit_hit() break if scraped_amount > 0: diff --git a/backend/utils/request.py b/backend/utils/request.py index 99067e92..3b5c0f2d 100644 --- a/backend/utils/request.py +++ b/backend/utils/request.py @@ -95,6 +95,7 @@ def ping(url: str, timeout=10, additional_headers=None): def get( url: str, timeout=10, + data=None, additional_headers=None, retry_if_failed=True, response_type=SimpleNamespace, @@ -103,6 +104,7 @@ def get( return _make_request( "GET", url, + data=data, timeout=timeout, additional_headers=additional_headers, retry_if_failed=retry_if_failed, @@ -160,7 +162,36 @@ class RateLimitExceeded(Exception): pass +import time +from threading import Lock + class RateLimiter: + """ + A rate limiter class that limits the number of calls within a specified period. + + Args: + max_calls (int): The maximum number of calls allowed within the specified period. + period (float): The time period (in seconds) within which the calls are limited. + raise_on_limit (bool, optional): Whether to raise an exception when the rate limit is exceeded. + Defaults to False. + + Attributes: + max_calls (int): The maximum number of calls allowed within the specified period. + period (float): The time period (in seconds) within which the calls are limited. + tokens (int): The number of available tokens for making calls. + last_call (float): The timestamp of the last call made. + lock (threading.Lock): A lock used for thread-safety. + raise_on_limit (bool): Whether to raise an exception when the rate limit is exceeded. + + Methods: + limit_hit(): Resets the token count to 0, indicating that the rate limit has been hit. + __enter__(): Enters the rate limiter context and checks if a call can be made. + __exit__(): Exits the rate limiter context. + + Raises: + RateLimitExceeded: If the rate limit is exceeded and `raise_on_limit` is set to True. + """ + def __init__(self, max_calls, period, raise_on_limit=False): self.max_calls = max_calls self.period = period @@ -169,7 +200,16 @@ def __init__(self, max_calls, period, raise_on_limit=False): self.lock = Lock() self.raise_on_limit = raise_on_limit + def limit_hit(self): + """ + Resets the token count to 0, indicating that the rate limit has been hit. + """ + self.tokens = 0 + def __enter__(self): + """ + Enters the rate limiter context and checks if a call can be made. + """ with self.lock: current_time = time.time() time_since_last_call = current_time - self.last_call @@ -190,4 +230,7 @@ def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): + """ + Exits the rate limiter context. + """ pass