Torrentio rate limiter improvements, realdebrid downloading improvements

backend/program/__init__.py

@@ -56,6 +56,7 @@ class Program:
     """Program class"""
 
     def __init__(self):
+        logger.info("Iceberg initializing...")
         self.settings = settings_manager.get_all()
         self.media_items = MediaItemContainer(items=[])
         self.data_path = get_data_path()
@@ -67,7 +68,7 @@ def __init__(self):
             Symlinker(self.media_items),
             Scraping(self.media_items),
         ]
-        logger.info("Iceberg initialized")
+        logger.info("Iceberg initialized!")
 
     def start(self):
         for thread in self.threads:

backend/program/debrid/realdebrid.py

@@ -4,6 +4,7 @@
 import threading
 import time
 import requests
+import PTN
 from requests import ConnectTimeout
 from utils.logger import logger
 from utils.request import get, post, ping
@@ -40,7 +41,7 @@ def __init__(self, media_items: MediaItemContainer):
             if self._validate_settings():
                 self._torrents = {}
                 break
-            logger.error("Realdebrid settings incorrect, retrying in 2...")
+            logger.error("Realdebrid settings incorrect or not premium, retrying in 2...")
             time.sleep(2)
 
     def _validate_settings(self):
@@ -49,7 +50,9 @@ def _validate_settings(self):
                 "https://api.real-debrid.com/rest/1.0/user",
                 additional_headers=self.auth_headers,
             )
-            return response.ok
+            if response.ok:
+                json = response.json()
+                return json["premium"] > 0
         except ConnectTimeout:
             return False
 
@@ -93,11 +96,21 @@ def download(self):
 
     def _download(self, item):
         """Download movie from real-debrid.com"""
-        self.check_stream_availability(item)
+        self._check_stream_availability(item)
         self._determine_best_stream(item)
-        self._download_item(item)
-        # item.change_state(MediaItemState.DOWNLOAD)
-        return 1
+        if not self._is_downloaded(item):
+            self._download_item(item)
+            return 1
+        return 0
+
+    def _is_downloaded(self, item):
+        if not item.get("active_stream", None):
+            return False
+        torrents = self.get_torrents()
+        if any(torrent.hash == item.active_stream.get("hash") for torrent in torrents):
+            logger.debug("Torrent already downloaded")
+            return True
+        return False
 
     def _download_item(self, item):
         if not item.get("active_stream", None):
@@ -167,12 +180,12 @@ def _determine_best_stream(self, item) -> bool:
             item.streams = {}
         return False
 
-    def check_stream_availability(self, item: MediaItem):
+    def _check_stream_availability(self, item: MediaItem):
         if len(item.streams) == 0:
             return
         streams = "/".join(
             list(item.streams)
-        )  # THIS IT TO SLOW, LETS CHECK ONE STREAM AT A TIME
+        )
         response = get(
             f"https://api.real-debrid.com/rest/1.0/torrents/instantAvailability/{streams}/",
             additional_headers=self.auth_headers,
@@ -184,12 +197,25 @@ def check_stream_availability(self, item: MediaItem):
                 continue
             for containers in provider_list.values():
                 for container in containers:
-                    wanted_files = {
-                        file_id: file
-                        for file_id, file in container.items()
-                        if os.path.splitext(file["filename"])[1] in WANTED_FORMATS
-                        and file["filesize"] > 50000000
-                    }
+                    wanted_files = None
+                    if item.type in ["movie", "season"]:
+                        wanted_files = {
+                            file_id: file
+                            for file_id, file in container.items()
+                            if os.path.splitext(file["filename"])[1] in WANTED_FORMATS
+                            and file["filesize"] > 50000000
+                        }
+                    if item.type == "episode":
+                        for file_id, file in container.items():
+                            parse = PTN.parse(file["filename"])
+                            episode = parse.get("episode")
+                            if type(episode) == list:
+                                if item.number in episode:
+                                    wanted_files = {file_id: file}
+                                    break
+                            elif item.number == episode:
+                                wanted_files = {file_id: file}
+                                break
                     if wanted_files:
                         cached = False
                         if item.type == "season":
@@ -246,6 +272,20 @@ def add_magnet(self, item: MediaItem) -> str:
             return response.data.id
         return None
 
+    def get_torrents(self) -> str:
+        """Add magnet link to real-debrid.com"""
+        response = get(
+            "https://api.real-debrid.com/rest/1.0/torrents/",
+            data = {
+                "offset": 0,
+                "limit": 2500
+            },
+            additional_headers=self.auth_headers,
+        )
+        if response.is_ok:
+            return response.data
+        return None
+
     def select_files(self, request_id, item) -> bool:
         """Select files from real-debrid.com"""
         files = item.active_stream.get("files")

backend/program/scrapers/torrentio.py

@@ -21,7 +21,7 @@ def __init__(self, media_items: MediaItemContainer):
         self.last_scrape = 0
         self.filters = self.class_settings["filter"]
         self.minute_limiter = RateLimiter(
-            max_calls=140, period=60 * 5, raise_on_limit=True
+            max_calls=60, period=60, raise_on_limit=True
         )
         self.second_limiter = RateLimiter(max_calls=1, period=1)
         self.initialized = True
@@ -37,11 +37,11 @@ def run(self):
                     scraped_amount += self._scrape_items([item])
                 else:
                     scraped_amount += self._scrape_show(item)
-            except RequestException as exception:
-                logger.error("%s, trying again next cycle", exception)
+            except RequestException:
+                self.minute_limiter.limit_hit()
                 break
-            except RateLimitExceeded as exception:
-                logger.error("%s, trying again next cycle", exception)
+            except RateLimitExceeded:
+                self.minute_limiter.limit_hit()
                 break
 
         if scraped_amount > 0:

backend/utils/request.py

@@ -95,6 +95,7 @@ def ping(url: str, timeout=10, additional_headers=None):
 def get(
     url: str,
     timeout=10,
+    data=None,
     additional_headers=None,
     retry_if_failed=True,
     response_type=SimpleNamespace,
@@ -103,6 +104,7 @@ def get(
     return _make_request(
         "GET",
         url,
+        data=data,
         timeout=timeout,
         additional_headers=additional_headers,
         retry_if_failed=retry_if_failed,
@@ -160,7 +162,36 @@ class RateLimitExceeded(Exception):
     pass
 
 
+import time
+from threading import Lock
+
 class RateLimiter:
+    """
+    A rate limiter class that limits the number of calls within a specified period.
+
+    Args:
+        max_calls (int): The maximum number of calls allowed within the specified period.
+        period (float): The time period (in seconds) within which the calls are limited.
+        raise_on_limit (bool, optional): Whether to raise an exception when the rate limit is exceeded. 
+            Defaults to False.
+
+    Attributes:
+        max_calls (int): The maximum number of calls allowed within the specified period.
+        period (float): The time period (in seconds) within which the calls are limited.
+        tokens (int): The number of available tokens for making calls.
+        last_call (float): The timestamp of the last call made.
+        lock (threading.Lock): A lock used for thread-safety.
+        raise_on_limit (bool): Whether to raise an exception when the rate limit is exceeded.
+
+    Methods:
+        limit_hit(): Resets the token count to 0, indicating that the rate limit has been hit.
+        __enter__(): Enters the rate limiter context and checks if a call can be made.
+        __exit__(): Exits the rate limiter context.
+
+    Raises:
+        RateLimitExceeded: If the rate limit is exceeded and `raise_on_limit` is set to True.
+    """
+
     def __init__(self, max_calls, period, raise_on_limit=False):
         self.max_calls = max_calls
         self.period = period
@@ -169,7 +200,16 @@ def __init__(self, max_calls, period, raise_on_limit=False):
         self.lock = Lock()
         self.raise_on_limit = raise_on_limit
 
+    def limit_hit(self):
+        """
+        Resets the token count to 0, indicating that the rate limit has been hit.
+        """
+        self.tokens = 0
+
     def __enter__(self):
+        """
+        Enters the rate limiter context and checks if a call can be made.
+        """
         with self.lock:
             current_time = time.time()
             time_since_last_call = current_time - self.last_call
@@ -190,4 +230,7 @@ def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
+        """
+        Exits the rate limiter context.
+        """
         pass