From 2161e4f416099139b7d99c65370f2b7b79e1bdfe Mon Sep 17 00:00:00 2001 From: Gaisberg <93206976+Gaisberg@users.noreply.github.com> Date: Thu, 28 Dec 2023 08:41:57 +0200 Subject: [PATCH] State machine improvements (#94) * A new look at state machine * Include basic unittests for items controller * lets not ignore test folders now * Add extra details to /items. Minor tweaks here and there. Extra logging. * Raise plex timeout to 60, same as pd. Move extra data to extended dict. * Add Orionoid Scraper * Fixed stream data * Fix frontend CORS * Apply black formatting to Orionoid * realdebrid and state fixes * Add support for multiple scrapers, disabled orionoid for now... * Fix frontend after state changes * Add missing files * Workaround for quick shutdown, fixed ongoing season logic * Couple minor tweaks. Orionoid still disabled for now. --------- Co-authored-by: Gaisberg Co-authored-by: Spoked --- backend/.gitignore | 1 - backend/controllers/default.py | 3 +- backend/controllers/items.py | 4 +- backend/program/__init__.py | 46 +- backend/program/content/mdblist.py | 3 +- backend/program/content/overseerr.py | 2 +- backend/program/content/plex_watchlist.py | 6 +- backend/program/media.py | 374 ------------- backend/program/media/container.py | 95 ++++ backend/program/media/item.py | 268 ++++++++++ backend/program/media/state.py | 115 ++++ backend/program/{libraries => }/plex.py | 561 ++++++++++--------- backend/program/{debrid => }/realdebrid.py | 592 ++++++++++----------- backend/program/scrapers/__init__.py | 33 +- backend/program/scrapers/orionoid.py | 171 ++++++ backend/program/scrapers/torrentio.py | 106 ++-- backend/program/symlink.py | 197 +++---- backend/program/updaters/trakt.py | 31 +- backend/pytest.ini | 6 + backend/tests/items_test.py | 30 ++ backend/utils/default_settings.json | 5 + backend/utils/utils.py | 22 +- entrypoint.sh | 2 +- frontend/src/routes/status/+page.svelte | 15 +- 24 files changed, 1462 insertions(+), 1226 deletions(-) delete mode 100644 backend/program/media.py create mode 100644 backend/program/media/container.py create mode 100644 backend/program/media/item.py create mode 100644 backend/program/media/state.py rename backend/program/{libraries => }/plex.py (67%) rename backend/program/{debrid => }/realdebrid.py (64%) create mode 100644 backend/program/scrapers/orionoid.py create mode 100644 backend/pytest.ini create mode 100644 backend/tests/items_test.py diff --git a/backend/.gitignore b/backend/.gitignore index bc42917a..72868b82 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -11,7 +11,6 @@ __pycache__ settings.json *.log data -test* # User-specific stuff .idea/**/workspace.xml diff --git a/backend/controllers/default.py b/backend/controllers/default.py index 72f8f187..9c56465b 100644 --- a/backend/controllers/default.py +++ b/backend/controllers/default.py @@ -1,7 +1,6 @@ from fastapi import APIRouter, Request from utils.settings import settings_manager -import requests -from program.debrid.realdebrid import get_user +from program.realdebrid import get_user router = APIRouter( diff --git a/backend/controllers/items.py b/backend/controllers/items.py index a06d5f77..34fa2ae5 100644 --- a/backend/controllers/items.py +++ b/backend/controllers/items.py @@ -1,5 +1,5 @@ from fastapi import APIRouter, HTTPException, Request -from program.media import MediaItemState +from program.media.state import MediaItemStates router = APIRouter( @@ -13,7 +13,7 @@ async def get_states(request: Request): return { "success": True, - "states": [state.name for state in MediaItemState], + "states": [state.name for state in MediaItemStates], } diff --git a/backend/program/__init__.py b/backend/program/__init__.py index 30a29c7e..1c5cedf7 100644 --- a/backend/program/__init__.py +++ b/backend/program/__init__.py @@ -1,41 +1,31 @@ """Program main module""" -import os import threading import time from typing import Optional from pydantic import BaseModel, HttpUrl, Field -from program.symlink import Symlinker +from program.media.container import MediaItemContainer from utils.logger import logger, get_data_path from utils.settings import settings_manager -from program.media import MediaItemContainer -from program.libraries.plex import Library as Plex -from program.debrid.realdebrid import Debrid as RealDebrid +from program.plex import Plex +from program.plex import PlexConfig from program.content import Content -from program.scrapers import Scraping from utils.utils import Pickly - - -# Pydantic models for configuration -class PlexConfig(BaseModel): - user: str - token: str - address: HttpUrl - watchlist: Optional[HttpUrl] = None +import concurrent.futures class MdblistConfig(BaseModel): lists: list[str] = Field(default_factory=list) - api_key: str + api_key: Optional[str] update_interval: int = 80 class OverseerrConfig(BaseModel): - url: HttpUrl - api_key: str + url: Optional[HttpUrl] + api_key: Optional[str] class RealDebridConfig(BaseModel): - api_key: str + api_key: Optional[str] class TorrentioConfig(BaseModel): @@ -53,11 +43,13 @@ class Settings(BaseModel): realdebrid: RealDebridConfig -class Program: +class Program(threading.Thread): """Program class""" def __init__(self): logger.info("Iceberg initializing...") + super().__init__(name="Iceberg") + self.running = False self.settings = settings_manager.get_all() self.media_items = MediaItemContainer(items=[]) self.data_path = get_data_path() @@ -66,13 +58,21 @@ def __init__(self): self.threads = [ Content(self.media_items), # Content must be first Plex(self.media_items), - RealDebrid(self.media_items), - Symlinker(self.media_items), - Scraping(self.media_items), ] logger.info("Iceberg initialized!") + def run(self): + while self.running: + with concurrent.futures.ThreadPoolExecutor( + max_workers=10, thread_name_prefix="Worker" + ) as executor: + for item in self.media_items: + executor.submit(item.perform_action) + time.sleep(2) + def start(self): + self.running = True + super().start() for thread in self.threads: thread.start() @@ -80,3 +80,5 @@ def stop(self): for thread in self.threads: thread.stop() self.pickly.stop() + self.running = False + super().join() diff --git a/backend/program/content/mdblist.py b/backend/program/content/mdblist.py index 2d64f8c8..02671546 100644 --- a/backend/program/content/mdblist.py +++ b/backend/program/content/mdblist.py @@ -1,9 +1,8 @@ """Mdblist content module""" -import json from utils.settings import settings_manager from utils.logger import logger from utils.request import RateLimitExceeded, RateLimiter, get, ping -from program.media import MediaItemContainer +from program.media.container import MediaItemContainer from program.updaters.trakt import Updater as Trakt diff --git a/backend/program/content/overseerr.py b/backend/program/content/overseerr.py index 5d647fab..e9a904e5 100644 --- a/backend/program/content/overseerr.py +++ b/backend/program/content/overseerr.py @@ -3,7 +3,7 @@ from utils.settings import settings_manager from utils.logger import logger from utils.request import get, ping -from program.media import MediaItemContainer +from program.media.container import MediaItemContainer from program.updaters.trakt import Updater as Trakt diff --git a/backend/program/content/plex_watchlist.py b/backend/program/content/plex_watchlist.py index 3f8e9e97..99261c6a 100644 --- a/backend/program/content/plex_watchlist.py +++ b/backend/program/content/plex_watchlist.py @@ -3,7 +3,7 @@ from utils.request import get, ping from utils.logger import logger from utils.settings import settings_manager as settings -from program.media import MediaItemContainer +from program.media.container import MediaItemContainer from program.updaters.trakt import Updater as Trakt import json @@ -18,7 +18,7 @@ def __init__(self, media_items: MediaItemContainer): self.previous_added_items_count = 0 if not self.watchlist_url or not self._validate_settings(): logger.info( - "Plex watchlist RSS URL is not configured and will not be used." + "Plex Watchlist is not configured and will not be used." ) return self.updater = Trakt() @@ -28,7 +28,7 @@ def _validate_settings(self): try: response = ping( self.watchlist_url, - timeout=5, + timeout=10, ) return response.ok except ConnectTimeout: diff --git a/backend/program/media.py b/backend/program/media.py deleted file mode 100644 index ca338b14..00000000 --- a/backend/program/media.py +++ /dev/null @@ -1,374 +0,0 @@ -"""MediaItem module""" - -from enum import IntEnum -from typing import List, Optional -from datetime import datetime -import datetime -import threading -import dill -import PTN - - -class MediaItemState(IntEnum): - """MediaItem states""" - - UNKNOWN = 0 - CONTENT = 1 - SCRAPE = 2 - DOWNLOAD = 3 - SYMLINK = 4 - LIBRARY = 5 - LIBRARY_PARTIAL = 6 - - -class MediaItem: - """MediaItem class""" - - def __init__(self, item): - self._lock = threading.Lock() - self.itemid = item_id.get_next_value() - self.scraped_at = 0 - self.active_stream = item.get("active_stream", None) - self.streams = {} - self.symlinked = False - self.requested_at = item.get("requested_at", None) or datetime.datetime.now() - self.requested_by = item.get("requested_by", None) - - # Media related - self.title = item.get("title", None) - self.imdb_id = item.get("imdb_id", None) - if self.imdb_id: - self.imdb_link = f"https://www.imdb.com/title/{self.imdb_id}/" - self.aired_at = item.get("aired_at", None) - self.genres = item.get("genres", []) - - # Plex related - self.key = item.get("key", None) - self.guid = item.get("guid", None) - self.updated = None - - @property - def state(self): - if self.key: - return MediaItemState.LIBRARY - if self.symlinked: - return MediaItemState.SYMLINK - if self.active_stream or self.file: - return MediaItemState.DOWNLOAD - if len(self.streams) > 0: - return MediaItemState.SCRAPE - if self.title: - return MediaItemState.CONTENT - return MediaItemState.UNKNOWN - - def is_cached(self): - if self.streams: - return any(stream.get("cached", None) for stream in self.streams.values()) - return False - - def is_scraped(self): - return len(self.streams) > 0 - - def is_checked_for_availability(self): - if self.streams: - return all( - stream.get("cached", None) is not None - for stream in self.streams.values() - ) - return False - - def to_dict(self): - return { - "item_id": self.itemid, - "title": self.title, - "type": self.type, - "imdb_id": self.imdb_id, - "state": self.state.name, - "imdb_link": self.imdb_link if hasattr(self, "imdb_link") else None, - "aired_at": self.aired_at, - "genres": self.genres, - "guid": self.guid, - "requested_at": self.requested_at, - "requested_by": self.requested_by - } - - def to_extended_dict(self): - dict = self.to_dict() - if self.type == "show": - dict["seasons"] = [season.to_extended_dict() for season in self.seasons] - if self.type == "season": - dict["episodes"] = [episode.to_extended_dict() for episode in self.episodes] - return dict - - def is_not_cached(self): - return not self.is_cached() - - def __iter__(self): - for attr, _ in vars(self).items(): - yield attr - - def __eq__(self, other): - if isinstance(other, type(self)): - return self.imdb_id == other.imdb_id - return False - - def get(self, key, default=None): - """Get item attribute""" - return getattr(self, key, default) - - def set(self, key, value): - """Set item attribute""" - _set_nested_attr(self, key, value) - - -class Movie(MediaItem): - """Movie class""" - - def __init__(self, item): - super().__init__(item) - self.type = "movie" - self.file = item.get("file", None) - - def __repr__(self): - return f"Movie:{self.title}:{self.state.name}" - - -class Show(MediaItem): - """Show class""" - - def __init__(self, item): - super().__init__(item) - self.locations = item.get("locations", []) - self.seasons = item.get("seasons", []) - self.type = "show" - - @property - def state(self): - if all(season.state is MediaItemState.LIBRARY for season in self.seasons): - return MediaItemState.LIBRARY - if any( - season.state in [MediaItemState.LIBRARY, MediaItemState.LIBRARY_PARTIAL] - for season in self.seasons - ): - return MediaItemState.LIBRARY_PARTIAL - if any(season.state == MediaItemState.DOWNLOAD for season in self.seasons): - return MediaItemState.DOWNLOAD - if any(season.state == MediaItemState.SCRAPE for season in self.seasons): - return MediaItemState.SCRAPE - if any(season.state == MediaItemState.CONTENT for season in self.seasons): - return MediaItemState.CONTENT - return MediaItemState.UNKNOWN - - def __repr__(self): - return f"Show:{self.title}:{self.state.name}" - - def add_season(self, season): - """Add season to show""" - self.seasons.append(season) - season.parent = self - - -class Season(MediaItem): - """Season class""" - - def __init__(self, item): - super().__init__(item) - self.type = "season" - self.parent = None - self.number = item.get("number", None) - self.episodes = item.get("episodes", []) - - @property - def state(self): - if len(self.episodes) > 0: - if all( - episode.state == MediaItemState.LIBRARY for episode in self.episodes - ): - return MediaItemState.LIBRARY - if any( - episode.state == MediaItemState.LIBRARY for episode in self.episodes - ): - return MediaItemState.LIBRARY_PARTIAL - if all( - episode.state == MediaItemState.SYMLINK for episode in self.episodes - ): - return MediaItemState.SYMLINK - if self.active_stream: - return MediaItemState.DOWNLOAD - if self.is_scraped(): - return MediaItemState.SCRAPE - if any( - episode.state == MediaItemState.CONTENT for episode in self.episodes - ): - return MediaItemState.CONTENT - return MediaItemState.UNKNOWN - - def __eq__(self, other): - return self.number == other.number - - def __repr__(self): - return f"Season:{self.number}:{self.state.name}" - - def add_episode(self, episode): - """Add episode to season""" - with self._lock: - self.episodes.append(episode) - episode.parent = self - - -class Episode(MediaItem): - """Episode class""" - - def __init__(self, item): - super().__init__(item) - self.type = "episode" - self.parent = None - self.number = item.get("number", None) - self.file = item.get("file", None) - - @property - def state(self): - return super().state - - def __eq__(self, other): - return self.number == other.number - - def __repr__(self): - return f"Episode:{self.number}:{self.state.name}" - - def get_file_episodes(self): - parse = PTN.parse(self.file) - episode_number = parse.get("episode") - if type(episode_number) == int: - episode_number = [episode_number] - if parse.get("excess"): - excess_episodes = None - if type(parse["excess"]) == list: - for excess in parse["excess"]: - excess_parse = PTN.parse(excess) - if excess_parse.get("episode") is not None: - excess_episodes = excess_parse["episode"] - break - if type(parse["excess"]) == str: - excess_parse = PTN.parse(parse["excess"]) - if excess_parse.get("episode") is not None: - excess_episodes = excess_parse["episode"] - if excess_episodes: - episode_number = episode_number + excess_episodes - return episode_number - - -class MediaItemContainer: - """MediaItemContainer class""" - - def __init__(self, items: Optional[List[MediaItem]] = None): - self.items = items if items is not None else [] - self.lock = threading.Lock() - - def __iter__(self): - for item in self.items: - yield item - - def __iadd__(self, other): - if not isinstance(other, MediaItem) and other is not None: - raise TypeError("Cannot append non-MediaItem to MediaItemContainer") - if other not in self.items: - self.items.append(other) - return self - - def sort(self, by, reverse): - self.items.sort(key=lambda item: item.get(by), reverse=reverse) - - def __len__(self): - """Get length of container""" - return len(self.items) - - def append(self, item) -> bool: - """Append item to container""" - with self.lock: - self.items.append(item) - self.sort("requested_at", True) - - def get(self, item) -> MediaItem: - """Get item matching given item from container""" - for my_item in self.items: - if my_item == item: - return my_item - return None - - def get_item_by_id(self, itemid) -> MediaItem: - """Get item matching given item from container""" - for my_item in self.items: - if my_item.itemid == int(itemid): - return my_item - return None - - def get_item(self, attr, value) -> "MediaItemContainer": - """Get items that match given items""" - return next((item for item in self.items if getattr(item, attr) == value), None) - - def extend(self, items) -> "MediaItemContainer": - """Extend container with items""" - with self.lock: - added_items = MediaItemContainer() - for media_item in items: - if media_item not in self.items: - self.items.append(media_item) - added_items.append(media_item) - self.sort("requested_at", True) - return added_items - - def remove(self, item): - """Remove item from container""" - if item in self.items: - self.items.remove(item) - - def count(self, state) -> int: - """Count items with given state in container""" - return len(self.get_items_with_state(state)) - - def get_items_with_state(self, state): - """Get items that need to be updated""" - return MediaItemContainer([item for item in self.items if item.state == state]) - - def save(self, filename): - """Save container to file""" - with open(filename, "wb") as file: - dill.dump(self.items, file) - - def load(self, filename): - """Load container from file""" - try: - with open(filename, "rb") as file: - self.items = dill.load(file) - except FileNotFoundError: - self.items = [] - - -def _set_nested_attr(obj, key, value): - if "." in key: - parts = key.split(".", 1) - current_key, rest_of_keys = parts[0], parts[1] - - if not hasattr(obj, current_key): - raise AttributeError(f"Object does not have the attribute '{current_key}'.") - - current_obj = getattr(obj, current_key) - _set_nested_attr(current_obj, rest_of_keys, value) - else: - if isinstance(obj, dict): - obj[key] = value - else: - setattr(obj, key, value) - - -class ItemId: - value = 0 - - @classmethod - def get_next_value(cls): - cls.value += 1 - return cls.value - - -item_id = ItemId() diff --git a/backend/program/media/container.py b/backend/program/media/container.py new file mode 100644 index 00000000..d2b2c6b9 --- /dev/null +++ b/backend/program/media/container.py @@ -0,0 +1,95 @@ +import os +import threading +import dill +from typing import List, Optional +from program.media.item import MediaItem + + +class MediaItemContainer: + """MediaItemContainer class""" + + def __init__(self, items: Optional[List[MediaItem]] = None): + self.items = items if items is not None else [] + self.lock = threading.Lock() + + def __iter__(self): + for item in self.items: + yield item + + def __iadd__(self, other): + if not isinstance(other, MediaItem) and other is not None: + raise TypeError("Cannot append non-MediaItem to MediaItemContainer") + if other not in self.items: + self.items.append(other) + return self + + def sort(self, by, reverse): + self.items.sort(key=lambda item: item.get(by), reverse=reverse) + + def __len__(self): + """Get length of container""" + return len(self.items) + + def append(self, item) -> bool: + """Append item to container""" + with self.lock: + self.items.append(item) + self.sort("requested_at", True) + + def get(self, item) -> MediaItem: + """Get item matching given item from container""" + for my_item in self.items: + if my_item == item: + return my_item + return None + + def get_item_by_id(self, itemid) -> MediaItem: + """Get item matching given item from container""" + for my_item in self.items: + if my_item.itemid == int(itemid): + return my_item + return None + + def get_item(self, attr, value) -> "MediaItemContainer": + """Get items that match given items""" + return next((item for item in self.items if getattr(item, attr) == value), None) + + def extend(self, items) -> "MediaItemContainer": + """Extend container with items""" + with self.lock: + added_items = MediaItemContainer() + for media_item in items: + if media_item not in self.items: + self.items.append(media_item) + added_items.append(media_item) + self.sort("requested_at", True) + return added_items + + def remove(self, item): + """Remove item from container""" + if item in self.items: + self.items.remove(item) + + def count(self, state) -> int: + """Count items with given state in container""" + return len(self.get_items_with_state(state)) + + def get_items_with_state(self, state): + """Get items that need to be updated""" + return MediaItemContainer([item for item in self.items if item.state == state]) + + def save(self, filename): + """Save container to file""" + with open(filename, "wb") as file: + dill.dump(self.items, file) + + def load(self, filename): + """Load container from file""" + try: + with open(filename, "rb") as file: + self.items = dill.load(file) + except FileNotFoundError: + self.items = [] + except EOFError: + os.remove(filename) + self.items = [] \ No newline at end of file diff --git a/backend/program/media/item.py b/backend/program/media/item.py new file mode 100644 index 00000000..41d970b8 --- /dev/null +++ b/backend/program/media/item.py @@ -0,0 +1,268 @@ +from datetime import datetime +import threading +from program.media.state import ( + Unknown, + Content, + Scrape, + Download, + Symlink, + Library, + LibraryPartial, +) +from utils.utils import parser + + +class MediaItem: + """MediaItem class""" + + def __init__(self, item): + self._lock = threading.Lock() + self.itemid = item_id.get_next_value() + self.scraped_at = 0 + self.active_stream = item.get("active_stream", None) + self.streams = {} + self.symlinked = False + self.requested_at = item.get("requested_at", None) or datetime.now() + self.requested_by = item.get("requested_by", None) + self.file = None + self.folder = None + + # Media related + self.title = item.get("title", None) + self.imdb_id = item.get("imdb_id", None) + if self.imdb_id: + self.imdb_link = f"https://www.imdb.com/title/{self.imdb_id}/" + self.tvdb_id = item.get("tvdb_id", None) + self.tmdb_id = item.get("tmdb_id", None) + self.network = item.get("network", None) + self.country = item.get("country", None) + self.language = item.get("language", None) + self.aired_at = item.get("aired_at", None) + self.genres = item.get("genres", []) + + # Plex related + self.key = item.get("key", None) + self.guid = item.get("guid", None) + self.updated = None + + self.state.set_context(self) + + def perform_action(self): + with self._lock: + self.state.perform_action() + + @property + def state(self): + _state = self._determine_state() + _state.set_context(self) + return _state + + def _determine_state(self): + if self.key: + return Library() + if self.symlinked: + return Symlink() + if self.file and self.folder: + return Download() + if len(self.streams) > 0: + return Scrape() + if self.title: + return Content() + return Unknown() + + def is_cached(self): + if self.streams: + return any(stream.get("cached", None) for stream in self.streams.values()) + return False + + def is_scraped(self): + return len(self.streams) > 0 + + def is_checked_for_availability(self): + if self.streams: + return all( + stream.get("cached", None) is not None + for stream in self.streams.values() + ) + return False + + def to_dict(self): + return { + "item_id": self.itemid, + "title": self.title, + "type": self.type, + "imdb_id": self.imdb_id if hasattr(self, "imdb_id") else None, + "tvdb_id": self.tvdb_id if hasattr(self, "tvdb_id") else None, + "tmdb_id": self.tmdb_id if hasattr(self, "tmdb_id") else None, + "state": self.state.name, + "imdb_link": self.imdb_link if hasattr(self, "imdb_link") else None, + "aired_at": self.aired_at, + "genres": self.genres if hasattr(self, "genres") else None, + "guid": self.guid, + "requested_at": self.requested_at, + "requested_by": self.requested_by, + } + + def to_extended_dict(self): + dict = self.to_dict() + if self.type == "show": + dict["seasons"] = [season.to_extended_dict() for season in self.seasons] + if self.type == "season": + dict["episodes"] = [episode.to_extended_dict() for episode in self.episodes] + dict["language"] = (self.language if hasattr(self, "language") else None,) + dict["country"] = (self.country if hasattr(self, "country") else None,) + dict["network"] = (self.network if hasattr(self, "network") else None,) + return dict + + def is_not_cached(self): + return not self.is_cached() + + def __iter__(self): + for attr, _ in vars(self).items(): + yield attr + + def __eq__(self, other): + if isinstance(other, type(self)): + return self.imdb_id == other.imdb_id + return False + + def get(self, key, default=None): + """Get item attribute""" + return getattr(self, key, default) + + def set(self, key, value): + """Set item attribute""" + _set_nested_attr(self, key, value) + + +class Movie(MediaItem): + """Movie class""" + + def __init__(self, item): + self.type = "movie" + self.file = item.get("file", None) + super().__init__(item) + + def __repr__(self): + return f"Movie:{self.title}:{self.state.name}" + + +class Show(MediaItem): + """Show class""" + + def __init__(self, item): + self.locations = item.get("locations", []) + self.seasons = item.get("seasons", []) + self.type = "show" + super().__init__(item) + + def _determine_state(self): + if all(season.state == Library for season in self.seasons): + return Library() + if any( + season.state == Library or season.state == LibraryPartial + for season in self.seasons + ): + return LibraryPartial() + if any(season.state == Download for season in self.seasons): + return Download() + if any(season.state == Scrape for season in self.seasons): + return Scrape() + if any(season.state == Content for season in self.seasons): + return Content() + return Unknown() + + def __repr__(self): + return f"Show:{self.title}:{self.state.name}" + + def add_season(self, season): + """Add season to show""" + self.seasons.append(season) + season.parent = self + + +class Season(MediaItem): + """Season class""" + + def __init__(self, item): + self.type = "season" + self.parent = None + self.number = item.get("number", None) + self.episodes = item.get("episodes", []) + super().__init__(item) + + def _determine_state(self): + if len(self.episodes) > 0: + if all(episode.state == Library for episode in self.episodes): + return Library() + if any(episode.state == Library for episode in self.episodes): + return LibraryPartial() + if all(episode.state == Symlink for episode in self.episodes): + return Symlink() + if all(episode.file and episode.folder for episode in self.episodes): + return Download() + if self.is_scraped() or any(episode.state == Scrape for episode in self.episodes): + return Scrape() + if any(episode.state == Content for episode in self.episodes): + return Content() + return Unknown() + + def __eq__(self, other): + return self.number == other.number + + def __repr__(self): + return f"Season:{self.number}:{self.state.name}" + + def add_episode(self, episode): + """Add episode to season""" + self.episodes.append(episode) + episode.parent = self + + +class Episode(MediaItem): + """Episode class""" + + def __init__(self, item): + self.type = "episode" + self.parent = None + self.number = item.get("number", None) + self.file = item.get("file", None) + super().__init__(item) + + def __eq__(self, other): + return self.number == other.number + + def __repr__(self): + return f"Episode:{self.number}:{self.state.name}" + + def get_file_episodes(self): + return parser.episodes(self.file) + + +def _set_nested_attr(obj, key, value): + if "." in key: + parts = key.split(".", 1) + current_key, rest_of_keys = parts[0], parts[1] + + if not hasattr(obj, current_key): + raise AttributeError(f"Object does not have the attribute '{current_key}'.") + + current_obj = getattr(obj, current_key) + _set_nested_attr(current_obj, rest_of_keys, value) + else: + if isinstance(obj, dict): + obj[key] = value + else: + setattr(obj, key, value) + + +class ItemId: + value = 0 + + @classmethod + def get_next_value(cls): + cls.value += 1 + return cls.value + + +item_id = ItemId() diff --git a/backend/program/media/state.py b/backend/program/media/state.py new file mode 100644 index 00000000..9ab6c9ee --- /dev/null +++ b/backend/program/media/state.py @@ -0,0 +1,115 @@ +from enum import Enum +from program.scrapers import scraper as scrape +from program.realdebrid import debrid +from program.symlink import symlink + + +class MediaItemState: + def __eq__(self, other) -> bool: + if type(other) == type: + return type(self) == other + return type(self) == type(other) + + def set_context(self, context): + self.context = context + + def perform_action(self): + pass + + +class Unknown(MediaItemState): + def __init__(self): + self.name = "Unknown" + + def perform_action(self): + pass + + +class Content(MediaItemState): + def __init__(self) -> None: + self.name = "Content" + + def perform_action(self): + if self.context.type in ["movie", "season", "episode"]: + scrape.run(self.context) + if self.context.type == "show": + for season in self.context.seasons: + if season.aired_at: + season.state.perform_action() + else: + for episode in season.episodes: + episode.state.perform_action() + + +class Scrape(MediaItemState): + def __init__(self) -> None: + self.name = "Scrape" + + def perform_action(self): + if self.context.type in ["movie", "season", "episode"]: + debrid.run(self.context) + if self.context.type == "show": + for season in self.context.seasons: + if season.aired_at: + season.state.perform_action() + else: + for episode in season.episodes: + episode.state.perform_action() + if self.context.type == "season": + self.context.state.perform_action() + + +class Download(MediaItemState): + def __init__(self) -> None: + self.name = "Download" + + def perform_action(self): + if self.context.type in ["movie", "episode"]: + symlink.run(self.context) + if self.context.type == "show": + for season in self.context.seasons: + for episode in season.episodes: + episode.state.perform_action() + if self.context.type == "season": + for episode in self.context.episodes: + episode.state.perform_action() + + +class Symlink(MediaItemState): + def __init__(self) -> None: + self.name = "Symlink" + + def perform_action(self): + pass + + +class Library(MediaItemState): + def __init__(self) -> None: + self.name = "Library" + + def perform_action(self): + pass + + +class LibraryPartial(MediaItemState): + def __init__(self) -> None: + self.name = "LibraryPartial" + + def perform_action(self): + if self.context.type == "show": + for season in self.context.seasons: + season.state.perform_action() + if self.context.type == "season": + for episode in self.context.episodes: + episode.state.perform_action() + + +# This for api to get states, not for program +class MediaItemStates(Enum): + Unknown = Unknown() + Content = Content() + Scrape = Scrape() + Download = Download() + Symlink = Symlink() + Library = Library() + LibraryPartial = LibraryPartial() diff --git a/backend/program/libraries/plex.py b/backend/program/plex.py similarity index 67% rename from backend/program/libraries/plex.py rename to backend/program/plex.py index f10beea4..131e8c92 100644 --- a/backend/program/libraries/plex.py +++ b/backend/program/plex.py @@ -1,252 +1,309 @@ -"""Plex library module""" -import concurrent.futures -import os -import threading -import time -from typing import List, Optional -from plexapi import exceptions -from plexapi.server import PlexServer -import requests -from requests.exceptions import ConnectionError -from pydantic import BaseModel, HttpUrl -from utils.logger import logger -from utils.settings import settings_manager as settings -from program.media import ( - MediaItemContainer, - MediaItemState, - MediaItem, - Movie, - Show, - Season, - Episode, -) - - -class PlexSettings(BaseModel): - user: str - token: str - url: HttpUrl - user_watchlist_rss: Optional[str] = None - - -class Library(threading.Thread): - """Plex library class""" - - def __init__(self, media_items: MediaItemContainer): - super().__init__(name="Plex") - # Plex class library is a necessity - while True: - try: - temp_settings = settings.get("plex") - self.library_path = os.path.abspath( - os.path.join(settings.get("container_mount"), os.pardir, "library") - ) - self.plex = PlexServer( - temp_settings["url"], temp_settings["token"], timeout=15 - ) - self.settings = PlexSettings(**temp_settings) - self.running = False - self.media_items = media_items - self._update_items() - break - except exceptions.Unauthorized: - logger.error("Wrong plex token, retrying in 2...") - except ConnectionError: - logger.error("Couldnt connect to plex, retrying in 2...") - time.sleep(2) - - def run(self): - while self.running: - self._update_sections() - self._update_items() - time.sleep(1) - - def start(self): - self.running = True - super().start() - - def stop(self): - self.running = False - super().join() - - def _update_items(self): - items = [] - sections = self.plex.library.sections() - processed_sections = set() - - for section in sections: - if section.key in processed_sections and not self._is_wanted_section( - section - ): - continue - - try: - if not section.refreshing: - with concurrent.futures.ThreadPoolExecutor( - max_workers=5, thread_name_prefix="Plex" - ) as executor: - future_items = { - executor.submit(self._create_item, item) - for item in section.all() - } - for future in concurrent.futures.as_completed(future_items): - media_item = future.result() - if media_item: - items.append(media_item) - except requests.exceptions.ReadTimeout: - logger.error( - f"Timeout occurred when accessing section: {section.title}" - ) - continue - - processed_sections.add(section.key) - matched_items = self.match_items(items) - - if matched_items > 0: - logger.info(f"Found {matched_items} new items") - - def _update_sections(self): - """Update plex library section""" - for section in self.plex.library.sections(): - for item in self.media_items: - log_string = None - if section.type == item.type: - if item.type == "movie": - if ( - item.state is MediaItemState.SYMLINK - and item.get("update_folder") != "updated" - ): - section.update(item.update_folder) - item.set("update_folder", "updated") - log_string = item.title - break - if item.type == "show": - for season in item.seasons: - if ( - season.state is MediaItemState.SYMLINK - and season.get("update_folder") != "updated" - ): - section.update(season.episodes[0].update_folder) - season.set("update_folder", "updated") - log_string = f"{item.title} season {season.number}" - break - else: - for episode in season.episodes: - if ( - episode.state is MediaItemState.SYMLINK - and episode.get("update_folder") != "updated" - and episode.parent.get("update_folder") - != "updated" - ): - section.update(episode.update_folder) - episode.set("update_folder", "updated") - log_string = f"{item.title} season {season.number} episode {episode.number}" - break - if log_string: - logger.debug("Updated section %s for %s", section.title, log_string) - - def _create_item(self, item): - new_item = _map_item_from_data(item) - if new_item and item.type == "show": - for season in item.seasons(): - if season.seasonNumber != 0: - new_season = _map_item_from_data(season) - if new_season: - new_season_episodes = [] - for episode in season.episodes(): - new_episode = _map_item_from_data(episode) - if new_episode: - new_season_episodes.append(new_episode) - new_season.episodes = new_season_episodes - new_item.seasons.append(new_season) - return new_item - - def match_items(self, found_items: List[MediaItem]): - """Matches items in given mediacontainer that are not in library - to items that are in library""" - items_to_update = 0 - - for item in self.media_items: - if item.state not in [ - MediaItemState.LIBRARY, - MediaItemState.LIBRARY_PARTIAL, - ]: - for found_item in found_items: - if found_item.imdb_id == item.imdb_id: - self._update_item(item, found_item) - items_to_update += 1 - break - # Leaving this here as a reminder to not forget about deleting items that are removed from plex, needs to be revisited - # if item.state is MediaItemState.LIBRARY and item not in found_items: - # self.media_items.remove(item) - return items_to_update - - def _update_item(self, item: MediaItem, library_item: MediaItem): - """Internal method to use with match_items - It does some magic to update media items according to library - items found""" - item.set("guid", library_item.guid) - item.set("key", library_item.key) - if item.type == "show": - for season in item.seasons: - for episode in season.episodes: - for found_season in library_item.seasons: - if found_season.number == season.number: - for found_episode in found_season.episodes: - if found_episode.number == episode.number: - episode.set("guid", found_episode.guid) - episode.set("key", found_episode.key) - break - break - - def _is_wanted_section(self, section): - return any(self.library_path in location for location in section.locations) - - -def _map_item_from_data(item): - """Map Plex API data to MediaItemContainer.""" - file = None - guid = getattr(item, "guid", None) - if item.type in ["movie", "episode"]: - file = getattr(item, "locations", [None])[0].split("/")[-1] - genres = [genre.tag for genre in getattr(item, "genres", [])] - title = getattr(item, "title", None) - key = getattr(item, "key", None) - season_number = getattr(item, "seasonNumber", None) - episode_number = getattr(item, "episodeNumber", None) - art_url = getattr(item, "artUrl", None) - imdb_id = None - aired_at = None - - if item.type in ["movie", "show"]: - guids = getattr(item, "guids", []) - imdb_id = next( - (guid.id.split("://")[-1] for guid in guids if "imdb" in guid.id), None - ) - aired_at = getattr(item, "originallyAvailableAt", None) - - media_item_data = { - "title": title, - "imdb_id": imdb_id, - "aired_at": aired_at, - "genres": genres, - "key": key, - "guid": guid, - "art_url": art_url, - "file": file, - } - - # Instantiate the appropriate subclass based on 'item_type' - if item.type == "movie": - return Movie(media_item_data) - elif item.type == "show": - return Show(media_item_data) - elif item.type == "season": - media_item_data["number"] = season_number - return Season(media_item_data) - elif item.type == "episode": - media_item_data["number"] = episode_number - media_item_data["season_number"] = season_number - return Episode(media_item_data) - else: - return None +"""Plex library module""" +import concurrent.futures +import os +import threading +import time +import uuid +import requests +from typing import List, Optional +from plexapi import exceptions +from plexapi.server import PlexServer +from requests.exceptions import ConnectionError +from pydantic import BaseModel, HttpUrl +from utils.logger import logger +from utils.settings import settings_manager as settings +from program.updaters.trakt import get_imdbid_from_tvdb +from program.media.container import MediaItemContainer +from program.media.state import Symlink, Library +from utils.request import get, post +from program.media.item import ( + MediaItem, + Movie, + Show, + Season, + Episode, +) + + +class PlexConfig(BaseModel): + user: Optional[str] = None + token: Optional[str] = None + url: Optional[HttpUrl] = None + watchlist_url: Optional[str] = None + + +class Plex(threading.Thread): + """Plex library class""" + + def __init__(self, media_items: MediaItemContainer): + super().__init__(name="Plex") + # Plex class library is a necessity + while True: + try: + temp_settings = settings.get("plex") + self.library_path = os.path.abspath( + os.path.join(settings.get("container_mount"), os.pardir, "library") + ) + self.plex = PlexServer( + temp_settings["url"], temp_settings["token"], timeout=60 + ) + self.running = False + self.log_worker_count = False + self.media_items = media_items + self._update_items() + break + except exceptions.Unauthorized: + logger.error("Wrong plex token, retrying in 2...") + except ConnectionError: + logger.error("Couldnt connect to plex, retrying in 2...") + except TimeoutError as e: + logger.warn( + "Plex timed out: retrying in 2 seconds... %s", str(e), exc_info=True + ) + except Exception as e: + logger.error("Unknown error: %s",str(e) , exc_info=True) + time.sleep(2) + + def run(self): + while self.running: + self._update_sections() + self._update_items() + time.sleep(1) + + def start(self): + self.running = True + super().start() + + def stop(self): + self.running = False + super().join() + + def _update_items(self): + items = [] + sections = self.plex.library.sections() + processed_sections = set() + max_workers = os.cpu_count() + 1 + for section in sections: + if section.key in processed_sections and not self._is_wanted_section( + section + ): + continue + + try: + if not section.refreshing: + with concurrent.futures.ThreadPoolExecutor( + max_workers=max_workers, thread_name_prefix="Plex" + ) as executor: + future_items = { + executor.submit(self._create_item, item) + for item in section.all() + } + for future in concurrent.futures.as_completed(future_items): + media_item = future.result() + if media_item: + items.append(media_item) + except requests.exceptions.ReadTimeout as e: + logger.error( + "Timeout occurred when accessing section %s with Reason: %s", + section.title, str(e) + ) + continue + except requests.exceptions.ConnectionError as e: + logger.error( + "Connection aborted. Remote end closed connection with response: %s for item %s", + str(e), section.title + ) + continue + + processed_sections.add(section.key) + matched_items = self.match_items(items) + + if matched_items > 0: + logger.info(f"Found {matched_items} new items") + + def _update_sections(self): + """Update plex library section""" + for section in self.plex.library.sections(): + for item in self.media_items: + log_string = None + if section.type == item.type: + if item.type == "movie": + if ( + item.state == Symlink + and item.get("update_folder") != "updated" + ): + section.update(item.update_folder) + item.set("update_folder", "updated") + log_string = item.title + break + if item.type == "show": + for season in item.seasons: + if ( + season.state == Symlink + and season.get("update_folder") != "updated" + ): + section.update(season.episodes[0].update_folder) + season.set("update_folder", "updated") + log_string = f"{item.title} season {season.number}" + break + else: + for episode in season.episodes: + if ( + episode.state == Symlink + and episode.get("update_folder") != "updated" + and episode.parent.get("update_folder") + != "updated" + ): + section.update(episode.update_folder) + episode.set("update_folder", "updated") + log_string = f"{item.title} season {season.number} episode {episode.number}" + break + if log_string: + logger.debug("Updated section %s for %s", section.title, log_string) + + def _create_item(self, item): + new_item = _map_item_from_data(item) + if new_item and item.type == "show": + for season in item.seasons(): + if season.seasonNumber != 0: + new_season = _map_item_from_data(season) + if new_season: + new_season_episodes = [] + for episode in season.episodes(): + new_episode = _map_item_from_data(episode) + if new_episode: + new_season_episodes.append(new_episode) + new_season.episodes = new_season_episodes + new_item.seasons.append(new_season) + return new_item + + def match_items(self, found_items: List[MediaItem]): + """Matches items in given mediacontainer that are not in library + to items that are in library""" + items_to_update = 0 + + for item in self.media_items: + if type(item.state) != Library: + for found_item in found_items: + if found_item.imdb_id == item.imdb_id: + items_to_update += self._update_item(item, found_item) + break + # Leaving this here as a reminder to not forget about deleting items that are removed from plex, needs to be revisited + # if item.state is MediaItemState.LIBRARY and item not in found_items: + # self.media_items.remove(item) + return items_to_update + + def _update_item(self, item: MediaItem, library_item: MediaItem): + """Internal method to use with match_items + It does some magic to update media items according to library + items found""" + items_updated = 0 + item.set("guid", library_item.guid) + item.set("key", library_item.key) + if item.type == "show": + for season in item.seasons: + for episode in season.episodes: + if episode.state != Library: + for found_season in library_item.seasons: + if found_season.number == season.number: + for found_episode in found_season.episodes: + if found_episode.number == episode.number: + episode.set("guid", found_episode.guid) + episode.set("key", found_episode.key) + items_updated += 1 + break + break + return items_updated + + def _is_wanted_section(self, section): + return any(self.library_path in location for location in section.locations) + + def _oauth(self): + random_uuid = uuid.uuid4() + response = get( + url="https://plex.tv/api/v2/user", + additional_headers={ + "X-Plex-Product": "Iceberg", + "X-Plex-Client-Identifier": random_uuid, + "X-Plex-Token": settings.get("plex.token") + }, + ) + if not response.ok: + data = post( + url="https://plex.tv/api/v2/pins", + additional_headers={ + "strong": "true", + "X-Plex-Product": "Iceberg", + "X-Plex-Client-Identifier": random_uuid, + }, + ) + if data.ok: + pin = data.id + + +def _map_item_from_data(item): + """Map Plex API data to MediaItemContainer.""" + file = None + guid = getattr(item, "guid", None) + if item.type in ["movie", "episode"]: + file = getattr(item, "locations", [None])[0].split("/")[-1] + genres = [genre.tag for genre in getattr(item, "genres", [])] + title = getattr(item, "title", None) + key = getattr(item, "key", None) + season_number = getattr(item, "seasonNumber", None) + episode_number = getattr(item, "episodeNumber", None) + art_url = getattr(item, "artUrl", None) + imdb_id = None + tvdb_id = None + aired_at = None + + if item.type in ["movie", "show"]: + guids = getattr(item, "guids", []) + imdb_id = next( + (guid.id.split("://")[-1] for guid in guids if "imdb" in guid.id), None + ) + aired_at = getattr(item, "originallyAvailableAt", None) + + # All movies have imdb, but not all shows do. + # This is due to season 0 (specials) not having imdb ids. + # Attempt to get the imdb id from the tvdb id if we don't have it. + # Needs more testing.. + # if not imdb_id: + # logger.debug("Unable to find imdb, trying tvdb for %s", title) + # tvdb_id = next( + # (guid.id.split("://")[-1] for guid in guids if "tvdb" in guid.id), None + # ) + # if tvdb_id: + # logger.debug("Unable to find imdb, but found tvdb: %s", tvdb_id) + # imdb_id = get_imdbid_from_tvdb(tvdb_id) + # if imdb_id: + # logger.debug("Found imdb from tvdb: %s", imdb_id) + + media_item_data = { + "title": title, + "imdb_id": imdb_id, + "tvdb_id": tvdb_id, + "aired_at": aired_at, + "genres": genres, + "key": key, + "guid": guid, + "art_url": art_url, + "file": file, + } + + # Instantiate the appropriate subclass based on 'item_type' + if item.type == "movie": + return Movie(media_item_data) + elif item.type == "show": + return Show(media_item_data) + elif item.type == "season": + media_item_data["number"] = season_number + return Season(media_item_data) + elif item.type == "episode": + media_item_data["number"] = episode_number + media_item_data["season_number"] = season_number + return Episode(media_item_data) + else: + # Specials may end up here.. + logger.error("Unknown Item: %s with type %s", item.title, item.type) + return None diff --git a/backend/program/debrid/realdebrid.py b/backend/program/realdebrid.py similarity index 64% rename from backend/program/debrid/realdebrid.py rename to backend/program/realdebrid.py index 84255848..f5ce96ce 100644 --- a/backend/program/debrid/realdebrid.py +++ b/backend/program/realdebrid.py @@ -1,310 +1,282 @@ -"""Realdebrid module""" -import os -import re -import threading -import time -import requests -import PTN -from requests import ConnectTimeout -from utils.logger import logger -from utils.request import get, post, ping -from utils.settings import settings_manager -from program.media import MediaItem, MediaItemContainer, MediaItemState - - -WANTED_FORMATS = [".mkv", ".mp4", ".avi"] -RD_BASE_URL = "https://api.real-debrid.com/rest/1.0" - - -def get_user(): - api_key = settings_manager.get("realdebrid")["api_key"] - headers = {"Authorization": f"Bearer {api_key}"} - response = requests.get( - "https://api.real-debrid.com/rest/1.0/user", headers=headers - ) - return response.json() - - -class Debrid(threading.Thread): - """Real-Debrid API Wrapper""" - - def __init__(self, media_items: MediaItemContainer): - super().__init__(name="Debrid") - # Realdebrid class library is a necessity - while True: - self.settings = settings_manager.get("realdebrid") - self.media_items = media_items - self.auth_headers = {"Authorization": f'Bearer {self.settings["api_key"]}'} - self.running = False - if self._validate_settings(): - self._torrents = {} - break - logger.error( - "Realdebrid settings incorrect or not premium, retrying in 2..." - ) - time.sleep(2) - - def _validate_settings(self): - try: - response = ping( - "https://api.real-debrid.com/rest/1.0/user", - additional_headers=self.auth_headers, - ) - if response.ok: - json = response.json() - return json["premium"] > 0 - except ConnectTimeout: - return False - - def run(self): - while self.running: - self.download() - time.sleep(1) - - def start(self) -> None: - self.running = True - super().start() - - def stop(self) -> None: - self.running = False - super().join() - - def download(self): - """Download given media items from real-debrid.com""" - added_files = 0 - - items = [] - for item in self.media_items: - if item.state is not MediaItemState.LIBRARY: - if item.type == "movie" and item.state is MediaItemState.SCRAPE: - items.append(item) - if item.type == "show": - item._lock.acquire() - for season in item.seasons: - if season.state is MediaItemState.SCRAPE: - items.append(season) - else: - for episode in season.episodes: - if episode.state is MediaItemState.SCRAPE: - items.append(episode) - item._lock.release() - - for item in items: - added_files += self._download(item) - - if added_files > 0: - logger.info("Downloaded %s cached releases", added_files) - - def _download(self, item): - """Download movie from real-debrid.com""" - downloaded = 0 - self._check_stream_availability(item) - self._determine_best_stream(item) - if not self._is_downloaded(item): - downloaded = self._download_item(item) - self._update_torrent_info(item) - return downloaded - - def _is_downloaded(self, item): - if not item.get("active_stream", None): - return False - torrents = self.get_torrents() - for torrent in torrents: - if torrent.hash == item.active_stream.get("hash"): - item.set("active_stream.id", torrent.id) - logger.debug("Torrent already downloaded") - return True - return False - - def _update_torrent_info(self, item): - info = self.get_torrent_info(item.get("active_stream")["id"]) - item.active_stream["name"] = info.filename - - def _download_item(self, item): - request_id = self.add_magnet(item) - - time.sleep(0.3) - self.select_files(request_id, item) - item.set("active_stream.id", request_id) - - if item.type == "movie": - log_string = item.title - if item.type == "season": - log_string = f"{item.parent.title} S{item.number}" - if item.type == "episode": - log_string = ( - f"{item.parent.parent.title} S{item.parent.number}E{item.number}" - ) - - logger.debug("Downloaded %s", log_string) - return 1 - - def _get_torrent_info(self, request_id): - data = self.get_torrent_info(request_id) - if not data["id"] in self._torrents.keys(): - self._torrents[data["id"]] = data - - def _determine_best_stream(self, item) -> bool: - """Returns true if season stream found for episode""" - cached = [ - stream_hash - for stream_hash, stream_value in item.streams.items() - if stream_value.get("cached") - ] - for stream_hash, stream in item.streams.items(): - if item.type == "episode": - if stream.get("files") and self._real_episode_count( - stream["files"] - ) >= len(item.parent.episodes): - item.parent.set("active_stream", stream) - logger.debug( - "Found cached release for %s %s", - item.parent.parent.title, - item.parent.number, - ) - return True - if ( - stream.get("files") - and self._real_episode_count(stream["files"]) == 0 - ): - continue - if stream_hash in cached: - stream["hash"] = stream_hash - item.set("active_stream", stream) - break - match (item.type): - case "movie": - log_string = item.title - case "season": - log_string = f"{item.parent.title} season {item.number}" - case "episode": - log_string = f"{item.parent.parent.title} season {item.parent.number} episode {item.number}" - case _: - log_string = "" - - if item.get("active_stream", None): - logger.debug("Found cached release for %s", log_string) - else: - logger.debug("No cached release found for %s", log_string) - item.streams = {} - return False - - def _check_stream_availability(self, item: MediaItem): - if len(item.streams) == 0: - return - streams = "/".join(list(item.streams)) - response = get( - f"https://api.real-debrid.com/rest/1.0/torrents/instantAvailability/{streams}/", - additional_headers=self.auth_headers, - response_type=dict, - ) - cached = False - for stream_hash, provider_list in response.data.items(): - if len(provider_list) == 0: - continue - for containers in provider_list.values(): - for container in containers: - wanted_files = None - if item.type in ["movie", "season"]: - wanted_files = { - file_id: file - for file_id, file in container.items() - if os.path.splitext(file["filename"])[1] in WANTED_FORMATS - and file["filesize"] > 50000000 - } - if item.type == "episode": - for file_id, file in container.items(): - parse = PTN.parse(file["filename"]) - episode = parse.get("episode") - if type(episode) == list: - if item.number in episode: - wanted_files = {file_id: file} - break - elif item.number == episode: - wanted_files = {file_id: file} - break - if wanted_files: - cached = False - if item.type == "season": - if self._real_episode_count(wanted_files) >= len( - item.episodes - ): - cached = True - if item.type == "movie": - if len(wanted_files) == 1: - cached = True - if item.type == "episode": - if len(wanted_files) >= 1: - cached = True - item.streams[stream_hash]["files"] = wanted_files - item.streams[stream_hash]["cached"] = cached - if cached: - return - - def _real_episode_count(self, files): - def count_episodes(episode_numbers): - count = 0 - for episode in episode_numbers: - if "-" in episode: - start, end = map(int, episode.split("-")) - count += end - start + 1 - else: - count += 1 - return count - - total_count = 0 - for file in files.values(): - episode_numbers = re.findall( - r"E(\d{1,2}(?:-\d{1,2})?)", - file["filename"], - re.IGNORECASE, - ) - total_count += count_episodes(episode_numbers) - return total_count - - def add_magnet(self, item: MediaItem) -> str: - """Add magnet link to real-debrid.com""" - if not item.active_stream.get("hash"): - return None - response = post( - "https://api.real-debrid.com/rest/1.0/torrents/addMagnet", - { - "magnet": "magnet:?xt=urn:btih:" - + item.active_stream["hash"] - + "&dn=&tr=" - }, - additional_headers=self.auth_headers, - ) - if response.is_ok: - return response.data.id - return None - - def get_torrents(self) -> str: - """Add magnet link to real-debrid.com""" - response = get( - "https://api.real-debrid.com/rest/1.0/torrents/", - data={"offset": 0, "limit": 2500}, - additional_headers=self.auth_headers, - ) - if response.is_ok: - return response.data - return None - - def select_files(self, request_id, item) -> bool: - """Select files from real-debrid.com""" - files = item.active_stream.get("files") - response = post( - f"https://api.real-debrid.com/rest/1.0/torrents/selectFiles/{request_id}", - {"files": ",".join(files.keys())}, - additional_headers=self.auth_headers, - ) - return response.is_ok - - def get_torrent_info(self, request_id): - """Get torrent info from real-debrid.com""" - response = get( - f"https://api.real-debrid.com/rest/1.0/torrents/info/{request_id}", - additional_headers=self.auth_headers, - ) - if response.is_ok: - return response.data +"""Realdebrid module""" +import os +import re +import time +import requests +from requests import ConnectTimeout +from utils.logger import logger +from utils.request import get, post, ping +from utils.settings import settings_manager +from utils.utils import parser + + +WANTED_FORMATS = [".mkv", ".mp4", ".avi"] +RD_BASE_URL = "https://api.real-debrid.com/rest/1.0" + + +def get_user(): + api_key = settings_manager.get("realdebrid")["api_key"] + headers = {"Authorization": f"Bearer {api_key}"} + response = requests.get( + "https://api.real-debrid.com/rest/1.0/user", headers=headers + ) + return response.json() + + +class Debrid: + """Real-Debrid API Wrapper""" + + def __init__(self): + # Realdebrid class library is a necessity + while True: + self.settings = settings_manager.get("realdebrid") + self.auth_headers = {"Authorization": f'Bearer {self.settings["api_key"]}'} + self.running = False + if self._validate_settings(): + self._torrents = {} + break + logger.error( + "Realdebrid settings incorrect or not premium, retrying in 2..." + ) + time.sleep(2) + + def _validate_settings(self): + try: + response = ping( + "https://api.real-debrid.com/rest/1.0/user", + additional_headers=self.auth_headers, + ) + if response.ok: + json = response.json() + return json["premium"] > 0 + except ConnectTimeout: + return False + + def run(self, item): + self.download(item) + + def download(self, item): + """Download given media items from real-debrid.com""" + self._download(item) + + def _download(self, item): + """Download movie from real-debrid.com""" + downloaded = 0 + self._check_stream_availability(item) + self._determine_best_stream(item) + if not self._is_downloaded(item): + downloaded = self._download_item(item) + self._update_torrent_info(item) + self._set_file_paths(item) + return downloaded + + def _is_downloaded(self, item): + if not item.get("active_stream", None): + return False + torrents = self.get_torrents() + for torrent in torrents: + if torrent.hash == item.active_stream.get("hash"): + item.set("active_stream.id", torrent.id) + logger.debug("Torrent already downloaded") + return True + return False + + def _update_torrent_info(self, item): + info = self.get_torrent_info(item.get("active_stream")["id"]) + item.active_stream["name"] = info.original_filename + + def _download_item(self, item): + request_id = self.add_magnet(item) + + time.sleep(0.3) + self.select_files(request_id, item) + item.set("active_stream.id", request_id) + + if item.type == "movie": + log_string = item.title + if item.type == "season": + log_string = f"{item.parent.title} S{item.number}" + if item.type == "episode": + log_string = ( + f"{item.parent.parent.title} S{item.parent.number}E{item.number}" + ) + + logger.debug("Downloaded %s", log_string) + return 1 + + def _get_torrent_info(self, request_id): + data = self.get_torrent_info(request_id) + if not data["id"] in self._torrents.keys(): + self._torrents[data["id"]] = data + + def _determine_best_stream(self, item) -> bool: + """Returns true if season stream found for episode""" + for hash, stream in item.streams.items(): + if stream.get("cached"): + item.set("active_stream", stream) + item.set("active_stream.hash", hash) + break + match (item.type): + case "movie": + log_string = item.title + case "season": + log_string = f"{item.parent.title} season {item.number}" + case "episode": + log_string = f"{item.parent.parent.title} season {item.parent.number} episode {item.number}" + case _: + log_string = "" + + if item.get("active_stream", None): + logger.debug("Found cached release for %s", log_string) + else: + logger.debug("No cached release found for %s", log_string) + item.streams = {} + return False + + def _check_stream_availability(self, item): + if len(item.streams) == 0: + return + streams = "/".join(list(item.streams)) + response = get( + f"https://api.real-debrid.com/rest/1.0/torrents/instantAvailability/{streams}/", + additional_headers=self.auth_headers, + response_type=dict, + ) + cached = False + for stream_hash, provider_list in response.data.items(): + if len(provider_list) == 0: + continue + for containers in provider_list.values(): + for container in containers: + wanted_files = { + file_id: file + for file_id, file in container.items() + if os.path.splitext(file["filename"])[1] in WANTED_FORMATS + } + if len(wanted_files) >= 1: + cached = False + if item.type == "season": + episodes = [] + for file in wanted_files.values(): + episodes += parser.episodes_in_season( + item.number, file["filename"] + ) + if len(episodes) >= len(item.episodes): + cached = True + if item.type == "movie": + if len(wanted_files) == 1: + cached = True + if item.type == "episode": + for file in wanted_files.values(): + episodes = parser.episodes_in_season( + item.parent.number, file["filename"] + ) + if item.number in episodes: + cached = True + break + item.streams[stream_hash]["files"] = wanted_files + item.streams[stream_hash]["cached"] = cached + if cached: + return + + def _real_episode_count(self, files): + def count_episodes(episode_numbers): + count = 0 + for episode in episode_numbers: + if "-" in episode: + start, end = map(int, episode.split("-")) + count += end - start + 1 + else: + count += 1 + return count + + total_count = 0 + for file in files.values(): + episode_numbers = re.findall( + r"E(\d{1,2}(?:-\d{1,2})?)", + file["filename"], + re.IGNORECASE, + ) + total_count += count_episodes(episode_numbers) + return total_count + + def _set_file_paths(self, item): + if item.type == "movie": + self._handle_movie_paths(item) + if item.type == "season": + self._handle_season_paths(item) + if item.type == "episode": + self._handle_episode_paths(item) + + def _handle_movie_paths(self, item): + item.set("folder", item.active_stream.get("name")) + item.set( + "file", + next(iter(item.active_stream["files"].values())).get("filename"), + ) + + def _handle_season_paths(self, season): + for file in season.active_stream["files"].values(): + for episode in parser.episodes_in_season(season.number, file["filename"]): + if episode - 1 in range(len(season.episodes)): + season.episodes[episode - 1].set( + "folder", season.active_stream.get("name") + ) + season.episodes[episode - 1].set("file", file["filename"]) + + def _handle_episode_paths(self, episode): + for file in episode.active_stream["files"].values(): + for episode_number in parser.episodes(file["filename"]): + if episode.number == episode_number: + episode.set("folder", episode.active_stream.get("name")) + episode.set("file", file["filename"]) + + def add_magnet(self, item) -> str: + """Add magnet link to real-debrid.com""" + if not item.active_stream.get("hash"): + return None + response = post( + "https://api.real-debrid.com/rest/1.0/torrents/addMagnet", + { + "magnet": "magnet:?xt=urn:btih:" + + item.active_stream["hash"] + + "&dn=&tr=" + }, + additional_headers=self.auth_headers, + ) + if response.is_ok: + return response.data.id + return None + + def get_torrents(self) -> str: + """Add magnet link to real-debrid.com""" + response = get( + "https://api.real-debrid.com/rest/1.0/torrents/", + data={"offset": 0, "limit": 2500}, + additional_headers=self.auth_headers, + ) + if response.is_ok: + return response.data + return None + + def select_files(self, request_id, item) -> bool: + """Select files from real-debrid.com""" + files = item.active_stream.get("files") + response = post( + f"https://api.real-debrid.com/rest/1.0/torrents/selectFiles/{request_id}", + {"files": ",".join(files.keys())}, + additional_headers=self.auth_headers, + ) + return response.is_ok + + def get_torrent_info(self, request_id): + """Get torrent info from real-debrid.com""" + response = get( + f"https://api.real-debrid.com/rest/1.0/torrents/info/{request_id}", + additional_headers=self.auth_headers, + ) + if response.is_ok: + return response.data + + +debrid = Debrid() diff --git a/backend/program/scrapers/__init__.py b/backend/program/scrapers/__init__.py index 491b8643..c75bdb7d 100644 --- a/backend/program/scrapers/__init__.py +++ b/backend/program/scrapers/__init__.py @@ -1,17 +1,13 @@ -import threading +from datetime import datetime import time - from utils.logger import logger from .torrentio import Torrentio +from .orionoid import Orionoid -class Scraping(threading.Thread): - def __init__(self, media_items): - super().__init__(name="Scraping") - self.media_items = media_items - self.services = [Torrentio(self.media_items)] - self.running = False - self.valid = False +class Scraping(): + def __init__(self): + self.services = [Torrentio(), ]#Orionoid()] while not self.validate(): logger.error( "You have no scraping services enabled, please enable at least one!" @@ -21,18 +17,11 @@ def __init__(self, media_items): def validate(self): return any(service.initialized for service in self.services) - def run(self) -> None: - while self.running: - for service in self.services: - if service.initialized: - service.run() - time.sleep(1) - + def run(self, item) -> None: + for service in self.services: + if service.initialized: + service.run(item) + item.set("scraped_at", datetime.now().timestamp()) - def start(self) -> None: - self.running = True - super().start() - def stop(self) -> None: - self.running = False - super().join() +scraper = Scraping() \ No newline at end of file diff --git a/backend/program/scrapers/orionoid.py b/backend/program/scrapers/orionoid.py new file mode 100644 index 00000000..909101c8 --- /dev/null +++ b/backend/program/scrapers/orionoid.py @@ -0,0 +1,171 @@ +""" Orionoid scraper module """ +from datetime import datetime +from typing import Optional +from pydantic import BaseModel +from requests.exceptions import RequestException +from utils.logger import logger +from utils.request import RateLimitExceeded, RateLimiter, get +from utils.settings import settings_manager +from utils.utils import parser + + +class OrionoidConfig(BaseModel): + api_key: Optional[str] + movie_filter: Optional[str] + tv_filter: Optional[str] + + +class Orionoid: + """Scraper for Orionoid""" + + def __init__(self): + self.settings = "orionoid" + self.class_settings = OrionoidConfig(**settings_manager.get(self.settings)) + self.keyapp = "D3CH6HMX9KD9EMD68RXRCDUNBDJV5HRR" + self.keyuser = self.class_settings.api_key + self.is_premium = False + self.initialized = False + + if self.validate_settings(): + self.is_premium = self.check_premium() + max_calls = 50 if not self.is_premium else 2500 + self.minute_limiter = RateLimiter(max_calls=max_calls, period=86400, raise_on_limit=True) + self.second_limiter = RateLimiter(max_calls=1, period=1) + self.initialized = True + + def validate_settings(self) -> bool: + """Validate the Orionoid class_settings.""" + if self.class_settings.api_key: + return True + logger.info("Orionoid is not configured and will not be used.") + return False + + def check_premium(self) -> bool: + """ + Check the user's status with the Orionoid API. + Returns True if the user is active, has a premium account, and has RealDebrid service enabled. + """ + url = f"https://api.orionoid.com?keyapp={self.keyapp}&keyuser={self.keyuser}&mode=user&action=retrieve" + response = get(url, retry_if_failed=False) + if response.is_ok: + active = True if response.data.data.status == "active" else False + premium = response.data.data.subscription.package.premium + debrid = response.data.data.service.realdebrid + if active and premium and debrid: + logger.info("Orionoid Premium Account Detected.") + return True + else: + logger.error(f"Orionoid Free Account Detected.") + return False + + def run(self, item): + """Scrape the Orionoid site for the given media items + and update the object with scraped streams""" + if self._can_we_scrape(item): + try: + self._scrape_item(item) + except RequestException: + self.minute_limiter.limit_hit() + return + except RateLimitExceeded: + self.minute_limiter.limit_hit() + return + + def _scrape_item(self, item): + """Scrape the Orionoid site for the given media item and log the results.""" + data = self.api_scrape(item) + log_string = item.title + if item.type == "season": + log_string = f"{item.parent.title} S{item.number}" + if item.type == "episode": + log_string = ( + f"{item.parent.parent.title} S{item.parent.number}E{item.number}" + ) + if len(data) > 0: + item.streams.update(data) + logger.debug("Found %s streams for %s", len(data), log_string) + else: + logger.debug("Could not find streams for %s", log_string) + + def construct_url(self, media_type, imdb_id, season=None, episode=1) -> str: + """Construct the URL for the Orionoid API.""" + base_url = "https://api.orionoid.com" + params = { + "keyapp": self.keyapp, + "keyuser": self.keyuser, + "mode": "stream", + "action": "retrieve", + "type": media_type, + "idimdb": imdb_id[2:], + "protocoltorrent": "magnet", + "access": "realdebrid", + "debridlookup": "realdebrid", + "filename": "true", + "fileunknown": "false", + "limitcount": "10", + "video3d": "false", + "videoquality": "sd,hd720,hd1080,hd2k,hd4k", + "sortorder": "descending", + "sortvalue": "best" if self.is_premium else "popularity", + "metarelease": "bdrip,bdrmx,bluray,webdl,ppv,dvdrip", + } + + if media_type == "show": + params["numberseason"] = season if season is not None else "1" + params["numberepisode"] = str(episode) + + custom_filters = ( + self.class_settings.movie_filter + if media_type == "movie" + else self.class_settings.tv_filter + ) + custom_filters = custom_filters.lstrip("&") if custom_filters else "" + url = f"{base_url}?{'&'.join([f'{key}={value}' for key, value in params.items()])}" + if custom_filters: + url += f"&{custom_filters}" + return url + + def _can_we_scrape(self, item) -> bool: + logger.debug("Checking if we can scrape %s", item.title) + logger.debug("Is released: %s", self._is_released(item)) + logger.debug("Needs new scrape: %s", self._needs_new_scrape(item)) + return self._is_released(item) and self._needs_new_scrape(item) + + def _needs_new_scrape(self, item) -> bool: + """Determine if a new scrape is needed based on the last scrape time.""" + current_time = datetime.now().timestamp() + scrape_interval = ( + 60 * 60 if self.is_premium else 60 * 60 * 24 + ) # 1 hour for premium, 1 day for non-premium + return current_time - item.scraped_at > scrape_interval or item.scraped_at == 0 + + def api_scrape(self, item): + """Wrapper for Orionoid scrape method""" + with self.minute_limiter: + if item.type == "season": + imdb_id = item.parent.imdb_id + url = self.construct_url("show", imdb_id, season=item.number) + elif item.type == "episode": + imdb_id = item.parent.parent.imdb_id + url = self.construct_url( + "show", imdb_id, season=item.parent.number, episode=item.number or 1 + ) + else: # item.type == "movie" + imdb_id = item.imdb_id + url = self.construct_url("movie", imdb_id) + + with self.second_limiter: + response = self.get(url, retry_if_failed=False, timeout=60) + item.set("scraped_at", datetime.now().timestamp()) + if response.is_ok: + data = {} + for stream in response.data.data.streams: + title = stream.file.name + infoHash = stream.file.hash + if parser.parse(title) and infoHash: + data[infoHash] = { + "name": title, + } + if len(data) > 0: + return data + return {} diff --git a/backend/program/scrapers/torrentio.py b/backend/program/scrapers/torrentio.py index b50f7e26..1c4a5693 100644 --- a/backend/program/scrapers/torrentio.py +++ b/backend/program/scrapers/torrentio.py @@ -5,18 +5,13 @@ from utils.request import RateLimitExceeded, get, RateLimiter from utils.settings import settings_manager from utils.utils import parser -from program.media import ( - MediaItem, - MediaItemContainer, - MediaItemState, -) +import program.media.state as states class Torrentio: """Scraper for torrentio""" - def __init__(self, media_items: MediaItemContainer): - self.media_items = media_items + def __init__(self): self.settings = "torrentio" self.class_settings = settings_manager.get(self.settings) self.last_scrape = 0 @@ -25,84 +20,44 @@ def __init__(self, media_items: MediaItemContainer): self.second_limiter = RateLimiter(max_calls=1, period=1) self.initialized = True - def run(self): + def run(self, item): """Scrape the torrentio site for the given media items and update the object with scraped streams""" - scraped_amount = 0 - items = [item for item in self.media_items if self._can_we_scrape(item)] - for item in items: + if self._can_we_scrape(item): try: - if item.type == "movie": - scraped_amount += self._scrape_items([item]) - else: - scraped_amount += self._scrape_show(item) + self._scrape_item(item) except RequestException: self.minute_limiter.limit_hit() - break + return except RateLimitExceeded: self.minute_limiter.limit_hit() - break + return - if scraped_amount > 0: - logger.info("Scraped %s streams", scraped_amount) + def _scrape_item(self, item): + data = self.api_scrape(item) + log_string = item.title + if item.type == "season": + log_string = f"{item.parent.title} S{item.number}" + if item.type == "episode": + log_string = f"{item.parent.parent.title} S{item.parent.number}E{item.number}" + if len(data) > 0: + item.streams.update(data) + logger.debug("Found %s streams for %s", len(data), log_string) + else: + logger.debug("Could not find streams for %s", log_string) - def _scrape_show(self, item: MediaItem): - scraped_amount = 0 - seasons = [season for season in item.seasons if self._can_we_scrape(season)] - scraped_amount += self._scrape_items(seasons) - episodes = [ - episode - for season in item.seasons - for episode in season.episodes - if not season.is_scraped() and self._can_we_scrape(episode) - ] - scraped_amount += self._scrape_items(episodes) - return scraped_amount + def _can_we_scrape(self, item) -> bool: + return self._is_released(item) and self._needs_new_scrape(item) - def _scrape_items(self, items: list): - amount_scraped = 0 - for item in items: - data = self.api_scrape(item) - log_string = item.title - if item.type == "season": - log_string = f"{item.parent.title} S{item.number}" - if item.type == "episode": - log_string = f"{item.parent.parent.title} S{item.parent.number}E{item.number}" - if len(data) > 0: - item.set("streams", data) - logger.debug("Found %s streams for %s", len(data), log_string) - amount_scraped += 1 - else: - logger.debug("Could not find streams for %s", log_string) - return amount_scraped - - def _can_we_scrape(self, item: MediaItem) -> bool: - def is_released(): - return item.aired_at is not None and item.aired_at < datetime.now() - - def needs_new_scrape(): - return ( - datetime.now().timestamp() - item.scraped_at - > 60 * 30 # 30 minutes between scrapes - or item.scraped_at == 0 - ) - - if item.type == "show" and item.state in [ - MediaItemState.CONTENT, - MediaItemState.LIBRARY_PARTIAL, - ]: - return True - - if item.type in ["movie", "season", "episode"] and is_released(): - valid_states = { - "movie": [MediaItemState.CONTENT], - "season": [MediaItemState.CONTENT], - "episode": [MediaItemState.CONTENT], - } - if item.state in valid_states[item.type]: - return needs_new_scrape() - - return False + def _is_released(self, item) -> bool: + return item.aired_at is not None and item.aired_at < datetime.now() + + def _needs_new_scrape(self, item) -> bool: + return ( + datetime.now().timestamp() - item.scraped_at + > 60 * 30 # 30 minutes between scrapes + or item.scraped_at == 0 + ) def api_scrape(self, item): """Wrapper for torrentio scrape method""" @@ -128,7 +83,6 @@ def api_scrape(self, item): url += f"{identifier}" with self.second_limiter: response = get(f"{url}.json", retry_if_failed=False) - item.set("scraped_at", datetime.now().timestamp()) if response.is_ok: data = {} for stream in response.data.streams: diff --git a/backend/program/symlink.py b/backend/program/symlink.py index b1e21ec6..9dd612c9 100644 --- a/backend/program/symlink.py +++ b/backend/program/symlink.py @@ -2,11 +2,8 @@ import os import threading import time -import PTN from utils.settings import settings_manager as settings from utils.logger import logger -from utils.utils import parser -from program.media import MediaItemState, MediaItemContainer class Symlinker(threading.Thread): @@ -23,14 +20,11 @@ class Symlinker(threading.Thread): cache_thread (ThreadRunner): The thread runner for updating the cache. """ - def __init__(self, media_items: MediaItemContainer): + def __init__(self): # Symlinking is required super().__init__(name="Symlinker") while True: - self.running = False - self.media_items = media_items - self.cache = {} self.mount_path = os.path.abspath(settings.get("container_mount")) self.host_path = os.path.abspath(settings.get("host_mount")) if os.path.exists(self.host_path): @@ -46,19 +40,8 @@ def __init__(self, media_items: MediaItemContainer): logger.error("Rclone mount not found, retrying in 2...") time.sleep(2) - def run(self): - while self.running: - self._run() - time.sleep(1) - - - def start(self): - self.running = True - super().start() - - def stop(self): - self.running = False - super().join() + def run(self, item): + self._run(item) def _determine_file_name(self, item): filename = None @@ -80,118 +63,70 @@ def _determine_file_name(self, item): filename = f"{showname} ({showyear}) - s{str(item.parent.number).zfill(2)}{episode_string} - {item.title}" return filename - def _run(self): - items = [] - for item in self.media_items: - if item.type == "movie" and item.state is MediaItemState.DOWNLOAD: - self._handle_movie_paths(item) - if os.path.exists(os.path.join(self.host_path, item.folder, item.file)): - items.append(item) - if item.type == "show" and item.state in [ - MediaItemState.DOWNLOAD, - MediaItemState.LIBRARY_PARTIAL, - ]: - for season in item.seasons: - if season.state is MediaItemState.DOWNLOAD: - self._handle_season_paths(season) - for episode in season.episodes: - if episode.state is MediaItemState.DOWNLOAD: - if os.path.exists( - os.path.join( - self.host_path, episode.folder, episode.file - ) - ): - items.append(episode) - else: - for episode in season.episodes: - if episode.state is MediaItemState.DOWNLOAD: - self._handle_episode_paths(episode) - if os.path.exists( - os.path.join( - self.host_path, episode.folder, episode.file - ) - ): - items.append(episode) - - for item in items: - extension = item.file.split(".")[-1] - symlink_filename = f"{self._determine_file_name(item)}.{extension}" + def _run(self, item): + if os.path.exists(os.path.join(self.host_path, item.folder, item.file)): + self._symlink(item) - if item.type == "movie": - movie_folder = ( - f"{item.title} ({item.aired_at.year}) " - + "{imdb-" - + item.imdb_id - + "}" - ) - symlink_folder_path = os.path.join( - self.symlink_path, "movies", movie_folder - ) - if not os.path.exists(symlink_folder_path): - os.mkdir(symlink_folder_path) - symlink_path = os.path.join(symlink_folder_path, symlink_filename) - update_folder = os.path.join( - self.mount_path, os.pardir, "library", "movies", movie_folder - ) - if item.type == "episode": - show = item.parent.parent - symlink_show_folder = ( - f"{show.title} ({show.aired_at.year})" + " {" + show.imdb_id + "}" - ) - symlink_show_path = os.path.join( - self.symlink_path, "shows", symlink_show_folder - ) - if not os.path.exists(symlink_show_path): - os.mkdir(symlink_show_path) - season = item.parent - symlink_season_folder = f"Season {str(season.number).zfill(2)}" - season_path = os.path.join(symlink_show_path, symlink_season_folder) - if not os.path.exists(season_path): - os.mkdir(season_path) - symlink_path = os.path.join(season_path, symlink_filename) - update_folder = os.path.join( - self.mount_path, - os.pardir, - "library", - "shows", - symlink_show_folder, - symlink_season_folder, - ) - - if symlink_path: - try: - os.remove(symlink_path) - except FileNotFoundError: - pass - os.symlink( - os.path.join(self.mount_path, item.folder, item.file), symlink_path - ) - item.set("update_folder", update_folder) - log_string = item.title - if item.type == "episode": - log_string = f"{item.parent.parent.title} season {item.parent.number} episode {item.number}" - logger.debug("Created symlink for %s", log_string) - item.symlinked = True + def _symlink(self, item): + extension = item.file.split(".")[-1] + symlink_filename = f"{self._determine_file_name(item)}.{extension}" - def _handle_movie_paths(self, item): - item.set("folder", item.active_stream.get("name")) - item.set( - "file", - next(iter(item.active_stream["files"].values())).get("filename"), - ) + if item.type == "movie": + movie_folder = ( + f"{item.title} ({item.aired_at.year}) " + + "{imdb-" + + item.imdb_id + + "}" + ) + symlink_folder_path = os.path.join( + self.symlink_path, "movies", movie_folder + ) + if not os.path.exists(symlink_folder_path): + os.mkdir(symlink_folder_path) + symlink_path = os.path.join(symlink_folder_path, symlink_filename) + update_folder = os.path.join( + self.mount_path, os.pardir, "library", "movies", movie_folder + ) + if item.type == "episode": + show = item.parent.parent + symlink_show_folder = ( + f"{show.title} ({show.aired_at.year})" + " {" + show.imdb_id + "}" + ) + symlink_show_path = os.path.join( + self.symlink_path, "shows", symlink_show_folder + ) + if not os.path.exists(symlink_show_path): + os.mkdir(symlink_show_path) + season = item.parent + symlink_season_folder = f"Season {str(season.number).zfill(2)}" + season_path = os.path.join(symlink_show_path, symlink_season_folder) + if not os.path.exists(season_path): + os.mkdir(season_path) + symlink_path = os.path.join(season_path, symlink_filename) + update_folder = os.path.join( + self.mount_path, + os.pardir, + "library", + "shows", + symlink_show_folder, + symlink_season_folder, + ) - def _handle_season_paths(self, season): - for file in season.active_stream["files"].values(): - for episode in parser.episodes(file["filename"]): - if episode in range(len(season.episodes)): - season.episodes[episode - 1].set( - "folder", season.active_stream.get("name") - ) - season.episodes[episode - 1].set("file", file["filename"]) + if symlink_path: + try: + os.remove(symlink_path) + except FileNotFoundError: + pass + os.symlink( + os.path.join(self.mount_path, item.folder, item.file), symlink_path + ) + item.set("update_folder", update_folder) + log_string = item.title + if item.type == "episode": + log_string = f"{item.parent.parent.title} season {item.parent.number} episode {item.number}" + logger.debug("Created symlink for %s", log_string) + item.symlinked = True + else: + logger.debug("Could not create symlink for %s in path %s", item.title, symlink_path) - def _handle_episode_paths(self, episode): - for file in episode.active_stream["files"].values(): - for episode_number in parser.episodes(file["filename"]): - if episode.number == episode_number: - episode.set("folder", episode.active_stream.get("name")) - episode.set("file", file["filename"]) +symlink = Symlinker() \ No newline at end of file diff --git a/backend/program/updaters/trakt.py b/backend/program/updaters/trakt.py index de21b6fa..63993f32 100644 --- a/backend/program/updaters/trakt.py +++ b/backend/program/updaters/trakt.py @@ -3,14 +3,8 @@ from os import path from utils.logger import get_data_path, logger from utils.request import get -from program.media import ( - Episode, - MediaItemContainer, - MediaItemState, - Movie, - Season, - Show, -) +from program.media.container import MediaItemContainer +from program.media.item import Movie, Show, Season, Episode CLIENT_ID = "0183a05ad97098d87287fe46da4ae286f434f32e8e951caad4cc147c947d79a3" @@ -73,13 +67,18 @@ def _map_item_from_data(data, item_type): released_at = data.released formatted_aired_at = datetime.strptime(released_at, "%Y-%m-%d") item = { - "state": MediaItemState.CONTENT, - "title": getattr(data, "title", None), - "year": getattr(data, "year", None), - "imdb_id": getattr(data.ids, "imdb", None), - "aired_at": formatted_aired_at, - "genres": getattr(data, "genres", None), - "requested_at": datetime.now(), + "title": getattr(data, "title", None), # 'Game of Thrones' + "year": getattr(data, "year", None), # 2011 + "status": getattr(data, "status", None), # 'ended', 'released', 'returning series' + "aired_at": formatted_aired_at, # datetime.datetime(2011, 4, 17, 0, 0) + "imdb_id": getattr(data.ids, "imdb", None), # 'tt0496424' + "tvdb_id": getattr(data.ids, "tvdb", None), # 79488 + "tmdb_id": getattr(data.ids, "tmdb", None), # 1399 + "genres": getattr(data, "genres", None), # ['Action', 'Adventure', 'Drama', 'Fantasy'] + "network": getattr(data, "network", None), # 'HBO' + "country": getattr(data, "country", None), # 'US' + "language": getattr(data, "language", None), # 'en' + "requested_at": datetime.now(), # datetime.datetime(2021, 4, 17, 0, 0) } match item_type: case "movie": @@ -131,7 +130,7 @@ def create_item_from_imdb_id(imdb_id: str): return _map_item_from_data(data, media_type) return None -def get_imdb_id_from_tvdb(tvdb_id: str) -> str: +def get_imdbid_from_tvdb(tvdb_id: str) -> str: """Get IMDb ID from TVDB ID in Trakt""" url = f"https://api.trakt.tv/search/tvdb/{tvdb_id}?extended=full" response = get( diff --git a/backend/pytest.ini b/backend/pytest.ini new file mode 100644 index 00000000..a82a97bc --- /dev/null +++ b/backend/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +minversion = 7.0 +; addopts = --cov=tests/. +pythonpath = . +testpaths = + tests \ No newline at end of file diff --git a/backend/tests/items_test.py b/backend/tests/items_test.py new file mode 100644 index 00000000..2d1f7f73 --- /dev/null +++ b/backend/tests/items_test.py @@ -0,0 +1,30 @@ +from starlette.testclient import TestClient +from fastapi import FastAPI +from program.media.state import MediaItemStates +import controllers.items as items +from program.media.container import MediaItemContainer +from unittest.mock import MagicMock + +app = FastAPI() +app.include_router(items.router) +app.program = MagicMock() +app.program.media_items = MediaItemContainer(items=[]) + +client = TestClient(app) + + +def test_get_states(): + response = client.get("/items/states") + assert response.status_code == 200 + assert response.json() == { + "success": True, + "states": [state.name for state in MediaItemStates], + } + + +def test_get_items(): + response = client.get("/items/") + assert response.status_code == 200 + assert isinstance(response.json(), dict) + assert response.json()["success"] == True + assert isinstance(response.json()["items"], list) diff --git a/backend/utils/default_settings.json b/backend/utils/default_settings.json index f721d0b5..ec748f41 100644 --- a/backend/utils/default_settings.json +++ b/backend/utils/default_settings.json @@ -22,6 +22,11 @@ "torrentio": { "filter": "sort=qualitysize%7Cqualityfilter=480p,other,scr,cam,unknown" }, + "orionoid": { + "api_key": "", + "movie_filter": "&audiolanguages=en,ja,kr,fr,es,ge&audiotype=standard,dubbed&limitretry=25", + "tv_filter": "&audiolanguages=en,ja,kr,fr,es,ge&audiotype=standard,dubbed&limitretry=25" + }, "realdebrid": { "api_key": "" } diff --git a/backend/utils/utils.py b/backend/utils/utils.py index 9b6e0a22..5f650289 100644 --- a/backend/utils/utils.py +++ b/backend/utils/utils.py @@ -3,11 +3,9 @@ import time import PTN -from program.media import MediaItemContainer - class Pickly(threading.Thread): - def __init__(self, media_items: MediaItemContainer, data_path: str): + def __init__(self, media_items, data_path: str): super().__init__(name="Pickly") self.media_items = media_items self.data_path = data_path @@ -16,6 +14,9 @@ def __init__(self, media_items: MediaItemContainer, data_path: str): def start(self) -> None: self.load() self.running = True + for item in self.media_items: + if item._lock.locked(): + item._lock.release() return super().start() def stop(self) -> None: @@ -32,7 +33,11 @@ def save(self) -> None: def run(self): while self.running: self.save() - time.sleep(10) + # workaround for quick shutdown, we should use threading.Event instead + for i in range(10): + if not self.running: + break + time.sleep(i) class Parser: @@ -53,6 +58,8 @@ def _parse(self, string): else: episodes.append(int(episode)) + season = parse.get("season") + resolution = parse.get("resolution") quality = parse.get("quality") language = parse.get("language") @@ -65,12 +72,19 @@ def _parse(self, string): "resolution": resolution or [], "language": language or [], "extended": extended, + "season": season, } def episodes(self, string): parse = self._parse(string) return parse["episodes"] + def episodes_in_season(self, season, string): + parse = self._parse(string) + if parse["season"] == season: + return parse["episodes"] + return [] + def parse(self, string): parse = self._parse(string) return ( diff --git a/entrypoint.sh b/entrypoint.sh index fc7eb5ea..45f4d36b 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -31,4 +31,4 @@ fi chown -R ${USERNAME}:${GROUPNAME} /iceberg echo "Container Initialization complete." -exec su -m $USERNAME -c 'cd backend && source /venv/bin/activate && exec python /iceberg/backend/main.py & ORIGIN=http://localhost:3000 node /iceberg/frontend/build' \ No newline at end of file +exec su -m $USERNAME -c 'cd backend && source /venv/bin/activate && exec python /iceberg/backend/main.py & ORIGIN=http://localhost:3000 node /iceberg/frontend/build' diff --git a/frontend/src/routes/status/+page.svelte b/frontend/src/routes/status/+page.svelte index 98b99e2e..a57b18b7 100644 --- a/frontend/src/routes/status/+page.svelte +++ b/frontend/src/routes/status/+page.svelte @@ -22,39 +22,40 @@ } const statusInfo: StatusInfo = { - UNKNOWN: { + Unknown: { color: 'text-red-500', bg: 'bg-red-500', description: 'Unknown status' }, - CONTENT: { + Content: { text: 'Requested', color: 'text-purple-500', bg: 'bg-purple-500', description: 'Item is requested from external service' }, - SCRAPE: { + Scrape: { color: 'text-yellow-500', bg: 'bg-yellow-500', description: 'Item is scraped and will be downloaded' }, - DOWNLOAD: { + Download: { color: 'text-yellow-500', bg: 'bg-yellow-500', description: 'Item is currently downloading' }, - SYMLINK: { + Symlink: { color: 'text-yellow-500', bg: 'bg-yellow-500', description: 'Item is currently being symmlinked' }, - LIBRARY: { + Library: { text: 'In Library', color: 'text-green-400', bg: 'bg-green-400', description: 'Item is in your library' }, - LIBRARY_PARTIAL: { + LibraryPartial: { + text: "In Library (Partial)", color: 'text-blue-400', bg: 'bg-blue-400', description: 'Item is in your library and is ongoing'