Skip to content

Commit

Permalink
Add tweaks, add some kind of parsing for torrents etc.
Browse files Browse the repository at this point in the history
  • Loading branch information
Gaisberg authored and AyushSehrawat committed Dec 22, 2023
1 parent cf26a2f commit 090fe56
Show file tree
Hide file tree
Showing 9 changed files with 270 additions and 183 deletions.
8 changes: 6 additions & 2 deletions backend/program/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
"""Program main module"""
import os
import threading
import time
from typing import Optional
from pydantic import BaseModel, HttpUrl, Field
from program.symlink import Symlinker
Expand All @@ -10,6 +12,7 @@
from program.debrid.realdebrid import Debrid as RealDebrid
from program.content import Content
from program.scrapers import Scraping
from utils.utils import Pickly


# Pydantic models for configuration
Expand Down Expand Up @@ -58,7 +61,8 @@ def __init__(self):
self.settings = settings_manager.get_all()
self.media_items = MediaItemContainer(items=[])
self.data_path = get_data_path()
self.media_items.load(os.path.join(self.data_path, "media.pkl"))
self.pickly = Pickly(self.media_items, self.data_path)
self.pickly.start()
self.threads = [
Content(self.media_items), # Content must be first
Plex(self.media_items),
Expand All @@ -75,4 +79,4 @@ def start(self):
def stop(self):
for thread in self.threads:
thread.stop()
self.media_items.save(os.path.join(self.data_path, "media.pkl"))
self.pickly.stop()
18 changes: 10 additions & 8 deletions backend/program/content/mdblist.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ def __init__(self, media_items: MediaItemContainer):
self.requests_per_2_minutes = self._calculate_request_time()
self.rate_limiter = RateLimiter(self.requests_per_2_minutes, 120, True)
self.initialized = True
self.last_items = []

def _validate_settings(self):
response = ping(
Expand All @@ -39,14 +40,15 @@ def run(self):
items += self._get_items_from_list(
list_id, self.settings["api_key"]
)

new_items = [item for item in items if item not in self.media_items]
container = self.updater.create_items(new_items)
for item in container:
item.set("requested_by", "Mdblist")
added_items = self.media_items.extend(container)
if len(added_items) > 0:
logger.info("Added %s items", len(added_items))
if len(items) != len(self.last_items):
self.last_items = items
new_items = [item for item in items if item not in self.media_items]
container = self.updater.create_items(new_items)
for item in container:
item.set("requested_by", "Mdblist")
added_items = self.media_items.extend(container)
if len(added_items) > 0:
logger.info("Added %s items", len(added_items))
except RateLimitExceeded:
pass

Expand Down
17 changes: 10 additions & 7 deletions backend/program/content/overseerr.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ def __init__(self, media_items: MediaItemContainer):
return
self.updater = Trakt()
self.not_found_ids = []
self.last_items = []
self.initialized = True

def _validate_settings(self):
Expand All @@ -36,13 +37,15 @@ def run(self):
"""Fetch media from overseerr and add them to media_items attribute
if they are not already there"""
items = self._get_items_from_overseerr(10000)
new_items = [item for item in items if item not in self.media_items]
container = self.updater.create_items(new_items)
for item in container:
item.set("requested_by", "Overseerr")
added_items = self.media_items.extend(container)
if len(added_items) > 0:
logger.info("Added %s items", len(added_items))
if len(items) != len(self.last_items):
self.last_items = items
new_items = [item for item in items if item not in self.media_items]
container = self.updater.create_items(new_items)
for item in container:
item.set("requested_by", "Overseerr")
added_items = self.media_items.extend(container)
if len(added_items) > 0:
logger.info("Added %s items", len(added_items))

def _get_items_from_overseerr(self, amount: int):
"""Fetch media from overseerr"""
Expand Down
39 changes: 20 additions & 19 deletions backend/program/debrid/realdebrid.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,7 @@ def get_user():
return response.json()


class Debrid(
threading.Thread
): # TODO CHECK TORRENTS LIST BEFORE DOWNLOAD, IF DOWNLOADED AND NOT IN LIBRARY CHOOSE ANOTHER TORRENT
class Debrid(threading.Thread):
"""Real-Debrid API Wrapper"""

def __init__(self, media_items: MediaItemContainer):
Expand All @@ -41,7 +39,9 @@ def __init__(self, media_items: MediaItemContainer):
if self._validate_settings():
self._torrents = {}
break
logger.error("Realdebrid settings incorrect or not premium, retrying in 2...")
logger.error(
"Realdebrid settings incorrect or not premium, retrying in 2..."
)
time.sleep(2)

def _validate_settings(self):
Expand Down Expand Up @@ -96,29 +96,35 @@ def download(self):

def _download(self, item):
"""Download movie from real-debrid.com"""
downloaded = 0
self._check_stream_availability(item)
self._determine_best_stream(item)
if not self._is_downloaded(item):
self._download_item(item)
return 1
return 0
downloaded = self._download_item(item)
self._update_torrent_info(item)
return downloaded

def _is_downloaded(self, item):
if not item.get("active_stream", None):
return False
torrents = self.get_torrents()
if any(torrent.hash == item.active_stream.get("hash") for torrent in torrents):
logger.debug("Torrent already downloaded")
return True
for torrent in torrents:
if torrent.hash == item.active_stream.get("hash"):
item.set("active_stream.id", torrent.id)
logger.debug("Torrent already downloaded")
return True
return False

def _update_torrent_info(self, item):
info = self.get_torrent_info(item.get("active_stream")["id"])
item.active_stream["name"] = info.filename

def _download_item(self, item):
if not item.get("active_stream", None):
return 0
request_id = self.add_magnet(item)

time.sleep(0.3)
self.select_files(request_id, item)
item.set("active_stream.id", request_id)

if item.type == "movie":
log_string = item.title
Expand Down Expand Up @@ -183,9 +189,7 @@ def _determine_best_stream(self, item) -> bool:
def _check_stream_availability(self, item: MediaItem):
if len(item.streams) == 0:
return
streams = "/".join(
list(item.streams)
)
streams = "/".join(list(item.streams))
response = get(
f"https://api.real-debrid.com/rest/1.0/torrents/instantAvailability/{streams}/",
additional_headers=self.auth_headers,
Expand Down Expand Up @@ -276,10 +280,7 @@ def get_torrents(self) -> str:
"""Add magnet link to real-debrid.com"""
response = get(
"https://api.real-debrid.com/rest/1.0/torrents/",
data = {
"offset": 0,
"limit": 2500
},
data={"offset": 0, "limit": 2500},
additional_headers=self.auth_headers,
)
if response.is_ok:
Expand Down
110 changes: 64 additions & 46 deletions backend/program/libraries/plex.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
"""Plex library module"""
import concurrent.futures
import os
import threading
import time
from typing import List, Optional
from plexapi import exceptions
from plexapi.server import PlexServer
import requests
from requests.exceptions import ReadTimeout, ConnectionError
from requests.exceptions import ConnectionError
from pydantic import BaseModel, HttpUrl
from utils.logger import logger
from utils.settings import settings_manager as settings
Expand Down Expand Up @@ -37,7 +38,9 @@ def __init__(self, media_items: MediaItemContainer):
while True:
try:
temp_settings = settings.get("plex")
self.library_path = os.path.abspath(os.path.join(settings.get("container_mount"), os.pardir, "library"))
self.library_path = os.path.abspath(
os.path.join(settings.get("container_mount"), os.pardir, "library")
)
self.plex = PlexServer(
temp_settings["url"], temp_settings["token"], timeout=15
)
Expand Down Expand Up @@ -71,15 +74,24 @@ def _update_items(self):
processed_sections = set()

for section in sections:
if section.key in processed_sections and not self._is_wanted_section(section):
if section.key in processed_sections and not self._is_wanted_section(
section
):
continue

try:
if not section.refreshing:
for item in section.all():
media_item = self._create_item(item)
if media_item:
items.append(media_item)
with concurrent.futures.ThreadPoolExecutor(
max_workers=5, thread_name_prefix="Plex"
) as executor:
future_items = {
executor.submit(self._create_item, item)
for item in section.all()
}
for future in concurrent.futures.as_completed(future_items):
media_item = future.result()
if media_item:
items.append(media_item)
except requests.exceptions.ReadTimeout:
logger.error(
f"Timeout occurred when accessing section: {section.title}"
Expand All @@ -97,39 +109,42 @@ def _update_items(self):
def _update_sections(self):
"""Update plex library section"""
for section in self.plex.library.sections():
if not self._is_wanted_section(section):
continue
movie_items = [
item
for item in self.media_items
if item.type == "movie"
and item.state is MediaItemState.SYMLINK
and item.update_folder != "updated"
]
episodes = [
episode
for item in self.media_items
if item.type == "show"
for season in item.seasons
for episode in season.episodes
if episode.state is MediaItemState.SYMLINK
and episode.update_folder != "updated"
]
items = movie_items + episodes

for item in items:
if (
item.type == section.type
or item.type in ["season", "episode"]
and section.type == "show"
):
section.update(item.update_folder)
item.set("update_folder", "updated")
log_string = item.title
if item.type == "episode":
log_string = f"{item.parent.parent.title} season {item.parent.number} episode {item.number}"
for item in self.media_items:
log_string = None
if section.type == item.type:
if item.type == "movie":
if (
item.state is MediaItemState.SYMLINK
and item.get("update_folder") != "updated"
):
section.update(item.update_folder)
item.set("update_folder", "updated")
log_string = item.title
break
if item.type == "show":
for season in item.seasons:
if (
season.state is MediaItemState.SYMLINK
and season.get("update_folder") != "updated"
):
section.update(season.episodes[0].update_folder)
season.set("update_folder", "updated")
log_string = f"{item.title} season {season.number}"
break
else:
for episode in season.episodes:
if (
episode.state is MediaItemState.SYMLINK
and episode.get("update_folder") != "updated"
and episode.parent.get("update_folder")
!= "updated"
):
section.update(episode.update_folder)
episode.set("update_folder", "updated")
log_string = f"{item.title} season {season.number} episode {episode.number}"
break
if log_string:
logger.debug("Updated section %s for %s", section.title, log_string)
break

def _create_item(self, item):
new_item = _map_item_from_data(item, item.type)
Expand Down Expand Up @@ -196,25 +211,28 @@ def _update_item(self, item: MediaItem, library_item: MediaItem):
def _is_wanted_section(self, section):
return any(self.library_path in location for location in section.locations)


def _map_item_from_data(item, item_type):
"""Map Plex API data to MediaItemContainer."""
guid = getattr(item, "guid", None)
file = None
guid = getattr(item, "guid", None)
if item_type in ["movie", "episode"]:
file = getattr(item, "locations", [None])[0].split("/")[-1]
genres = [genre.tag for genre in getattr(item, "genres", [])]
available_at = getattr(item, "originallyAvailableAt", None)
title = getattr(item, "title", None)
guids = getattr(item, "guids", [])
key = getattr(item, "key", None)
season_number = getattr(item, "seasonNumber", None)
episode_number = getattr(item, "episodeNumber", None)
art_url = getattr(item, "artUrl", None)
imdb_id = None
aired_at = None

imdb_id = next(
(guid.id.split("://")[-1] for guid in guids if "imdb" in guid.id), None
)
aired_at = available_at or None
if item_type != "season":
guids = getattr(item, "guids", [])
imdb_id = next(
(guid.id.split("://")[-1] for guid in guids if "imdb" in guid.id), None
)
aired_at = getattr(item, "originallyAvailableAt", None)

media_item_data = {
"title": title,
Expand Down
Loading

0 comments on commit 090fe56

Please sign in to comment.