merge(release/1.2.3): enable challenge solver video record optional

MonitorrentInstaller/MonitorrentInstaller/Product.wxs

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
-	<Product Id="*" Name="Monitorrent" Language="1033" Version="1.2.2.100" Manufacturer="Monitorrent Team" UpgradeCode="dd4cf505-1e44-4311-a8f2-efcf097175a7">
+	<Product Id="*" Name="Monitorrent" Language="1033" Version="1.2.3.100" Manufacturer="Monitorrent Team" UpgradeCode="dd4cf505-1e44-4311-a8f2-efcf097175a7">
 		<Package InstallerVersion="200" Compressed="yes" InstallScope="perMachine" />
 
 		<MajorUpgrade DowngradeErrorMessage="A newer version of [ProductName] is already installed." AllowSameVersionUpgrades="yes"/>

README.md

@@ -56,14 +56,14 @@ Multiplatform image (`arm64` and `amd64` (`arm` is not supported anymore, until
 [https://hub.docker.com/r/werwolfby/monitorrent/](https://hub.docker.com/r/werwolfby/monitorrent/)
 
 ### Windows Installer:
-https://github.com/werwolfby/monitorrent/releases/download/1.2.2/MonitorrentInstaller-1.2.2.msi
+https://github.com/werwolfby/monitorrent/releases/download/1.2.3/MonitorrentInstaller-1.2.3.msi
 
 ### Manual Install
 
 Requirements:
   - Python 3.9+ and pip
 
-Download latest build: https://github.com/werwolfby/monitorrent/releases/download/1.2.2/monitorrent-1.2.2.zip
+Download latest build: https://github.com/werwolfby/monitorrent/releases/download/1.2.3/monitorrent-1.2.3.zip
 Extract into **monitorent** folder
  * pip install -r requirements.txt
  * playwright --with-deps install firefox

monitorrent/__init__.py

@@ -1 +1 @@
-__version__ = '1.2.2'
+__version__ = '1.2.3'

monitorrent/plugin_managers.py

@@ -61,22 +61,22 @@ def get_settings(self, name):
 
     def set_settings(self, name, settings):
         tracker = self.get_tracker(name)
+        tracker_settings = self.settings_manager.tracker_settings
+        tracker.init(tracker_settings)
         if hasattr(tracker, 'update_credentials'):
             tracker.update_credentials(settings)
             return True
         return False
 
     def check_connection(self, name):
         tracker = self.get_tracker(name)
-        if not isinstance(tracker, WithCredentialsMixin):
-            return False
         tracker_settings = self.settings_manager.tracker_settings
-        # get_tracker returns PluginTrackerBase
-        # noinspection PyUnresolvedReferences
         tracker.init(tracker_settings)
+        if not isinstance(tracker, WithCredentialsMixin):
+            return False
         return tracker.verify()
 
-    def get_tracker(self, name):
+    def get_tracker(self, name: str) -> TrackerPluginBase:
         if name not in self.trackers:
             raise KeyError('Tracker {} not found'.format(name))
         return self.trackers[name]

monitorrent/plugins/trackers/__init__.py

@@ -29,10 +29,33 @@
 from future.utils import with_metaclass
 
 
+class CloudflareChallengeSolverSettings(object):
+    def __init__(self, debug, timeout, record_video, record_har, keep_records):
+        self.debug = debug
+        self.timeout = timeout
+        self.record_video = record_video
+        self.record_har = record_har
+        self.keep_records = keep_records
+
+    def get_new_context_kwargs(self, video_folder):
+        if not self.debug or video_folder is None:
+            return {}
+
+        kwargs = {}
+        if self.record_video:
+            kwargs['record_video_dir'] = video_folder
+            kwargs['record_video_size'] = {"width": 1024, "height": 768}
+        if self.record_har:
+            kwargs['record_har_path'] = path.join(video_folder, 'challenge.har')
+
+        return kwargs
+
+
 class TrackerSettings(object):
-    def __init__(self, requests_timeout, proxies):
+    def __init__(self, requests_timeout, proxies, cloudflare_challenge_solver_settings):
         self.requests_timeout = requests_timeout
         self.proxies = proxies
+        self.cloudflare_challenge_solver_settings = cloudflare_challenge_solver_settings
 
     def get_requests_kwargs(self):
         return {'timeout': self.requests_timeout, 'proxies': self.proxies}
@@ -361,7 +384,22 @@ def _execute_login(self, engine):
         return True
 
 
-def extract_cloudflare_credentials_and_headers(url: str, headers: dict, cookies: dict, timeout: int = 120000):
+def update_headers_and_cookies_mixin(self, url):
+    if not hasattr(self, 'headers') or not hasattr(self, 'cookies') or not hasattr(self, 'headers_cookies_updater'):
+        raise Exception('headers, cookies and headers_cookies_updater should be defined in object')
+
+    headers, cookies = extract_cloudflare_credentials_and_headers(url, self.headers, self.cookies,
+                                                                  self.tracker_settings.cloudflare_challenge_solver_settings)
+    if headers != self.headers or cookies != self.cookies:
+        self.headers = headers
+        self.cookies = cookies
+
+        self.headers_cookies_updater(self.headers, self.cookies)
+
+    return headers, cookies
+
+
+def extract_cloudflare_credentials_and_headers(url: str, headers: dict, cookies: dict, settings: CloudflareChallengeSolverSettings):
     scrapper = cloudscraper.create_scraper()
 
     try:
@@ -372,16 +410,14 @@ def extract_cloudflare_credentials_and_headers(url: str, headers: dict, cookies:
         # If page doesn't have cloudflare, don't send new cookies and headers
         return headers, cookies
     except CloudflareException:
-        return asyncio.run(solve_challenge(url, timeout=timeout))
-
+        return asyncio.run(solve_challenge(url, settings))
 
-async def solve_challenge(url, timeout=120000):
-    video_folder = path.join('webapp', 'challenges', datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
-    os.makedirs(video_folder, exist_ok=True)
 
-    # keep only 3 last challenges, delete others
-    for challenge_folder in sorted(glob.glob(path.join('webapp', 'challenges', '*')), reverse=True, key=path.getctime)[3:]:
-        shutil.rmtree(challenge_folder)
+async def solve_challenge(url, settings: CloudflareChallengeSolverSettings):
+    video_folder = None
+    if settings.debug:
+        video_folder = path.join('webapp', 'challenges', datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
+        os.makedirs(video_folder, exist_ok=True)
 
     browser_launch_kwargs = get_browser_launch_kwargs()
     ws_endpoint = browser_launch_kwargs.pop('ws_endpoint', '')
@@ -391,7 +427,7 @@ async def solve_challenge(url, timeout=120000):
             browser = await p.firefox.connect(ws_endpoint=ws_endpoint)
         else:
             browser = await p.firefox.launch(**browser_launch_kwargs)
-        context = await browser.new_context(record_har_path=path.join(video_folder, 'challenge.har'), record_video_dir=video_folder, record_video_size={"width": 1024, "height": 768})
+        context = await browser.new_context(**settings.get_new_context_kwargs(video_folder))
         try:
             page = await context.new_page()
 
@@ -407,9 +443,9 @@ async def on_request(req):
             await page.goto(url)
 
             features = [
-                asyncio.create_task(page.locator('input[type="button"]').click(timeout=timeout)),
-                asyncio.create_task(page.frame_locator("iframe").locator("input").click(timeout=timeout)),
-                asyncio.create_task(page.wait_for_selector('.left-side > .menu', timeout=timeout)),
+                asyncio.create_task(page.locator('input[type="button"]').click(timeout=settings.timeout)),
+                asyncio.create_task(page.frame_locator("iframe").locator("input").click(timeout=settings.timeout)),
+                asyncio.create_task(page.wait_for_selector('.left-side > .menu', timeout=settings.timeout)),
             ]
 
             done, rest = await asyncio.wait(features, return_when=asyncio.FIRST_COMPLETED)
@@ -438,6 +474,10 @@ async def on_request(req):
             await context.close()
             await browser.close()
 
+            # keep only settings.keep_records last challenges, delete others
+            for challenge_folder in sorted(glob.glob(path.join('webapp', 'challenges', '*')), reverse=True, key=path.getctime)[settings.keep_records:]:
+                shutil.rmtree(challenge_folder)
+
 
 async def wait_for_iframe_input(page, timeout=120000):
     await page.frame_locator("iframe").locator("input").click(timeout=timeout)

monitorrent/plugins/trackers/lostfilm.py

@@ -16,8 +16,8 @@
 from monitorrent.utils.downloader import download
 from monitorrent.plugins import Topic
 from monitorrent.plugins.status import Status
-from monitorrent.plugins.trackers import TrackerPluginBase, WithCredentialsMixin, LoginResult, \
-    extract_cloudflare_credentials_and_headers
+from monitorrent.plugins.trackers import TrackerPluginBase, WithCredentialsMixin, LoginResult, TrackerSettings, \
+    update_headers_and_cookies_mixin
 from monitorrent.plugins.clients import TopicSettings
 import html
 
@@ -454,7 +454,7 @@ def __init__(self, quality, download_url):
 
 
 class LostFilmTVTracker(object):
-    tracker_settings = None
+    tracker_settings: TrackerSettings = None
     _season_title_info = re.compile(u'^(?P<season>\d+)(\.(?P<season_fraction>\d+))?\s+сезон' +
                                     u'(\s+((\d+)-)?(?P<episode>\d+)\s+серия)?$')
     _follow_show_re = re.compile(r'^FollowSerial\((?P<cat>\d+)\)$', re.UNICODE)
@@ -481,7 +481,7 @@ def setup(self, session=None, headers=None, cookies=None):
     def login(self, email, password, headers=None, cookies=None):
         self.headers = headers or {}
         self.cookies = cookies or {}
-        headers, cookies = self._update_headers_and_cookies("https://" + self.netloc)
+        headers, cookies = update_headers_and_cookies_mixin(self, "https://" + self.netloc)
 
         params = {"act": "users", "type": "login", "mail": email, "pass": password, "rem": 1, "need_captcha": "", "captcha": ""}
         response = requests.post("https://www.lostfilm.tv/ajaxik.users.php", params, headers=headers, cookies=cookies)
@@ -499,7 +499,7 @@ def verify(self):
         if not cookies:
             return False
         my_settings_url = 'https://www.lostfilm.tv/my_settings'
-        self._update_headers_and_cookies(my_settings_url)
+        update_headers_and_cookies_mixin(self, my_settings_url)
         r1 = requests.get(my_settings_url, headers=self.headers, cookies=self.get_cookies(),
                           **self.tracker_settings.get_requests_kwargs())
         return r1.url == my_settings_url and '<meta http-equiv="refresh" content="0; url=/">' not in r1.text
@@ -521,7 +521,7 @@ def parse_url(self, url, parse_series=False):
         if url is None:
             return None
 
-        self._update_headers_and_cookies(url)
+        update_headers_and_cookies_mixin(self, url)
 
         response = requests.get(url, headers=self.headers, cookies=self.get_cookies(), allow_redirects=False,
                                 **self.tracker_settings.get_requests_kwargs())
@@ -596,7 +596,7 @@ def parse_download(table):
 
             return LostFileDownloadInfo(LostFilmQuality.parse(quality), download_url)
 
-        self._update_headers_and_cookies(url)
+        update_headers_and_cookies_mixin(self, url)
 
         download_redirect_url = self.download_url_pattern.format(cat=cat, season=season, episode=episode)
         session = requests.session()
@@ -613,16 +613,6 @@ def parse_download(table):
         soup = get_soup(download_page.text)
         return list(map(parse_download, soup.find_all('div', class_='inner-box--item')))
 
-    def _update_headers_and_cookies(self, url):
-        headers, cookies = extract_cloudflare_credentials_and_headers(url, self.headers, self.cookies)
-        if headers != self.headers or cookies != self.cookies:
-            self.headers = headers
-            self.cookies = cookies
-
-            self.headers_cookies_updater(self.headers, self.cookies)
-
-        return headers, cookies
-
 
 class LostFilmPlugin(WithCredentialsMixin, TrackerPluginBase):
     credentials_class = LostFilmTVCredentials
@@ -759,9 +749,7 @@ def login(self):
             if not username or not password:
                 return LoginResult.CredentialsNotSpecified
         try:
-            self.tracker.login(username, password,
-                               headers or self.tracker.headers,
-                               cookies or self.tracker.cookies)
+            self.tracker.login(username, password, headers, cookies)
             with DBSession() as db:
                 cred = db.query(self.credentials_class).first()
                 cred.session = self.tracker.session

monitorrent/plugins/trackers/rutracker.py

@@ -11,7 +11,7 @@
 from monitorrent.plugin_managers import register_plugin
 from monitorrent.utils.soup import get_soup
 from monitorrent.plugins.trackers import TrackerPluginBase, WithCredentialsMixin, ExecuteWithHashChangeMixin, \
-    LoginResult, extract_cloudflare_credentials_and_headers
+    LoginResult, TrackerSettings, update_headers_and_cookies_mixin
 
 PLUGIN_NAME = 'rutracker.org'
 
@@ -67,7 +67,7 @@ def get_current_version(engine):
 
 
 class RutrackerTracker(object):
-    tracker_settings = None
+    tracker_settings: TrackerSettings = None
     login_url = "https://rutracker.org/forum/login.php"
     profile_page = "https://rutracker.org/forum/privmsg.php?folder=inbox"
     _regex = re.compile(six.text_type(r'^https?://w*\.*rutracker.org/forum/viewtopic.php\?t=(\d+)(/.*)?$'))
@@ -109,7 +109,7 @@ def parse_url(self, url):
     def login(self, username, password, headers=None, cookies=None):
         self.headers = headers
         self.cookies = cookies
-        self._update_headers_and_cookies("https://rutracker.org/forum/index.php")
+        update_headers_and_cookies_mixin(self, "https://rutracker.org/forum/index.php")
 
         username_q = username.encode('windows-1251')
         password_q = password.encode('windows-1251')
@@ -164,16 +164,6 @@ def get_download_url(self, url):
 
         return "https://rutracker.org/forum/dl.php?t=" + id
 
-    def _update_headers_and_cookies(self, url):
-        headers, cookies = extract_cloudflare_credentials_and_headers(url, self.headers, self.cookies)
-        if headers != self.headers or cookies != self.cookies:
-            self.headers = headers
-            self.cookies = cookies
-
-            self.headers_cookies_updater(self.headers, self.cookies)
-
-        return headers, cookies
-
 
 class RutrackerPlugin(WithCredentialsMixin, ExecuteWithHashChangeMixin, TrackerPluginBase):
     tracker = RutrackerTracker()

monitorrent/rest/challenge_logs.py

@@ -17,16 +17,17 @@ def on_get(self, req, resp):
         # scan folder webapp/challenges
         challenges_folder = path.join('webapp', 'challenges')
         challenges = []
-        for folder in sorted(os.listdir(challenges_folder), reverse=True):
-            challenge_folder = path.join(challenges_folder, folder)
-            if not os.path.isdir(challenge_folder):
-                continue
-            challenge = {
-                'folder': folder,
-                'video': self._get_video(challenge_folder),
-                'har': self._get_har(challenge_folder),
-            }
-            challenges.append(challenge)
+        if os.path.exists(challenges_folder):
+            for folder in sorted(os.listdir(challenges_folder), reverse=True):
+                challenge_folder = path.join(challenges_folder, folder)
+                if not os.path.isdir(challenge_folder):
+                    continue
+                challenge = {
+                    'folder': folder,
+                    'video': self._get_video(challenge_folder),
+                    'har': self._get_har(challenge_folder),
+                }
+                challenges.append(challenge)
 
         resp.json = challenges
 

monitorrent/rest/settings_cloudflare_challenge_solver.py

@@ -0,0 +1,59 @@
+import falcon
+import six
+from monitorrent.settings_manager import SettingsManager
+from monitorrent.new_version_checker import NewVersionChecker
+
+
+# noinspection PyUnusedLocal
+class SettingsCloudflareChallengeSolver(object):
+    def __init__(self, settings_manager):
+        """
+        :type settings_manager: SettingsManager
+        """
+        self.settings_manager = settings_manager
+
+    def on_get(self, req, resp):
+        resp.json = {
+            'debug': self.settings_manager.cloudflare_challenge_solver_debug,
+            'record_video': self.settings_manager.cloudflare_challenge_solver_record_video,
+            'record_har': self.settings_manager.cloudflare_challenge_solver_record_har,
+            'keep_records': self.settings_manager.cloudflare_challenge_solver_keep_records,
+        }
+
+    def on_patch(self, req, resp):
+        if req.json is None or len(req.json) == 0:
+            raise falcon.HTTPBadRequest('BodyRequired', 'Expecting not empty JSON body')
+
+        debug = req.json.get('debug')
+        if debug is not None and not isinstance(debug, bool):
+            raise falcon.HTTPBadRequest('WrongValue', '"debug" have to be bool')
+
+        record_video = req.json.get('record_video')
+        if record_video is not None and not isinstance(record_video, bool):
+            raise falcon.HTTPBadRequest('WrongValue', '"record_video" have to be bool')
+
+        record_har = req.json.get('record_har')
+        if record_har is not None and not isinstance(record_har, bool):
+            raise falcon.HTTPBadRequest('WrongValue', '"record_har" have to be bool')
+
+        keep_records = req.json.get('keep_records')
+        if keep_records is not None and not isinstance(keep_records, int):
+            raise falcon.HTTPBadRequest('WrongValue', '"keep_records" have to be int')
+
+        if debug is not None:
+            if self.settings_manager.cloudflare_challenge_solver_debug != debug:
+                self.settings_manager.cloudflare_challenge_solver_debug = debug
+
+        if record_video is not None:
+            if self.settings_manager.cloudflare_challenge_solver_record_video != record_video:
+                self.settings_manager.cloudflare_challenge_solver_record_video = record_video
+
+        if record_har is not None:
+            if self.settings_manager.cloudflare_challenge_solver_record_har != record_har:
+                self.settings_manager.cloudflare_challenge_solver_record_har = record_har
+
+        if keep_records is not None:
+            if self.settings_manager.cloudflare_challenge_solver_keep_records != keep_records:
+                self.settings_manager.cloudflare_challenge_solver_keep_records = keep_records
+
+        resp.status = falcon.HTTP_NO_CONTENT