Misc bug fixes

redditrepostsleuth/core/celery/admin_tasks.py

@@ -126,7 +126,7 @@ def update_subreddit_config_from_database(self, monitored_sub: MonitoredSub, use
 
 @celery.task(bind=True, base=AdminTask, autoretry_for=(UtilApiException,ConnectionError,TooManyRequests), retry_kwards={'max_retries': 3})
 def check_user_for_only_fans(self, username: str) -> None:
-    skip_names = ['[deleted]']
+    skip_names = ['[deleted]', 'AutoModerator']
 
     if username in skip_names:
         log.info('Skipping name %s', username)

redditrepostsleuth/core/celery/app.py

@@ -4,6 +4,7 @@
 from celery import Celery, signals
 from celery.signals import after_setup_logger
 from kombu.serialization import registry
+from prawcore import TooManyRequests
 
 from redditrepostsleuth.core.exception import IngestHighMatchMeme, ImageConversionException
 
@@ -28,7 +29,7 @@ def init_sentry(**_kwargs):
                     monitor_beat_tasks=True,
                 ),
             ],
-            ignore_errors=[IngestHighMatchMeme, ImageConversionException, WorkerLostError]
+            ignore_errors=[IngestHighMatchMeme, ImageConversionException, WorkerLostError, TooManyRequests]
         )
 
 @after_setup_logger.connect

redditrepostsleuth/core/celery/task_logic/scheduled_task_logic.py

@@ -143,8 +143,9 @@ def token_checker() -> None:
     if token_res.status_code != 200:
         log.error('Failed to get new token')
         return
-    decoded_token = jwt.decode(json.loads(token_res.text), '', algorithms=["HS256"], options={"verify_signature": False})
-    redis_client.set('prof_token', decoded_token['sub'])
+    #decoded_token = jwt.decode(json.loads(token_res.text), '', algorithms=["HS256"], options={"verify_signature": False})
+    new_token = json.loads(token_res.text)
+    redis_client.set('prof_token', new_token)
     log.info('New token set in Redis')
 
 def update_monitored_sub_data(

redditrepostsleuth/core/celery/tasks/repost_tasks.py

@@ -0,0 +1,123 @@
+from typing import NoReturn
+
+import requests
+from redlock import RedLockError
+from requests.exceptions import ConnectTimeout
+from sqlalchemy import func
+
+from redditrepostsleuth.core.celery import celery
+from redditrepostsleuth.core.celery.basetasks import AnnoyTask, RedditTask, RepostTask
+from redditrepostsleuth.core.celery.task_logic.repost_image import repost_watch_notify, check_for_post_watch
+from redditrepostsleuth.core.db.databasemodels import Post, RepostWatch
+from redditrepostsleuth.core.exception import NoIndexException, IngestHighMatchMeme, IndexApiException
+from redditrepostsleuth.core.logfilters import ContextFilter
+from redditrepostsleuth.core.logging import log, configure_logger
+from redditrepostsleuth.core.model.search.search_match import SearchMatch
+from redditrepostsleuth.core.util.helpers import get_default_link_search_settings, get_default_image_search_settings, \
+    get_default_text_search_settings
+from redditrepostsleuth.core.util.repost.repost_helpers import filter_search_results
+from redditrepostsleuth.core.util.repost.repost_search import text_search_by_post, image_search_by_post, \
+    link_search
+
+log = configure_logger(
+    name='redditrepostsleuth',
+    format='%(asctime)s - %(module)s:%(funcName)s:%(lineno)d - Trace_ID=%(trace_id)s Post_ID=%(post_id)s Subreddit=%(subreddit)s Service=%(service)s Level=%(levelname)s Message=%(message)s',
+    filters=[ContextFilter()]
+)
+
+
+@celery.task(bind=True, base=AnnoyTask, serializer='pickle', ignore_results=True, autoretry_for=(RedLockError, NoIndexException, IngestHighMatchMeme), retry_kwargs={'max_retries': 20, 'countdown': 300})
+def check_image_repost_save(self, post: Post) -> NoReturn:
+
+    try:
+        r = requests.head(post.url)
+        if r.status_code != 200:
+            log.info('Skipping image that is deleted %s', post.url)
+            celery.send_task('redditrepostsleuth.core.celery.admin_tasks.delete_post_task', args=[post.post_id])
+            return
+
+        search_settings = get_default_image_search_settings(self.config)
+        search_settings.max_matches = 75
+        search_results = self.dup_service.check_image(
+            post.url,
+            post=post,
+            search_settings=search_settings,
+            source='ingest'
+        )
+        with self.uowm.start() as uow:
+            search_results = image_search_by_post(
+                post,
+                uow,
+                self.dup_service,
+                search_settings,
+                'ingest',
+                high_match_meme_check=True
+            )
+
+            if search_results.matches:
+                watches = check_for_post_watch(search_results.matches, uow)
+                if watches and self.config.enable_repost_watch:
+                    notify_watch.apply_async((watches, post), queue='watch_notify')
+
+    except (RedLockError, NoIndexException, IngestHighMatchMeme):
+        raise
+    except (ConnectTimeout):
+        log.warning('Failed to validate image url at %s', post.url)
+    except Exception as e:
+        log.exception('')
+
+
+@celery.task(bind=True, base=RepostTask, ignore_results=True, serializer='pickle')
+def link_repost_check(self, post):
+
+    try:
+        with self.uowm.start() as uow:
+
+            log.debug('Checking URL for repost: %s', post.url)
+            search_results = link_search(post.url, uow,
+                                         get_default_link_search_settings(self.config),
+                                         'ingest',
+                                         post=post,
+                                         filter_function=filter_search_results
+                                         )
+
+            search_results.search_times.stop_timer('total_search_time')
+            log.info('Link Query Time: %s', search_results.search_times.query_time)
+
+
+
+    except Exception as e:
+        log.exception('')
+
+
+@celery.task(bind=True, base=RepostTask, ignore_results=True, serializer='pickle')
+def check_for_text_repost_task(self, post: Post) -> None:
+    log.debug('Checking post for repost: %s', post.post_id)
+    try:
+        with self.uowm.start() as uow:
+            search_results = text_search_by_post(
+                post,
+                uow,
+                get_default_text_search_settings(self.config),
+                filter_function=filter_search_results,
+                source='ingest'
+            )
+            log.info('Found %s matching text posts', len(search_results.matches))
+    except IndexApiException as e:
+        log.warning(e, exc_info=False)
+        raise
+    except Exception as e:
+        log.exception('Unknown exception during test repost check')
+
+
+@celery.task(bind=True, base=RedditTask, ignore_results=True)
+def notify_watch(self, watches: list[dict[SearchMatch, RepostWatch]], repost: Post):
+    repost_watch_notify(watches, self.reddit, self.response_handler, repost)
+    with self.uowm.start() as uow:
+        for w in watches:
+            w['watch'].last_detection = func.utc_timestamp()
+            uow.repostwatch.update(w['watch'])
+            try:
+                uow.commit()
+            except Exception as e:
+                log.exception('Failed to save repost watch %s', w['watch'].id, exc_info=True)

redditrepostsleuth/core/util/replytemplates.py

@@ -1,6 +1,8 @@
 
 UNSUPPORTED_POST_TYPE = 'Sorry, I don\'t support this post type ({post_type}) right now.  Feel free to check back in the future!'
 
+UNKNOWN_SUBMISSION_TYPE = 'Sorry, I\'m not able to determine the type of post and cannot process your request. '
+
 WIKI_STATS = '### Submission Index Stats\n\n**Total Posts:** {post_count}\n\n**Image Posts:** {images}\n\n**Link Posts:** {links}\n\n**Video Posts:** {video}\n\n **Text Posts:** {text}\n\n **Oldest Post:** {oldest}\n\n### Repost Statistics\n\n**Image Reposts:** {image_reposts}\n\n**Times Summoned:** {summoned}'
 
 

redditrepostsleuth/summonssvc/summonshandler.py

@@ -23,7 +23,7 @@
     get_default_link_search_settings, get_default_text_search_settings
 from redditrepostsleuth.core.util.replytemplates import UNSUPPORTED_POST_TYPE, WATCH_ENABLED, \
     WATCH_ALREADY_ENABLED, WATCH_DISABLED_NOT_FOUND, WATCH_DISABLED, \
-    SUMMONS_ALREADY_RESPONDED, BANNED_SUB_MSG, OVER_LIMIT_BAN
+    SUMMONS_ALREADY_RESPONDED, BANNED_SUB_MSG, OVER_LIMIT_BAN, UNKNOWN_SUBMISSION_TYPE
 from redditrepostsleuth.core.util.repost.repost_helpers import filter_search_results, \
     save_image_repost_result
 from redditrepostsleuth.core.util.repost.repost_search import image_search_by_post, link_search, text_search_by_post
@@ -81,7 +81,13 @@ def process_summons(self, summons: Summons) -> None:
         if self.summons_disabled:
             self._send_summons_disable_msg(summons)
 
-        if summons.post.post_type.name is None or summons.post.post_type.name not in self.config.supported_post_types:
+        if summons.post.post_type is None:
+            log.warning('Post %s has no post type.  Cannot process summons', summons.post.post_id)
+            response = SummonsResponse(summons=summons, message=UNKNOWN_SUBMISSION_TYPE)
+            self._send_response(response)
+            return
+
+        if summons.post.post_type.name not in self.config.supported_post_types:
             log.warning('Post %s: Type %s not supported', f'https://redd.it/{summons.post.post_id}', summons.post.post_type.name)
             self._send_unsupported_msg(summons)
             return