Fixing errors in Subreddit config and data updating

docker-compose.yml

@@ -10,6 +10,7 @@ services:
         environment:
             - db_user=ingest
             - LOG_LEVEL=INFO
+            - RUN_ENV=production
         env_file:
             - .env
         entrypoint: python -u /src/ingestsvc.py
@@ -24,6 +25,7 @@ services:
         env_file:
             - .env
         environment:
+            - RUN_ENV=production
             - db_user=scheduled_task
             - LOG_LEVEL=INFO
             - CELERY_IMPORTS=redditrepostsleuth.core.celery.admin_tasks,redditrepostsleuth.core.celery.tasks.scheduled_tasks
@@ -40,6 +42,7 @@ services:
         env_file:
             - .env
         environment:
+            - RUN_ENV=production
             - LOG_LEVEL=INFO
             - C_FORCE_ROOT=True
             - CELERY_IMPORTS=redditrepostsleuth.core.celery.admin_tasks
@@ -53,6 +56,7 @@ services:
         env_file:
             - .env
         environment:
+            - RUN_ENV=production
             - LOG_LEVEL=INFO
             - QUERY_LIMIT=1000000
             - CELERY_IMPORTS=redditrepostsleuth.core.celery.admin_tasks
@@ -67,6 +71,7 @@ services:
         env_file:
             - .env
         environment:
+            - RUN_ENV=production
             - LOG_LEVEL=INFO
             - CELERY_IMPORTS=redditrepostsleuth.core.celery.admin_tasks
         entrypoint: celery -A redditrepostsleuth.core.celery worker -Q post_delete -n deleted_post_worker -c 16
@@ -83,6 +88,7 @@ services:
         env_file:
             - .env
         environment:
+            - RUN_ENV=production
             - LOG_LEVEL=WARN
         entrypoint: python -u /src/queue_monitor.py
 
@@ -96,6 +102,7 @@ services:
         env_file:
             - .env
         environment:
+            - RUN_ENV=production
             - LOG_LEVEL=WARN
         entrypoint: python -u /src/summons_monitor.py
 
@@ -109,6 +116,7 @@ services:
         env_file:
             - .env
         environment:
+            - RUN_ENV=production
             - db_user=sub_monitor
             - LOG_LEVEL=INFO
             - CELERY_IMPORTS=redditrepostsleuth.core.celery.response_tasks
@@ -124,6 +132,7 @@ services:
         env_file:
             - .env
         environment:
+            - RUN_ENV=production
             - db_user=ingest
             - LOG_LEVEL=ERROR
             - CELERY_IMPORTS=redditrepostsleuth.core.celery.ingesttasks
@@ -140,7 +149,8 @@ services:
             - .env
         environment:
             - db_user=repost_link
-            - LOG_LEVEL=INFO
+            - LOG_LEVEL=WARN
+            - RUN_ENV=production
             - CELERY_IMPORTS=redditrepostsleuth.core.celery.tasks.repost_tasks
         entrypoint: celery -A redditrepostsleuth.core.celery worker -Q repost_link -n repost_link_worker --autoscale=6,1
 
@@ -154,6 +164,7 @@ services:
         env_file:
             - .env
         environment:
+            - RUN_ENV=production
             - db_user=repost_image
             - LOG_LEVEL=INFO
             - CELERY_IMPORTS=redditrepostsleuth.core.celery.tasks.repost_tasks
@@ -169,6 +180,7 @@ services:
         env_file:
             - .env
         environment:
+            - RUN_ENV=production
             - LOG_LEVEL=INFO
             - CELERY_IMPORTS=redditrepostsleuth.core.celery.admin_tasks
         entrypoint: celery -A redditrepostsleuth.core.celery worker -Q onlyfans_check -n onlyfans_worker --autoscale=8,5
@@ -183,8 +195,9 @@ services:
         env_file:
             - .env
         environment:
+            - RUN_ENV=production
             - LOG_LEVEL=INFO
-            - CELERY_IMPORTS=redditrepostsleuth.core.celery.tasks.scheduled_tasks
-        entrypoint: celery -A redditrepostsleuth.core.celery worker -Q subreddit_config_updates,monitored_sub_config_update -n subreddit_config_update_worker --autoscale=8,1
+            - CELERY_IMPORTS=redditrepostsleuth.core.celery.tasks.scheduled_tasks,redditrepostsleuth.core.celery.admin_tasks
+        entrypoint: celery -A redditrepostsleuth.core.celery worker -Q subreddit_config_updates,update_wiki_from_database -n subreddit_config_update_worker --autoscale=8,1
 
 

redditrepostsleuth/adminsvc/new_activation_monitor.py

@@ -45,6 +45,8 @@ def activate_sub(self, msg: Message):
         # TODO: API Reduction - No need to call Reddit API after seeing int exists in DB
         try:
             monitored_sub = self._create_monitored_sub_in_db(msg)
+        except ValueError:
+            return
         except Exception as e:
             log.exception('Failed to save new monitored sub', exc_info=True)
             return
@@ -98,7 +100,8 @@ def _create_monitored_sub_in_db(self, msg: Message) -> MonitoredSub:
         with self.uowm.start() as uow:
             existing = uow.monitored_sub.get_by_sub(msg.subreddit.display_name)
             if existing:
-                return existing
+                log.info('Monitored sub %s already exists, skipping activation', msg.subreddit.display_name)
+                raise ValueError(f'Monitored Sub already in database: {msg.subreddit.display_name}')
             monitored_sub = MonitoredSub(**{**DEFAULT_CONFIG_VALUES, **{'name': msg.subreddit.display_name}})
             uow.monitored_sub.add(monitored_sub)
             try:

redditrepostsleuth/core/celery/admin_tasks.py

@@ -9,6 +9,7 @@
 from sqlalchemy.exc import IntegrityError
 
 from redditrepostsleuth.core.celery import celery
+from redditrepostsleuth.core.celery.basetasks import AdminTask
 from redditrepostsleuth.core.config import Config
 from redditrepostsleuth.core.db.databasemodels import MonitoredSub, Post, UserReview
 from redditrepostsleuth.core.db.db_utils import get_db_engine
@@ -26,13 +27,6 @@
 )
 
 
-
-class AdminTask(Task):
-    def __init__(self):
-        self.config = Config()
-        self.uowm = UnitOfWorkManager(get_db_engine(self.config))
-        self.event_logger = EventLogging()
-
 class PyMysqlTask(Task):
     def __init__(self):
         self.config = Config()
@@ -121,10 +115,13 @@ def update_last_deleted_check(self, post_ids: list[int]) -> None:
 
 @celery.task(bind=True, base=AdminTask)
 def update_subreddit_config_from_database(self, monitored_sub: MonitoredSub, user_data: dict) -> NoReturn:
-    self.config_updater.update_wiki_config_from_database(monitored_sub, notify=True)
+    try:
+        self.config_updater.update_wiki_config_from_database(monitored_sub, notify=False)
+    except Exception as e:
+        log.exception('')
     self.config_updater.notification_svc.send_notification(
-        f'r/{monitored_sub.name} config updated on site by {user_data["name"]}',
-        subject='**Config updated on repostsleuth.com**'
+        f'[r/{monitored_sub.name}](https://reddit.com/r/{monitored_sub.name}) config updated on site by [u/{user_data["name"]}](https://reddit.com/u/{user_data["name"]})',
+        subject='Config updated on repostsleuth.com'
     )
 
 

redditrepostsleuth/core/celery/basetasks.py

@@ -53,15 +53,15 @@ def __init__(self):
 class AdminTask(Task):
     def __init__(self):
         self.config = Config()
-        self.reddit = RedditManager(get_reddit_instance(self.config))
+        self.reddit = get_reddit_instance(self.config)
         self.uowm = UnitOfWorkManager(get_db_engine(self.config))
         self.event_logger = EventLogging(config=self.config)
         self.response_handler = ResponseHandler(self.reddit, self.uowm, self.event_logger,
                                                 live_response=self.config.live_responses)
         self.notification_svc = NotificationService(self.config)
         self.config_updater = SubredditConfigUpdater(
             self.uowm,
-            self.reddit.reddit,
+            self.reddit,
             self.response_handler,
             self.config,
             notification_svc=self.notification_svc

redditrepostsleuth/core/celery/celeryconfig.py

@@ -29,6 +29,7 @@
     'redditrepostsleuth.core.celery.admin_tasks.update_proxies_job': {'queue': 'scheduled_tasks'},
     'redditrepostsleuth.core.celery.response_tasks.process_summons':  {'queue': 'summons'},
     'redditrepostsleuth.core.celery.admin_tasks.check_user_for_only_fans': {'queue': 'onlyfans_check'},
+    'redditrepostsleuth.core.celery.admin_tasks.update_subreddit_config_from_database': {'queue': 'update_wiki_from_database'}
 
 
 }

redditrepostsleuth/core/celery/task_logic/scheduled_task_logic.py

@@ -10,7 +10,7 @@
 import requests
 from praw import Reddit
 from praw.exceptions import PRAWException
-from prawcore import NotFound, Forbidden
+from prawcore import NotFound, Forbidden, Redirect
 from sqlalchemy import text, func
 
 from redditrepostsleuth.core.config import Config
@@ -168,7 +168,7 @@ def update_monitored_sub_data(
     if monitored_sub.is_mod:
         if monitored_sub.failed_admin_check_count > 0:
             notification_svc.send_notification(
-                f'Failed admin check for r/{monitored_sub.name} reset',
+                f'Failed admin check for [r/{monitored_sub.name}](https://reddit.com/r/{monitored_sub.name}) reset',
                 subject='Failed Admin Check Reset'
             )
         monitored_sub.failed_admin_check_count = 0
@@ -178,10 +178,10 @@ def update_monitored_sub_data(
         monitored_sub.active = False
         uow.commit()
         notification_svc.send_notification(
-            f'Failed admin check for https://reddit.com/r/{monitored_sub.name} increased to {monitored_sub.failed_admin_check_count}.',
+            f'Failed admin check for [r/{monitored_sub.name}](https://reddit.com/r/{monitored_sub.name}) increased to {monitored_sub.failed_admin_check_count}.',
             subject='Failed Admin Check Increased'
         )
-        return
+
 
     if monitored_sub.failed_admin_check_count == 2:
         subreddit = reddit.subreddit(monitored_sub.name)
@@ -198,12 +198,14 @@ def update_monitored_sub_data(
         return
     elif monitored_sub.failed_admin_check_count >= 4 and monitored_sub.name.lower() != 'dankmemes':
         notification_svc.send_notification(
-            f'Sub r/{monitored_sub.name} failed admin check 4 times.  Removing',
+            f'[r/{monitored_sub.name}](https://reddit.com/r/{monitored_sub.name}) failed admin check {monitored_sub.failed_admin_check_count} times',
             subject='Removing Monitored Subreddit'
         )
         uow.monitored_sub.remove(monitored_sub)
         uow.commit()
         return
+    elif monitored_sub.failed_admin_check_count > 0:
+        log.info('Subreddit %s failed admin check, skipping remaining checks', monitored_sub.name)
 
     try:
         monitored_sub.subscribers = subreddit.subscribers
@@ -212,6 +214,8 @@ def update_monitored_sub_data(
         uow.monitored_sub.remove(monitored_sub)
         uow.commit()
         return
+    except Redirect as e:
+        log.exception('')
 
     monitored_sub.is_private = True if subreddit.subreddit_type == 'private' else False
     monitored_sub.nsfw = True if subreddit.over18 else False

redditrepostsleuth/core/celery/tasks/scheduled_tasks.py

@@ -1,5 +1,5 @@
 from praw.exceptions import PRAWException
-from prawcore import TooManyRequests, Redirect
+from prawcore import TooManyRequests, Redirect, ServerError
 
 from redditrepostsleuth.adminsvc.bot_comment_monitor import BotCommentMonitor
 from redditrepostsleuth.adminsvc.inbox_monitor import InboxMonitor
@@ -166,8 +166,8 @@ def update_daily_stats(self):
             daily_stats.link_reposts_24h = uow.repost.get_count(hours=24, post_type=3)
             daily_stats.link_reposts_total = uow.repost.get_count(post_type=3)
             daily_stats.image_reposts_24h = uow.repost.get_count(hours=24, post_type=2)
-            daily_stats.image_reposts_total = uow.repost.get_count(hours=24, post_type=2)
-            daily_stats.monitored_subreddit_count = uow.monitored_sub.count()
+            daily_stats.image_reposts_total = uow.repost.get_count(post_type=2)
+            daily_stats.monitored_subreddit_count = uow.monitored_sub.get_count()
             uow.stat_daily_count.add(daily_stats)
             uow.commit()
             log.info('[Daily Stat Update] Finished')
@@ -216,6 +216,8 @@ def update_monitored_sub_stats_task(self, sub_name: str) -> None:
             )
     except TooManyRequests:
         raise
+    except ServerError as e:
+        log.warning('Server error checking %s', sub_name)
     except Exception as e:
         log.exception('')
 

redditrepostsleuth/core/services/duplicateimageservice.py

@@ -383,7 +383,7 @@ def _final_meme_filter(self,
                     meme_hashes = get_image_hashes(match.post.url, hash_size=self.config.default_meme_filter_hash_size)
                     match_hash = meme_hashes['dhash_h']
                 except ImageConversionException:
-                    log.error('Failed to get meme hash for %s.  Sending to delete queue', match.post.post_id)
+                    log.warning('Failed to get meme hash for %s.  Sending to delete queue', match.post.post_id)
                     delete_post_task.apply_async((match.post.post_id,))
                     continue
                 except Exception:

redditrepostsleuth/core/util/imagehashing.py

@@ -33,13 +33,17 @@ def generate_img_by_post(post: Post) -> Image:
 
 def generate_img_by_url(url: str) -> Image:
 
-    req = request.Request(
-        url,
-        data=None,
-        headers={
-            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36'
-        }
-    )
+    try:
+        req = request.Request(
+            url,
+            data=None,
+            headers={
+                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36'
+            }
+        )
+    except ValueError as e:
+        log.warning('Problem with URL: %s', e)
+        raise ImageConversionException(str(e))
 
     try:
         response = request.urlopen(req, timeout=10)

redditrepostsleuth/repostsleuthsiteapi/endpoints/monitored_sub.py

@@ -141,7 +141,7 @@ def on_patch(self, req: Request, resp: Response, subreddit: Text):
                 raise HTTPInternalServerError(title='Problem Saving Config', description='Something went tits up when saving the config')
 
         celery.send_task('redditrepostsleuth.core.celery.admin_tasks.update_subreddit_config_from_database', args=[monitored_sub, user_data],
-                         queue='monitored_sub_config_update')
+                         queue='update_wiki_from_database')