Skip to content

Commit

Permalink
Misc bug fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
barrycarey committed Oct 13, 2023
1 parent 42d5fd6 commit fd613f1
Show file tree
Hide file tree
Showing 6 changed files with 139 additions and 6 deletions.
2 changes: 1 addition & 1 deletion redditrepostsleuth/core/celery/admin_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ def update_subreddit_config_from_database(self, monitored_sub: MonitoredSub, use

@celery.task(bind=True, base=AdminTask, autoretry_for=(UtilApiException,ConnectionError,TooManyRequests), retry_kwards={'max_retries': 3})
def check_user_for_only_fans(self, username: str) -> None:
skip_names = ['[deleted]']
skip_names = ['[deleted]', 'AutoModerator']

if username in skip_names:
log.info('Skipping name %s', username)
Expand Down
3 changes: 2 additions & 1 deletion redditrepostsleuth/core/celery/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from celery import Celery, signals
from celery.signals import after_setup_logger
from kombu.serialization import registry
from prawcore import TooManyRequests

from redditrepostsleuth.core.exception import IngestHighMatchMeme, ImageConversionException

Expand All @@ -28,7 +29,7 @@ def init_sentry(**_kwargs):
monitor_beat_tasks=True,
),
],
ignore_errors=[IngestHighMatchMeme, ImageConversionException, WorkerLostError]
ignore_errors=[IngestHighMatchMeme, ImageConversionException, WorkerLostError, TooManyRequests]
)

@after_setup_logger.connect
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -143,8 +143,9 @@ def token_checker() -> None:
if token_res.status_code != 200:
log.error('Failed to get new token')
return
decoded_token = jwt.decode(json.loads(token_res.text), '', algorithms=["HS256"], options={"verify_signature": False})
redis_client.set('prof_token', decoded_token['sub'])
#decoded_token = jwt.decode(json.loads(token_res.text), '', algorithms=["HS256"], options={"verify_signature": False})
new_token = json.loads(token_res.text)
redis_client.set('prof_token', new_token)
log.info('New token set in Redis')

def update_monitored_sub_data(
Expand Down
123 changes: 123 additions & 0 deletions redditrepostsleuth/core/celery/tasks/repost_tasks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
from typing import NoReturn

import requests
from redlock import RedLockError
from requests.exceptions import ConnectTimeout
from sqlalchemy import func

from redditrepostsleuth.core.celery import celery
from redditrepostsleuth.core.celery.basetasks import AnnoyTask, RedditTask, RepostTask
from redditrepostsleuth.core.celery.task_logic.repost_image import repost_watch_notify, check_for_post_watch
from redditrepostsleuth.core.db.databasemodels import Post, RepostWatch
from redditrepostsleuth.core.exception import NoIndexException, IngestHighMatchMeme, IndexApiException
from redditrepostsleuth.core.logfilters import ContextFilter
from redditrepostsleuth.core.logging import log, configure_logger
from redditrepostsleuth.core.model.search.search_match import SearchMatch
from redditrepostsleuth.core.util.helpers import get_default_link_search_settings, get_default_image_search_settings, \
get_default_text_search_settings
from redditrepostsleuth.core.util.repost.repost_helpers import filter_search_results
from redditrepostsleuth.core.util.repost.repost_search import text_search_by_post, image_search_by_post, \
link_search

log = configure_logger(
name='redditrepostsleuth',
format='%(asctime)s - %(module)s:%(funcName)s:%(lineno)d - Trace_ID=%(trace_id)s Post_ID=%(post_id)s Subreddit=%(subreddit)s Service=%(service)s Level=%(levelname)s Message=%(message)s',
filters=[ContextFilter()]
)


@celery.task(bind=True, base=AnnoyTask, serializer='pickle', ignore_results=True, autoretry_for=(RedLockError, NoIndexException, IngestHighMatchMeme), retry_kwargs={'max_retries': 20, 'countdown': 300})
def check_image_repost_save(self, post: Post) -> NoReturn:

try:
r = requests.head(post.url)
if r.status_code != 200:
log.info('Skipping image that is deleted %s', post.url)
celery.send_task('redditrepostsleuth.core.celery.admin_tasks.delete_post_task', args=[post.post_id])
return

search_settings = get_default_image_search_settings(self.config)
search_settings.max_matches = 75
search_results = self.dup_service.check_image(
post.url,
post=post,
search_settings=search_settings,
source='ingest'
)
with self.uowm.start() as uow:
search_results = image_search_by_post(
post,
uow,
self.dup_service,
search_settings,
'ingest',
high_match_meme_check=True
)

if search_results.matches:
watches = check_for_post_watch(search_results.matches, uow)
if watches and self.config.enable_repost_watch:
notify_watch.apply_async((watches, post), queue='watch_notify')

except (RedLockError, NoIndexException, IngestHighMatchMeme):
raise
except (ConnectTimeout):
log.warning('Failed to validate image url at %s', post.url)
except Exception as e:
log.exception('')


@celery.task(bind=True, base=RepostTask, ignore_results=True, serializer='pickle')
def link_repost_check(self, post):

try:
with self.uowm.start() as uow:

log.debug('Checking URL for repost: %s', post.url)
search_results = link_search(post.url, uow,
get_default_link_search_settings(self.config),
'ingest',
post=post,
filter_function=filter_search_results
)

search_results.search_times.stop_timer('total_search_time')
log.info('Link Query Time: %s', search_results.search_times.query_time)



except Exception as e:
log.exception('')


@celery.task(bind=True, base=RepostTask, ignore_results=True, serializer='pickle')
def check_for_text_repost_task(self, post: Post) -> None:
log.debug('Checking post for repost: %s', post.post_id)
try:
with self.uowm.start() as uow:
search_results = text_search_by_post(
post,
uow,
get_default_text_search_settings(self.config),
filter_function=filter_search_results,
source='ingest'
)
log.info('Found %s matching text posts', len(search_results.matches))
except IndexApiException as e:
log.warning(e, exc_info=False)
raise
except Exception as e:
log.exception('Unknown exception during test repost check')


@celery.task(bind=True, base=RedditTask, ignore_results=True)
def notify_watch(self, watches: list[dict[SearchMatch, RepostWatch]], repost: Post):
repost_watch_notify(watches, self.reddit, self.response_handler, repost)
with self.uowm.start() as uow:
for w in watches:
w['watch'].last_detection = func.utc_timestamp()
uow.repostwatch.update(w['watch'])
try:
uow.commit()
except Exception as e:
log.exception('Failed to save repost watch %s', w['watch'].id, exc_info=True)
2 changes: 2 additions & 0 deletions redditrepostsleuth/core/util/replytemplates.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@

UNSUPPORTED_POST_TYPE = 'Sorry, I don\'t support this post type ({post_type}) right now. Feel free to check back in the future!'

UNKNOWN_SUBMISSION_TYPE = 'Sorry, I\'m not able to determine the type of post and cannot process your request. '

WIKI_STATS = '### Submission Index Stats\n\n**Total Posts:** {post_count}\n\n**Image Posts:** {images}\n\n**Link Posts:** {links}\n\n**Video Posts:** {video}\n\n **Text Posts:** {text}\n\n **Oldest Post:** {oldest}\n\n### Repost Statistics\n\n**Image Reposts:** {image_reposts}\n\n**Times Summoned:** {summoned}'


Expand Down
10 changes: 8 additions & 2 deletions redditrepostsleuth/summonssvc/summonshandler.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
get_default_link_search_settings, get_default_text_search_settings
from redditrepostsleuth.core.util.replytemplates import UNSUPPORTED_POST_TYPE, WATCH_ENABLED, \
WATCH_ALREADY_ENABLED, WATCH_DISABLED_NOT_FOUND, WATCH_DISABLED, \
SUMMONS_ALREADY_RESPONDED, BANNED_SUB_MSG, OVER_LIMIT_BAN
SUMMONS_ALREADY_RESPONDED, BANNED_SUB_MSG, OVER_LIMIT_BAN, UNKNOWN_SUBMISSION_TYPE
from redditrepostsleuth.core.util.repost.repost_helpers import filter_search_results, \
save_image_repost_result
from redditrepostsleuth.core.util.repost.repost_search import image_search_by_post, link_search, text_search_by_post
Expand Down Expand Up @@ -81,7 +81,13 @@ def process_summons(self, summons: Summons) -> None:
if self.summons_disabled:
self._send_summons_disable_msg(summons)

if summons.post.post_type.name is None or summons.post.post_type.name not in self.config.supported_post_types:
if summons.post.post_type is None:
log.warning('Post %s has no post type. Cannot process summons', summons.post.post_id)
response = SummonsResponse(summons=summons, message=UNKNOWN_SUBMISSION_TYPE)
self._send_response(response)
return

if summons.post.post_type.name not in self.config.supported_post_types:
log.warning('Post %s: Type %s not supported', f'https://redd.it/{summons.post.post_id}', summons.post.post_type.name)
self._send_unsupported_msg(summons)
return
Expand Down

0 comments on commit fd613f1

Please sign in to comment.