Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dev merge #379

Merged
merged 11 commits into from
May 27, 2024
2 changes: 1 addition & 1 deletion docker-compose-infra.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ services:
- '6379:6379'
volumes:
- /config/redis:/data
command: redis-server --save 60 1
command: redis-server --save 550 1 --maxmemory 40gb

influxdb:
image: influxdb:latest
Expand Down
26 changes: 21 additions & 5 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,22 @@ services:
- CELERY_IMPORTS=redditrepostsleuth.core.celery.tasks.monitored_sub_tasks
entrypoint: celery -A redditrepostsleuth.core.celery worker -Q submonitor -n submonitor_worker --autoscale=6,2

reddit_actions_worker:
container_name: reddit-actions-worker
restart: unless-stopped
user: '1001'
build:
context: .
dockerfile: docker/WorkerDockerFile
env_file:
- .env
environment:
- RUN_ENV=production
- db_user=sub_monitor
- LOG_LEVEL=INFO
- CELERY_IMPORTS=redditrepostsleuth.core.celery.tasks.reddit_action_tasks
entrypoint: celery -A redditrepostsleuth.core.celery worker -Q reddit_actions -n reddit_actions --autoscale=3,2

ingest_worker:
restart: unless-stopped
container_name: ingest-worker
Expand All @@ -134,9 +150,9 @@ services:
environment:
- RUN_ENV=production
- db_user=ingest
- LOG_LEVEL=ERROR
- LOG_LEVEL=INFO
- CELERY_IMPORTS=redditrepostsleuth.core.celery.tasks.ingest_tasks
entrypoint: celery -A redditrepostsleuth.core.celery worker -Q post_ingest -n ingest_worker --autoscale=3,16
entrypoint: celery -A redditrepostsleuth.core.celery worker -Q post_ingest -n ingest_worker --autoscale=16,1

link_repost_worker:
container_name: link-repost-worker
Expand Down Expand Up @@ -181,9 +197,9 @@ services:
- .env
environment:
- RUN_ENV=production
- LOG_LEVEL=INFO
- LOG_LEVEL=WARNING
- CELERY_IMPORTS=redditrepostsleuth.core.celery.admin_tasks
entrypoint: celery -A redditrepostsleuth.core.celery worker -Q onlyfans_check -n onlyfans_worker --autoscale=8,5
entrypoint: celery -A redditrepostsleuth.core.celery worker -Q onlyfans_check -n onlyfans_worker --autoscale=10,5

subreddit_config_update_worker:
container_name: subreddit-config-update-worker
Expand All @@ -198,6 +214,6 @@ services:
- RUN_ENV=production
- LOG_LEVEL=INFO
- CELERY_IMPORTS=redditrepostsleuth.core.celery.tasks.scheduled_tasks,redditrepostsleuth.core.celery.admin_tasks
entrypoint: celery -A redditrepostsleuth.core.celery worker -Q subreddit_config_updates,update_wiki_from_database -n subreddit_config_update_worker --autoscale=8,1
entrypoint: celery -A redditrepostsleuth.core.celery worker -Q batch_delete_searches,subreddit_config_updates,update_wiki_from_database -n subreddit_config_update_worker --autoscale=8,1


5 changes: 5 additions & 0 deletions docs/dev_docs/modifying_monitored_sub.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@

### Adding or Removing Config Values
* Add / Remove config values in core/db/databasemodels.py
* Add/Remove in core/util/default_bot_config.py
* Update sub_monitor_exposed_config_options in the config json
3 changes: 2 additions & 1 deletion redditrepostsleuth/adminsvc/inbox_monitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ def check_inbox(self):
print('[Scheduled Job] Checking Inbox Start')
for msg in self.reddit.inbox.messages(limit=500):
if msg.author != 'RepostSleuthBot' and msg.subject.lower() in ['false negative', 'false positive']:
self._process_user_report(msg)
#self._process_user_report(msg)
pass
elif TOP_POST_WATCH_SUBJECT.lower() in msg.subject.lower():
self._process_watch_request(msg)

Expand Down
17 changes: 10 additions & 7 deletions redditrepostsleuth/adminsvc/new_activation_monitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from praw.models import Subreddit, Message
from prawcore import TooManyRequests

from redditrepostsleuth.core.celery.tasks.reddit_action_tasks import send_modmail_task
from redditrepostsleuth.core.config import Config
from redditrepostsleuth.core.db.databasemodels import MonitoredSub
from redditrepostsleuth.core.db.db_utils import get_db_engine
Expand Down Expand Up @@ -36,7 +37,7 @@ def __init__(
def check_for_new_invites(self):
for msg in self.reddit.inbox.messages(limit=1000):
if 'invitation to moderate' in msg.subject:
log.info('Found invitation for %s', msg.subreddit.display_name)
log.debug('Found invitation for %s', msg.subreddit.display_name)
self.activate_sub(msg)


Expand Down Expand Up @@ -78,11 +79,13 @@ def _notify_added(self, subreddit: Subreddit) -> NoReturn:
log.info('Sending success PM to %s', subreddit.display_name)
wiki_url = f'https://www.reddit.com/r/{subreddit.display_name}/wiki/repost_sleuth_config'
try:
self.response_handler.send_mod_mail(
subreddit.display_name,
MONITORED_SUB_ADDED.format(wiki_config=wiki_url),
'Repost Sleuth Activated',
source='activation'
send_modmail_task.apply_async(
(
subreddit.display_name,
MONITORED_SUB_ADDED.format(wiki_config=wiki_url),
'Repost Sleuth Activated',
),
{'source': 'activation'}
)
monitored_sub.activation_notification_sent = True
except RedditAPIException as e:
Expand All @@ -100,7 +103,7 @@ def _create_monitored_sub_in_db(self, msg: Message) -> MonitoredSub:
with self.uowm.start() as uow:
existing = uow.monitored_sub.get_by_sub(msg.subreddit.display_name)
if existing:
log.info('Monitored sub %s already exists, skipping activation', msg.subreddit.display_name)
log.debug('Monitored sub %s already exists, skipping activation', msg.subreddit.display_name)
raise ValueError(f'Monitored Sub already in database: {msg.subreddit.display_name}')
monitored_sub = MonitoredSub(**{**DEFAULT_CONFIG_VALUES, **{'name': msg.subreddit.display_name}})
uow.monitored_sub.add(monitored_sub)
Expand Down
56 changes: 51 additions & 5 deletions redditrepostsleuth/core/celery/admin_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@
from redditrepostsleuth.core.exception import UtilApiException, UserNotFound
from redditrepostsleuth.core.logfilters import ContextFilter
from redditrepostsleuth.core.logging import log, configure_logger
from redditrepostsleuth.core.util.onlyfans_handling import check_user_for_promoter_links
from redditrepostsleuth.core.util.helpers import chunk_list
from redditrepostsleuth.core.util.onlyfans_handling import check_user_for_promoter_links, \
check_user_comments_for_promoter_links

log = configure_logger(
name='redditrepostsleuth',
Expand Down Expand Up @@ -124,27 +126,28 @@ def update_subreddit_config_from_database(self, monitored_sub: MonitoredSub, use
)


@celery.task(bind=True, base=AdminTask, autoretry_for=(UtilApiException,ConnectionError,TooManyRequests), retry_kwards={'max_retries': 3})
@celery.task(bind=True, base=AdminTask, autoretry_for=(UtilApiException,ConnectionError,TooManyRequests), retry_kwards={'max_retries': 3, 'retry_backoff': True})
def check_user_for_only_fans(self, username: str) -> None:
skip_names = ['[deleted]', 'AutoModerator']

if username in skip_names:
log.info('Skipping name %s', username)
return

try:
with self.uowm.start() as uow:
user = uow.user_review.get_by_username(username)

if user:
delta = datetime.utcnow() - user.last_checked
if delta.days < 30:
log.debug('Skipping existing user %s, last check was %s days ago', username, delta.days)
log.info('Skipping existing user %s, last check was %s days ago', username, delta.days)
return
user.content_links_found = False
user.notes = None
user.last_checked = func.utc_timestamp()

log.debug('Checking user %s', username)
log.info('Checking user %s', username)
if not user:
user = UserReview(username=username)
try:
Expand All @@ -164,4 +167,47 @@ def check_user_for_only_fans(self, username: str) -> None:
except IntegrityError:
pass
except Exception as e:
log.exception('')
log.exception('')


@celery.task(bind=True, base=AdminTask, autoretry_for=(UtilApiException,ConnectionError,TooManyRequests), retry_kwards={'max_retries': 3})
def check_user_comments_for_only_fans(self, username: str) -> None:
"""
This should be run after the profile check so we don't do any timeframe checking
:param self:
:param username:
:return:
"""
skip_names = ['[deleted]', 'AutoModerator']

if username in skip_names:
log.info('Skipping name %s', username)
return

try:
with self.uowm.start() as uow:
user = uow.user_review.get_by_username(username)

if not user:
log.error('User not found: %s', username)

try:
result = check_user_comments_for_promoter_links(username)
except UserNotFound as e:
log.warning(e)
return

if result:
log.info('Promoter found: %s - %s', username, str(result))
user.content_links_found = True
user.notes = str(result)
uow.user_review.add(user)
uow.commit()
except (UtilApiException, ConnectionError, TooManyRequests) as e:
raise e
except IntegrityError:
pass
except Exception as e:
log.exception('')


1 change: 0 additions & 1 deletion redditrepostsleuth/core/celery/basetasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
from redditrepostsleuth.core.db.uow.unitofworkmanager import UnitOfWorkManager
from redditrepostsleuth.core.notification.notification_service import NotificationService
from redditrepostsleuth.core.services.eventlogging import EventLogging
from redditrepostsleuth.core.services.reddit_manager import RedditManager
from redditrepostsleuth.core.services.response_handler import ResponseHandler
from redditrepostsleuth.core.services.subreddit_config_updater import SubredditConfigUpdater
from redditrepostsleuth.core.util.helpers import get_reddit_instance
Expand Down
42 changes: 24 additions & 18 deletions redditrepostsleuth/core/celery/celeryconfig.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,19 @@
'redditrepostsleuth.core.celery.tasks.scheduled_tasks.*': {'queue': 'scheduled_tasks'},
'redditrepostsleuth.core.celery.admin_tasks.update_proxies_job': {'queue': 'scheduled_tasks'},
'redditrepostsleuth.core.celery.admin_tasks.check_user_for_only_fans': {'queue': 'onlyfans_check'},
'redditrepostsleuth.core.celery.admin_tasks.update_subreddit_config_from_database': {'queue': 'update_wiki_from_database'}
'redditrepostsleuth.core.celery.admin_tasks.update_subreddit_config_from_database': {'queue': 'update_wiki_from_database'},
'redditrepostsleuth.core.celery.admin_tasks.delete_search_batch': {'queue': 'batch_delete_searches'},
'redditrepostsleuth.core.celery.tasks.reddit_action_tasks.*': {'queue': 'reddit_actions'},


}


beat_schedule = {
'update-proxy-list': {
'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_proxies_task',
'schedule': 3600
},
# 'update-proxy-list': {
# 'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_proxies_task',
# 'schedule': 3600
# },
'check-inbox': {
'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.check_inbox_task',
'schedule': 300
Expand All @@ -60,7 +62,7 @@
'schedule': 300
},
'update-top-reposts': {
'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_top_reposts_task',
'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_all_top_reposts_task',
'schedule': 86400
},
'update-top-reposters': {
Expand All @@ -71,26 +73,30 @@
'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_daily_top_reposters_task',
'schedule': 900
},
'send-reports-to-meme-voting': {
'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.send_reports_to_meme_voting_task',
'schedule': 3600
},
'check-meme-template-potential-votes': {
'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.check_meme_template_potential_votes_task',
'schedule': 1800
},
# 'send-reports-to-meme-voting': {
# 'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.send_reports_to_meme_voting_task',
# 'schedule': 3600
# },
# 'check-meme-template-potential-votes': {
# 'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.check_meme_template_potential_votes_task',
# 'schedule': 1800
# },
'monitored-sub-config-update': {
'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.queue_config_updates_task',
'schedule': 3600
},
'update-profile-token': {
'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_profile_token_task',
'schedule': 120
},
# 'update-profile-token': {
# 'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_profile_token_task',
# 'schedule': 120
# },
'update-daily-stats': {
'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_daily_stats',
'schedule': 86400
},
'search-history-cleanup': {
'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.queue_search_history_cleanup',
'schedule': 3600
},

}

Expand Down
Loading
Loading