Fix bug & Update requirements.txt & Update docker-compose.yml & Updat…

…e .env & Sanitize all domain names Fixed a minor bug affecting the export alert functionality on TheHive. - Updated the requirements.txt file to reflect the latest dependencies. - Updated docker-compose.yml configuration. - Updated .env file to ensure correct environment variable setup. - Sanitized all domain names to follow consistent formatting and ensure compatibility.
thalesgroup-cert · Dec 20, 2024 · ab4dbe7 · ab4dbe7
1 parent 824082e
commit ab4dbe7
Show file tree

Hide file tree

Showing 16 changed files with 83 additions and 97 deletions.
diff --git a/.env b/.env
@@ -36,16 +36,16 @@ EMAIL_USE_SSL=False
 EMAIL_HOST_USER=
 EMAIL_HOST_PASSWORD=
 WATCHER_URL=https://example.watcher.local
-WATCHER_LOGO=https://raw.githubusercontent.com/thalesgroup-cert/Watcher/master/Watcher/static/Watcher-logo-simple.png
-EMAIL_SUBJECT_TAG_SITE_MONITORING=INCIDENT
+WATCHER_LOGO=
+GITHUB_LOGO=
+SUBJECT_TAG_SITE_MONITORING=INCIDENT
 EMAIL_CLASSIFICATION=INTERNAL
 
 # THE HIVE Setup
 THE_HIVE_URL=
 THE_HIVE_VERIFY_SSL=False
 THE_HIVE_KEY=
-THE_HIVE_CASE_ASSIGNEE=watcher
-THE_HIVE_TAGS=Watcher,Impersonation,Malicious Domain,Typosquatting
+# Ensure the custom field referenced here is CREATED IN THEHIVE. Otherwise, Alert exports to TheHive will be impacted
 THE_HIVE_CUSTOM_FIELD=watcher-id
 THE_HIVE_EMAIL_SENDER=[email protected]
 

diff --git a/Watcher/Watcher/common/core.py b/Watcher/Watcher/common/core.py
@@ -63,7 +63,7 @@ def generate_ref():
     },
     'website_monitoring': {
         'content_template': (
-            "*[SITE MONITORING - INCIDENT #{ticket_id}] 🔔 {alert_type} on {domain_name} 🔔*\n\n"
+            f"*[{settings.SUBJECT_TAG_SITE_MONITORING}" "{ticket_id}] 🔔 {alert_type} on {domain_name_sanitized} 🔔*\n\n"
             "Dear team,\n\n"
             "Please find the new incident detected below:\n\n"
             "*• Difference Score:* {difference_score}\n"
@@ -82,10 +82,10 @@ def generate_ref():
     },
     'dns_finder': {
         'content_template': (
-            "*[DNS FINDER - ALERT #{alert.pk}] 🚨 Suspicious DNS Detected: {alert.dns_twisted.domain_name} 🚨*\n\n"
+            "*[DNS FINDER - ALERT #{alert.pk}] 🚨 Suspicious DNS Detected: {dns_domain_name_sanitized} 🚨*\n\n"
             "Dear team,\n\n"
             "New Twisted DNS found: \n\n"
-            "*• Twisted DNS:* {alert.dns_twisted.domain_name}\n"
+            "*• Twisted DNS:* {dns_domain_name_sanitized}\n"
             "*• Corporate Keyword:* {alert.dns_twisted.keyword_monitored}\n"
             "*• Corporate DNS:* {alert.dns_twisted.dns_monitored}\n"
             "*• Fuzzer:* {alert.dns_twisted.fuzzer}\n\n"
@@ -98,15 +98,14 @@ def generate_ref():
         'content_template': (
             "*[{alerts_number} ALERTS] 🚨 DNS Finder 🚨*\n\n"
             "Dear team,\n\n"
-            "*{alerts_number}* New DNS Twisted Alerts for *{dns_monitored.domain_name}* asset.\n\n"
+            "*{alerts_number}* New DNS Twisted Alerts for *{dns_domain_name_sanitized_group}* asset.\n\n"
             "Please, find more details <{details_url}|here>."
         ),
         'channel': settings.SLACK_CHANNEL,
         'url_suffix': '#/dns_finder/',
     },
 }
 
-
 # Configuration for Citadel
 APP_CONFIG_CITADEL = {
     'threats_watcher': {
@@ -146,7 +145,7 @@ def generate_ref():
     },
     'website_monitoring': {
         'content_template': (
-            "<p><strong><h4>[SITE MONITORING - INCIDENT #{ticket_id}] 🔔 {alert_type} on {domain_name} 🔔</h4></strong></p>"
+            f"<p><strong><h4>[{settings.SUBJECT_TAG_SITE_MONITORING}{{ticket_id}}] 🔔 {{alert_type}} on {{domain_name_sanitized}} 🔔</h4></strong></p>"
             "<p>Dear team,</p>"
             "<p>Please find the new incident detected below:</p>"
             "<ul>"
@@ -167,11 +166,11 @@ def generate_ref():
     },
     'dns_finder': {
         'content_template': (
-            "<p><strong><h4>[DNS FINDER - ALERT #{alert.pk}] 🚨 Suspicious DNS Detected: {alert.dns_twisted.domain_name} 🚨</h4></strong></p>"
+            "<p><strong><h4>[DNS FINDER - ALERT #{alert.pk}] 🚨 Suspicious DNS Detected: {dns_domain_name_sanitized} 🚨</h4></strong></p>"
             "<p>Dear team,</p>"
             "<p>New Twisted DNS found:</p>"
             "<ul>"
-            "<li><strong>Twisted DNS:</strong> {alert.dns_twisted.domain_name}</li>"
+            "<li><strong>Twisted DNS:</strong> {dns_domain_name_sanitized}</li>"
             "<li><strong>Corporate Keyword:</strong> {alert.dns_twisted.keyword_monitored}</li>"
             "<li><strong>Corporate DNS:</strong> {alert.dns_twisted.dns_monitored}</li>"
             "<li><strong>Fuzzer:</strong> {alert.dns_twisted.fuzzer}</li>"
@@ -185,7 +184,7 @@ def generate_ref():
         'content_template': (
             "<p><strong><h4>[{alerts_number} ALERTS] 🚨 DNS Finder 🚨</h4></strong></p>"
             "<p>Dear team,</p>"
-            "<p><strong>{alerts_number}</strong> New DNS Twisted Alerts for <strong>{dns_monitored.domain_name}</strong> asset.</p>"
+            "<p><strong>{alerts_number}</strong> New DNS Twisted Alerts for <strong>{dns_domain_name_sanitized_group}</strong> asset.</p>"
             "<p>Please, find more details <a href='{details_url}'>here</a>.</p>"
         ),
         'citadel_room_id': settings.CITADEL_ROOM_ID,
@@ -218,12 +217,12 @@ def generate_ref():
         'tags': ["Data Leak", "Watcher", "Sensitive Data", "Leak Detection"]
     },
     'website_monitoring': {
-        'title': "Website Monitoring Detected - {alert_type} on {domain_name}",
+        'title': "Website Monitoring Detected - {alert_type} on {domain_name_sanitized}",
         'description_template': (
             "**Alert:**\n"
             "**New website monitoring incident detected:**\n"
             "*Type of alert:* {alert_type}\n"
-            "*Domain name:* {domain_name}\n"
+            "*Domain name:* {domain_name_sanitized}\n"
             "*• Difference Score:* {difference_score}\n"
             "*• New Ip:* {new_ip}\n"
             "*• Old Ip:* {old_ip}\n"
@@ -235,7 +234,7 @@ def generate_ref():
             "*• Old Mail Server:* {old_mail_A_record_ip}\n"
         ),
         'severity': 2,
-        'tags': ["Website Monitoring", "Watcher", "Incident", "Website", "Domain Name"]
+        'tags': ["Website Monitoring", "Watcher", "Incident", "Website", "Domain Name", "Impersonation" , "Malicious Domain", "Typosquatting"]
     },
     'dns_finder': {
         'title': "New Twisted DNS found - {alert.dns_twisted.domain_name}",
@@ -248,7 +247,7 @@ def generate_ref():
             "*Fuzzer:* {alert.dns_twisted.fuzzer}\n"
         ),
         'severity': 3,
-        'tags': ["DNS Finder", "Watcher", "Twisted DNS", "Corporate Keywords", "Corporate DNS Assets"]
+        'tags': ["DNS Finder", "Watcher", "Twisted DNS", "Corporate Keywords", "Corporate DNS Assets", "Impersonation" , "Malicious Domain", "Typosquatting"]
     },
 }
 
@@ -268,7 +267,7 @@ def generate_ref():
         'template_func': get_data_leak_group_template,
     },
     'website_monitoring': {
-        'subject': '[' + settings.EMAIL_SUBJECT_TAG_SITE_MONITORING + ' #{ticket_id}] Website Monitoring Detected',
+        'subject': '[' + settings.SUBJECT_TAG_SITE_MONITORING + '{ticket_id}] {alert_type} on {domain_name_sanitized}',
         'template_func': get_site_monitoring_template,
     },
     'dns_finder': {
@@ -293,11 +292,13 @@ def collect_observables(app_name, context_data):
     observables = []
 
     if app_name == 'threats_watcher':
-        for word in context_data.get('email_words', []):
-            if re.match(r'^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$', word): 
-                observables.append({"dataType": "domain", "data": word})
+        email_words = context_data.get('email_words', [])
+        for word in email_words:
+            clean_word = re.sub(r'<[^>]*>', '', word) 
+            if re.match(r'^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$', clean_word): 
+                observables.append({"dataType": "domain", "data": clean_word})
             else:
-                observables.append({"dataType": "other", "data": word})
+                observables.append({"dataType": "other", "data": clean_word})
 
     elif app_name == 'website_monitoring':
         site = context_data.get('site')
@@ -399,9 +400,10 @@ def send_notification(channel, content_template, subscribers_filter, send_func,
             if not site:
                 return
             common_data = {
-                'ticket_id': site.ticket_id,
+                'ticket_id': f" - #{site.ticket_id}" if site.ticket_id else '',
                 'alert_type': alert_data.get('type', None),
                 'domain_name': site.domain_name,
+                'domain_name_sanitized': site.domain_name.replace('.', '[.]'),
                 'difference_score': alert_data.get('difference_score', None),
                 'new_ip': alert_data.get('new_ip', None),
                 'old_ip': alert_data.get('old_ip', None),
@@ -454,6 +456,7 @@ def send_notification(channel, content_template, subscribers_filter, send_func,
 
             common_data = {
                 'alert': alert,
+                'dns_domain_name_sanitized': alert.dns_twisted.domain_name.replace('.', '[.]'),
                 'details_url': settings.WATCHER_URL + app_config_slack['url_suffix'],
                 'app_name': 'dns_finder'
             }
@@ -591,7 +594,7 @@ def send_notification(channel, content_template, subscribers_filter, send_func,
 
             common_data = {
                 'alerts_number': alerts_number,
-                'dns_monitored': dns_monitored,
+                'dns_domain_name_sanitized_group': dns_monitored.domain_name.replace('.', '[.]'),
                 'details_url': settings.WATCHER_URL + app_config_slack['url_suffix'],
                 'app_name': 'dns_finder_group',
             }

diff --git a/Watcher/Watcher/common/mail_template/data_leak_group_template.py b/Watcher/Watcher/common/mail_template/data_leak_group_template.py
@@ -114,7 +114,7 @@ def get_data_leak_group_template(keyword, alerts_number):
                 <tr>
                     <!-- Header -->
                     <td class="header" colspan="2">
-                        <img src=""" + str(settings.WATCHER_LOGO_BASE64) + """ " alt="Watcher Logo">
+                        <img src=""" + str(settings.WATCHER_LOGO) + """ " alt="Watcher Logo">
                         <h1>Data Leak Group Alerts</h1>
                     </td>
                 </tr>

diff --git a/Watcher/Watcher/common/mail_template/data_leak_template.py b/Watcher/Watcher/common/mail_template/data_leak_template.py
@@ -111,7 +111,7 @@ def get_data_leak_template(alert):
                 <tr>
                     <!-- Header -->
                     <td class="header" colspan="2">
-                        <img src=\"""" + str(settings.WATCHER_LOGO_BASE64) + """ " alt="Threats Watcher Logo">
+                        <img src=\"""" + str(settings.WATCHER_LOGO) + """ " alt="Threats Watcher Logo">
                         <h1>Data Leak Alert #""" + str(alert.pk) + """</h1>
                     </td>
                 </tr>

diff --git a/Watcher/Watcher/common/mail_template/dns_finder_cert_transparency.py b/Watcher/Watcher/common/mail_template/dns_finder_cert_transparency.py
@@ -114,7 +114,7 @@ def get_dns_finder_cert_transparency_template(alert):
                 <tr>
                     <!-- Header -->
                     <td class="header" colspan="2">
-                        <img src=\"""" + str(settings.WATCHER_LOGO_BASE64) + """ " alt="Threats Watcher Logo">
+                        <img src=\"""" + str(settings.WATCHER_LOGO) + """ " alt="Threats Watcher Logo">
                         <h1>DNS Finder: Alert #""" + str(alert.pk) + """</h1>
                     </td>
                 </tr>

diff --git a/Watcher/Watcher/common/mail_template/dns_finder_group_template.py b/Watcher/Watcher/common/mail_template/dns_finder_group_template.py
@@ -123,7 +123,7 @@ def get_dns_finder_group_template(dns_monitored, alerts_number):
                 <tr>
                     <!-- Header -->
                     <td class="header" colspan="2">
-                        <img src="{settings.WATCHER_LOGO_BASE64}" alt="DNS Finder Logo">
+                        <img src="{settings.WATCHER_LOGO}" alt="DNS Finder Logo">
                         <h1>DNS Finder Group Alerts</h1>
                     </td>
                 </tr>

diff --git a/Watcher/Watcher/common/mail_template/dns_finder_template.py b/Watcher/Watcher/common/mail_template/dns_finder_template.py
@@ -116,7 +116,7 @@ def get_dns_finder_template(alert):
                 <tr>
                     <!-- Header -->
                     <td class="header" colspan="2">
-                        <img src="{settings.WATCHER_LOGO_BASE64}" alt="Watcher Logo">
+                        <img src="{settings.WATCHER_LOGO}" alt="Watcher Logo">
                         <h1>DNS Finder: Alert #{alert.pk}</h1>
                     </td>
                 </tr>

diff --git a/Watcher/Watcher/common/mail_template/site_monitoring_template.py b/Watcher/Watcher/common/mail_template/site_monitoring_template.py
@@ -130,7 +130,7 @@ def get_site_monitoring_template(website_status, website_url, alert_id):
                 <tr>
                     <!-- Header -->
                     <td class="header" colspan="2">
-                        <img src=\"""" + str(settings.WATCHER_LOGO_BASE64) + """ " alt="Watcher Logo">
+                        <img src=\"""" + str(settings.WATCHER_LOGO) + """ " alt="Watcher Logo">
                         <h1>Website Monitoring</h1>
                     </td>
                 </tr>

diff --git a/Watcher/Watcher/common/mail_template/threats_watcher_template.py b/Watcher/Watcher/common/mail_template/threats_watcher_template.py
@@ -114,15 +114,15 @@ def get_threats_watcher_template(words_occurrence, email_words):
                 <tr>
                     <!-- Header -->
                     <td class="header" colspan="2">
-                        <img src=\"""" + str(settings.WATCHER_LOGO_BASE64) + """ " alt="Threats Watcher Logo">
+                        <img src=\"""" + str(settings.WATCHER_LOGO) + """ " alt="Threats Watcher Logo">
                         <h1>Threats Watcher</h1>
                     </td>
                 </tr>
                 <!-- Content -->
                 <tr>
                     <td class="content" colspan="2">
                         <p>Dear team,</p>
-                        <p>Please find below trendy word(s) that match at least <strong>""" + str(words_occurrence) + """</strong> times:</p>
+                        <p>Please find the new trendy word(s) detected below:</p>
                         <div class="word-list">
                             """ + "<p>".join(email_words) + """
                         </div>

diff --git a/Watcher/Watcher/common/utils/send_thehive_alerts.py b/Watcher/Watcher/common/utils/send_thehive_alerts.py
@@ -80,6 +80,11 @@ def send_thehive_alert(title, description, severity, tags, app_name, domain_name
     if ticket_id is None:
         ticket_id = generate_ref()
 
+    if app_name == 'website_monitoring' and domain_name:
+        site = Site.objects.get(domain_name=domain_name)
+        site.ticket_id = ticket_id 
+        site.save()
+
     if app_name == 'website_monitoring' and not ticket_id:
         return
 

diff --git a/Watcher/Watcher/common/utils/update_thehive.py b/Watcher/Watcher/common/utils/update_thehive.py
@@ -100,12 +100,16 @@ def create_observables(observables):
     :return: A list of formatted observables ready for TheHive.
     :rtype: list
     """
+
+    current_time = timezone.now().strftime("%H:%M:%S")
+    current_date = timezone.now().strftime("%d/%m/%y")
+
     observables_data = []
     for obs in observables:
         observable_data = {
             "dataType": obs['dataType'],
             "data": obs['data'],
-            "message": f"Observable added: {obs['data']} on {timezone.now()}",
+            "message": f"An observable was added on {current_date} at {current_time}.",
             "ioc": True,
             "sighted": True,
             "tlp": 2
@@ -130,7 +134,6 @@ def update_existing_alert_case(item_type, existing_item, observables, comment, t
 
     if observables:
         observables_data = create_observables(observables)
-        print(f"{timezone.now()} - Adding observables to {item_type} with ID {item_id}...")
         add_observables_to_item(item_type, item_id, observables_data, thehive_url, api_key)
 
     if comment:
@@ -224,10 +227,10 @@ def handle_alert_or_case(ticket_id, observables, comment, title, description, se
     case_type, case_item = search_thehive_for_ticket_id(ticket_id, thehive_url, api_key, item_type="case")
 
     if case_item:
-        print(f"{timezone.now()} - Case found for {settings.THE_HIVE_CUSTOM_FIELD} {ticket_id}. Updating...")
+        print(f"{timezone.now()} - Case found for {settings.THE_HIVE_CUSTOM_FIELD} {ticket_id}. Proceeding with update.")
         update_existing_alert_case("case", case_item, observables, comment, thehive_url, api_key)
     elif alert_item:
-        print(f"{timezone.now()} - Alert found for {settings.THE_HIVE_CUSTOM_FIELD} {ticket_id}. Updating...")
+        print(f"{timezone.now()} - Alert found for {settings.THE_HIVE_CUSTOM_FIELD} {ticket_id}. Proceeding with update.")
         update_existing_alert_case("alert", alert_item, observables, comment, thehive_url, api_key)
     else:
         create_new_alert(

diff --git a/Watcher/Watcher/dns_finder/core.py b/Watcher/Watcher/dns_finder/core.py
@@ -22,8 +22,7 @@ def start_scheduler():
         - Fire main_certificate_transparency from Monday to Sunday: every hour.
     """
     scheduler = BackgroundScheduler(timezone=str(tzlocal.get_localzone()))
-    # scheduler.add_job(main_dns_twist, 'cron', day_of_week='mon-sun', hour='*/2', id='main_dns_twist',
-    scheduler.add_job(main_dns_twist, 'cron', day_of_week='mon-sun', minute='*/2', id='main_dns_twist',
+    scheduler.add_job(main_dns_twist, 'cron', day_of_week='mon-sun', hour='*/2', id='main_dns_twist',
                       max_instances=10,
                       replace_existing=True)
     scheduler.add_job(main_certificate_transparency, 'cron', day_of_week='mon-sun', hour='*/1',

diff --git a/Watcher/Watcher/threats_watcher/core.py b/Watcher/Watcher/threats_watcher/core.py
@@ -11,9 +11,6 @@
 import feedparser
 import requests
 import re
-from email.mime.multipart import MIMEMultipart
-from email.mime.text import MIMEText
-import smtplib
 from django.db import close_old_connections
 from common.core import send_app_specific_notifications
 from django.db.models import Q