Skip mailto: links

aws-samples · Feb 22, 2024 · 75bb7bd · 75bb7bd
1 parent d73f057
commit 75bb7bd
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/lib/shared/layers/python-sdk/python/genai_core/websites/crawler.py b/lib/shared/layers/python-sdk/python/genai_core/websites/crawler.py
@@ -119,6 +119,10 @@ def parse_url(url: str):
 
     for idx in range(len(links)):
         link = links[idx]
+        lowercase_link = link.lower().strip()
+        if lowercase_link.startswith("mailto:"):
+            continue
+
         current = urlparse(link)
         if not current.netloc:
             local_links.append(f"{base_url}{link}")