Merge pull request #273 from massi-ang/fix_crawler

fix(crawler): only process "text/html" Content-Type pages
aws-samples · Dec 16, 2023 · f9277a7 · f9277a7
2 parents 7a3e918 + 17d6da4
commit f9277a7
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/lib/shared/layers/python-sdk/python/genai_core/websites/crawler.py b/lib/shared/layers/python-sdk/python/genai_core/websites/crawler.py
@@ -101,6 +101,8 @@ def parse_url(url: str):
     base_url = f"{root_url_parse.scheme}://{root_url_parse.netloc}"
 
     response = requests.get(url, timeout=20)
+    if response.headers["Content-Type"] != "text/html":
+        raise Exception(f"Invalid content type {response.headers['Content-Type']}")
     soup = BeautifulSoup(response.content, "html.parser")
     content = soup.text
     content = re.sub(r"[ \n]+", " ", content)