Skip to content

Commit

Permalink
Restart on urllib3 ReadTimeoutError exceptions too
Browse files Browse the repository at this point in the history
Started running into these in Chrome scans for some reason.
  • Loading branch information
ghostwords committed Nov 8, 2024
1 parent a33a030 commit 782d00e
Showing 1 changed file with 3 additions and 3 deletions.
6 changes: 3 additions & 3 deletions crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from fnmatch import fnmatch
from pprint import pformat
from shutil import copytree
from urllib3.exceptions import MaxRetryError, ProtocolError
from urllib3.exceptions import MaxRetryError, ProtocolError, ReadTimeoutError
from urllib.parse import urljoin, urlparse

from selenium import webdriver
Expand Down Expand Up @@ -609,7 +609,7 @@ def _load_ext_page():
try:
self.handle_alerts_and(_load_ext_page)
break
except (MaxRetryError, ProtocolError) as e:
except (MaxRetryError, ProtocolError, ReadTimeoutError) as e:
self.logger.warning("Error loading extension page:\n%s", str(e))
self.restart_browser()
except TimeoutException:
Expand Down Expand Up @@ -1043,7 +1043,7 @@ def crawl(self):
self.logger.info("Visiting %d: %s", i + 1, domain)
url = self.get_domain(domain)
visited.append(url)
except (MaxRetryError, ProtocolError) as e:
except (MaxRetryError, ProtocolError, ReadTimeoutError) as e:
self.logger.warning("Error loading %s:\n%s", domain, str(e))
self.restart_browser()
except TimeoutException:
Expand Down

0 comments on commit 782d00e

Please sign in to comment.