Adding retry to requests get

bukosabino · Jul 8, 2024 · 19d9a78 · 19d9a78
1 parent df47000
commit 19d9a78
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/src/etls/boa/scrapper.py b/src/etls/boa/scrapper.py
@@ -125,7 +125,7 @@ def download_day(self, day: date) -> tp.List[BOAMetadataDocument]:
                      # 'SECC-C':'BOA%2Bo%2BDisposiciones%2Bo%2BPersonal%2Bo%2BAcuerdos%2Bo%2BJusticia%2Bo%2BAnuncios' 
                      }
             session = create_retry_session(retries=5)
-            response = session.get(self.base_url, params=params)
+            response = session.get(self.base_url, params=params, timeout=10)
             raw_result = response.text
             if '<span class="titulo">No se han recuperado documentos</span>' in raw_result:
                 logger.info(f"No hay contenido disponible para el día {day}")

diff --git a/src/etls/bopv/scrapper.py b/src/etls/bopv/scrapper.py
@@ -135,7 +135,7 @@ def download_document(self, url: str) -> BOPVMetadataDocument:
         logger.info("Scrapping document: %s", url)
         try:
             session = create_retry_session(retries=5)
-            response = session.get(url, headers=self.headers)
+            response = session.get(url, headers=self.headers, timeout=10)
             if response.status_code != 200:
                 response.raise_for_status() 
             soup = BeautifulSoup(response.content, "html.parser")

diff --git a/src/etls/bopz/scrapper.py b/src/etls/bopz/scrapper.py
@@ -124,7 +124,7 @@ def download_document(self, url: str, metadata: BeautifulSoup) -> BOPZMetadataDo
         logger = lg.getLogger(self.download_document.__name__)
         logger.info("Scrapping document: %s", url)
         session = create_retry_session(retries=5)
-        response = session.get(url)
+        response = session.get(url, timeout=10)
         response.raise_for_status()
         soup = BeautifulSoup(response.text, "lxml")
         with tempfile.NamedTemporaryFile("w", delete=False) as fn: