diff --git a/src/etls/boa/scrapper.py b/src/etls/boa/scrapper.py index 7c54ff7..e67d925 100644 --- a/src/etls/boa/scrapper.py +++ b/src/etls/boa/scrapper.py @@ -125,7 +125,7 @@ def download_day(self, day: date) -> tp.List[BOAMetadataDocument]: # 'SECC-C':'BOA%2Bo%2BDisposiciones%2Bo%2BPersonal%2Bo%2BAcuerdos%2Bo%2BJusticia%2Bo%2BAnuncios' } session = create_retry_session(retries=5) - response = session.get(self.base_url, params=params) + response = session.get(self.base_url, params=params, timeout=10) raw_result = response.text if 'No se han recuperado documentos' in raw_result: logger.info(f"No hay contenido disponible para el día {day}") diff --git a/src/etls/bopv/scrapper.py b/src/etls/bopv/scrapper.py index a96dbf8..176c312 100644 --- a/src/etls/bopv/scrapper.py +++ b/src/etls/bopv/scrapper.py @@ -135,7 +135,7 @@ def download_document(self, url: str) -> BOPVMetadataDocument: logger.info("Scrapping document: %s", url) try: session = create_retry_session(retries=5) - response = session.get(url, headers=self.headers) + response = session.get(url, headers=self.headers, timeout=10) if response.status_code != 200: response.raise_for_status() soup = BeautifulSoup(response.content, "html.parser") diff --git a/src/etls/bopz/scrapper.py b/src/etls/bopz/scrapper.py index 8b820eb..c844486 100644 --- a/src/etls/bopz/scrapper.py +++ b/src/etls/bopz/scrapper.py @@ -124,7 +124,7 @@ def download_document(self, url: str, metadata: BeautifulSoup) -> BOPZMetadataDo logger = lg.getLogger(self.download_document.__name__) logger.info("Scrapping document: %s", url) session = create_retry_session(retries=5) - response = session.get(url) + response = session.get(url, timeout=10) response.raise_for_status() soup = BeautifulSoup(response.text, "lxml") with tempfile.NamedTemporaryFile("w", delete=False) as fn: