Update to ETOS Library 4.0.0

requirements.txt

@@ -16,4 +16,4 @@ falcon~=3.1
 jsontas~=1.3
 packageurl-python~=0.11
 etcd3gw~=2.3
-etos_lib==3.2.2
+etos_lib==4.0.0

setup.cfg

@@ -35,7 +35,7 @@ install_requires =
     jsontas~=1.3
     packageurl-python~=0.11
     etcd3gw~=2.3
-    etos_lib==3.2.2
+    etos_lib==4.0.0
 
 python_requires = >=3.8
 

src/environment_provider/lib/config.py

@@ -193,16 +193,13 @@ def test_suite(self) -> list[dict]:
                 if batch is not None:
                     self.__test_suite = batch
                 elif batch_uri is not None:
-                    json_header = {"Accept": "application/json"}
-                    json_response = self.etos.http.wait_for_request(
+                    response = self.etos.http.get(
                         batch_uri,
                         timeout=self.etos.config.get("TEST_SUITE_TIMEOUT"),
-                        headers=json_header,
+                        headers={"Accept": "application/json"},
                     )
-                    response = {}
-                    for response in json_response:
-                        break
-                    self.__test_suite = response
+                    response.raise_for_status()
+                    self.__test_suite = response.json()
             except AttributeError:
                 pass
         return self.__test_suite if self.__test_suite else []

src/environment_provider/lib/log_area.py

@@ -16,18 +16,17 @@
 """Environment provider log area handler."""
 import logging
 import os
-import time
 import traceback
 from copy import deepcopy
 from json.decoder import JSONDecodeError
-from typing import IO, Iterator, Optional, Union
+from typing import IO, Optional, Union
 
 from cryptography.fernet import Fernet
 from etos_lib import ETOS
 from requests import Response
 from requests.auth import HTTPBasicAuth, HTTPDigestAuth
+from requests.exceptions import ConnectionError as RequestsConnectionError
 from requests.exceptions import HTTPError
-from urllib3.exceptions import MaxRetryError, NewConnectionError
 
 # pylint:disable=too-many-arguments
 
@@ -68,32 +67,30 @@ def upload(self, log: str, name: str, folder: str) -> str:
 
         with open(log, "rb") as log_file:
             for _ in range(3):
-                request_generator = self.__retry_upload(log_file=log_file, **upload)
                 try:
-                    for response in request_generator:
-                        self.logger.debug("%r", response)
-                        if not upload.get("as_json", True):
-                            self.logger.debug("%r", response.text)
-                        self.logger.info("Uploaded log %r.", log)
-                        self.logger.info("Upload URI          %r", upload["url"])
-                        self.logger.info("Data:               %r", data)
-                        break
+                    response = self.__upload(log_file=log_file, **upload)
+                    self.logger.debug("%r", response)
+                    if not upload.get("as_json", True):
+                        self.logger.debug("%r", response.text)
+                    self.logger.info("Uploaded log %r.", log)
+                    self.logger.info("Upload URI          %r", upload["url"])
+                    self.logger.info("Data:               %r", data)
                     break
                 except:  # noqa pylint:disable=bare-except
                     self.logger.error("%r", traceback.format_exc())
                     self.logger.error("Failed to upload log!")
                     self.logger.error("Attempted upload of %r", log)
         return upload["url"]
 
-    def __retry_upload(
+    def __upload(
         self,
         verb: str,
         url: str,
         log_file: IO,
         timeout: Optional[int] = None,
         as_json: bool = True,
         **requests_kwargs: dict,
-    ) -> Iterator[Union[Response, dict]]:
+    ) -> Union[Response, dict]:
         """Attempt to connect to url for x time.
 
         :param verb: Which HTTP verb to use. GET, PUT, POST
@@ -107,30 +104,18 @@ def __retry_upload(
         """
         if timeout is None:
             timeout = self.etos.debug.default_http_timeout
-        end_time = time.time() + timeout
         self.logger.debug("Retrying URL %s for %d seconds with a %s request.", url, timeout, verb)
-        iteration = 0
-        while time.time() < end_time:
-            iteration += 1
-            self.logger.debug("Iteration: %d", iteration)
-            try:
-                # Seek back to the start of the file so that the uploaded file
-                # is not 0 bytes in size.
-                log_file.seek(0)
-                yield self.etos.http.request(verb, url, as_json, data=log_file, **requests_kwargs)
-                break
-            except (
-                ConnectionError,
-                HTTPError,
-                NewConnectionError,
-                MaxRetryError,
-                TimeoutError,
-                JSONDecodeError,
-            ):
-                self.logger.warning("%r", traceback.format_exc())
-                time.sleep(2)
-        else:
-            raise ConnectionError(f"Unable to {verb} {url} with params {requests_kwargs}")
+
+        method = getattr(self.etos.http, verb.lower())
+        try:
+            response = method(url, data=log_file, timeout=timeout, **requests_kwargs)
+            response.raise_for_status()
+            if as_json:
+                return response.json()
+            return response
+        except (HTTPError, RequestsConnectionError, JSONDecodeError) as error:
+            # pylint:disable=broad-exception-raised
+            raise Exception("Failed to upload test suite to {url!r}") from error
 
     def __decrypt(self, password: Union[str, dict]) -> str:
         """Decrypt a password using an encryption key.

src/environment_provider_api/backend/configure.py

@@ -127,9 +127,9 @@ def get_configuration(provider_registry: ProviderRegistry) -> dict:
     dataset = provider_registry.dataset()
     return {
         "iut_provider": iut_provider.ruleset if iut_provider else None,
-        "execution_space_provider": execution_space_provider.ruleset
-        if execution_space_provider
-        else None,
+        "execution_space_provider": (
+            execution_space_provider.ruleset if execution_space_provider else None
+        ),
         "log_area_provider": log_area_provider.ruleset if log_area_provider else None,
         "dataset": dataset,
     }

src/execution_space_provider/utilities/external_provider.py

@@ -22,8 +22,11 @@
 
 import requests
 from etos_lib import ETOS
+from etos_lib.lib.http import Http
 from jsontas.jsontas import JsonTas
 from packageurl import PackageURL
+from requests.exceptions import HTTPError
+from urllib3.util import Retry
 
 from environment_provider.lib.encrypt import encrypt
 
@@ -69,6 +72,18 @@ def __init__(self, etos: ETOS, jsontas: JsonTas, ruleset: dict) -> None:
         self.id = self.ruleset.get("id")  # pylint:disable=invalid-name
         self.context = self.etos.config.get("environment_provider_context")
         self.identifier = self.etos.config.get("SUITE_ID")
+        self.http = Http(
+            retry=Retry(
+                total=None,
+                read=0,
+                connect=10,  # With 1 as backoff_factor, will retry for 1023s
+                status=10,  # With 1 as backoff_factor, will retry for 1023s
+                backoff_factor=1,
+                other=0,
+                allowed_methods=["POST", "GET"],
+                status_forcelist=Retry.RETRY_AFTER_STATUS_CODES,  # 413, 429, 503
+            )
+        )
         self.logger.info("Initialized external execution space provider %r", self.id)
 
     @property
@@ -172,19 +187,16 @@ def start(self, minimum_amount: int, maximum_amount: int) -> str:
             "dataset": self.dataset.get("dataset"),
             "context": self.dataset.get("context"),
         }
-        response_iterator = self.etos.http.retry(
-            "POST",
-            self.ruleset.get("start", {}).get("host"),
-            json=data,
-            headers={"X-ETOS-ID": self.identifier},
-        )
         try:
-            for response in response_iterator:
-                return response.get("id")
-        except ConnectionError as http_error:
-            self.logger.error("Could not start external provider due to a connection error")
-            raise TimeoutError(f"Unable to start external provider {self.id!r}") from http_error
-        raise TimeoutError(f"Unable to start external provider {self.id!r}")
+            response = self.http.post(
+                self.ruleset.get("start", {}).get("host"),
+                json=data,
+                headers={"X-ETOS-ID": self.identifier},
+            )
+            response.raise_for_status()
+            return response.json().get("id")
+        except (HTTPError, JSONDecodeError) as error:
+            raise Exception(f"Could not start external provider {self.id!r}") from error
 
     def wait(self, provider_id: str) -> dict:
         """Wait for external execution space provider to finish its request.

src/iut_provider/utilities/external_provider.py

@@ -22,8 +22,11 @@
 
 import requests
 from etos_lib import ETOS
+from etos_lib.lib.http import Http
 from jsontas.jsontas import JsonTas
 from packageurl import PackageURL
+from requests.exceptions import HTTPError
+from urllib3.util import Retry
 
 from ..exceptions import IutCheckinFailed, IutCheckoutFailed, IutNotAvailable
 from ..iut import Iut
@@ -63,6 +66,18 @@ def __init__(self, etos: ETOS, jsontas: JsonTas, ruleset: dict) -> None:
         self.id = self.ruleset.get("id")  # pylint:disable=invalid-name
         self.context = self.etos.config.get("environment_provider_context")
         self.identifier = self.etos.config.get("SUITE_ID")
+        self.http = Http(
+            retry=Retry(
+                total=None,
+                read=0,
+                connect=10,  # With 1 as backoff_factor, will retry for 1023s
+                status=10,  # With 1 as backoff_factor, will retry for 1023s
+                backoff_factor=1,
+                other=0,
+                allowed_methods=["POST", "GET"],
+                status_forcelist=Retry.RETRY_AFTER_STATUS_CODES,  # 413, 429, 503
+            )
+        )
         self.logger.info("Initialized external IUT provider %r", self.id)
 
     @property
@@ -139,19 +154,16 @@ def start(self, minimum_amount: int, maximum_amount: int) -> str:
             "dataset": self.dataset.get("dataset"),
             "context": self.dataset.get("context"),
         }
-        response_iterator = self.etos.http.retry(
-            "POST",
-            self.ruleset.get("start", {}).get("host"),
-            json=data,
-            headers={"X-ETOS-ID": self.identifier},
-        )
         try:
-            for response in response_iterator:
-                return response.get("id")
-        except ConnectionError as http_error:
-            self.logger.error("Could not start external provider due to a connection error")
-            raise TimeoutError(f"Unable to start external provider {self.id!r}") from http_error
-        raise TimeoutError(f"Unable to start external provider {self.id!r}")
+            response = self.http.post(
+                self.ruleset.get("start", {}).get("host"),
+                json=data,
+                headers={"X-ETOS-ID": self.identifier},
+            )
+            response.raise_for_status()
+            return response.json().get("id")
+        except (HTTPError, JSONDecodeError) as error:
+            raise Exception(f"Could not start external provider {self.id!r}") from error
 
     def wait(self, provider_id: str) -> dict:
         """Wait for external IUT provider to finish its request.

src/log_area_provider/utilities/external_provider.py

@@ -22,8 +22,11 @@
 
 import requests
 from etos_lib import ETOS
+from etos_lib.lib.http import Http
 from jsontas.jsontas import JsonTas
 from packageurl import PackageURL
+from requests.exceptions import HTTPError
+from urllib3.util import Retry
 
 from ..exceptions import LogAreaCheckinFailed, LogAreaCheckoutFailed, LogAreaNotAvailable
 from ..log_area import LogArea
@@ -63,6 +66,18 @@ def __init__(self, etos: ETOS, jsontas: JsonTas, ruleset: dict) -> None:
         self.id = self.ruleset.get("id")  # pylint:disable=invalid-name
         self.context = self.etos.config.get("environment_provider_context")
         self.identifier = self.etos.config.get("SUITE_ID")
+        self.http = Http(
+            retry=Retry(
+                total=None,
+                read=0,
+                connect=10,  # With 1 as backoff_factor, will retry for 1023s
+                status=10,  # With 1 as backoff_factor, will retry for 1023s
+                backoff_factor=1,
+                other=0,
+                allowed_methods=["POST", "GET"],
+                status_forcelist=Retry.RETRY_AFTER_STATUS_CODES,  # 413, 429, 503
+            )
+        )
         self.logger.info("Initialized external log area provider %r", self.id)
 
     @property
@@ -143,19 +158,16 @@ def start(self, minimum_amount: int, maximum_amount: int) -> str:
             "dataset": self.dataset.get("dataset"),
             "context": self.dataset.get("context"),
         }
-        response_iterator = self.etos.http.retry(
-            "POST",
-            self.ruleset.get("start", {}).get("host"),
-            json=data,
-            headers={"X-ETOS-ID": self.identifier},
-        )
         try:
-            for response in response_iterator:
-                return response.get("id")
-        except ConnectionError as http_error:
-            self.logger.error("Could not start external provider due to a connection error")
-            raise TimeoutError(f"Unable to start external provider {self.id!r}") from http_error
-        raise TimeoutError(f"Unable to start external provider {self.id!r}")
+            response = self.http.post(
+                self.ruleset.get("start", {}).get("host"),
+                json=data,
+                headers={"X-ETOS-ID": self.identifier},
+            )
+            response.raise_for_status()
+            return response.json().get("id")
+        except (HTTPError, JSONDecodeError) as error:
+            raise Exception(f"Could not start external provider {self.id!r}") from error
 
     def wait(self, provider_id: str) -> dict:
         """Wait for external log area provider to finish its request.