From 515a1ff0f5645624392debafc1eaf2ece2de2c4d Mon Sep 17 00:00:00 2001 From: Tobias Persson Date: Mon, 5 Feb 2024 09:01:08 +0100 Subject: [PATCH] Update to ETOS Library 4.0.0 --- requirements.txt | 2 +- setup.cfg | 2 +- src/environment_provider/lib/config.py | 11 ++-- src/environment_provider/lib/log_area.py | 59 +++++++------------ .../backend/configure.py | 6 +- .../utilities/external_provider.py | 36 +++++++---- .../utilities/external_provider.py | 36 +++++++---- .../utilities/external_provider.py | 36 +++++++---- .../test_external_execution_space.py | 28 ++++++--- tests/external_iut/test_external_iut.py | 28 ++++++--- .../test_external_log_area.py | 28 ++++++--- tox.ini | 6 +- 12 files changed, 171 insertions(+), 107 deletions(-) diff --git a/requirements.txt b/requirements.txt index e97bed5..3b0edae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,4 @@ falcon~=3.1 jsontas~=1.3 packageurl-python~=0.11 etcd3gw~=2.3 -etos_lib==3.2.2 +etos_lib==4.0.0 diff --git a/setup.cfg b/setup.cfg index a88f10e..765984d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ install_requires = jsontas~=1.3 packageurl-python~=0.11 etcd3gw~=2.3 - etos_lib==3.2.2 + etos_lib==4.0.0 python_requires = >=3.8 diff --git a/src/environment_provider/lib/config.py b/src/environment_provider/lib/config.py index 899f448..f6efe76 100644 --- a/src/environment_provider/lib/config.py +++ b/src/environment_provider/lib/config.py @@ -193,16 +193,13 @@ def test_suite(self) -> list[dict]: if batch is not None: self.__test_suite = batch elif batch_uri is not None: - json_header = {"Accept": "application/json"} - json_response = self.etos.http.wait_for_request( + response = self.etos.http.get( batch_uri, timeout=self.etos.config.get("TEST_SUITE_TIMEOUT"), - headers=json_header, + headers={"Accept": "application/json"}, ) - response = {} - for response in json_response: - break - self.__test_suite = response + response.raise_for_status() + self.__test_suite = response.json() except AttributeError: pass return self.__test_suite if self.__test_suite else [] diff --git a/src/environment_provider/lib/log_area.py b/src/environment_provider/lib/log_area.py index fc009ea..da9ce64 100644 --- a/src/environment_provider/lib/log_area.py +++ b/src/environment_provider/lib/log_area.py @@ -16,18 +16,17 @@ """Environment provider log area handler.""" import logging import os -import time import traceback from copy import deepcopy from json.decoder import JSONDecodeError -from typing import IO, Iterator, Optional, Union +from typing import IO, Optional, Union from cryptography.fernet import Fernet from etos_lib import ETOS from requests import Response from requests.auth import HTTPBasicAuth, HTTPDigestAuth +from requests.exceptions import ConnectionError as RequestsConnectionError from requests.exceptions import HTTPError -from urllib3.exceptions import MaxRetryError, NewConnectionError # pylint:disable=too-many-arguments @@ -68,16 +67,14 @@ def upload(self, log: str, name: str, folder: str) -> str: with open(log, "rb") as log_file: for _ in range(3): - request_generator = self.__retry_upload(log_file=log_file, **upload) try: - for response in request_generator: - self.logger.debug("%r", response) - if not upload.get("as_json", True): - self.logger.debug("%r", response.text) - self.logger.info("Uploaded log %r.", log) - self.logger.info("Upload URI %r", upload["url"]) - self.logger.info("Data: %r", data) - break + response = self.__upload(log_file=log_file, **upload) + self.logger.debug("%r", response) + if not upload.get("as_json", True): + self.logger.debug("%r", response.text) + self.logger.info("Uploaded log %r.", log) + self.logger.info("Upload URI %r", upload["url"]) + self.logger.info("Data: %r", data) break except: # noqa pylint:disable=bare-except self.logger.error("%r", traceback.format_exc()) @@ -85,7 +82,7 @@ def upload(self, log: str, name: str, folder: str) -> str: self.logger.error("Attempted upload of %r", log) return upload["url"] - def __retry_upload( + def __upload( self, verb: str, url: str, @@ -93,7 +90,7 @@ def __retry_upload( timeout: Optional[int] = None, as_json: bool = True, **requests_kwargs: dict, - ) -> Iterator[Union[Response, dict]]: + ) -> Union[Response, dict]: """Attempt to connect to url for x time. :param verb: Which HTTP verb to use. GET, PUT, POST @@ -107,30 +104,18 @@ def __retry_upload( """ if timeout is None: timeout = self.etos.debug.default_http_timeout - end_time = time.time() + timeout self.logger.debug("Retrying URL %s for %d seconds with a %s request.", url, timeout, verb) - iteration = 0 - while time.time() < end_time: - iteration += 1 - self.logger.debug("Iteration: %d", iteration) - try: - # Seek back to the start of the file so that the uploaded file - # is not 0 bytes in size. - log_file.seek(0) - yield self.etos.http.request(verb, url, as_json, data=log_file, **requests_kwargs) - break - except ( - ConnectionError, - HTTPError, - NewConnectionError, - MaxRetryError, - TimeoutError, - JSONDecodeError, - ): - self.logger.warning("%r", traceback.format_exc()) - time.sleep(2) - else: - raise ConnectionError(f"Unable to {verb} {url} with params {requests_kwargs}") + + method = getattr(self.etos.http, verb.lower()) + try: + response = method(url, data=log_file, timeout=timeout, **requests_kwargs) + response.raise_for_status() + if as_json: + return response.json() + return response + except (HTTPError, RequestsConnectionError, JSONDecodeError) as error: + # pylint:disable=broad-exception-raised + raise Exception("Failed to upload test suite to {url!r}") from error def __decrypt(self, password: Union[str, dict]) -> str: """Decrypt a password using an encryption key. diff --git a/src/environment_provider_api/backend/configure.py b/src/environment_provider_api/backend/configure.py index bd36552..e567c4b 100644 --- a/src/environment_provider_api/backend/configure.py +++ b/src/environment_provider_api/backend/configure.py @@ -127,9 +127,9 @@ def get_configuration(provider_registry: ProviderRegistry) -> dict: dataset = provider_registry.dataset() return { "iut_provider": iut_provider.ruleset if iut_provider else None, - "execution_space_provider": execution_space_provider.ruleset - if execution_space_provider - else None, + "execution_space_provider": ( + execution_space_provider.ruleset if execution_space_provider else None + ), "log_area_provider": log_area_provider.ruleset if log_area_provider else None, "dataset": dataset, } diff --git a/src/execution_space_provider/utilities/external_provider.py b/src/execution_space_provider/utilities/external_provider.py index 180dada..289a647 100644 --- a/src/execution_space_provider/utilities/external_provider.py +++ b/src/execution_space_provider/utilities/external_provider.py @@ -22,8 +22,11 @@ import requests from etos_lib import ETOS +from etos_lib.lib.http import Http from jsontas.jsontas import JsonTas from packageurl import PackageURL +from requests.exceptions import HTTPError +from urllib3.util import Retry from environment_provider.lib.encrypt import encrypt @@ -69,6 +72,18 @@ def __init__(self, etos: ETOS, jsontas: JsonTas, ruleset: dict) -> None: self.id = self.ruleset.get("id") # pylint:disable=invalid-name self.context = self.etos.config.get("environment_provider_context") self.identifier = self.etos.config.get("SUITE_ID") + self.http = Http( + retry=Retry( + total=None, + read=0, + connect=10, # With 1 as backoff_factor, will retry for 1023s + status=10, # With 1 as backoff_factor, will retry for 1023s + backoff_factor=1, + other=0, + allowed_methods=["POST", "GET"], + status_forcelist=Retry.RETRY_AFTER_STATUS_CODES, # 413, 429, 503 + ) + ) self.logger.info("Initialized external execution space provider %r", self.id) @property @@ -172,19 +187,16 @@ def start(self, minimum_amount: int, maximum_amount: int) -> str: "dataset": self.dataset.get("dataset"), "context": self.dataset.get("context"), } - response_iterator = self.etos.http.retry( - "POST", - self.ruleset.get("start", {}).get("host"), - json=data, - headers={"X-ETOS-ID": self.identifier}, - ) try: - for response in response_iterator: - return response.get("id") - except ConnectionError as http_error: - self.logger.error("Could not start external provider due to a connection error") - raise TimeoutError(f"Unable to start external provider {self.id!r}") from http_error - raise TimeoutError(f"Unable to start external provider {self.id!r}") + response = self.http.post( + self.ruleset.get("start", {}).get("host"), + json=data, + headers={"X-ETOS-ID": self.identifier}, + ) + response.raise_for_status() + return response.json().get("id") + except (HTTPError, JSONDecodeError) as error: + raise Exception(f"Could not start external provider {self.id!r}") from error def wait(self, provider_id: str) -> dict: """Wait for external execution space provider to finish its request. diff --git a/src/iut_provider/utilities/external_provider.py b/src/iut_provider/utilities/external_provider.py index 25368cf..122a5d3 100644 --- a/src/iut_provider/utilities/external_provider.py +++ b/src/iut_provider/utilities/external_provider.py @@ -22,8 +22,11 @@ import requests from etos_lib import ETOS +from etos_lib.lib.http import Http from jsontas.jsontas import JsonTas from packageurl import PackageURL +from requests.exceptions import HTTPError +from urllib3.util import Retry from ..exceptions import IutCheckinFailed, IutCheckoutFailed, IutNotAvailable from ..iut import Iut @@ -63,6 +66,18 @@ def __init__(self, etos: ETOS, jsontas: JsonTas, ruleset: dict) -> None: self.id = self.ruleset.get("id") # pylint:disable=invalid-name self.context = self.etos.config.get("environment_provider_context") self.identifier = self.etos.config.get("SUITE_ID") + self.http = Http( + retry=Retry( + total=None, + read=0, + connect=10, # With 1 as backoff_factor, will retry for 1023s + status=10, # With 1 as backoff_factor, will retry for 1023s + backoff_factor=1, + other=0, + allowed_methods=["POST", "GET"], + status_forcelist=Retry.RETRY_AFTER_STATUS_CODES, # 413, 429, 503 + ) + ) self.logger.info("Initialized external IUT provider %r", self.id) @property @@ -139,19 +154,16 @@ def start(self, minimum_amount: int, maximum_amount: int) -> str: "dataset": self.dataset.get("dataset"), "context": self.dataset.get("context"), } - response_iterator = self.etos.http.retry( - "POST", - self.ruleset.get("start", {}).get("host"), - json=data, - headers={"X-ETOS-ID": self.identifier}, - ) try: - for response in response_iterator: - return response.get("id") - except ConnectionError as http_error: - self.logger.error("Could not start external provider due to a connection error") - raise TimeoutError(f"Unable to start external provider {self.id!r}") from http_error - raise TimeoutError(f"Unable to start external provider {self.id!r}") + response = self.http.post( + self.ruleset.get("start", {}).get("host"), + json=data, + headers={"X-ETOS-ID": self.identifier}, + ) + response.raise_for_status() + return response.json().get("id") + except (HTTPError, JSONDecodeError) as error: + raise Exception(f"Could not start external provider {self.id!r}") from error def wait(self, provider_id: str) -> dict: """Wait for external IUT provider to finish its request. diff --git a/src/log_area_provider/utilities/external_provider.py b/src/log_area_provider/utilities/external_provider.py index 98aff5b..4d8db82 100644 --- a/src/log_area_provider/utilities/external_provider.py +++ b/src/log_area_provider/utilities/external_provider.py @@ -22,8 +22,11 @@ import requests from etos_lib import ETOS +from etos_lib.lib.http import Http from jsontas.jsontas import JsonTas from packageurl import PackageURL +from requests.exceptions import HTTPError +from urllib3.util import Retry from ..exceptions import LogAreaCheckinFailed, LogAreaCheckoutFailed, LogAreaNotAvailable from ..log_area import LogArea @@ -63,6 +66,18 @@ def __init__(self, etos: ETOS, jsontas: JsonTas, ruleset: dict) -> None: self.id = self.ruleset.get("id") # pylint:disable=invalid-name self.context = self.etos.config.get("environment_provider_context") self.identifier = self.etos.config.get("SUITE_ID") + self.http = Http( + retry=Retry( + total=None, + read=0, + connect=10, # With 1 as backoff_factor, will retry for 1023s + status=10, # With 1 as backoff_factor, will retry for 1023s + backoff_factor=1, + other=0, + allowed_methods=["POST", "GET"], + status_forcelist=Retry.RETRY_AFTER_STATUS_CODES, # 413, 429, 503 + ) + ) self.logger.info("Initialized external log area provider %r", self.id) @property @@ -143,19 +158,16 @@ def start(self, minimum_amount: int, maximum_amount: int) -> str: "dataset": self.dataset.get("dataset"), "context": self.dataset.get("context"), } - response_iterator = self.etos.http.retry( - "POST", - self.ruleset.get("start", {}).get("host"), - json=data, - headers={"X-ETOS-ID": self.identifier}, - ) try: - for response in response_iterator: - return response.get("id") - except ConnectionError as http_error: - self.logger.error("Could not start external provider due to a connection error") - raise TimeoutError(f"Unable to start external provider {self.id!r}") from http_error - raise TimeoutError(f"Unable to start external provider {self.id!r}") + response = self.http.post( + self.ruleset.get("start", {}).get("host"), + json=data, + headers={"X-ETOS-ID": self.identifier}, + ) + response.raise_for_status() + return response.json().get("id") + except (HTTPError, JSONDecodeError) as error: + raise Exception(f"Could not start external provider {self.id!r}") from error def wait(self, provider_id: str) -> dict: """Wait for external log area provider to finish its request. diff --git a/tests/external_execution_space/test_external_execution_space.py b/tests/external_execution_space/test_external_execution_space.py index 6dedc4a..19399a0 100644 --- a/tests/external_execution_space/test_external_execution_space.py +++ b/tests/external_execution_space/test_external_execution_space.py @@ -21,6 +21,7 @@ from etos_lib import ETOS from jsontas.jsontas import JsonTas from packageurl import PackageURL +from requests.exceptions import RetryError from execution_space_provider.exceptions import ( ExecutionSpaceCheckinFailed, @@ -71,6 +72,7 @@ def test_provider_start(self): ruleset = {"id": "test_provider_start", "start": {"host": server.host}} self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a start request.") start_id = provider.start(1, 2) self.logger.info("STEP: Verify that the ID from the start request is returned.") @@ -102,13 +104,14 @@ def test_provider_start_http_exception(self): ) expected_start_id = "123" - with FakeServer(["bad_request", "ok"], [{}, {"id": expected_start_id}]) as server: + with FakeServer(["service_unavailable", "ok"], [{}, {"id": expected_start_id}]) as server: ruleset = { "id": "test_provider_start_http_exception", "start": {"host": server.host}, } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a start request that fails.") start_id = provider.start(1, 2) self.logger.info("STEP: Verify that the start method tries again on HTTP errors.") @@ -116,15 +119,15 @@ def test_provider_start_http_exception(self): self.assertEqual(start_id, expected_start_id) def test_provider_start_timeout(self): - """Test that the start method raises a TimeoutError. + """Test that the start method raises a RetryError. Approval criteria: - - The start method shall raise TimeoutError if the timeout is reached. + - The start method shall raise RetryError if the timeout is reached. Test steps:: 1. Initialize an external provider. 2. Send a start request which will never finish. - 3. Verify that the start method raises TimeoutError. + 3. Verify that the start method raises RetryError. """ etos = ETOS("testing_etos", "testing_etos", "testing_etos") jsontas = JsonTas() @@ -141,17 +144,18 @@ def test_provider_start_timeout(self): ) os.environ["ETOS_DEFAULT_HTTP_TIMEOUT"] = "1" - with FakeServer("bad_request", {}) as server: + with FakeServer("service_unavailable", {}) as server: ruleset = { "id": "test_provider_start_timeout", "start": {"host": server.host}, } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a start request which will never finish.") - with self.assertRaises(TimeoutError): - self.logger.info("STEP: Verify that the start method raises TimeoutError.") + with self.assertRaises(RetryError): + self.logger.info("STEP: Verify that the start method raises RetryError.") provider.start(1, 2) def test_provider_stop(self): @@ -185,6 +189,7 @@ def test_provider_stop(self): ruleset = {"id": "test_provider_stop", "stop": {"host": server.host}} self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a stop request for a single execution space.") provider.checkin(execution_space) self.logger.info("STEP: Verify that the stop endpoint is called.") @@ -229,6 +234,7 @@ def test_provider_stop_many(self): ruleset = {"id": "test_provider_stop_many", "stop": {"host": server.host}} self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a stop request for multiple execution spaces.") provider.checkin_all() self.logger.info("STEP: Verify that the stop endpoint is called.") @@ -266,6 +272,7 @@ def test_provider_stop_failed(self): ruleset = {"id": "test_provider_stop_failed", "stop": {"host": server.host}} self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a stop request that fails.") with self.assertRaises(ExecutionSpaceCheckinFailed): self.logger.info( @@ -307,6 +314,7 @@ def test_provider_stop_timeout(self): } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a stop request that fails.") with self.assertRaises(TimeoutError): self.logger.info("STEP: Verify that the checkin method raises a TimeoutError.") @@ -342,6 +350,7 @@ def test_provider_status(self): ruleset = {"id": "test_provider_status", "status": {"host": server.host}} self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a status request for a started execution space provider.") response = provider.wait("1") self.logger.info("STEP: Verify that the wait method return response on DONE.") @@ -380,6 +389,7 @@ def test_provider_status_pending(self): } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a status request for a started execution space provider.") provider.wait("1") self.logger.info("STEP: Verify that the wait method waits on PENDING.") @@ -418,6 +428,7 @@ def test_provider_status_failed(self): } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a status request for a started execution space provider.") with self.assertRaises(ExecutionSpaceCheckoutFailed): self.logger.info( @@ -464,6 +475,7 @@ def test_provider_status_http_exceptions(self): } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info( "STEP: Send a status request for a started execution space provider." ) @@ -505,6 +517,7 @@ def test_provider_status_timeout(self): } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a status request that times out.") with self.assertRaises(TimeoutError): self.logger.info("STEP: Verify that the wait method raises TimeoutError.") @@ -559,6 +572,7 @@ def test_request_and_wait(self): } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info( "STEP: Send a checkout request via the external execution space provider." ) diff --git a/tests/external_iut/test_external_iut.py b/tests/external_iut/test_external_iut.py index 2aeda00..b61ccba 100644 --- a/tests/external_iut/test_external_iut.py +++ b/tests/external_iut/test_external_iut.py @@ -21,6 +21,7 @@ from etos_lib import ETOS from jsontas.jsontas import JsonTas from packageurl import PackageURL +from requests.exceptions import RetryError from iut_provider.exceptions import IutCheckinFailed, IutCheckoutFailed, IutNotAvailable from iut_provider.iut import Iut @@ -67,6 +68,7 @@ def test_provider_start(self): ruleset = {"id": "test_provider_start", "start": {"host": server.host}} self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a start request.") start_id = provider.start(1, 2) self.logger.info("STEP: Verify that the ID from the start request is returned.") @@ -98,13 +100,14 @@ def test_provider_start_http_exception(self): ) expected_start_id = "123" - with FakeServer(["bad_request", "ok"], [{}, {"id": expected_start_id}]) as server: + with FakeServer(["service_unavailable", "ok"], [{}, {"id": expected_start_id}]) as server: ruleset = { "id": "test_provider_start_http_exception", "start": {"host": server.host}, } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a start request that fails.") start_id = provider.start(1, 2) self.logger.info("STEP: Verify that the start method tries again on HTTP errors.") @@ -112,15 +115,15 @@ def test_provider_start_http_exception(self): self.assertEqual(start_id, expected_start_id) def test_provider_start_timeout(self): - """Test that the start method raises a TimeoutError. + """Test that the start method raises a RetryError. Approval criteria: - - The start method shall raise TimeoutError if the timeout is reached. + - The start method shall raise RetryError if the timeout is reached. Test steps:: 1. Initialize an external provider. 2. Send a start request which will never finish. - 3. Verify that the start method raises TimeoutError. + 3. Verify that the start method raises RetryError. """ etos = ETOS("testing_etos", "testing_etos", "testing_etos") jsontas = JsonTas() @@ -137,17 +140,18 @@ def test_provider_start_timeout(self): ) os.environ["ETOS_DEFAULT_HTTP_TIMEOUT"] = "1" - with FakeServer("bad_request", {}) as server: + with FakeServer("service_unavailable", {}) as server: ruleset = { "id": "test_provider_start_timeout", "start": {"host": server.host}, } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a start request which will never finish.") - with self.assertRaises(TimeoutError): - self.logger.info("STEP: Verify that the start method raises TimeoutError.") + with self.assertRaises(RetryError): + self.logger.info("STEP: Verify that the start method raises RetryError.") provider.start(1, 2) def test_provider_stop(self): @@ -181,6 +185,7 @@ def test_provider_stop(self): ruleset = {"id": "test_provider_stop", "stop": {"host": server.host}} self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a stop request for a single IUT.") provider.checkin(iut) self.logger.info("STEP: Verify that the stop endpoint is called.") @@ -220,6 +225,7 @@ def test_provider_stop_many(self): ruleset = {"id": "test_provider_stop_many", "stop": {"host": server.host}} self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a stop request for multiple IUTs.") provider.checkin_all() self.logger.info("STEP: Verify that the stop endpoint is called.") @@ -257,6 +263,7 @@ def test_provider_stop_failed(self): ruleset = {"id": "test_provider_stop_failed", "stop": {"host": server.host}} self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a stop request that fails.") with self.assertRaises(IutCheckinFailed): self.logger.info( @@ -297,6 +304,7 @@ def test_provider_stop_timeout(self): } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a stop request that fails.") with self.assertRaises(TimeoutError): self.logger.info("STEP: Verify that the checkin method raises a TimeoutError.") @@ -332,6 +340,7 @@ def test_provider_status(self): ruleset = {"id": "test_provider_status", "status": {"host": server.host}} self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a status request for a started IUT provider.") response = provider.wait("1") self.logger.info("STEP: Verify that the wait method return response on DONE.") @@ -370,6 +379,7 @@ def test_provider_status_pending(self): } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a status request for a started IUT provider.") provider.wait("1") self.logger.info("STEP: Verify that the wait method waits on PENDING.") @@ -408,6 +418,7 @@ def test_provider_status_failed(self): } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a status request for a started IUT provider.") with self.assertRaises(IutCheckoutFailed): self.logger.info("STEP: Verify that the wait method raises IutCheckoutFailed.") @@ -452,6 +463,7 @@ def test_provider_status_http_exceptions(self): } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a status request for a started IUT provider.") with self.assertRaises(exception): self.logger.info( @@ -491,6 +503,7 @@ def test_provider_status_timeout(self): } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a status request that times out.") with self.assertRaises(TimeoutError): self.logger.info("STEP: Verify that the wait method raises TimeoutError.") @@ -540,6 +553,7 @@ def test_request_and_wait(self): } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a checkout request via the external IUT provider.") iuts = provider.request_and_wait_for_iuts() self.logger.info("STEP: Verify that the provider returns a list of checked out IUTs.") diff --git a/tests/external_log_area/test_external_log_area.py b/tests/external_log_area/test_external_log_area.py index 8df0e05..46b1744 100644 --- a/tests/external_log_area/test_external_log_area.py +++ b/tests/external_log_area/test_external_log_area.py @@ -21,6 +21,7 @@ from etos_lib import ETOS from jsontas.jsontas import JsonTas from packageurl import PackageURL +from requests.exceptions import RetryError from log_area_provider.exceptions import ( LogAreaCheckinFailed, @@ -71,6 +72,7 @@ def test_provider_start(self): ruleset = {"id": "test_provider_start", "start": {"host": server.host}} self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a start request.") start_id = provider.start(1, 2) self.logger.info("STEP: Verify that the ID from the start request is returned.") @@ -102,13 +104,14 @@ def test_provider_start_http_exception(self): ) expected_start_id = "123" - with FakeServer(["bad_request", "ok"], [{}, {"id": expected_start_id}]) as server: + with FakeServer(["service_unavailable", "ok"], [{}, {"id": expected_start_id}]) as server: ruleset = { "id": "test_provider_start_http_exception", "start": {"host": server.host}, } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a start request that fails.") start_id = provider.start(1, 2) self.logger.info("STEP: Verify that the start method tries again on HTTP errors.") @@ -116,15 +119,15 @@ def test_provider_start_http_exception(self): self.assertEqual(start_id, expected_start_id) def test_provider_start_timeout(self): - """Test that the start method raises a TimeoutError. + """Test that the start method raises a RetryError. Approval criteria: - - The start method shall raise TimeoutError if the timeout is reached. + - The start method shall raise RetryError if the timeout is reached. Test steps:: 1. Initialize an external provider. 2. Send a start request which will never finish. - 3. Verify that the start method raises TimeoutError. + 3. Verify that the start method raises RetryError. """ etos = ETOS("testing_etos", "testing_etos", "testing_etos") jsontas = JsonTas() @@ -141,17 +144,18 @@ def test_provider_start_timeout(self): ) os.environ["ETOS_DEFAULT_HTTP_TIMEOUT"] = "1" - with FakeServer("bad_request", {}) as server: + with FakeServer("service_unavailable", {}) as server: ruleset = { "id": "test_provider_start_timeout", "start": {"host": server.host}, } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a start request which will never finish.") - with self.assertRaises(TimeoutError): - self.logger.info("STEP: Verify that the start method raises TimeoutError.") + with self.assertRaises(RetryError): + self.logger.info("STEP: Verify that the start method raises RetryError.") provider.start(1, 2) def test_provider_stop(self): @@ -185,6 +189,7 @@ def test_provider_stop(self): ruleset = {"id": "test_provider_stop", "stop": {"host": server.host}} self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a stop request for a single log area.") provider.checkin(log_area) self.logger.info("STEP: Verify that the stop endpoint is called.") @@ -229,6 +234,7 @@ def test_provider_stop_many(self): ruleset = {"id": "test_provider_stop_many", "stop": {"host": server.host}} self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a stop request for multiple log areas.") provider.checkin_all() self.logger.info("STEP: Verify that the stop endpoint is called.") @@ -266,6 +272,7 @@ def test_provider_stop_failed(self): ruleset = {"id": "test_provider_stop_failed", "stop": {"host": server.host}} self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a stop request that fails.") with self.assertRaises(LogAreaCheckinFailed): self.logger.info( @@ -306,6 +313,7 @@ def test_provider_stop_timeout(self): } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a stop request that fails.") with self.assertRaises(TimeoutError): self.logger.info("STEP: Verify that the checkin method raises a TimeoutError.") @@ -341,6 +349,7 @@ def test_provider_status(self): ruleset = {"id": "test_provider_status", "status": {"host": server.host}} self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a status request for a started log area provider.") response = provider.wait("1") self.logger.info("STEP: Verify that the wait method return response on DONE.") @@ -379,6 +388,7 @@ def test_provider_status_pending(self): } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a status request for a started log area provider.") provider.wait("1") self.logger.info("STEP: Verify that the wait method waits on PENDING.") @@ -417,6 +427,7 @@ def test_provider_status_failed(self): } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a status request for a started log area provider.") with self.assertRaises(LogAreaCheckoutFailed): self.logger.info("STEP: Verify that the wait method raises LogAreaCheckoutFailed.") @@ -461,6 +472,7 @@ def test_provider_status_http_exceptions(self): } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a status request for a started log area provider.") with self.assertRaises(exception): self.logger.info( @@ -500,6 +512,7 @@ def test_provider_status_timeout(self): } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a status request that times out.") with self.assertRaises(TimeoutError): self.logger.info("STEP: Verify that the wait method raises TimeoutError.") @@ -554,6 +567,7 @@ def test_request_and_wait(self): } self.logger.info("STEP: Initialize an external provider.") provider = ExternalProvider(etos, jsontas, ruleset) + provider.http.adapter.max_retries.status = 1 self.logger.info("STEP: Send a checkout request via the external log area provider.") log_areas = provider.request_and_wait_for_log_areas() self.logger.info( diff --git a/tox.ini b/tox.ini index dfab9be..9ff533c 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,11 @@ envlist = py3,black,pylint,pydocstyle [testenv] -setenv = ETOS_ENABLE_SENDING_LOGS=false +setenv = + ETOS_ENABLE_SENDING_LOGS=false + ETOS_GRAPHQL_SERVER=http://localhost/no + ETOS_API=http://localhost/nah + ETOS_ENVIRONMENT_PROVIDER=http://localhost/nuhuh deps = -r{toxinidir}/test-requirements.txt commands =