From 8436d3d58ed67797037193700576f9d109964e03 Mon Sep 17 00:00:00 2001 From: Jan Range Date: Fri, 1 Mar 2024 20:50:58 +0100 Subject: [PATCH 1/7] async pytest dep --- requirements/tests.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/tests.txt b/requirements/tests.txt index 45b26c5..e12bdbb 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -2,5 +2,6 @@ -r common.txt pytest pytest-cov +pytest-asyncio tox selenium==3.141.0 From 1b21b4a156b9d6427585f9bee78a48f7f89d7b86 Mon Sep 17 00:00:00 2001 From: Jan Range Date: Fri, 1 Mar 2024 20:51:27 +0100 Subject: [PATCH 2/7] async context manager and requests --- src/pyDataverse/api.py | 162 ++++++++++++++++++++++++++++------------- 1 file changed, 111 insertions(+), 51 deletions(-) diff --git a/src/pyDataverse/api.py b/src/pyDataverse/api.py index 14c4826..f3c9d38 100644 --- a/src/pyDataverse/api.py +++ b/src/pyDataverse/api.py @@ -1,6 +1,7 @@ """Dataverse API wrapper for all it's API's.""" import json +from typing import Any, Dict, Optional import httpx import subprocess as sp from urllib.parse import urljoin @@ -37,7 +38,10 @@ class Api: """ def __init__( - self, base_url: str, api_token: str = None, api_version: str = "latest" + self, + base_url: str, + api_token: str = None, + api_version: str = "latest", ): """Init an Api() class. @@ -64,6 +68,7 @@ def __init__( raise ApiUrlError("base_url {0} is not a string.".format(base_url)) self.base_url = base_url + self.client = None if not isinstance(api_version, ("".__class__, "".__class__)): raise ApiUrlError("api_version {0} is not a string.".format(api_version)) @@ -120,28 +125,17 @@ def get_request(self, url, params=None, auth=False): if self.api_token: params["key"] = str(self.api_token) - try: - url = urljoin(self.base_url_api, url) - resp = httpx.get(url, params=params) - if resp.status_code == 401: - error_msg = resp.json()["message"] - raise ApiAuthorizationError( - "ERROR: GET - Authorization invalid {0}. MSG: {1}.".format( - url, error_msg - ) - ) - elif resp.status_code >= 300: - if resp.text: - error_msg = resp.text - raise OperationFailedError( - "ERROR: GET HTTP {0} - {1}. MSG: {2}".format( - resp.status_code, url, error_msg - ) - ) - return resp - except ConnectError: - raise ConnectError( - "ERROR: GET - Could not establish connection to api {0}.".format(url) + if self.client is None: + return self._sync_request( + method=httpx.get, + url=url, + params=params, + ) + else: + return self._async_request( + method=self.client.get, + url=url, + params=params, ) def post_request(self, url, data=None, auth=False, params=None, files=None): @@ -174,19 +168,21 @@ def post_request(self, url, data=None, auth=False, params=None, files=None): if self.api_token: params["key"] = self.api_token - try: - resp = httpx.post(url, data=data, params=params, files=files) - if resp.status_code == 401: - error_msg = resp.json()["message"] - raise ApiAuthorizationError( - "ERROR: POST HTTP 401 - Authorization error {0}. MSG: {1}".format( - url, error_msg - ) - ) - return resp - except ConnectError: - raise ConnectError( - "ERROR: POST - Could not establish connection to API: {0}".format(url) + if self.client is None: + return self._sync_request( + method=httpx.post, + url=url, + data=data, + params=params, + files=files, + ) + else: + return self._async_request( + method=self.client.post, + url=url, + data=data, + params=params, + files=files, ) def put_request(self, url, data=None, auth=False, params=None): @@ -215,19 +211,19 @@ def put_request(self, url, data=None, auth=False, params=None): if self.api_token: params["key"] = self.api_token - try: - resp = httpx.put(url, data=data, params=params) - if resp.status_code == 401: - error_msg = resp.json()["message"] - raise ApiAuthorizationError( - "ERROR: PUT HTTP 401 - Authorization error {0}. MSG: {1}".format( - url, error_msg - ) - ) - return resp - except ConnectError: - raise ConnectError( - "ERROR: PUT - Could not establish connection to api '{0}'.".format(url) + if self.client is None: + return self._sync_request( + method=httpx.put, + url=url, + data=data, + params=params, + ) + else: + return self._async_request( + method=self.client.put, + url=url, + data=data, + params=params, ) def delete_request(self, url, auth=False, params=None): @@ -254,13 +250,77 @@ def delete_request(self, url, auth=False, params=None): if self.api_token: params["key"] = self.api_token + if self.client is None: + return self._sync_request( + method=httpx.delete, + url=url, + params=params, + ) + else: + return self._async_request( + method=self.client.delete, + url=url, + params=params, + ) + + def _sync_request( + self, + method, + **kwargs, + ): + assert "url" in kwargs, "URL is required for a request." + + kwargs = {k: v for k, v in kwargs.items() if v is not None} + try: - return httpx.delete(url, params=params) + resp = method(**kwargs) + + if resp.status_code == 401: + error_msg = resp.json()["message"] + raise ApiAuthorizationError( + "ERROR: HTTP 401 - Authorization error {0}. MSG: {1}".format( + kwargs["url"], error_msg + ) + ) + + return resp + except ConnectError: raise ConnectError( - "ERROR: DELETE could not establish connection to api {}.".format(url) + "ERROR - Could not establish connection to api '{0}'.".format( + kwargs["url"] + ) ) + async def _async_request( + self, + method, + **kwargs, + ): + return await method(**kwargs) + + async def __aenter__(self): + """ + Context manager method that initializes an instance of httpx.AsyncClient. + + Returns: + httpx.AsyncClient: An instance of httpx.AsyncClient. + """ + self.client = httpx.AsyncClient() + + async def __aexit__(self, exc_type, exc_value, traceback): + """ + Closes the client connection when exiting a context manager. + + Args: + exc_type (type): The type of the exception raised, if any. + exc_value (Exception): The exception raised, if any. + traceback (traceback): The traceback object associated with the exception, if any. + """ + + await self.client.aclose() + self.client = None + class DataAccessApi(Api): """Class to access Dataverse's Data Access API. From 0fd58ee68e7a820adc51a238fc5ce0c92da9cb91 Mon Sep 17 00:00:00 2001 From: Jan Range Date: Fri, 1 Mar 2024 20:51:36 +0100 Subject: [PATCH 3/7] test async functions --- tests/api/test_async_api.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 tests/api/test_async_api.py diff --git a/tests/api/test_async_api.py b/tests/api/test_async_api.py new file mode 100644 index 0000000..9cb8232 --- /dev/null +++ b/tests/api/test_async_api.py @@ -0,0 +1,18 @@ +import asyncio +import pytest + +from pyDataverse.api import NativeApi + + +class TestAsyncAPI: + + @pytest.mark.asyncio + async def test_async_api(self, native_api): + + async with native_api: + tasks = [native_api.get_info_version() for _ in range(10)] + responses = await asyncio.gather(*tasks) + + assert len(responses) == 10 + for response in responses: + assert response.status_code == 200, "Request failed." From 2b4c740ea46c8baeab5ffc04ba525a2e9b2e44da Mon Sep 17 00:00:00 2001 From: Jan Range Date: Fri, 1 Mar 2024 23:37:20 +0100 Subject: [PATCH 4/7] add docs and refactor kwarg filter --- src/pyDataverse/api.py | 68 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/src/pyDataverse/api.py b/src/pyDataverse/api.py index f3c9d38..a952268 100644 --- a/src/pyDataverse/api.py +++ b/src/pyDataverse/api.py @@ -268,9 +268,23 @@ def _sync_request( method, **kwargs, ): + """ + Sends a synchronous request to the specified URL using the specified HTTP method. + + Args: + method (function): The HTTP method to use for the request. + **kwargs: Additional keyword arguments to be passed to the method. + + Returns: + requests.Response: The response object returned by the request. + + Raises: + ApiAuthorizationError: If the response status code is 401 (Authorization error). + ConnectError: If a connection to the API cannot be established. + """ assert "url" in kwargs, "URL is required for a request." - kwargs = {k: v for k, v in kwargs.items() if v is not None} + kwargs = self._filter_kwargs(kwargs) try: resp = method(**kwargs) @@ -297,7 +311,57 @@ async def _async_request( method, **kwargs, ): - return await method(**kwargs) + """ + Sends an asynchronous request to the specified URL using the specified HTTP method. + + Args: + method (callable): The HTTP method to use for the request. + **kwargs: Additional keyword arguments to be passed to the method. + + Raises: + ApiAuthorizationError: If the response status code is 401 (Authorization error). + ConnectError: If a connection to the API cannot be established. + + Returns: + The response object. + + """ + assert "url" in kwargs, "URL is required for a request." + + kwargs = self._filter_kwargs(kwargs) + + try: + resp = await method(**kwargs) + + if resp.status_code == 401: + error_msg = resp.json()["message"] + raise ApiAuthorizationError( + "ERROR: HTTP 401 - Authorization error {0}. MSG: {1}".format( + kwargs["url"], error_msg + ) + ) + + return resp + + except ConnectError: + raise ConnectError( + "ERROR - Could not establish connection to api '{0}'.".format( + kwargs["url"] + ) + ) + + @staticmethod + def _filter_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: + """ + Filters out any keyword arguments that are `None` from the specified dictionary. + + Args: + kwargs (Dict[str, Any]): The dictionary to filter. + + Returns: + Dict[str, Any]: The filtered dictionary. + """ + return {k: v for k, v in kwargs.items() if v is not None} async def __aenter__(self): """ From 733d5c3eb5e26814f2fff0f95e6cd5682038f6b9 Mon Sep 17 00:00:00 2001 From: Jan Range Date: Fri, 1 Mar 2024 23:51:17 +0100 Subject: [PATCH 5/7] remove unused import --- tests/api/test_async_api.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/api/test_async_api.py b/tests/api/test_async_api.py index 9cb8232..61b1910 100644 --- a/tests/api/test_async_api.py +++ b/tests/api/test_async_api.py @@ -1,8 +1,6 @@ import asyncio import pytest -from pyDataverse.api import NativeApi - class TestAsyncAPI: From cdc897723c8129a409289534c681b81330ca33fd Mon Sep 17 00:00:00 2001 From: Jan Range Date: Wed, 10 Apr 2024 13:06:29 +0200 Subject: [PATCH 6/7] add `Optional` typing for `api_token` --- src/pyDataverse/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyDataverse/api.py b/src/pyDataverse/api.py index a952268..1b79d70 100644 --- a/src/pyDataverse/api.py +++ b/src/pyDataverse/api.py @@ -40,7 +40,7 @@ class Api: def __init__( self, base_url: str, - api_token: str = None, + api_token: Optional[str] = None, api_version: str = "latest", ): """Init an Api() class. From 6f0e89d34e23b78d896183f0ba199b7eeae88e74 Mon Sep 17 00:00:00 2001 From: Jan Range Date: Mon, 22 Apr 2024 11:44:36 +0200 Subject: [PATCH 7/7] add missing `urllib3` dependency --- requirements/common.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/common.txt b/requirements/common.txt index d17ab22..82884e0 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -1,2 +1,3 @@ httpx>=0.26.0 jsonschema>=3.2.0 +urllib3>=2.2.1