From 49023a4ab48d907ddb7589e2ea54a7bc0f05468f Mon Sep 17 00:00:00 2001 From: Moritz Schott Date: Fri, 17 Nov 2023 17:16:52 +0100 Subject: [PATCH] feat: add curl command to logging closes: #42 Co-authored-by: Christina Ludwig --- CHANGELOG.md | 1 + ohsome/exceptions.py | 11 ++++ .../test_exceptions/test_log_curl.yaml | 57 +++++++++++++++++++ ohsome/test/test_exceptions.py | 34 ++++++++++- poetry.lock | 13 ++++- pyproject.toml | 1 + 6 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 ohsome/test/cassettes/test_exceptions/test_log_curl.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 24f141b..1957777 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - support for python 3.12 - custom [retry](https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.Retry) configuration - start and end timestamp meta information of the client are now datetime objects + - if a request fails a bash script containing the respective `curl` command is logged (if possible). This allows for easier debugging and sharing of failed requests. ### Removed diff --git a/ohsome/exceptions.py b/ohsome/exceptions.py index e5725fb..efc38d1 100644 --- a/ohsome/exceptions.py +++ b/ohsome/exceptions.py @@ -7,6 +7,8 @@ import json from pathlib import Path +from curlify2 import Curlify + class OhsomeException(Exception): """Exception to handle ohsome API errors""" @@ -33,8 +35,17 @@ def log(self, log_dir: Path): self.log_bpolys(log_dir, log_file_name) self.log_parameter(log_dir, log_file_name) if self.response is not None: + self.log_curl(log_dir, log_file_name) self.log_response(log_dir, log_file_name) + def log_curl(self, log_dir: Path, log_file_name: str) -> None: + """Log the respective curl command for the request for easy debugging and sharing.""" + log_file = log_dir / f"{log_file_name}_curl.sh" + curl = Curlify(self.response.request) + curl_command = curl.to_curl() + with log_file.open(mode="w") as dst: + dst.write(curl_command) + def log_response(self, log_dir: Path, log_file_name: str): """ Log raw response. This may duplicate much data but is helpful for debugging to know the exact raw answer by the diff --git a/ohsome/test/cassettes/test_exceptions/test_log_curl.yaml b/ohsome/test/cassettes/test_exceptions/test_log_curl.yaml new file mode 100644 index 0000000..270b745 --- /dev/null +++ b/ohsome/test/cassettes/test_exceptions/test_log_curl.yaml @@ -0,0 +1,57 @@ +interactions: +- request: + body: bboxes=8.67555%2C49.39885%2C8.69637%2C49.41122&timeout=0.001 + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '60' + Content-Type: + - application/x-www-form-urlencoded + user-agent: + - ohsome-py/0.2.0 + method: POST + uri: https://api.ohsome.org/v1/elements/count + response: + body: + string: "{\n \"timestamp\" : \"2023-11-17T12:28:28.06766606\",\n \"status\" + : 413,\n \"message\" : \"The given query is too large in respect to the given + timeout. Please use a smaller region and/or coarser time period.\",\n \"requestUrl\" + : \"https://api.ohsome.org/v1/elements/count\"\n}" + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - Origin,Accept,X-Requested-With,Content-Type,Access-Control-Request-Method,Access-Control-Request-Headers,Authorization + Access-Control-Allow-Methods: + - POST, GET + Access-Control-Allow-Origin: + - '*' + Access-Control-Max-Age: + - '3600' + Cache-Control: + - no-cache, no-store, must-revalidate + Connection: + - close + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Fri, 17 Nov 2023 12:28:27 GMT + Server: + - Apache + Strict-Transport-Security: + - max-age=63072000; includeSubdomains; + Transfer-Encoding: + - chunked + vary: + - accept-encoding + status: + code: 413 + message: '' +version: 1 diff --git a/ohsome/test/test_exceptions.py b/ohsome/test/test_exceptions.py index 8787051..995f112 100644 --- a/ohsome/test/test_exceptions.py +++ b/ohsome/test/test_exceptions.py @@ -118,7 +118,12 @@ def test_log_bpolys(base_client_without_log, tmpdir): base_client_without_log.elements.count.post( bpolys=bpolys, time=time, filter=fltr, timeout=timeout ) - log_file_patterns = ["ohsome_*_bpolys.geojson", "ohsome_*.json", "ohsome_*raw.txt"] + log_file_patterns = [ + "ohsome_*_bpolys.geojson", + "ohsome_*_curl.sh", + "ohsome_*.json", + "ohsome_*raw.txt", + ] for p in log_file_patterns: log_file = list(Path(base_client_without_log.log_dir).glob(p)) assert len(log_file) == 1, f"Log file {p} not found" @@ -126,6 +131,33 @@ def test_log_bpolys(base_client_without_log, tmpdir): log_file[0].unlink() +@pytest.mark.vcr +def test_log_curl(base_client_without_log, tmpdir): + """ + Test whether log file containing curl command is created when request fails + :return: + """ + + base_client_without_log.log = True + base_client_without_log.log_dir = tmpdir.mkdir("logs").strpath + + bboxes = [8.67555, 49.39885, 8.69637, 49.41122] + timeout = 0.001 + + with pytest.raises(ohsome.OhsomeException): + base_client_without_log.elements.count.post(bboxes=bboxes, timeout=timeout) + + log_file = list(Path(base_client_without_log.log_dir).glob("ohsome_*_curl.sh")) + with open(log_file[0]) as file: + assert file.read() == ( + 'curl -X POST -H "user-agent: ohsome-py/0.2.0" -H "Accept-Encoding: gzip, ' + 'deflate" -H "Accept: */*" -H "Connection: keep-alive" -H "Content-Length: 60" ' + '-H "Content-Type: application/x-www-form-urlencoded" ' + "-d 'bboxes=8.67555%2C49.39885%2C8.69637%2C49.41122&timeout=0.001' " + "https://api.ohsome.org/v1/elements/count" + ) + + def test_metadata_invalid_baseurl(custom_client_with_wrong_url): """ Throw exception if the ohsome API is not available diff --git a/poetry.lock b/poetry.lock index 7362e71..da16bd2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -514,6 +514,17 @@ traitlets = ">=4" [package.extras] test = ["pytest"] +[[package]] +name = "curlify2" +version = "2.0.0" +description = "Library to convert python requests and httpx object to curl command." +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "curlify2-2.0.0-py3-none-any.whl", hash = "sha256:23878b13fa33ed92ae5dff0476e5b881b39fc79a5fcb37e6726f96239a136474"}, + {file = "curlify2-2.0.0.tar.gz", hash = "sha256:6c3a98dc8603b76da990f58754b18ec4e9bfe5b881fc52d33ddf9d9096568809"}, +] + [[package]] name = "debugpy" version = "1.8.0" @@ -3005,4 +3016,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "a88b70fa86e8b500ab8e2ab90a07bbf00091c7dbfb75eaa1410c96ef8574aa71" +content-hash = "1f9dcaf124ff252719359a924e1dc99c248a0839434b194fdaabda7c5662e65b" diff --git a/pyproject.toml b/pyproject.toml index b124e47..d3e96d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ pandas = "^2.1.3" numpy = "^1.20.0" geopandas = "^0.14.1" urllib3 = "^2.1.0" +curlify2 = "^2.0.0" [tool.poetry.group.test.dependencies] pytest = "^7.4.3"