Merge branch 'check_timestamp_with_TZ' of https://github.com/GIScienc…

…e/ohsome-py into check_timestamp_with_TZ
GIScience · Nov 20, 2023 · de01d70 · de01d70
2 parents fd700b5 + 19d16d9
commit de01d70
Show file tree

Hide file tree

Showing 45 changed files with 304 additions and 95 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,8 @@
  - support for python 3.12
  - custom [retry](https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.Retry) configuration
  - start and end timestamp meta information of the client are now datetime objects
+ - accept shapely Polygon and MultiPolygon for `bpolys` input parameter
+ - if a request fails a bash script containing the respective `curl` command is logged (if possible). This allows for easier debugging and sharing of failed requests.
 
 ### Removed
 

diff --git a/ohsome/clients.py b/ohsome/clients.py
@@ -5,11 +5,14 @@
 import datetime as dt
 import json
 from pathlib import Path
-from typing import Union, Optional
+from typing import Union, Optional, List
 from urllib.parse import urljoin
 import datetime as dt
 
+import geopandas as gpd
+import pandas as pd
 import requests
+import shapely
 from requests import Session
 from requests.adapters import HTTPAdapter
 from requests.exceptions import RetryError
@@ -235,22 +238,53 @@ def __init__(
 
     def post(
         self,
-        bboxes=None,
-        bcircles=None,
-        bpolys=None,
-        time=None,
-        filter=None,
-        filter2=None,
-        format=None,
-        showMetadata=None,
-        timeout=None,
-        groupByKey=None,
-        groupByKeys=None,
-        groupByValues=None,
-        properties=None,
-        clipGeometry=None,
-        endpoint=None,
-    ):
+        bboxes: Optional[
+            Union[
+                str,
+                dict,
+                pd.DataFrame,
+                List[str],
+                List[float],
+                List[List[str]],
+                List[List[float]],
+            ]
+        ] = None,
+        bcircles: Optional[
+            Union[
+                str,
+                List[str],
+                List[float],
+                List[List[str]],
+                List[List[float]],
+                dict,
+                gpd.GeoDataFrame,
+                pd.DataFrame,
+            ]
+        ] = None,
+        bpolys: Optional[
+            Union[
+                gpd.GeoDataFrame,
+                gpd.GeoSeries,
+                shapely.Polygon,
+                shapely.MultiPolygon,
+                str,
+            ]
+        ] = None,
+        time: Optional[
+            Union[str, dt.datetime, dt.date, list, pd.DatetimeIndex, pd.Series]
+        ] = None,
+        filter: Optional[str] = None,
+        filter2: Optional[str] = None,
+        format: Optional[str] = None,
+        showMetadata: Optional[bool] = None,
+        timeout: Optional[int] = None,
+        groupByKey: Optional[str] = None,
+        groupByKeys: Optional[Union[str, List[str]]] = None,
+        groupByValues: Optional[Union[str, List[str]]] = None,
+        properties: Optional[Union[str, List[str]]] = None,
+        clipGeometry: Optional[bool] = None,
+        endpoint: Optional[str] = None,
+    ) -> OhsomeResponse:
         """
         Sends request to ohsome API
 
@@ -267,7 +301,7 @@ def post(
         - pandas.DataFrame with columns 'lon', 'lat' and 'radius'
         - geopandas.GeoDataFrame with geometry column with Point geometries only and a column 'radius'.
 
-        :param bpolys: Polygons given as geopandas.GeoDataFrame, GeoJSON FeatureCollection or str
+        :param bpolys: Polygons given as geopandas.GeoDataFrame, geopandas.GeoSeries, shapely.Polygon, shapely.MultiPolygon, GeoJSON FeatureCollection or str
         e.g. "8.65821,49.41129,8.65821,49.41825,8.70053,8.65821|8.67817,49.42147,8.67817,49.4342,8.67817"
 
         :param time: One or more ISO-8601 conform timestring(s) given as str, list, pandas.Series, pandas.DateTimeIndex,

diff --git a/ohsome/exceptions.py b/ohsome/exceptions.py
@@ -7,6 +7,8 @@
 import json
 from pathlib import Path
 
+from curlify2 import Curlify
+
 
 class OhsomeException(Exception):
     """Exception to handle ohsome API errors"""
@@ -33,8 +35,17 @@ def log(self, log_dir: Path):
         self.log_bpolys(log_dir, log_file_name)
         self.log_parameter(log_dir, log_file_name)
         if self.response is not None:
+            self.log_curl(log_dir, log_file_name)
             self.log_response(log_dir, log_file_name)
 
+    def log_curl(self, log_dir: Path, log_file_name: str) -> None:
+        """Log the respective curl command for the request for easy debugging and sharing."""
+        log_file = log_dir / f"{log_file_name}_curl.sh"
+        curl = Curlify(self.response.request)
+        curl_command = curl.to_curl()
+        with log_file.open(mode="w") as dst:
+            dst.write(curl_command)
+
     def log_response(self, log_dir: Path, log_file_name: str):
         """
         Log raw response. This may duplicate much data but is helpful for debugging to know the exact raw answer by the

diff --git a/ohsome/helper.py b/ohsome/helper.py
@@ -4,12 +4,14 @@
 """Ohsome utility functions"""
 
 import datetime
+import json
 import re
-from typing import Tuple
+from typing import Tuple, Union, List
 
 import geopandas as gpd
 import numpy as np
 import pandas as pd
+import shapely
 
 from ohsome import OhsomeException
 
@@ -73,7 +75,18 @@ def format_boundary(params: dict) -> dict:
     return params
 
 
-def format_bcircles(bcircles):
+def format_bcircles(
+    bcircles: Union[
+        str,
+        List[str],
+        List[float],
+        List[List[str]],
+        List[List[float]],
+        dict,
+        gpd.GeoDataFrame,
+        pd.DataFrame,
+    ]
+) -> str:
     """
     Formats bcircles parameter to comply with ohsome API
     :param
@@ -103,9 +116,12 @@ def format_bcircles(bcircles):
             ]
         )
     elif isinstance(bcircles, gpd.GeoDataFrame):
-        if bcircles.geometry.geom_type.unique() != ["Point"]:
+        if (bcircles.geometry.geom_type.unique() != ["Point"]) or (
+            "radius" not in bcircles.columns
+        ):
             raise OhsomeException(
-                message="The geometry of the 'bcircles' GeoDataFrame may only include 'Point' geometry types."
+                message="The geometry of the 'bcircles' GeoDataFrame may only include 'Point' geometry types and "
+                "requires a 'radius' column."
             )
         formatted = bcircles.apply(
             lambda r: f"{int(r.name)}:{r.geometry.x},{r.geometry.y},{r['radius']}",
@@ -127,7 +143,17 @@ def format_bcircles(bcircles):
         raise OhsomeException(message="'bcircles' parameter has invalid format.")
 
 
-def format_bboxes(bboxes):
+def format_bboxes(
+    bboxes: Union[
+        str,
+        dict,
+        pd.DataFrame,
+        List[str],
+        List[float],
+        List[List[str]],
+        List[List[float]],
+    ]
+) -> str:
     """
     Formats bboxes parameter to comply with ohsome API
     :param
@@ -177,17 +203,31 @@ def format_bboxes(bboxes):
         )
 
 
-def format_bpolys(bpolys):
+def format_bpolys(
+    bpolys: Union[gpd.GeoDataFrame, shapely.Polygon, shapely.MultiPolygon, str]
+) -> str:
     """
     Formats bpolys parameter to comply with ohsome API
     :param
     bpolys: Polygons given as geopandas.GeoDataFrame or string formatted as GeoJSON FeatureCollection.
     :return:
     """
-    if isinstance(bpolys, gpd.GeoDataFrame):
+    if isinstance(bpolys, gpd.GeoDataFrame) or isinstance(bpolys, gpd.GeoSeries):
         return bpolys.to_json(na="drop")
+    elif isinstance(bpolys, shapely.Polygon) or isinstance(
+        bpolys, shapely.MultiPolygon
+    ):
+        return format_bpolys(gpd.GeoDataFrame(geometry=[bpolys]))
+    elif isinstance(bpolys, str):
+        try:
+            return format_bpolys(gpd.GeoDataFrame.from_features(json.loads(bpolys)))
+        except Exception as e:
+            raise OhsomeException(message="Invalid geojson.") from e
     else:
-        return bpolys
+        raise OhsomeException(
+            message="bpolys must be a geojson string, a shapely polygonal object or a geopandas "
+            "object"
+        )
 
 
 def format_list_parameters(parameters: dict) -> dict:

diff --git a/ohsome/test/cassettes/test_client/test_check_time_parameter_datetime.yaml b/ohsome/test/cassettes/test_client/test_check_time_parameter_datetime.yaml
@@ -43,7 +43,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Fri, 17 Nov 2023 11:28:23 GMT
+      - Fri, 17 Nov 2023 15:37:04 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:

diff --git a/ohsome/test/cassettes/test_client/test_check_time_parameter_list.yaml b/ohsome/test/cassettes/test_client/test_check_time_parameter_list.yaml
@@ -44,7 +44,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Fri, 17 Nov 2023 11:28:23 GMT
+      - Fri, 17 Nov 2023 15:37:04 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:

diff --git a/ohsome/test/cassettes/test_client/test_end_timestamp_as_time_input.yaml b/ohsome/test/cassettes/test_client/test_end_timestamp_as_time_input.yaml
@@ -42,7 +42,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Fri, 17 Nov 2023 11:28:26 GMT
+      - Fri, 17 Nov 2023 15:37:06 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:

diff --git a/ohsome/test/cassettes/test_client/test_format_bboxes_dataframe.yaml b/ohsome/test/cassettes/test_client/test_format_bboxes_dataframe.yaml
@@ -42,7 +42,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Fri, 17 Nov 2023 11:28:32 GMT
+      - Fri, 17 Nov 2023 15:37:13 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:

diff --git a/ohsome/test/cassettes/test_client/test_format_bboxes_list.yaml b/ohsome/test/cassettes/test_client/test_format_bboxes_list.yaml
@@ -42,7 +42,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Fri, 17 Nov 2023 11:28:32 GMT
+      - Fri, 17 Nov 2023 15:37:13 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:
@@ -99,7 +99,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Fri, 17 Nov 2023 11:28:34 GMT
+      - Fri, 17 Nov 2023 15:37:14 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:
@@ -156,7 +156,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Fri, 17 Nov 2023 11:28:34 GMT
+      - Fri, 17 Nov 2023 15:37:14 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:
@@ -213,7 +213,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Fri, 17 Nov 2023 11:28:36 GMT
+      - Fri, 17 Nov 2023 15:37:16 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:
@@ -270,7 +270,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Fri, 17 Nov 2023 11:28:36 GMT
+      - Fri, 17 Nov 2023 15:37:16 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:

diff --git a/ohsome/test/cassettes/test_client/test_format_bcircles_dataframe.yaml b/ohsome/test/cassettes/test_client/test_format_bcircles_dataframe.yaml
@@ -45,7 +45,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Fri, 17 Nov 2023 11:28:26 GMT
+      - Fri, 17 Nov 2023 15:37:06 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:

diff --git a/ohsome/test/cassettes/test_client/test_format_bcircles_list.yaml b/ohsome/test/cassettes/test_client/test_format_bcircles_list.yaml
@@ -42,7 +42,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Fri, 17 Nov 2023 11:28:27 GMT
+      - Fri, 17 Nov 2023 15:37:08 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:
@@ -102,7 +102,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Fri, 17 Nov 2023 11:28:27 GMT
+      - Fri, 17 Nov 2023 15:37:08 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:
@@ -159,7 +159,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Fri, 17 Nov 2023 11:28:29 GMT
+      - Fri, 17 Nov 2023 15:37:09 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:
@@ -216,7 +216,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Fri, 17 Nov 2023 11:28:29 GMT
+      - Fri, 17 Nov 2023 15:37:09 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:

diff --git a/...nt/test_format_bcircles_geodataframe.yaml → ...t_client/test_format_bcircles_pandas.yaml b/...nt/test_format_bcircles_geodataframe.yaml → ...t_client/test_format_bcircles_pandas.yaml
@@ -51,7 +51,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Fri, 17 Nov 2023 11:28:31 GMT
+      - Fri, 17 Nov 2023 15:37:11 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:

diff --git a/ohsome/test/cassettes/test_client/test_format_bpolys.yaml b/ohsome/test/cassettes/test_client/test_format_bpolys.yaml
@@ -1,6 +1,6 @@
 interactions:
 - request:
-    body: bpolys=%7B%22type%22%3A+%22FeatureCollection%22%2C+%22features%22%3A+%5B%7B%22id%22%3A+%220%22%2C+%22type%22%3A+%22Feature%22%2C+%22properties%22%3A+%7B%7D%2C+%22geometry%22%3A+%7B%22type%22%3A+%22Polygon%22%2C+%22coordinates%22%3A+%5B%5B%5B8.695077896118164%2C+49.408711468953854%5D%2C+%5B8.699712753295898%2C+49.408711468953854%5D%2C+%5B8.699712753295898%2C+49.41155955732304%5D%2C+%5B8.695077896118164%2C+49.41155955732304%5D%2C+%5B8.695077896118164%2C+49.408711468953854%5D%5D%5D%7D%7D%2C+%7B%22id%22%3A+%221%22%2C+%22type%22%3A+%22Feature%22%2C+%22properties%22%3A+%7B%7D%2C+%22geometry%22%3A+%7B%22type%22%3A+%22Polygon%22%2C+%22coordinates%22%3A+%5B%5B%5B8.677010536193848%2C+49.41370947536709%5D%2C+%5B8.682074546813965%2C+49.41370947536709%5D%2C+%5B8.682074546813965%2C+49.416641030041134%5D%2C+%5B8.677010536193848%2C+49.416641030041134%5D%2C+%5B8.677010536193848%2C+49.41370947536709%5D%5D%5D%7D%7D%5D%7D&time=2018-01-01&filter=amenity%3Drestaurant+and+type%3Anode
+    body: bpolys=%7B%22type%22%3A+%22FeatureCollection%22%2C+%22features%22%3A+%5B%7B%22id%22%3A+%220%22%2C+%22type%22%3A+%22Feature%22%2C+%22properties%22%3A+%7B%22id%22%3A+%220%22%7D%2C+%22geometry%22%3A+%7B%22type%22%3A+%22Polygon%22%2C+%22coordinates%22%3A+%5B%5B%5B8.695077896118164%2C+49.408711468953854%5D%2C+%5B8.699712753295898%2C+49.408711468953854%5D%2C+%5B8.699712753295898%2C+49.41155955732304%5D%2C+%5B8.695077896118164%2C+49.41155955732304%5D%2C+%5B8.695077896118164%2C+49.408711468953854%5D%5D%5D%7D%7D%2C+%7B%22id%22%3A+%221%22%2C+%22type%22%3A+%22Feature%22%2C+%22properties%22%3A+%7B%22id%22%3A+%221%22%7D%2C+%22geometry%22%3A+%7B%22type%22%3A+%22Polygon%22%2C+%22coordinates%22%3A+%5B%5B%5B8.677010536193848%2C+49.41370947536709%5D%2C+%5B8.682074546813965%2C+49.41370947536709%5D%2C+%5B8.682074546813965%2C+49.416641030041134%5D%2C+%5B8.677010536193848%2C+49.416641030041134%5D%2C+%5B8.677010536193848%2C+49.41370947536709%5D%5D%5D%7D%7D%5D%7D&time=2018-01-01&filter=amenity%3Drestaurant+and+type%3Anode
     headers:
       Accept:
       - '*/*'
@@ -9,7 +9,7 @@ interactions:
       Connection:
       - keep-alive
       Content-Length:
-      - '975'
+      - '1013'
       Content-Type:
       - application/x-www-form-urlencoded
       user-agent:
@@ -42,7 +42,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Fri, 17 Nov 2023 11:28:31 GMT
+      - Fri, 17 Nov 2023 15:37:11 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server: