From 96d480b07e799beb296d9b6e943aea052a435796 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Thu, 6 Jun 2024 12:34:59 +0100 Subject: [PATCH] refactor: use default line length 88 (over 132) --- fmtm_splitter/db.py | 18 +++-- fmtm_splitter/fmtm_splitter_buildings.py | 23 ++++-- fmtm_splitter/overpass.py | 20 ++++-- fmtm_splitter/splitter.py | 89 +++++++++++++++++++----- tests/conftest.py | 9 ++- tests/test_splitter.py | 25 ++++++- 6 files changed, 147 insertions(+), 37 deletions(-) diff --git a/fmtm_splitter/db.py b/fmtm_splitter/db.py index 22ff1d8..499e52a 100644 --- a/fmtm_splitter/db.py +++ b/fmtm_splitter/db.py @@ -34,7 +34,9 @@ log = logging.getLogger(__name__) -def create_connection(db: Union[str, psycopg2.extensions.connection]) -> psycopg2.extensions.connection: +def create_connection( + db: Union[str, psycopg2.extensions.connection], +) -> psycopg2.extensions.connection: """Get db connection from existing psycopg2 connection, or URL string. Args: @@ -59,7 +61,10 @@ def create_connection(db: Union[str, psycopg2.extensions.connection]) -> psycopg elif _sqlalchemy_import and isinstance(db, sqlalchemy.orm.session.Session): conn = db.connection().connection else: - msg = "The `db` variable is not a valid string, psycopg connection, " "or SQLAlchemy Session." + msg = ( + "The `db` variable is not a valid string, psycopg connection, " + "or SQLAlchemy Session." + ) log.error(msg) raise ValueError(msg) @@ -106,7 +111,9 @@ def create_tables(conn: psycopg2.extensions.connection): tags JSONB ); """ - log.debug("Running tables create command for 'project_aoi', 'ways_poly', 'ways_line'") + log.debug( + "Running tables create command for 'project_aoi', 'ways_poly', 'ways_line'" + ) cur = conn.cursor() cur.execute(create_cmd) @@ -171,5 +178,8 @@ def insert_geom(cur: psycopg2.extensions.cursor, table_name: str, **kwargs) -> N Returns: None """ - query = f"INSERT INTO {table_name}(geom,osm_id,tags) " "VALUES (%(geom)s,%(osm_id)s,%(tags)s)" + query = ( + f"INSERT INTO {table_name}(geom,osm_id,tags) " + "VALUES (%(geom)s,%(osm_id)s,%(tags)s)" + ) cur.execute(query, kwargs) diff --git a/fmtm_splitter/fmtm_splitter_buildings.py b/fmtm_splitter/fmtm_splitter_buildings.py index 1ac0199..fa6399f 100755 --- a/fmtm_splitter/fmtm_splitter_buildings.py +++ b/fmtm_splitter/fmtm_splitter_buildings.py @@ -37,7 +37,9 @@ def split_by_buildings( ): """Split the polygon by buildings in the database using an SQL query.""" dbstring = f"PG:host={dbd[0]} dbname={dbd[1]} " f"user={dbd[2]} password={dbd[3]}" - dbshell = psycopg2.connect(host=dbd[0], database=dbd[1], user=dbd[2], password=dbd[3]) + dbshell = psycopg2.connect( + host=dbd[0], database=dbd[1], user=dbd[2], password=dbd[3] + ) dbshell.autocommit = True dbcursor = dbshell.cursor() dbcursor.execute("DROP TABLE IF EXISTS aoi;") @@ -72,9 +74,16 @@ def split_by_buildings( """, ) p.add_argument("-b", "--boundary", required=True, help="Polygon AOI GeoJSON file") - p.add_argument("-n", "--numfeatures", default=20, help="Number of features on average desired per task") + p.add_argument( + "-n", + "--numfeatures", + default=20, + help="Number of features on average desired per task", + ) p.add_argument("-v", "--verbose", action="store_true", help="verbose output") - p.add_argument("-o", "--outfile", default="fmtm.geojson", help="Output file from splitting") + p.add_argument( + "-o", "--outfile", default="fmtm.geojson", help="Output file from splitting" + ) p.add_argument("-ho", "--host", help="Database host", default="localhost") p.add_argument("-db", "--database", help="Database to use") p.add_argument("-u", "--user", help="Database username") @@ -86,7 +95,9 @@ def split_by_buildings( quit() # if verbose, dump to the terminal. - formatter = logging.Formatter("%(threadName)10s - %(name)s - %(levelname)s - %(message)s") + formatter = logging.Formatter( + "%(threadName)10s - %(name)s - %(levelname)s - %(message)s" + ) level = logging.DEBUG if args.verbose: log.setLevel(level) @@ -114,6 +125,8 @@ def split_by_buildings( modularqueries = [] for sqlfile in modularsqlfiles: with open(os.path.join(modulardir, sqlfile), "r") as sql: - modularqueries.append(sql.read().replace("{%numfeatures%}", str(args.numfeatures))) + modularqueries.append( + sql.read().replace("{%numfeatures%}", str(args.numfeatures)) + ) dbdetails = [args.host, args.database, args.user, args.password] features = split_by_buildings(aoi, modularqueries, dbdetails) diff --git a/fmtm_splitter/overpass.py b/fmtm_splitter/overpass.py index 77a6212..6ab39e3 100755 --- a/fmtm_splitter/overpass.py +++ b/fmtm_splitter/overpass.py @@ -28,11 +28,17 @@ def query(query_string, overpass_url): except Exception: print("overpass did not want to answer that one\n") if response.status_code == 200: - print(f"The overpass API at {overpass_url} accepted the query and " f"returned something.") + print( + f"The overpass API at {overpass_url} accepted the query and " + f"returned something." + ) return response.text else: print(response) - print("Yeah, that didn't work. We reached the Overpass API but " "something went wrong on the server side.") + print( + "Yeah, that didn't work. We reached the Overpass API but " + "something went wrong on the server side." + ) def dbpush(infile, dbd): @@ -55,7 +61,10 @@ def dbpush(infile, dbd): p = subprocess.run(pg, capture_output=True, encoding="utf-8") response = p.stdout error = p.stderr - print(f"osm2pgsql seems to have accepted {infile} and " f"returned {response} \nand\n{error}") + print( + f"osm2pgsql seems to have accepted {infile} and " + f"returned {response} \nand\n{error}" + ) return response except Exception as e: print(e) @@ -70,7 +79,10 @@ def dbpush(infile, dbd): p.add_argument("-q", "--query", help="Text file in overpass query language") p.add_argument("-b", "--boundary", help="AOI as GeoJSON file") p.add_argument( - "-url", "--overpass_url", help="Overpass API server URL", default="https://overpass.kumi.systems/api/interpreter" + "-url", + "--overpass_url", + help="Overpass API server URL", + default="https://overpass.kumi.systems/api/interpreter", ) p.add_argument("-ho", "--host", help="Database host", default="localhost") p.add_argument("-db", "--database", help="Database to use") diff --git a/fmtm_splitter/splitter.py b/fmtm_splitter/splitter.py index 2a694c8..7a33c73 100755 --- a/fmtm_splitter/splitter.py +++ b/fmtm_splitter/splitter.py @@ -33,7 +33,14 @@ from shapely.geometry import Polygon, shape from shapely.ops import unary_union -from fmtm_splitter.db import aoi_to_postgis, close_connection, create_connection, create_tables, drop_tables, insert_geom +from fmtm_splitter.db import ( + aoi_to_postgis, + close_connection, + create_connection, + create_tables, + drop_tables, + insert_geom, +) from osm_rawdata.postgres import PostgresClient # Instantiate logger @@ -65,34 +72,48 @@ def __init__( self.split_features = None @staticmethod - def input_to_geojson(input_data: Union[str, FeatureCollection, dict], merge: bool = False) -> GeoJSON: + def input_to_geojson( + input_data: Union[str, FeatureCollection, dict], merge: bool = False + ) -> GeoJSON: """Parse input data consistently to a GeoJSON obj.""" log.info(f"Parsing GeoJSON from type {type(input_data)}") - if isinstance(input_data, str) and len(input_data) < 250 and Path(input_data).is_file(): + if ( + isinstance(input_data, str) + and len(input_data) < 250 + and Path(input_data).is_file() + ): # Impose restriction for path lengths <250 chars with open(input_data, "r") as jsonfile: try: parsed_geojson = geojson.load(jsonfile) except json.decoder.JSONDecodeError as e: - raise IOError(f"File exists, but content is invalid JSON: {input_data}") from e + raise IOError( + f"File exists, but content is invalid JSON: {input_data}" + ) from e elif isinstance(input_data, FeatureCollection): parsed_geojson = input_data elif isinstance(input_data, dict): parsed_geojson = geojson.loads(geojson.dumps(input_data)) elif isinstance(input_data, str): - geojson_truncated = input_data if len(input_data) < 250 else f"{input_data[:250]}..." + geojson_truncated = ( + input_data if len(input_data) < 250 else f"{input_data[:250]}..." + ) log.debug(f"GeoJSON string passed: {geojson_truncated}") parsed_geojson = geojson.loads(input_data) else: - err = f"The specified AOI is not valid (must be geojson or str): {input_data}" + err = ( + f"The specified AOI is not valid (must be geojson or str): {input_data}" + ) log.error(err) raise ValueError(err) return parsed_geojson @staticmethod - def geojson_to_featcol(geojson: Union[FeatureCollection, Feature, dict]) -> FeatureCollection: + def geojson_to_featcol( + geojson: Union[FeatureCollection, Feature, dict], + ) -> FeatureCollection: """Standardise any geojson type to FeatureCollection.""" # Parse and unparse geojson to extract type if isinstance(geojson, FeatureCollection): @@ -107,7 +128,9 @@ def geojson_to_featcol(geojson: Union[FeatureCollection, Feature, dict]) -> Feat return FeatureCollection(features) @staticmethod - def geojson_to_shapely_polygon(geojson: Union[FeatureCollection, Feature, dict]) -> Polygon: + def geojson_to_shapely_polygon( + geojson: Union[FeatureCollection, Feature, dict], + ) -> Polygon: """Parse GeoJSON and return shapely Polygon. The GeoJSON may be of type FeatureCollection, Feature, or Polygon, @@ -154,12 +177,16 @@ def splitBySquare( # noqa: N802 polygons = [] for x in cols[:-1]: for y in rows[:-1]: - grid_polygon = Polygon([(x, y), (x + width, y), (x + width, y + length), (x, y + length)]) + grid_polygon = Polygon( + [(x, y), (x + width, y), (x + width, y + length), (x, y + length)] + ) clipped_polygon = grid_polygon.intersection(self.aoi) if not clipped_polygon.is_empty: polygons.append(clipped_polygon) - self.split_features = FeatureCollection([Feature(geometry=poly) for poly in polygons]) + self.split_features = FeatureCollection( + [Feature(geometry=poly) for poly in polygons] + ) return self.split_features def splitBySQL( # noqa: N802 @@ -198,7 +225,9 @@ def splitBySQL( # noqa: N802 # Run custom SQL if not buildings or not osm_extract: - log.info("No `buildings` or `osm_extract` params passed, executing custom SQL") + log.info( + "No `buildings` or `osm_extract` params passed, executing custom SQL" + ) # FIXME untested conn = create_connection(db) splitter_cursor = conn.cursor() @@ -326,7 +355,9 @@ def splitByFeature( # noqa: N802 # Clip the multi_polygon by the AOI boundary clipped_multi_polygon = multi_polygon.intersection(self.aoi) - polygon_features = [Feature(geometry=polygon) for polygon in list(clipped_multi_polygon.geoms)] + polygon_features = [ + Feature(geometry=polygon) for polygon in list(clipped_multi_polygon.geoms) + ] # Convert the Polygon Features into a FeatureCollection self.split_features = FeatureCollection(features=polygon_features) @@ -514,7 +545,9 @@ def split_by_sql( split_features = FeatureCollection(features) else: splitter = FMTMSplitter(aoi_featcol) - split_features = splitter.splitBySQL(query, db, num_buildings, osm_extract=extract_geojson) + split_features = splitter.splitBySQL( + query, db, num_buildings, osm_extract=extract_geojson + ) if not split_features: msg = "Failed to generate split features." log.error(msg) @@ -638,16 +671,31 @@ def main(args_list: list[str] | None = None): ) # The default SQL query for feature splitting parser.add_argument("-v", "--verbose", action="store_true", help="verbose output") - parser.add_argument("-o", "--outfile", default="fmtm.geojson", help="Output file from splitting") - parser.add_argument("-m", "--meters", nargs="?", const=50, help="Size in meters if using square splitting") - parser.add_argument("-number", "--number", nargs="?", const=5, help="Number of buildings in a task") + parser.add_argument( + "-o", "--outfile", default="fmtm.geojson", help="Output file from splitting" + ) + parser.add_argument( + "-m", + "--meters", + nargs="?", + const=50, + help="Size in meters if using square splitting", + ) + parser.add_argument( + "-number", "--number", nargs="?", const=5, help="Number of buildings in a task" + ) parser.add_argument("-b", "--boundary", required=True, help="Polygon AOI") parser.add_argument("-s", "--source", help="Source data, Geojson or PG:[dbname]") parser.add_argument("-c", "--custom", help="Custom SQL query for database") parser.add_argument( - "-db", "--dburl", default="postgresql://fmtm:dummycipassword@db:5432/splitter", help="The database url string to custom sql" + "-db", + "--dburl", + default="postgresql://fmtm:dummycipassword@db:5432/splitter", + help="The database url string to custom sql", + ) + parser.add_argument( + "-e", "--extract", help="The OSM data extract for fmtm splitter" ) - parser.add_argument("-e", "--extract", help="The OSM data extract for fmtm splitter") # Accept command line args, or func params args = parser.parse_args(args_list) @@ -658,7 +706,10 @@ def main(args_list: list[str] | None = None): # Set logger logging.basicConfig( level="DEBUG" if args.verbose else "INFO", - format=("%(asctime)s.%(msecs)03d [%(levelname)s] " "%(name)s | %(funcName)s:%(lineno)d | %(message)s"), + format=( + "%(asctime)s.%(msecs)03d [%(levelname)s] " + "%(name)s | %(funcName)s:%(lineno)d | %(message)s" + ), datefmt="%y-%m-%d %H:%M:%S", stream=sys.stdout, ) diff --git a/tests/conftest.py b/tests/conftest.py index 50d604c..aabc10a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,7 +30,10 @@ logging.basicConfig( level="DEBUG", - format=("%(asctime)s.%(msecs)03d [%(levelname)s] " "%(name)s | %(funcName)s:%(lineno)d | %(message)s"), + format=( + "%(asctime)s.%(msecs)03d [%(levelname)s] " + "%(name)s | %(funcName)s:%(lineno)d | %(message)s" + ), datefmt="%y-%m-%d %H:%M:%S", stream=sys.stdout, ) @@ -80,7 +83,9 @@ def aoi_multi_json(): square_maxy = miny + (j + 1) * height # Create Polygon for each square - square_geojson = json.loads(to_geojson(box(square_minx, square_miny, square_maxx, square_maxy))) + square_geojson = json.loads( + to_geojson(box(square_minx, square_miny, square_maxx, square_maxy)) + ) squares.append(geojson.Feature(geometry=square_geojson)) return geojson.FeatureCollection(features=squares) diff --git a/tests/test_splitter.py b/tests/test_splitter.py index 740561d..ab9452c 100644 --- a/tests/test_splitter.py +++ b/tests/test_splitter.py @@ -26,7 +26,13 @@ import geojson import pytest -from fmtm_splitter.splitter import FMTMSplitter, main, split_by_features, split_by_sql, split_by_square +from fmtm_splitter.splitter import ( + FMTMSplitter, + main, + split_by_features, + split_by_sql, + split_by_square, +) log = logging.getLogger(__name__) @@ -175,7 +181,11 @@ def test_split_by_sql_fmtm_multi_geom(extract_json): assert isinstance(features.get("features")[0], dict) assert len(features.get("features")) == 35 - polygons = [feature for feature in features.get("features", []) if feature.get("geometry").get("type") == "Polygon"] + polygons = [ + feature + for feature in features.get("features", []) + if feature.get("geometry").get("type") == "Polygon" + ] assert len(polygons) == 35 polygon_feat = geojson.loads(json.dumps(polygons[0])) @@ -218,7 +228,16 @@ def test_split_by_features_cli(): split_geojson = Path(__file__).parent / "testdata" / "kathmandu_split.geojson" try: - main(["--boundary", str(infile), "--source", str(split_geojson), "--outfile", str(outfile)]) + main( + [ + "--boundary", + str(infile), + "--source", + str(split_geojson), + "--outfile", + str(outfile), + ] + ) except SystemExit: pass