From a3c7364d49a7fe423172d38ab8b9bd28fb8b6e94 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Fri, 4 May 2018 16:10:57 -0700 Subject: [PATCH 1/5] Planet extractor updates --- planetutils/osm_planet_extract.py | 13 +++++++++--- planetutils/planet.py | 35 +++++++++++++++++++++++++------ tests/test_planet.py | 23 ++++++++++++++++---- 3 files changed, 58 insertions(+), 13 deletions(-) diff --git a/planetutils/osm_planet_extract.py b/planetutils/osm_planet_extract.py index 2547bd7..1261af4 100644 --- a/planetutils/osm_planet_extract.py +++ b/planetutils/osm_planet_extract.py @@ -12,8 +12,17 @@ def main(): parser.add_argument('--geojson', help='Path to GeoJSON file: bbox for each feature is extracted.') parser.add_argument('--name', help='Name to give to extract file.') parser.add_argument('--bbox', help='Bounding box for extract file. Format for coordinates: left,bottom,right,top') + parser.add_argument('--toolchain', help='OSM toolchain', default='osmosis') + parser.add_argument('--commands', help='Output a command list instead of performing action, e.g. for parallel usage', action='store_true') args = parser.parse_args() - p = Planet(args.osmpath) + + + if args.toolchain == 'osmosis': + p = PlanetExtractorOsmosis(args.osmpath) + elif args.toolchain == 'osmctools': + p = PlanetExtractorOsmconvert(args.osmpath) + else: + parser.error('unknown toolchain: %s'%args.toolchain) bboxes = {} if args.csv: @@ -24,8 +33,6 @@ def main(): bboxes[args.name] = bbox.bbox_string(args.bbox) else: parser.error('must specify --csv, --geojson, or --bbox and --name') - print bboxes - p.extract_bboxes(bboxes, outpath=args.outpath) if __name__ == '__main__': diff --git a/planetutils/planet.py b/planetutils/planet.py index 96cb5a4..b504f2d 100644 --- a/planetutils/planet.py +++ b/planetutils/planet.py @@ -46,12 +46,17 @@ def get_timestamp(self): ][0] return timestamp.strip() - def download_planet(self): - raise NotImplementedError +class Planet(PlanetBase): + pass - def update_planet(self, outpath, grain='hour', changeset_url=None): +class PlanetExtractor(PlanetBase): + def extract_bboxes(self, bboxes, workers=1, outpath='.'): raise NotImplementedError + def extract_bbox(self, name, bbox, workers=1, outpath='.'): + return self.extract_bboxes({name: bbox}, outpath=outpath, workers=workers) + +class PlanetExtractorOsmosis(PlanetExtractor): def extract_bboxes(self, bboxes, workers=1, outpath='.'): args = [] args += ['--read-pbf-fast', self.osmpath, 'workers=%s'%int(workers)] @@ -71,11 +76,24 @@ def extract_bboxes(self, bboxes, workers=1, outpath='.'): args += arg self.osmosis(*args) +class PlanetExtractorOsmconvert(PlanetExtractor): + def extract_bboxes(self, bboxes, workers=1, outpath='.'): + for name, bbox in bboxes.items(): + self.extract_bbox(name, bbox) + def extract_bbox(self, name, bbox, workers=1, outpath='.'): - return self.extract_bboxes({name: bbox}, outpath=outpath, workers=workers) + validate_bbox(bbox) + left, bottom, right, top = bbox + args = [ + self.osmpath, + '-b=%s,%s,%s,%s'%(left, bottom, right, top), + '-o=%s'%os.path.join(outpath, '%s.osm.pbf'%name) + ] + self.osmconvert(*args) -class Planet(PlanetBase): - pass +class PlanetDownloader(PlanetBase): + def download_planet(self): + raise NotImplementedError class PlanetDownloaderHttp(PlanetBase): def _download(self, url, outpath): @@ -122,6 +140,11 @@ def _get_planets(self, bucket, prefix, match): objs.append(obj) return objs + +class PlanetUpdater(PlanetBase): + def update_planet(self, outpath, grain='hour', changeset_url=None): + raise NotImplementedError + class PlanetUpdaterOsmupdate(PlanetBase): pass diff --git a/tests/test_planet.py b/tests/test_planet.py index f1f8bb3..86f253d 100644 --- a/tests/test_planet.py +++ b/tests/test_planet.py @@ -8,6 +8,9 @@ TESTFILE_TIMESTAMP = '2018-02-02T22:34:43Z' TEST_BBOX = [-122.430439,37.766508,-122.379670,37.800052] +# import planetutils.log as log +# log.set_verbose() + class TestPlanetBase(unittest.TestCase): def test_osmosis(self): p = planet.PlanetBase(TESTFILE) @@ -27,11 +30,13 @@ def test_get_timestamp(self): p = planet.PlanetBase(TESTFILE) self.assertEquals(p.get_timestamp(), TESTFILE_TIMESTAMP) - def test_extract_bbox(self, name=None, bbox=None): - name = name or 'test' - bbox = bbox or TEST_BBOX +class TestPlanetExtractor(unittest.TestCase): + kls = None + def extract_bbox(self): + name = 'test' + bbox = TEST_BBOX d = tempfile.mkdtemp() - p = planet.PlanetBase(TESTFILE) + p = self.kls(TESTFILE) outfile = os.path.join(d, '%s.osm.pbf'%name) p.extract_bbox(name, bbox, outpath=d) self.assertTrue(os.path.exists(outfile)) @@ -40,6 +45,16 @@ def test_extract_bbox(self, name=None, bbox=None): os.unlink(outfile) os.rmdir(d) +class TestPlanetExtractorOsmconvert(TestPlanetExtractor): + kls = planet.PlanetExtractorOsmconvert + def test_extract_bbox(self): + self.extract_bbox() + +class TestPlanetExtractorOsmosis(TestPlanetExtractor): + kls = planet.PlanetExtractorOsmosis + def test_extract_bbox(self): + self.extract_bbox() + class TestPlanetDownloaderHttp(unittest.TestCase): def test_download_planet(self): p = planet.PlanetDownloaderHttp('test.osm.pbf') From 55c4c9bded5e3b369e763bd2e08b74add1969111 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Fri, 4 May 2018 17:10:35 -0700 Subject: [PATCH 2/5] extract_commands --- planetutils/osm_planet_extract.py | 6 +++++- planetutils/planet.py | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/planetutils/osm_planet_extract.py b/planetutils/osm_planet_extract.py index 1261af4..8d9b9a3 100644 --- a/planetutils/osm_planet_extract.py +++ b/planetutils/osm_planet_extract.py @@ -33,7 +33,11 @@ def main(): bboxes[args.name] = bbox.bbox_string(args.bbox) else: parser.error('must specify --csv, --geojson, or --bbox and --name') - p.extract_bboxes(bboxes, outpath=args.outpath) + + if args.commands: + print p.extract_commands(bboxes, outpath=args.outpath) + else: + p.extract_bboxes(bboxes, outpath=args.outpath) if __name__ == '__main__': main() diff --git a/planetutils/planet.py b/planetutils/planet.py index b504f2d..d405dfe 100644 --- a/planetutils/planet.py +++ b/planetutils/planet.py @@ -56,6 +56,13 @@ def extract_bboxes(self, bboxes, workers=1, outpath='.'): def extract_bbox(self, name, bbox, workers=1, outpath='.'): return self.extract_bboxes({name: bbox}, outpath=outpath, workers=workers) + def extract_commands(self, bboxes, outpath='.'): + args = [] + self.osmconvert = lambda *x:args.append(x) + self.osmosis = lambda *x:args.append(x) + self.extract_bboxes(bboxes, outpath=outpath) + return args + class PlanetExtractorOsmosis(PlanetExtractor): def extract_bboxes(self, bboxes, workers=1, outpath='.'): args = [] From 1a653a9c3d16327bd82e5bcc603c2c6c083fa6ae Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Tue, 8 May 2018 19:05:27 -0700 Subject: [PATCH 3/5] --commands --- planetutils/osm_planet_extract.py | 7 +++++-- planetutils/planet.py | 20 ++++++++------------ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/planetutils/osm_planet_extract.py b/planetutils/osm_planet_extract.py index 8fa7b2c..e2420d6 100644 --- a/planetutils/osm_planet_extract.py +++ b/planetutils/osm_planet_extract.py @@ -37,9 +37,12 @@ def main(): else: parser.error('must specify --csv, --geojson, or --bbox and --name') + print args.outpath if args.commands: - print p.extract_commands(bboxes, outpath=args.outpath) - else: + commands = p.extract_commands(bboxes, outpath=args.outpath) + for i in commands: + print " ".join(i) + else: p.extract_bboxes(bboxes, outpath=args.outpath) if __name__ == '__main__': diff --git a/planetutils/planet.py b/planetutils/planet.py index 35134bf..8c6f404 100644 --- a/planetutils/planet.py +++ b/planetutils/planet.py @@ -15,21 +15,18 @@ def __init__(self, osmpath=None, grain='hour', changeset_url=None, osmosis_workd d, p = os.path.split(osmpath) self.osmosis_workdir = osmosis_workdir or os.path.join(d, '%s.workdir'%p) - def osmosis(self, *args): - cmd = ['osmosis'] + list(args) - log.debug(' '.join(cmd)) + def command(self, args): + log.debug(args) return subprocess.check_output( cmd, shell=False ) + def osmosis(self, *args): + return self.command(['osmosis'] + list(args)) + def osmconvert(self, *args): - cmd = ['osmconvert'] + list(args) - log.debug(' '.join(cmd)) - return subprocess.check_output( - cmd, - shell=False - ) + return self.command(['osmconvert'] + list(args)) def get_timestamp(self): timestamp = self.osmconvert( @@ -60,8 +57,7 @@ def extract_bbox(self, name, bbox, workers=1, outpath='.'): def extract_commands(self, bboxes, outpath='.'): args = [] - self.osmconvert = lambda *x:args.append(x) - self.osmosis = lambda *x:args.append(x) + self.command = lambda x:args.append(x) self.extract_bboxes(bboxes, outpath=outpath) return args @@ -88,7 +84,7 @@ def extract_bboxes(self, bboxes, workers=1, outpath='.'): class PlanetExtractorOsmconvert(PlanetExtractor): def extract_bboxes(self, bboxes, workers=1, outpath='.'): for name, bbox in bboxes.items(): - self.extract_bbox(name, bbox) + self.extract_bbox(name, bbox, outpath=outpath) def extract_bbox(self, name, bbox, workers=1, outpath='.'): validate_bbox(bbox) From ef333ed459101db28dea479e41d86552c385f9f5 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Wed, 9 May 2018 18:14:40 -0700 Subject: [PATCH 4/5] Fix test --- planetutils/planet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planetutils/planet.py b/planetutils/planet.py index 8c6f404..f8b699d 100644 --- a/planetutils/planet.py +++ b/planetutils/planet.py @@ -18,7 +18,7 @@ def __init__(self, osmpath=None, grain='hour', changeset_url=None, osmosis_workd def command(self, args): log.debug(args) return subprocess.check_output( - cmd, + args, shell=False ) From 4dc8bd4ae8d899feac89d0234a06109afb3c4721 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Wed, 9 May 2018 18:17:00 -0700 Subject: [PATCH 5/5] Remove stray print --- planetutils/osm_planet_extract.py | 1 - 1 file changed, 1 deletion(-) diff --git a/planetutils/osm_planet_extract.py b/planetutils/osm_planet_extract.py index e2420d6..efade52 100644 --- a/planetutils/osm_planet_extract.py +++ b/planetutils/osm_planet_extract.py @@ -37,7 +37,6 @@ def main(): else: parser.error('must specify --csv, --geojson, or --bbox and --name') - print args.outpath if args.commands: commands = p.extract_commands(bboxes, outpath=args.outpath) for i in commands: