From e5437815109851583888441afe19bbb6ef82fc58 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Thu, 12 Jul 2018 14:12:31 -0700 Subject: [PATCH 01/10] Python3 support --- planetutils/elevation_tile_downloader.py | 2 +- planetutils/osm_planet_extract.py | 2 +- planetutils/osm_planet_get_timestamp.py | 2 +- planetutils/planet.py | 3 +-- setup.py | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/planetutils/elevation_tile_downloader.py b/planetutils/elevation_tile_downloader.py index c6a5b0d..ea7eb93 100644 --- a/planetutils/elevation_tile_downloader.py +++ b/planetutils/elevation_tile_downloader.py @@ -10,7 +10,7 @@ def makedirs(path): try: os.makedirs(path) - except OSError, e: + except OSError as e: pass class ElevationTileDownloader(object): diff --git a/planetutils/osm_planet_extract.py b/planetutils/osm_planet_extract.py index 0acd1f7..22babaf 100644 --- a/planetutils/osm_planet_extract.py +++ b/planetutils/osm_planet_extract.py @@ -42,7 +42,7 @@ def main(): if args.commands: commands = p.extract_commands(bboxes, outpath=args.outpath) for i in commands: - print " ".join(i) + print(" ".join(i)) else: p.extract_bboxes(bboxes, outpath=args.outpath) diff --git a/planetutils/osm_planet_get_timestamp.py b/planetutils/osm_planet_get_timestamp.py index 11f16b6..0a72749 100644 --- a/planetutils/osm_planet_get_timestamp.py +++ b/planetutils/osm_planet_get_timestamp.py @@ -9,7 +9,7 @@ def main(): args = parser.parse_args() p = Planet(args.osmpath) log.set_quiet() - print p.get_timestamp() + print(p.get_timestamp()) if __name__ == '__main__': main() diff --git a/planetutils/planet.py b/planetutils/planet.py index 4c7f297..186c26a 100644 --- a/planetutils/planet.py +++ b/planetutils/planet.py @@ -109,7 +109,6 @@ def extract_bboxes(self, bboxes, workers=1, outpath='.'): 'output_format': 'pbf', 'bbox': {'left': left, 'right': right, 'top': top, 'bottom':bottom} }) - print extracts[-1] config = {'directory': outpath, 'extracts': extracts} path = None with tempfile.NamedTemporaryFile(delete=False) as f: @@ -199,7 +198,7 @@ def _initialize(self): raise Exception('workdir exists and is not a directory: %s'%self.osmosis_workdir) try: os.makedirs(self.osmosis_workdir) - except OSError, e: + except OSError as e: pass self.osmosis( '--read-replication-interval-init', diff --git a/setup.py b/setup.py index 8889a4c..eb467b8 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,6 @@ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 2' + 'Programming Language :: Python :: 2.7' ] ) From 97db198509418d761d9726a1691f1c70f5719821 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Thu, 12 Jul 2018 14:14:08 -0700 Subject: [PATCH 02/10] Fix imports --- planetutils/download.py | 3 ++- planetutils/elevation_tile_download.py | 7 ++++--- planetutils/elevation_tile_downloader.py | 7 ++++--- planetutils/osm_extract_download.py | 7 ++++--- planetutils/osm_extract_downloader.py | 5 +++-- planetutils/osm_planet_extract.py | 8 +++++--- planetutils/osm_planet_get_timestamp.py | 6 ++++-- planetutils/osm_planet_update.py | 5 +++-- planetutils/planet.py | 5 +++-- planetutils/tilepack_download.py | 9 +++++---- planetutils/tilepack_downloader.py | 6 ++++-- planetutils/tilepack_list.py | 5 +++-- 12 files changed, 44 insertions(+), 29 deletions(-) diff --git a/planetutils/download.py b/planetutils/download.py index 5711f9a..747b5ec 100644 --- a/planetutils/download.py +++ b/planetutils/download.py @@ -1,6 +1,7 @@ +from __future__ import absolute_import import os import subprocess -import log +from . import log def download(url, outpath): pass diff --git a/planetutils/elevation_tile_download.py b/planetutils/elevation_tile_download.py index 8b24d8e..ceebe59 100644 --- a/planetutils/elevation_tile_download.py +++ b/planetutils/elevation_tile_download.py @@ -1,9 +1,10 @@ #!/usr/bin/env python +from __future__ import absolute_import import argparse -import log -from bbox import load_bboxes_csv, bbox_string -from elevation_tile_downloader import ElevationTileDownloader +from . import log +from .bbox import load_bboxes_csv, bbox_string +from .elevation_tile_downloader import ElevationTileDownloader def main(): parser = argparse.ArgumentParser() diff --git a/planetutils/elevation_tile_downloader.py b/planetutils/elevation_tile_downloader.py index ea7eb93..f24e334 100644 --- a/planetutils/elevation_tile_downloader.py +++ b/planetutils/elevation_tile_downloader.py @@ -1,11 +1,12 @@ #!/usr/bin/env python +from __future__ import absolute_import import os import subprocess import math -import download -import log -from bbox import validate_bbox +from . import download +from . import log +from .bbox import validate_bbox def makedirs(path): try: diff --git a/planetutils/osm_extract_download.py b/planetutils/osm_extract_download.py index 3daca50..ec3598e 100644 --- a/planetutils/osm_extract_download.py +++ b/planetutils/osm_extract_download.py @@ -1,10 +1,11 @@ #!/usr/bin/env python +from __future__ import absolute_import import os import argparse -import log -from bbox import bbox_string, load_bboxes_csv -from osm_extract_downloader import OsmExtractDownloader +from . import log +from .bbox import bbox_string, load_bboxes_csv +from .osm_extract_downloader import OsmExtractDownloader def main(): parser = argparse.ArgumentParser(usage="OSM Extract Download tool.") diff --git a/planetutils/osm_extract_downloader.py b/planetutils/osm_extract_downloader.py index abb06f9..3982fe8 100644 --- a/planetutils/osm_extract_downloader.py +++ b/planetutils/osm_extract_downloader.py @@ -1,11 +1,12 @@ +from __future__ import absolute_import import urllib import urlparse import subprocess import json import urllib2 -import log -import download +from . import log +from . import download class OsmExtractDownloader(object): HOST = 'https://app.interline.io' diff --git a/planetutils/osm_planet_extract.py b/planetutils/osm_planet_extract.py index 22babaf..3c7a3d0 100644 --- a/planetutils/osm_planet_extract.py +++ b/planetutils/osm_planet_extract.py @@ -1,8 +1,10 @@ #!/usr/bin/env python +from __future__ import print_function +from __future__ import absolute_import import argparse -from planet import * -import bbox -from bbox import bbox_string, load_bboxes_csv +from .planet import * +from . import bbox +from .bbox import bbox_string, load_bboxes_csv def main(): parser = argparse.ArgumentParser() diff --git a/planetutils/osm_planet_get_timestamp.py b/planetutils/osm_planet_get_timestamp.py index 0a72749..5b1cc8b 100644 --- a/planetutils/osm_planet_get_timestamp.py +++ b/planetutils/osm_planet_get_timestamp.py @@ -1,7 +1,9 @@ #!/usr/bin/env python +from __future__ import print_function +from __future__ import absolute_import import argparse -from planet import * -import log +from .planet import * +from . import log def main(): parser = argparse.ArgumentParser() diff --git a/planetutils/osm_planet_update.py b/planetutils/osm_planet_update.py index aef3cd1..b3a87a6 100644 --- a/planetutils/osm_planet_update.py +++ b/planetutils/osm_planet_update.py @@ -1,8 +1,9 @@ #!/usr/bin/env python +from __future__ import absolute_import import argparse -import log -from planet import * +from . import log +from .planet import * def main(): parser = argparse.ArgumentParser() diff --git a/planetutils/planet.py b/planetutils/planet.py index 186c26a..c396d0e 100644 --- a/planetutils/planet.py +++ b/planetutils/planet.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +from __future__ import absolute_import import re import os import subprocess @@ -8,8 +9,8 @@ import boto3 -import log -from bbox import validate_bbox +from . import log +from .bbox import validate_bbox class PlanetBase(object): def __init__(self, osmpath=None, grain='hour', changeset_url=None, osmosis_workdir=None): diff --git a/planetutils/tilepack_download.py b/planetutils/tilepack_download.py index 54c6eac..63a213e 100644 --- a/planetutils/tilepack_download.py +++ b/planetutils/tilepack_download.py @@ -1,11 +1,12 @@ #!/usr/bin/env python +from __future__ import absolute_import import os import argparse -import log -import tilepack -from bbox import bbox_string, load_bboxes_csv -from tilepack_downloader import TilepackDownloader +from . import log +from . import tilepack +from .bbox import bbox_string, load_bboxes_csv +from .tilepack_downloader import TilepackDownloader def main(): parser = argparse.ArgumentParser(usage="Valhalla Tilepack Download tool. If no Tilepack ID is provided, the latest Tilepack is used.") diff --git a/planetutils/tilepack_downloader.py b/planetutils/tilepack_downloader.py index 10f097d..376e918 100644 --- a/planetutils/tilepack_downloader.py +++ b/planetutils/tilepack_downloader.py @@ -1,3 +1,5 @@ +from __future__ import print_function +from __future__ import absolute_import import os import urllib import urlparse @@ -5,8 +7,8 @@ import json import urllib2 -import log -import download +from . import log +from . import download class TilepackDownloader(object): HOST = 'https://app.interline.io' diff --git a/planetutils/tilepack_list.py b/planetutils/tilepack_list.py index dff060c..b9f61c9 100644 --- a/planetutils/tilepack_list.py +++ b/planetutils/tilepack_list.py @@ -1,9 +1,10 @@ #!/usr/bin/env python +from __future__ import absolute_import import os import argparse -from bbox import bbox_string, load_bboxes_csv -from tilepack_downloader import TilepackDownloader +from .bbox import bbox_string, load_bboxes_csv +from .tilepack_downloader import TilepackDownloader def main(): parser = argparse.ArgumentParser(usage="List Valhalla Tilepacks.") From 07adea6a1d0b3f44488d3814fd089437f2a41077 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Thu, 12 Jul 2018 14:15:40 -0700 Subject: [PATCH 03/10] Python3 --- tests/test_bbox.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_bbox.py b/tests/test_bbox.py index 64e649c..4627b16 100644 --- a/tests/test_bbox.py +++ b/tests/test_bbox.py @@ -1,3 +1,4 @@ +from builtins import zip import tempfile import os import unittest From 58e947ad4c79e3125cd1ce4926d32fd23c38a234 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Thu, 12 Jul 2018 14:43:38 -0700 Subject: [PATCH 04/10] urllib --- planetutils/bbox.py | 1 + planetutils/download.py | 2 +- planetutils/elevation_tile_download.py | 2 +- planetutils/elevation_tile_downloader.py | 6 +++--- planetutils/osm_extract_download.py | 2 +- planetutils/osm_extract_downloader.py | 13 ++++++++----- planetutils/osm_planet_extract.py | 3 +-- planetutils/osm_planet_get_timestamp.py | 3 +-- planetutils/osm_planet_update.py | 2 +- planetutils/planet.py | 12 ++++++++---- planetutils/tilepack_download.py | 2 +- planetutils/tilepack_downloader.py | 15 +++++++++------ planetutils/tilepack_list.py | 2 +- setup.py | 2 +- tests/test_bbox.py | 9 +++++---- tests/test_elevation_tile_downloader.py | 11 ++++++----- tests/test_planet.py | 7 ++++--- 17 files changed, 53 insertions(+), 41 deletions(-) diff --git a/planetutils/bbox.py b/planetutils/bbox.py index 6794504..14226b1 100644 --- a/planetutils/bbox.py +++ b/planetutils/bbox.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +from __future__ import absolute_import, unicode_literals import json import os import csv diff --git a/planetutils/download.py b/planetutils/download.py index 747b5ec..c586aca 100644 --- a/planetutils/download.py +++ b/planetutils/download.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import os import subprocess from . import log diff --git a/planetutils/elevation_tile_download.py b/planetutils/elevation_tile_download.py index ceebe59..0728e80 100644 --- a/planetutils/elevation_tile_download.py +++ b/planetutils/elevation_tile_download.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import argparse from . import log diff --git a/planetutils/elevation_tile_downloader.py b/planetutils/elevation_tile_downloader.py index f24e334..fa243f8 100644 --- a/planetutils/elevation_tile_downloader.py +++ b/planetutils/elevation_tile_downloader.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import os import subprocess import math @@ -35,8 +35,8 @@ def get_bbox_tiles(self, bbox): max_y = int(math.ceil(top)) expect = (max_x - min_x + 1) * (max_y - min_y + 1) tiles = set() - for x in xrange(min_x, max_x): - for y in xrange(min_y, max_y): + for x in range(min_x, max_x): + for y in range(min_y, max_y): tiles.add((x,y)) return tiles diff --git a/planetutils/osm_extract_download.py b/planetutils/osm_extract_download.py index ec3598e..e9def02 100644 --- a/planetutils/osm_extract_download.py +++ b/planetutils/osm_extract_download.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import os import argparse diff --git a/planetutils/osm_extract_downloader.py b/planetutils/osm_extract_downloader.py index 3982fe8..2b54720 100644 --- a/planetutils/osm_extract_downloader.py +++ b/planetutils/osm_extract_downloader.py @@ -1,9 +1,12 @@ -from __future__ import absolute_import -import urllib -import urlparse +from __future__ import absolute_import, unicode_literals +from future.standard_library import install_aliases +install_aliases() +from urllib.parse import urlparse, urlencode +from urllib.request import urlopen + import subprocess import json -import urllib2 + from . import log from . import download @@ -23,7 +26,7 @@ def download(self, outpath, osm_extract_id, osm_extract_version='latest', api_to q['string_id'] = osm_extract_id if api_token: q['api_token'] = api_token - u[3] = urllib.urlencode(q) + u[3] = urlencode(q) url = urlparse.urlunsplit(u) # Download diff --git a/planetutils/osm_planet_extract.py b/planetutils/osm_planet_extract.py index 3c7a3d0..1974d9c 100644 --- a/planetutils/osm_planet_extract.py +++ b/planetutils/osm_planet_extract.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals, print_function import argparse from .planet import * from . import bbox diff --git a/planetutils/osm_planet_get_timestamp.py b/planetutils/osm_planet_get_timestamp.py index 5b1cc8b..5d38cd0 100644 --- a/planetutils/osm_planet_get_timestamp.py +++ b/planetutils/osm_planet_get_timestamp.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals, print_function import argparse from .planet import * from . import log diff --git a/planetutils/osm_planet_update.py b/planetutils/osm_planet_update.py index b3a87a6..a0de46b 100644 --- a/planetutils/osm_planet_update.py +++ b/planetutils/osm_planet_update.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import argparse from . import log diff --git a/planetutils/planet.py b/planetutils/planet.py index c396d0e..82e6770 100644 --- a/planetutils/planet.py +++ b/planetutils/planet.py @@ -1,9 +1,13 @@ #!/usr/bin/env python -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals +from future.standard_library import install_aliases +install_aliases() +from urllib.parse import urlparse, urlencode +from urllib.request import urlopen + import re import os import subprocess -import urllib2 import tempfile import json @@ -23,7 +27,7 @@ def command(self, args): return subprocess.check_output( args, shell=False - ) + ).decode('utf-8') def osmosis(self, *args): return self.command(['osmosis'] + list(args)) @@ -217,7 +221,7 @@ def _initialize_state(self): return timestamp = self.get_timestamp() url = 'https://replicate-sequences.osm.mazdermind.de/?%s'%timestamp - state = urllib2.urlopen(url).read() + state = urlopen(url).read() with open(statepath, 'w') as f: f.write(state) diff --git a/planetutils/tilepack_download.py b/planetutils/tilepack_download.py index 63a213e..1c84195 100644 --- a/planetutils/tilepack_download.py +++ b/planetutils/tilepack_download.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import os import argparse diff --git a/planetutils/tilepack_downloader.py b/planetutils/tilepack_downloader.py index 376e918..10d136a 100644 --- a/planetutils/tilepack_downloader.py +++ b/planetutils/tilepack_downloader.py @@ -1,11 +1,14 @@ -from __future__ import print_function -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals, print_function +from future.standard_library import install_aliases +install_aliases() +from urllib.parse import urlparse, urlencode +from urllib.request import urlopen + import os -import urllib import urlparse import subprocess import json -import urllib2 + from . import log from . import download @@ -22,14 +25,14 @@ def download(self, outpath, version='latest', api_token=None, compressed=False): q = urlparse.parse_qs(u[3]) if api_token: q['api_token'] = api_token - u[3] = urllib.urlencode(q) + u[3] = urlencode(q) url = urlparse.urlunsplit(u) # Download download.download_curl(url, outpath, compressed=compressed) def list(self): url = "%s/valhalla_planet_tilepacks.json"%(self.HOST) - contents = urllib2.urlopen(url).read() + contents = urlopen(url).read() tilepacks = json.loads(contents).get('data', []) tilepacks = sorted(tilepacks, key=lambda x:int(x.get('id'))) for tilepack in tilepacks: diff --git a/planetutils/tilepack_list.py b/planetutils/tilepack_list.py index b9f61c9..41e7e44 100644 --- a/planetutils/tilepack_list.py +++ b/planetutils/tilepack_list.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import os import argparse diff --git a/setup.py b/setup.py index eb467b8..abbe40c 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ author_email='ian@interline.io', license='MIT', packages=find_packages(exclude=['contrib', 'docs', 'tests']), - install_requires=['requests','boto3', 'osmium'], + install_requires=['requests','boto3'], tests_require=['nose'], test_suite = 'nose.collector', entry_points={ diff --git a/tests/test_bbox.py b/tests/test_bbox.py index 4627b16..1212e5d 100644 --- a/tests/test_bbox.py +++ b/tests/test_bbox.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, unicode_literals from builtins import zip import tempfile import os @@ -23,21 +24,21 @@ def test_bounds(self): self.assertRaises(AssertionError, bbox.validate_bbox, (0,0,0,-90)) def test_returns_array(self): - self.assertEquals(bbox.validate_bbox([1,2,3,4]), [1.0, 2.0, 3.0, 4.0]) + self.assertEqual(bbox.validate_bbox([1,2,3,4]), [1.0, 2.0, 3.0, 4.0]) class TestLoadBboxesCsv(unittest.TestCase): def test_load(self): - f = tempfile.NamedTemporaryFile(delete=False) + f = tempfile.NamedTemporaryFile(delete=False, mode='w') f.write("%s,%0.2f,%0.2f,%0.2f,%0.2f"%( 'CA', CA[0], CA[1], CA[2], CA[3] )) f.close() - self.assertEquals(bbox.load_bboxes_csv(f.name)['CA'], CA) + self.assertEqual(bbox.load_bboxes_csv(f.name)['CA'], CA) class TestBboxString(unittest.TestCase): def test_returns_array(self): - self.assertEquals(bbox.bbox_string('1.0,2.0,3.0,4.0'), [1.0,2.0,3.0,4.0]) + self.assertEqual(bbox.bbox_string('1.0,2.0,3.0,4.0'), [1.0,2.0,3.0,4.0]) def test_validates(self): self.assertRaises(AssertionError, bbox.bbox_string, ('10,-10,20,-20')) diff --git a/tests/test_elevation_tile_downloader.py b/tests/test_elevation_tile_downloader.py index 1b3ebcb..1e8e45d 100644 --- a/tests/test_elevation_tile_downloader.py +++ b/tests/test_elevation_tile_downloader.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, unicode_literals import tempfile import os import types @@ -15,15 +16,15 @@ def test_hgtpath(self): e = ElevationTileDownloader('.') expect = ('N122', 'N122E037.hgt') hgtpath = e.hgtpath(37, 122) - self.assertEquals(hgtpath[0], expect[0]) - self.assertEquals(hgtpath[1], expect[1]) + self.assertEqual(hgtpath[0], expect[0]) + self.assertEqual(hgtpath[1], expect[1]) def test_get_bbox_tiles(self): e = ElevationTileDownloader('.') tiles = e.get_bbox_tiles(CA) - self.assertEquals(len(tiles), 154) + self.assertEqual(len(tiles), 154) tiles = e.get_bbox_tiles([-180,-90,180,90]) - self.assertEquals(len(tiles), 64800) + self.assertEqual(len(tiles), 64800) def download_bbox(self, e, method, args, expect): COUNT = [] @@ -31,7 +32,7 @@ def c(self, bucket, prefix, x, y): COUNT.append([x,y]) e.download_hgt = types.MethodType(c, ElevationTileDownloader) method(*args) - self.assertEquals(len(COUNT), expect) + self.assertEqual(len(COUNT), expect) def test_download_planet(self): e = ElevationTileDownloader('.') diff --git a/tests/test_planet.py b/tests/test_planet.py index 86f253d..18dacd1 100644 --- a/tests/test_planet.py +++ b/tests/test_planet.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, unicode_literals import tempfile import types import os @@ -28,7 +29,7 @@ def test_osmconvert(self): def test_get_timestamp(self): p = planet.PlanetBase(TESTFILE) - self.assertEquals(p.get_timestamp(), TESTFILE_TIMESTAMP) + self.assertEqual(p.get_timestamp(), TESTFILE_TIMESTAMP) class TestPlanetExtractor(unittest.TestCase): kls = None @@ -41,7 +42,7 @@ def extract_bbox(self): p.extract_bbox(name, bbox, outpath=d) self.assertTrue(os.path.exists(outfile)) p2 = planet.PlanetBase(outfile) - self.assertEquals(p2.get_timestamp(), TESTFILE_TIMESTAMP) + self.assertEqual(p2.get_timestamp(), TESTFILE_TIMESTAMP) os.unlink(outfile) os.rmdir(d) @@ -64,7 +65,7 @@ def c(self, url, outpath): COUNT.append([url,outpath]) p._download = types.MethodType(c, planet.PlanetDownloaderHttp) p.download_planet() - self.assertEquals(COUNT[0], ['https://planet.openstreetmap.org/pbf/planet-latest.osm.pbf', 'test.osm.pbf']) + self.assertEqual(COUNT[0], ['https://planet.openstreetmap.org/pbf/planet-latest.osm.pbf', 'test.osm.pbf']) if __name__ == '__main__': unittest.main() From a1dc1c0919cf914da59852cea9d375e14972fd25 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Thu, 12 Jul 2018 14:43:55 -0700 Subject: [PATCH 05/10] include osmium --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index abbe40c..eb467b8 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ author_email='ian@interline.io', license='MIT', packages=find_packages(exclude=['contrib', 'docs', 'tests']), - install_requires=['requests','boto3'], + install_requires=['requests','boto3', 'osmium'], tests_require=['nose'], test_suite = 'nose.collector', entry_points={ From f6b774ab578a803a0a7e6aafed31206d5de089d9 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Thu, 12 Jul 2018 14:54:23 -0700 Subject: [PATCH 06/10] Use python3 for circleci --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 300624b..7880413 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2 jobs: build: docker: - - image: circleci/python:2.7.14-stretch + - image: circleci/python:3.6.1 steps: - checkout - run: From 394eef3f5d4620d6ce99f4a5c82a80dd30cb249e Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Thu, 12 Jul 2018 15:16:27 -0700 Subject: [PATCH 07/10] python 3.6.6 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7880413..c8d873b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2 jobs: build: docker: - - image: circleci/python:3.6.1 + - image: circleci/python:3.6.6 steps: - checkout - run: From 00254c67bd30f176aefebdb7559c455d255cd6cc Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Thu, 12 Jul 2018 15:19:57 -0700 Subject: [PATCH 08/10] future --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index eb467b8..73b07fd 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ author_email='ian@interline.io', license='MIT', packages=find_packages(exclude=['contrib', 'docs', 'tests']), - install_requires=['requests','boto3', 'osmium'], + install_requires=['future', 'requests','boto3', 'osmium'], tests_require=['nose'], test_suite = 'nose.collector', entry_points={ From 69c7c6e186ad849a63aae5e493a7b6b6c908dd1c Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Thu, 12 Jul 2018 20:41:19 -0700 Subject: [PATCH 09/10] Strategy arguments --- planetutils/osm_planet_extract.py | 5 +++-- planetutils/planet.py | 16 ++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/planetutils/osm_planet_extract.py b/planetutils/osm_planet_extract.py index 1974d9c..7882539 100644 --- a/planetutils/osm_planet_extract.py +++ b/planetutils/osm_planet_extract.py @@ -15,6 +15,7 @@ def main(): parser.add_argument('--bbox', help='Bounding box for extract file. Format for coordinates: left,bottom,right,top') parser.add_argument('--verbose', help="Verbose output", action='store_true') parser.add_argument('--toolchain', help='OSM toolchain', default='osmosis') + parser.add_argument('--strategy', help='Osmium extract strategy: simple, complete_ways, or smart', default='complete_ways') parser.add_argument('--commands', help='Output a command list instead of performing action, e.g. for parallel usage', action='store_true') args = parser.parse_args() @@ -41,11 +42,11 @@ def main(): parser.error('must specify --csv, --geojson, or --bbox and --name') if args.commands: - commands = p.extract_commands(bboxes, outpath=args.outpath) + commands = p.extract_commands(bboxes, outpath=args.outpath, strategy=args.strategy) for i in commands: print(" ".join(i)) else: - p.extract_bboxes(bboxes, outpath=args.outpath) + p.extract_bboxes(bboxes, outpath=args.outpath, strategy=args.strategy) if __name__ == '__main__': main() diff --git a/planetutils/planet.py b/planetutils/planet.py index 82e6770..716a813 100644 --- a/planetutils/planet.py +++ b/planetutils/planet.py @@ -62,14 +62,14 @@ def extract_bboxes(self, bboxes, workers=1, outpath='.'): def extract_bbox(self, name, bbox, workers=1, outpath='.'): return self.extract_bboxes({name: bbox}, outpath=outpath, workers=workers) - def extract_commands(self, bboxes, outpath='.'): + def extract_commands(self, bboxes, outpath='.', **kw): args = [] self.command = lambda x:args.append(x) - self.extract_bboxes(bboxes, outpath=outpath) + self.extract_bboxes(bboxes, outpath=outpath, **kw) return args class PlanetExtractorOsmosis(PlanetExtractor): - def extract_bboxes(self, bboxes, workers=1, outpath='.'): + def extract_bboxes(self, bboxes, workers=1, outpath='.', **kw): args = [] args += ['--read-pbf-fast', self.osmpath, 'workers=%s'%int(workers)] args += ['--tee', str(len(bboxes))] @@ -89,11 +89,11 @@ def extract_bboxes(self, bboxes, workers=1, outpath='.'): self.osmosis(*args) class PlanetExtractorOsmconvert(PlanetExtractor): - def extract_bboxes(self, bboxes, workers=1, outpath='.'): + def extract_bboxes(self, bboxes, workers=1, outpath='.', **kw): for name, bbox in bboxes.items(): self.extract_bbox(name, bbox, outpath=outpath) - def extract_bbox(self, name, bbox, workers=1, outpath='.'): + def extract_bbox(self, name, bbox, workers=1, outpath='.', **kw): validate_bbox(bbox) left, bottom, right, top = bbox args = [ @@ -104,7 +104,7 @@ def extract_bbox(self, name, bbox, workers=1, outpath='.'): self.osmconvert(*args) class PlanetExtractorOsmium(PlanetExtractor): - def extract_bboxes(self, bboxes, workers=1, outpath='.'): + def extract_bboxes(self, bboxes, workers=1, outpath='.', strategy='complete_ways', **kw): extracts = [] for name, bbox in bboxes.items(): validate_bbox(bbox) @@ -116,10 +116,10 @@ def extract_bboxes(self, bboxes, workers=1, outpath='.'): }) config = {'directory': outpath, 'extracts': extracts} path = None - with tempfile.NamedTemporaryFile(delete=False) as f: + with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: json.dump(config, f) path = f.name - self.command(['osmium', 'extract', '-c', path, self.osmpath]) + self.command(['osmium', 'extract', '-s', strategy, '-c', path, self.osmpath]) os.unlink(path) class PlanetDownloader(PlanetBase): From dccb81f853d196562baad9c80c3fc06c56c6eca4 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Thu, 12 Jul 2018 20:46:38 -0700 Subject: [PATCH 10/10] Add size argument --- planetutils/osm_planet_update.py | 4 ++-- planetutils/planet.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/planetutils/osm_planet_update.py b/planetutils/osm_planet_update.py index a0de46b..c063af1 100644 --- a/planetutils/osm_planet_update.py +++ b/planetutils/osm_planet_update.py @@ -13,6 +13,7 @@ def main(): parser.add_argument('--s3', action='store_true', help='Download using S3 client from AWS Public Datasets program. AWS credentials required.') parser.add_argument('--workdir', help="Osmosis replication workingDirectory.", default='.') parser.add_argument('--verbose', help="Verbose output", action='store_true') + parser.add_argument('--size', help='Osmium update memory limit', default='1024') args = parser.parse_args() if args.verbose: @@ -26,7 +27,6 @@ def main(): d = PlanetDownloaderHttp(args.osmpath) d.download_planet() - if args.toolchain == 'osmosis': p = PlanetUpdaterOsmosis(args.osmpath) elif args.toolchain == 'osmium': @@ -34,7 +34,7 @@ def main(): else: parser.error('unknown toolchain: %s'%args.toolchain) - p.update_planet(args.outpath) + p.update_planet(args.outpath, size=args.size) if __name__ == '__main__': main() diff --git a/planetutils/planet.py b/planetutils/planet.py index 716a813..50a17bf 100644 --- a/planetutils/planet.py +++ b/planetutils/planet.py @@ -173,20 +173,20 @@ def _get_planets(self, bucket, prefix, match): class PlanetUpdater(PlanetBase): - def update_planet(self, outpath, grain='hour', changeset_url=None): + def update_planet(self, outpath, grain='hour', changeset_url=None, **kw): raise NotImplementedError class PlanetUpdaterOsmupdate(PlanetBase): pass class PlanetUpdaterOsmium(PlanetBase): - def update_planet(self, outpath, grain='minute', changeset_url=None): + def update_planet(self, outpath, grain='minute', changeset_url=None, size='1024', **kw): if not os.path.exists(self.osmpath): raise Exception('planet file does not exist: %s'%self.osmpath) - self.command(['pyosmium-up-to-date', '-s', '8000', '-v', self.osmpath, '-o', outpath]) + self.command(['pyosmium-up-to-date', '-s', size, '-v', self.osmpath, '-o', outpath]) class PlanetUpdaterOsmosis(PlanetBase): - def update_planet(self, outpath, grain='minute', changeset_url=None): + def update_planet(self, outpath, grain='minute', changeset_url=None, **kw): if not os.path.exists(self.osmpath): raise Exception('planet file does not exist: %s'%self.osmpath) self.changeset_url = changeset_url or 'https://planet.openstreetmap.org/replication/%s'%grain