diff --git a/.circleci/config.yml b/.circleci/config.yml index 300624b..c8d873b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2 jobs: build: docker: - - image: circleci/python:2.7.14-stretch + - image: circleci/python:3.6.6 steps: - checkout - run: diff --git a/planetutils/bbox.py b/planetutils/bbox.py index 6794504..14226b1 100644 --- a/planetutils/bbox.py +++ b/planetutils/bbox.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +from __future__ import absolute_import, unicode_literals import json import os import csv diff --git a/planetutils/download.py b/planetutils/download.py index 5711f9a..c586aca 100644 --- a/planetutils/download.py +++ b/planetutils/download.py @@ -1,6 +1,7 @@ +from __future__ import absolute_import, unicode_literals import os import subprocess -import log +from . import log def download(url, outpath): pass diff --git a/planetutils/elevation_tile_download.py b/planetutils/elevation_tile_download.py index 8b24d8e..0728e80 100644 --- a/planetutils/elevation_tile_download.py +++ b/planetutils/elevation_tile_download.py @@ -1,9 +1,10 @@ #!/usr/bin/env python +from __future__ import absolute_import, unicode_literals import argparse -import log -from bbox import load_bboxes_csv, bbox_string -from elevation_tile_downloader import ElevationTileDownloader +from . import log +from .bbox import load_bboxes_csv, bbox_string +from .elevation_tile_downloader import ElevationTileDownloader def main(): parser = argparse.ArgumentParser() diff --git a/planetutils/elevation_tile_downloader.py b/planetutils/elevation_tile_downloader.py index c6a5b0d..fa243f8 100644 --- a/planetutils/elevation_tile_downloader.py +++ b/planetutils/elevation_tile_downloader.py @@ -1,16 +1,17 @@ #!/usr/bin/env python +from __future__ import absolute_import, unicode_literals import os import subprocess import math -import download -import log -from bbox import validate_bbox +from . import download +from . import log +from .bbox import validate_bbox def makedirs(path): try: os.makedirs(path) - except OSError, e: + except OSError as e: pass class ElevationTileDownloader(object): @@ -34,8 +35,8 @@ def get_bbox_tiles(self, bbox): max_y = int(math.ceil(top)) expect = (max_x - min_x + 1) * (max_y - min_y + 1) tiles = set() - for x in xrange(min_x, max_x): - for y in xrange(min_y, max_y): + for x in range(min_x, max_x): + for y in range(min_y, max_y): tiles.add((x,y)) return tiles diff --git a/planetutils/osm_extract_download.py b/planetutils/osm_extract_download.py index 3daca50..e9def02 100644 --- a/planetutils/osm_extract_download.py +++ b/planetutils/osm_extract_download.py @@ -1,10 +1,11 @@ #!/usr/bin/env python +from __future__ import absolute_import, unicode_literals import os import argparse -import log -from bbox import bbox_string, load_bboxes_csv -from osm_extract_downloader import OsmExtractDownloader +from . import log +from .bbox import bbox_string, load_bboxes_csv +from .osm_extract_downloader import OsmExtractDownloader def main(): parser = argparse.ArgumentParser(usage="OSM Extract Download tool.") diff --git a/planetutils/osm_extract_downloader.py b/planetutils/osm_extract_downloader.py index abb06f9..2b54720 100644 --- a/planetutils/osm_extract_downloader.py +++ b/planetutils/osm_extract_downloader.py @@ -1,11 +1,15 @@ -import urllib -import urlparse +from __future__ import absolute_import, unicode_literals +from future.standard_library import install_aliases +install_aliases() +from urllib.parse import urlparse, urlencode +from urllib.request import urlopen + import subprocess import json -import urllib2 -import log -import download + +from . import log +from . import download class OsmExtractDownloader(object): HOST = 'https://app.interline.io' @@ -22,7 +26,7 @@ def download(self, outpath, osm_extract_id, osm_extract_version='latest', api_to q['string_id'] = osm_extract_id if api_token: q['api_token'] = api_token - u[3] = urllib.urlencode(q) + u[3] = urlencode(q) url = urlparse.urlunsplit(u) # Download diff --git a/planetutils/osm_planet_extract.py b/planetutils/osm_planet_extract.py index 0acd1f7..7882539 100644 --- a/planetutils/osm_planet_extract.py +++ b/planetutils/osm_planet_extract.py @@ -1,8 +1,9 @@ #!/usr/bin/env python +from __future__ import absolute_import, unicode_literals, print_function import argparse -from planet import * -import bbox -from bbox import bbox_string, load_bboxes_csv +from .planet import * +from . import bbox +from .bbox import bbox_string, load_bboxes_csv def main(): parser = argparse.ArgumentParser() @@ -14,6 +15,7 @@ def main(): parser.add_argument('--bbox', help='Bounding box for extract file. Format for coordinates: left,bottom,right,top') parser.add_argument('--verbose', help="Verbose output", action='store_true') parser.add_argument('--toolchain', help='OSM toolchain', default='osmosis') + parser.add_argument('--strategy', help='Osmium extract strategy: simple, complete_ways, or smart', default='complete_ways') parser.add_argument('--commands', help='Output a command list instead of performing action, e.g. for parallel usage', action='store_true') args = parser.parse_args() @@ -40,11 +42,11 @@ def main(): parser.error('must specify --csv, --geojson, or --bbox and --name') if args.commands: - commands = p.extract_commands(bboxes, outpath=args.outpath) + commands = p.extract_commands(bboxes, outpath=args.outpath, strategy=args.strategy) for i in commands: - print " ".join(i) + print(" ".join(i)) else: - p.extract_bboxes(bboxes, outpath=args.outpath) + p.extract_bboxes(bboxes, outpath=args.outpath, strategy=args.strategy) if __name__ == '__main__': main() diff --git a/planetutils/osm_planet_get_timestamp.py b/planetutils/osm_planet_get_timestamp.py index 11f16b6..5d38cd0 100644 --- a/planetutils/osm_planet_get_timestamp.py +++ b/planetutils/osm_planet_get_timestamp.py @@ -1,7 +1,8 @@ #!/usr/bin/env python +from __future__ import absolute_import, unicode_literals, print_function import argparse -from planet import * -import log +from .planet import * +from . import log def main(): parser = argparse.ArgumentParser() @@ -9,7 +10,7 @@ def main(): args = parser.parse_args() p = Planet(args.osmpath) log.set_quiet() - print p.get_timestamp() + print(p.get_timestamp()) if __name__ == '__main__': main() diff --git a/planetutils/osm_planet_update.py b/planetutils/osm_planet_update.py index aef3cd1..c063af1 100644 --- a/planetutils/osm_planet_update.py +++ b/planetutils/osm_planet_update.py @@ -1,8 +1,9 @@ #!/usr/bin/env python +from __future__ import absolute_import, unicode_literals import argparse -import log -from planet import * +from . import log +from .planet import * def main(): parser = argparse.ArgumentParser() @@ -12,6 +13,7 @@ def main(): parser.add_argument('--s3', action='store_true', help='Download using S3 client from AWS Public Datasets program. AWS credentials required.') parser.add_argument('--workdir', help="Osmosis replication workingDirectory.", default='.') parser.add_argument('--verbose', help="Verbose output", action='store_true') + parser.add_argument('--size', help='Osmium update memory limit', default='1024') args = parser.parse_args() if args.verbose: @@ -25,7 +27,6 @@ def main(): d = PlanetDownloaderHttp(args.osmpath) d.download_planet() - if args.toolchain == 'osmosis': p = PlanetUpdaterOsmosis(args.osmpath) elif args.toolchain == 'osmium': @@ -33,7 +34,7 @@ def main(): else: parser.error('unknown toolchain: %s'%args.toolchain) - p.update_planet(args.outpath) + p.update_planet(args.outpath, size=args.size) if __name__ == '__main__': main() diff --git a/planetutils/planet.py b/planetutils/planet.py index 4c7f297..50a17bf 100644 --- a/planetutils/planet.py +++ b/planetutils/planet.py @@ -1,15 +1,20 @@ #!/usr/bin/env python +from __future__ import absolute_import, unicode_literals +from future.standard_library import install_aliases +install_aliases() +from urllib.parse import urlparse, urlencode +from urllib.request import urlopen + import re import os import subprocess -import urllib2 import tempfile import json import boto3 -import log -from bbox import validate_bbox +from . import log +from .bbox import validate_bbox class PlanetBase(object): def __init__(self, osmpath=None, grain='hour', changeset_url=None, osmosis_workdir=None): @@ -22,7 +27,7 @@ def command(self, args): return subprocess.check_output( args, shell=False - ) + ).decode('utf-8') def osmosis(self, *args): return self.command(['osmosis'] + list(args)) @@ -57,14 +62,14 @@ def extract_bboxes(self, bboxes, workers=1, outpath='.'): def extract_bbox(self, name, bbox, workers=1, outpath='.'): return self.extract_bboxes({name: bbox}, outpath=outpath, workers=workers) - def extract_commands(self, bboxes, outpath='.'): + def extract_commands(self, bboxes, outpath='.', **kw): args = [] self.command = lambda x:args.append(x) - self.extract_bboxes(bboxes, outpath=outpath) + self.extract_bboxes(bboxes, outpath=outpath, **kw) return args class PlanetExtractorOsmosis(PlanetExtractor): - def extract_bboxes(self, bboxes, workers=1, outpath='.'): + def extract_bboxes(self, bboxes, workers=1, outpath='.', **kw): args = [] args += ['--read-pbf-fast', self.osmpath, 'workers=%s'%int(workers)] args += ['--tee', str(len(bboxes))] @@ -84,11 +89,11 @@ def extract_bboxes(self, bboxes, workers=1, outpath='.'): self.osmosis(*args) class PlanetExtractorOsmconvert(PlanetExtractor): - def extract_bboxes(self, bboxes, workers=1, outpath='.'): + def extract_bboxes(self, bboxes, workers=1, outpath='.', **kw): for name, bbox in bboxes.items(): self.extract_bbox(name, bbox, outpath=outpath) - def extract_bbox(self, name, bbox, workers=1, outpath='.'): + def extract_bbox(self, name, bbox, workers=1, outpath='.', **kw): validate_bbox(bbox) left, bottom, right, top = bbox args = [ @@ -99,7 +104,7 @@ def extract_bbox(self, name, bbox, workers=1, outpath='.'): self.osmconvert(*args) class PlanetExtractorOsmium(PlanetExtractor): - def extract_bboxes(self, bboxes, workers=1, outpath='.'): + def extract_bboxes(self, bboxes, workers=1, outpath='.', strategy='complete_ways', **kw): extracts = [] for name, bbox in bboxes.items(): validate_bbox(bbox) @@ -109,13 +114,12 @@ def extract_bboxes(self, bboxes, workers=1, outpath='.'): 'output_format': 'pbf', 'bbox': {'left': left, 'right': right, 'top': top, 'bottom':bottom} }) - print extracts[-1] config = {'directory': outpath, 'extracts': extracts} path = None - with tempfile.NamedTemporaryFile(delete=False) as f: + with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: json.dump(config, f) path = f.name - self.command(['osmium', 'extract', '-c', path, self.osmpath]) + self.command(['osmium', 'extract', '-s', strategy, '-c', path, self.osmpath]) os.unlink(path) class PlanetDownloader(PlanetBase): @@ -169,20 +173,20 @@ def _get_planets(self, bucket, prefix, match): class PlanetUpdater(PlanetBase): - def update_planet(self, outpath, grain='hour', changeset_url=None): + def update_planet(self, outpath, grain='hour', changeset_url=None, **kw): raise NotImplementedError class PlanetUpdaterOsmupdate(PlanetBase): pass class PlanetUpdaterOsmium(PlanetBase): - def update_planet(self, outpath, grain='minute', changeset_url=None): + def update_planet(self, outpath, grain='minute', changeset_url=None, size='1024', **kw): if not os.path.exists(self.osmpath): raise Exception('planet file does not exist: %s'%self.osmpath) - self.command(['pyosmium-up-to-date', '-s', '8000', '-v', self.osmpath, '-o', outpath]) + self.command(['pyosmium-up-to-date', '-s', size, '-v', self.osmpath, '-o', outpath]) class PlanetUpdaterOsmosis(PlanetBase): - def update_planet(self, outpath, grain='minute', changeset_url=None): + def update_planet(self, outpath, grain='minute', changeset_url=None, **kw): if not os.path.exists(self.osmpath): raise Exception('planet file does not exist: %s'%self.osmpath) self.changeset_url = changeset_url or 'https://planet.openstreetmap.org/replication/%s'%grain @@ -199,7 +203,7 @@ def _initialize(self): raise Exception('workdir exists and is not a directory: %s'%self.osmosis_workdir) try: os.makedirs(self.osmosis_workdir) - except OSError, e: + except OSError as e: pass self.osmosis( '--read-replication-interval-init', @@ -217,7 +221,7 @@ def _initialize_state(self): return timestamp = self.get_timestamp() url = 'https://replicate-sequences.osm.mazdermind.de/?%s'%timestamp - state = urllib2.urlopen(url).read() + state = urlopen(url).read() with open(statepath, 'w') as f: f.write(state) diff --git a/planetutils/tilepack_download.py b/planetutils/tilepack_download.py index 54c6eac..1c84195 100644 --- a/planetutils/tilepack_download.py +++ b/planetutils/tilepack_download.py @@ -1,11 +1,12 @@ #!/usr/bin/env python +from __future__ import absolute_import, unicode_literals import os import argparse -import log -import tilepack -from bbox import bbox_string, load_bboxes_csv -from tilepack_downloader import TilepackDownloader +from . import log +from . import tilepack +from .bbox import bbox_string, load_bboxes_csv +from .tilepack_downloader import TilepackDownloader def main(): parser = argparse.ArgumentParser(usage="Valhalla Tilepack Download tool. If no Tilepack ID is provided, the latest Tilepack is used.") diff --git a/planetutils/tilepack_downloader.py b/planetutils/tilepack_downloader.py index 10f097d..10d136a 100644 --- a/planetutils/tilepack_downloader.py +++ b/planetutils/tilepack_downloader.py @@ -1,12 +1,17 @@ +from __future__ import absolute_import, unicode_literals, print_function +from future.standard_library import install_aliases +install_aliases() +from urllib.parse import urlparse, urlencode +from urllib.request import urlopen + import os -import urllib import urlparse import subprocess import json -import urllib2 -import log -import download + +from . import log +from . import download class TilepackDownloader(object): HOST = 'https://app.interline.io' @@ -20,14 +25,14 @@ def download(self, outpath, version='latest', api_token=None, compressed=False): q = urlparse.parse_qs(u[3]) if api_token: q['api_token'] = api_token - u[3] = urllib.urlencode(q) + u[3] = urlencode(q) url = urlparse.urlunsplit(u) # Download download.download_curl(url, outpath, compressed=compressed) def list(self): url = "%s/valhalla_planet_tilepacks.json"%(self.HOST) - contents = urllib2.urlopen(url).read() + contents = urlopen(url).read() tilepacks = json.loads(contents).get('data', []) tilepacks = sorted(tilepacks, key=lambda x:int(x.get('id'))) for tilepack in tilepacks: diff --git a/planetutils/tilepack_list.py b/planetutils/tilepack_list.py index dff060c..41e7e44 100644 --- a/planetutils/tilepack_list.py +++ b/planetutils/tilepack_list.py @@ -1,9 +1,10 @@ #!/usr/bin/env python +from __future__ import absolute_import, unicode_literals import os import argparse -from bbox import bbox_string, load_bboxes_csv -from tilepack_downloader import TilepackDownloader +from .bbox import bbox_string, load_bboxes_csv +from .tilepack_downloader import TilepackDownloader def main(): parser = argparse.ArgumentParser(usage="List Valhalla Tilepacks.") diff --git a/setup.py b/setup.py index 8889a4c..73b07fd 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ author_email='ian@interline.io', license='MIT', packages=find_packages(exclude=['contrib', 'docs', 'tests']), - install_requires=['requests','boto3', 'osmium'], + install_requires=['future', 'requests','boto3', 'osmium'], tests_require=['nose'], test_suite = 'nose.collector', entry_points={ @@ -35,6 +35,6 @@ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 2' + 'Programming Language :: Python :: 2.7' ] ) diff --git a/tests/test_bbox.py b/tests/test_bbox.py index 64e649c..1212e5d 100644 --- a/tests/test_bbox.py +++ b/tests/test_bbox.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import, unicode_literals +from builtins import zip import tempfile import os import unittest @@ -22,21 +24,21 @@ def test_bounds(self): self.assertRaises(AssertionError, bbox.validate_bbox, (0,0,0,-90)) def test_returns_array(self): - self.assertEquals(bbox.validate_bbox([1,2,3,4]), [1.0, 2.0, 3.0, 4.0]) + self.assertEqual(bbox.validate_bbox([1,2,3,4]), [1.0, 2.0, 3.0, 4.0]) class TestLoadBboxesCsv(unittest.TestCase): def test_load(self): - f = tempfile.NamedTemporaryFile(delete=False) + f = tempfile.NamedTemporaryFile(delete=False, mode='w') f.write("%s,%0.2f,%0.2f,%0.2f,%0.2f"%( 'CA', CA[0], CA[1], CA[2], CA[3] )) f.close() - self.assertEquals(bbox.load_bboxes_csv(f.name)['CA'], CA) + self.assertEqual(bbox.load_bboxes_csv(f.name)['CA'], CA) class TestBboxString(unittest.TestCase): def test_returns_array(self): - self.assertEquals(bbox.bbox_string('1.0,2.0,3.0,4.0'), [1.0,2.0,3.0,4.0]) + self.assertEqual(bbox.bbox_string('1.0,2.0,3.0,4.0'), [1.0,2.0,3.0,4.0]) def test_validates(self): self.assertRaises(AssertionError, bbox.bbox_string, ('10,-10,20,-20')) diff --git a/tests/test_elevation_tile_downloader.py b/tests/test_elevation_tile_downloader.py index 1b3ebcb..1e8e45d 100644 --- a/tests/test_elevation_tile_downloader.py +++ b/tests/test_elevation_tile_downloader.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, unicode_literals import tempfile import os import types @@ -15,15 +16,15 @@ def test_hgtpath(self): e = ElevationTileDownloader('.') expect = ('N122', 'N122E037.hgt') hgtpath = e.hgtpath(37, 122) - self.assertEquals(hgtpath[0], expect[0]) - self.assertEquals(hgtpath[1], expect[1]) + self.assertEqual(hgtpath[0], expect[0]) + self.assertEqual(hgtpath[1], expect[1]) def test_get_bbox_tiles(self): e = ElevationTileDownloader('.') tiles = e.get_bbox_tiles(CA) - self.assertEquals(len(tiles), 154) + self.assertEqual(len(tiles), 154) tiles = e.get_bbox_tiles([-180,-90,180,90]) - self.assertEquals(len(tiles), 64800) + self.assertEqual(len(tiles), 64800) def download_bbox(self, e, method, args, expect): COUNT = [] @@ -31,7 +32,7 @@ def c(self, bucket, prefix, x, y): COUNT.append([x,y]) e.download_hgt = types.MethodType(c, ElevationTileDownloader) method(*args) - self.assertEquals(len(COUNT), expect) + self.assertEqual(len(COUNT), expect) def test_download_planet(self): e = ElevationTileDownloader('.') diff --git a/tests/test_planet.py b/tests/test_planet.py index 86f253d..18dacd1 100644 --- a/tests/test_planet.py +++ b/tests/test_planet.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, unicode_literals import tempfile import types import os @@ -28,7 +29,7 @@ def test_osmconvert(self): def test_get_timestamp(self): p = planet.PlanetBase(TESTFILE) - self.assertEquals(p.get_timestamp(), TESTFILE_TIMESTAMP) + self.assertEqual(p.get_timestamp(), TESTFILE_TIMESTAMP) class TestPlanetExtractor(unittest.TestCase): kls = None @@ -41,7 +42,7 @@ def extract_bbox(self): p.extract_bbox(name, bbox, outpath=d) self.assertTrue(os.path.exists(outfile)) p2 = planet.PlanetBase(outfile) - self.assertEquals(p2.get_timestamp(), TESTFILE_TIMESTAMP) + self.assertEqual(p2.get_timestamp(), TESTFILE_TIMESTAMP) os.unlink(outfile) os.rmdir(d) @@ -64,7 +65,7 @@ def c(self, url, outpath): COUNT.append([url,outpath]) p._download = types.MethodType(c, planet.PlanetDownloaderHttp) p.download_planet() - self.assertEquals(COUNT[0], ['https://planet.openstreetmap.org/pbf/planet-latest.osm.pbf', 'test.osm.pbf']) + self.assertEqual(COUNT[0], ['https://planet.openstreetmap.org/pbf/planet-latest.osm.pbf', 'test.osm.pbf']) if __name__ == '__main__': unittest.main()