Merge branch 'master' of https://github.com/interline-io/planetutils

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: circleci/python:2.7.14-stretch
+      - image: circleci/python:3.6.6
     steps:
       - checkout
       - run:

planetutils/bbox.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import absolute_import, unicode_literals
 import json
 import os
 import csv

planetutils/download.py

@@ -1,6 +1,7 @@
+from __future__ import absolute_import, unicode_literals
 import os
 import subprocess
-import log
+from . import log
 
 def download(url, outpath):
     pass

planetutils/elevation_tile_download.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python
+from __future__ import absolute_import, unicode_literals
 import argparse
 
-import log
-from bbox import load_bboxes_csv, bbox_string
-from elevation_tile_downloader import ElevationTileDownloader
+from . import log
+from .bbox import load_bboxes_csv, bbox_string
+from .elevation_tile_downloader import ElevationTileDownloader
 
 def main():
     parser = argparse.ArgumentParser()

planetutils/elevation_tile_downloader.py

@@ -1,16 +1,17 @@
 #!/usr/bin/env python
+from __future__ import absolute_import, unicode_literals
 import os
 import subprocess
 import math
 
-import download
-import log
-from bbox import validate_bbox
+from . import download
+from . import log
+from .bbox import validate_bbox
 
 def makedirs(path):
     try:
         os.makedirs(path)
-    except OSError, e:
+    except OSError as e:
         pass
 
 class ElevationTileDownloader(object):
@@ -34,8 +35,8 @@ def get_bbox_tiles(self, bbox):
         max_y = int(math.ceil(top))
         expect = (max_x - min_x + 1) * (max_y - min_y + 1)
         tiles = set()
-        for x in xrange(min_x, max_x):
-            for y in xrange(min_y, max_y):
+        for x in range(min_x, max_x):
+            for y in range(min_y, max_y):
                 tiles.add((x,y))
         return tiles
 

planetutils/osm_extract_download.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python
+from __future__ import absolute_import, unicode_literals
 import os
 import argparse
 
-import log
-from bbox import bbox_string, load_bboxes_csv
-from osm_extract_downloader import OsmExtractDownloader
+from . import log
+from .bbox import bbox_string, load_bboxes_csv
+from .osm_extract_downloader import OsmExtractDownloader
 
 def main():
     parser = argparse.ArgumentParser(usage="OSM Extract Download tool.")

planetutils/osm_extract_downloader.py

@@ -1,11 +1,15 @@
-import urllib
-import urlparse
+from __future__ import absolute_import, unicode_literals
+from future.standard_library import install_aliases
+install_aliases()
+from urllib.parse import urlparse, urlencode
+from urllib.request import urlopen
+
 import subprocess
 import json
-import urllib2
 
-import log
-import download
+
+from . import log
+from . import download
 
 class OsmExtractDownloader(object):
     HOST = 'https://app.interline.io'
@@ -22,7 +26,7 @@ def download(self, outpath, osm_extract_id, osm_extract_version='latest', api_to
             q['string_id'] = osm_extract_id
         if api_token:
             q['api_token'] = api_token
-        u[3] = urllib.urlencode(q)
+        u[3] = urlencode(q)
         url = urlparse.urlunsplit(u)
 
         # Download

planetutils/osm_planet_extract.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python
+from __future__ import absolute_import, unicode_literals, print_function
 import argparse
-from planet import *
-import bbox
-from bbox import bbox_string, load_bboxes_csv
+from .planet import *
+from . import bbox
+from .bbox import bbox_string, load_bboxes_csv
 
 def main():
     parser = argparse.ArgumentParser()
@@ -14,6 +15,7 @@ def main():
     parser.add_argument('--bbox', help='Bounding box for extract file. Format for coordinates: left,bottom,right,top')
     parser.add_argument('--verbose', help="Verbose output", action='store_true')
     parser.add_argument('--toolchain', help='OSM toolchain', default='osmosis')
+    parser.add_argument('--strategy', help='Osmium extract strategy: simple, complete_ways, or smart', default='complete_ways')
     parser.add_argument('--commands', help='Output a command list instead of performing action, e.g. for parallel usage', action='store_true')
     args = parser.parse_args()
 
@@ -40,11 +42,11 @@ def main():
         parser.error('must specify --csv, --geojson, or --bbox and --name')
 
     if args.commands:
-        commands = p.extract_commands(bboxes, outpath=args.outpath)
+        commands = p.extract_commands(bboxes, outpath=args.outpath, strategy=args.strategy)
         for i in commands:
-            print " ".join(i)
+            print(" ".join(i))
     else:
-        p.extract_bboxes(bboxes, outpath=args.outpath)
+        p.extract_bboxes(bboxes, outpath=args.outpath, strategy=args.strategy)
 
 if __name__ == '__main__':
     main()

planetutils/osm_planet_get_timestamp.py

@@ -1,15 +1,16 @@
 #!/usr/bin/env python
+from __future__ import absolute_import, unicode_literals, print_function
 import argparse
-from planet import *
-import log
+from .planet import *
+from . import log
 
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('osmpath', help='OSM file')
     args = parser.parse_args()
     p = Planet(args.osmpath)
     log.set_quiet()
-    print p.get_timestamp()
+    print(p.get_timestamp())
 
 if __name__ == '__main__':
     main()

planetutils/osm_planet_update.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python
+from __future__ import absolute_import, unicode_literals
 import argparse
 
-import log
-from planet import *
+from . import log
+from .planet import *
 
 def main():
     parser = argparse.ArgumentParser()
@@ -12,6 +13,7 @@ def main():
     parser.add_argument('--s3', action='store_true', help='Download using S3 client from AWS Public Datasets program. AWS credentials required.')
     parser.add_argument('--workdir', help="Osmosis replication workingDirectory.", default='.')
     parser.add_argument('--verbose', help="Verbose output", action='store_true')
+    parser.add_argument('--size', help='Osmium update memory limit', default='1024')
     args = parser.parse_args()
 
     if args.verbose:
@@ -25,15 +27,14 @@ def main():
             d = PlanetDownloaderHttp(args.osmpath)
         d.download_planet()
 
-
     if args.toolchain == 'osmosis':
         p = PlanetUpdaterOsmosis(args.osmpath)
     elif args.toolchain == 'osmium':
         p = PlanetUpdaterOsmium(args.osmpath)
     else:
         parser.error('unknown toolchain: %s'%args.toolchain)
 
-    p.update_planet(args.outpath)
+    p.update_planet(args.outpath, size=args.size)
 
 if __name__ == '__main__':
     main()

planetutils/planet.py

@@ -1,15 +1,20 @@
 #!/usr/bin/env python
+from __future__ import absolute_import, unicode_literals
+from future.standard_library import install_aliases
+install_aliases()
+from urllib.parse import urlparse, urlencode
+from urllib.request import urlopen
+
 import re
 import os
 import subprocess
-import urllib2
 import tempfile
 import json
 
 import boto3
 
-import log
-from bbox import validate_bbox
+from . import log
+from .bbox import validate_bbox
 
 class PlanetBase(object):
     def __init__(self, osmpath=None, grain='hour', changeset_url=None, osmosis_workdir=None):
@@ -22,7 +27,7 @@ def command(self, args):
         return subprocess.check_output(
             args,
             shell=False
-        )
+        ).decode('utf-8')
 
     def osmosis(self, *args):
         return self.command(['osmosis'] + list(args))
@@ -57,14 +62,14 @@ def extract_bboxes(self, bboxes, workers=1, outpath='.'):
     def extract_bbox(self, name, bbox, workers=1, outpath='.'):
         return self.extract_bboxes({name: bbox}, outpath=outpath, workers=workers)
 
-    def extract_commands(self, bboxes, outpath='.'):
+    def extract_commands(self, bboxes, outpath='.', **kw):
         args = []
         self.command = lambda x:args.append(x)
-        self.extract_bboxes(bboxes, outpath=outpath)
+        self.extract_bboxes(bboxes, outpath=outpath, **kw)
         return args
 
 class PlanetExtractorOsmosis(PlanetExtractor):
-    def extract_bboxes(self, bboxes, workers=1, outpath='.'):
+    def extract_bboxes(self, bboxes, workers=1, outpath='.', **kw):
         args = []
         args += ['--read-pbf-fast', self.osmpath, 'workers=%s'%int(workers)]
         args += ['--tee', str(len(bboxes))]
@@ -84,11 +89,11 @@ def extract_bboxes(self, bboxes, workers=1, outpath='.'):
         self.osmosis(*args)
 
 class PlanetExtractorOsmconvert(PlanetExtractor):
-    def extract_bboxes(self, bboxes, workers=1, outpath='.'):
+    def extract_bboxes(self, bboxes, workers=1, outpath='.', **kw):
         for name, bbox in bboxes.items():
             self.extract_bbox(name, bbox, outpath=outpath)
 
-    def extract_bbox(self, name, bbox, workers=1, outpath='.'):
+    def extract_bbox(self, name, bbox, workers=1, outpath='.', **kw):
         validate_bbox(bbox)
         left, bottom, right, top = bbox
         args = [
@@ -99,7 +104,7 @@ def extract_bbox(self, name, bbox, workers=1, outpath='.'):
         self.osmconvert(*args)
 
 class PlanetExtractorOsmium(PlanetExtractor):
-    def extract_bboxes(self, bboxes, workers=1, outpath='.'):
+    def extract_bboxes(self, bboxes, workers=1, outpath='.', strategy='complete_ways', **kw):
         extracts = []
         for name, bbox in bboxes.items():
             validate_bbox(bbox)
@@ -109,13 +114,12 @@ def extract_bboxes(self, bboxes, workers=1, outpath='.'):
                 'output_format': 'pbf',
                 'bbox': {'left': left, 'right': right, 'top': top, 'bottom':bottom}
             })
-            print extracts[-1]
         config = {'directory': outpath, 'extracts': extracts}
         path = None
-        with tempfile.NamedTemporaryFile(delete=False) as f:
+        with tempfile.NamedTemporaryFile(mode='w', delete=False) as f:
             json.dump(config, f)
             path = f.name
-        self.command(['osmium', 'extract', '-c', path, self.osmpath])
+        self.command(['osmium', 'extract', '-s', strategy, '-c', path, self.osmpath])
         os.unlink(path)
 
 class PlanetDownloader(PlanetBase):
@@ -169,20 +173,20 @@ def _get_planets(self, bucket, prefix, match):
 
 
 class PlanetUpdater(PlanetBase):
-    def update_planet(self, outpath, grain='hour', changeset_url=None):
+    def update_planet(self, outpath, grain='hour', changeset_url=None, **kw):
         raise NotImplementedError
 
 class PlanetUpdaterOsmupdate(PlanetBase):
     pass
 
 class PlanetUpdaterOsmium(PlanetBase):
-    def update_planet(self, outpath, grain='minute', changeset_url=None):
+    def update_planet(self, outpath, grain='minute', changeset_url=None, size='1024', **kw):
         if not os.path.exists(self.osmpath):
             raise Exception('planet file does not exist: %s'%self.osmpath)
-        self.command(['pyosmium-up-to-date', '-s', '8000', '-v', self.osmpath, '-o', outpath])
+        self.command(['pyosmium-up-to-date', '-s', size, '-v', self.osmpath, '-o', outpath])
 
 class PlanetUpdaterOsmosis(PlanetBase):
-    def update_planet(self, outpath, grain='minute', changeset_url=None):
+    def update_planet(self, outpath, grain='minute', changeset_url=None, **kw):
         if not os.path.exists(self.osmpath):
             raise Exception('planet file does not exist: %s'%self.osmpath)
         self.changeset_url = changeset_url or 'https://planet.openstreetmap.org/replication/%s'%grain
@@ -199,7 +203,7 @@ def _initialize(self):
             raise Exception('workdir exists and is not a directory: %s'%self.osmosis_workdir)
         try:
             os.makedirs(self.osmosis_workdir)
-        except OSError, e:
+        except OSError as e:
             pass
         self.osmosis(
             '--read-replication-interval-init',
@@ -217,7 +221,7 @@ def _initialize_state(self):
             return
         timestamp = self.get_timestamp()
         url = 'https://replicate-sequences.osm.mazdermind.de/?%s'%timestamp
-        state = urllib2.urlopen(url).read()
+        state = urlopen(url).read()
         with open(statepath, 'w') as f:
             f.write(state)
 

planetutils/tilepack_download.py

@@ -1,11 +1,12 @@
 #!/usr/bin/env python
+from __future__ import absolute_import, unicode_literals
 import os
 import argparse
 
-import log
-import tilepack
-from bbox import bbox_string, load_bboxes_csv
-from tilepack_downloader import TilepackDownloader
+from . import log
+from . import tilepack
+from .bbox import bbox_string, load_bboxes_csv
+from .tilepack_downloader import TilepackDownloader
 
 def main():
     parser = argparse.ArgumentParser(usage="Valhalla Tilepack Download tool. If no Tilepack ID is provided, the latest Tilepack is used.")

planetutils/tilepack_downloader.py

@@ -1,12 +1,17 @@
+from __future__ import absolute_import, unicode_literals, print_function
+from future.standard_library import install_aliases
+install_aliases()
+from urllib.parse import urlparse, urlencode
+from urllib.request import urlopen
+
 import os
-import urllib
 import urlparse
 import subprocess
 import json
-import urllib2
 
-import log
-import download
+
+from . import log
+from . import download
 
 class TilepackDownloader(object):
     HOST = 'https://app.interline.io'
@@ -20,14 +25,14 @@ def download(self, outpath, version='latest', api_token=None, compressed=False):
         q = urlparse.parse_qs(u[3])
         if api_token:
             q['api_token'] = api_token
-        u[3] = urllib.urlencode(q)
+        u[3] = urlencode(q)
         url = urlparse.urlunsplit(u)
         # Download
         download.download_curl(url, outpath, compressed=compressed)
 
     def list(self):
         url = "%s/valhalla_planet_tilepacks.json"%(self.HOST)
-        contents = urllib2.urlopen(url).read()
+        contents = urlopen(url).read()
         tilepacks = json.loads(contents).get('data', [])
         tilepacks = sorted(tilepacks, key=lambda x:int(x.get('id')))
         for tilepack in tilepacks: