Merge pull request #12 from interline-io/download-extracts

Refactoring and osm_extract_download

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "python.pythonPath": "${workspaceFolder}/virtualenv/bin/python"
+}

planetutils/download.py

@@ -0,0 +1,31 @@
+import os
+import subprocess
+import log
+
+def download(url, outpath):
+    pass
+
+def download_gzip(url, outpath):
+    with open(outpath, 'wb') as f:
+        ps1 = subprocess.Popen(['curl', '-L', '--fail', '-s', url], stdout=subprocess.PIPE)
+        ps2 = subprocess.Popen(['gzip', '-d'], stdin=ps1.stdout, stdout=f)
+        ps2.wait()
+
+def download_curl(url, outpath, compressed=False):
+    if os.path.exists(outpath):
+        log.warning("Warning: output path %s already exists."%outpath)
+    args = ['curl', '-L', '--fail', '-o', outpath, url]
+    if not compressed:
+        args.append('--compressed')
+
+    log.info("Downloading to %s"%outpath)
+    log.debug(url)
+    log.debug(' '.join(args))
+    p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    out, err = p.communicate()
+    e = p.wait()
+    if e != 0:
+        raise Exception("Error downloading: %s"%err.split("curl:")[-1])
+    else:
+        log.info("Done")
+

planetutils/elevation_tile_download.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python
 import argparse
-from elevation import *
 
+import log
 from bbox import load_bboxes_csv, bbox_string
+from elevation_tile_downloader import ElevationTileDownloader
 
 def main():
     parser = argparse.ArgumentParser()
@@ -15,7 +16,7 @@ def main():
     if args.verbose:
         log.set_verbose()
 
-    p = ElevationDownloader(args.outpath)
+    p = ElevationTileDownloader(args.outpath)
     if args.csv:
         p.download_bboxes(load_bboxes_csv(args.csv))
     elif args.bbox:

planetutils/elevation.py → planetutils/elevation_tile_downloader.py

@@ -3,6 +3,7 @@
 import subprocess
 import math
 
+import download
 import log
 from bbox import validate_bbox
 
@@ -12,13 +13,7 @@ def makedirs(path):
     except OSError, e:
         pass
 
-def download_gzip(url, path):
-    with open(path, 'wb') as f:
-        ps1 = subprocess.Popen(['curl', '-s', url], stdout=subprocess.PIPE)
-        ps2 = subprocess.Popen(['gzip', '-d'], stdin=ps1.stdout, stdout=f)
-        ps2.wait()    
-
-class ElevationDownloader(object):
+class ElevationTileDownloader(object):
     HGT_SIZE = (3601 * 3601 * 2)
 
     def __init__(self, outpath='.'):
@@ -75,5 +70,5 @@ def download_hgt(self, bucket, prefix, x, y):
         makedirs(os.path.join(self.outpath, od))
         url = 'http://s3.amazonaws.com/%s/%s/%s/%s.gz'%(bucket, prefix, od, key)
         log.info("downloading %s to %s"%(url, op))
-        download_gzip(url, op)
+        download.download_gzip(url, op)
 

planetutils/osm_extract_download.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+import os
+import argparse
+
+import log
+from bbox import bbox_string, load_bboxes_csv
+from osm_extract_downloader import OsmExtractDownloader
+
+def main():
+    parser = argparse.ArgumentParser(usage="OSM Extract Download tool.")
+    parser.add_argument('id', help='Extract ID')
+    # parser.add_argument('--osm-extract-version', help='OSM Extract version', default='latest')
+    parser.add_argument('--outpath', help='Output path for Extract; default is <name>.osm.pbf')
+    parser.add_argument('--api-token', help='Interline Auth Token; default is read from $INTERLINE_API_TOKEN')
+    parser.add_argument('--verbose', help="Verbose output", action='store_true')
+    args = parser.parse_args()
+
+    if args.verbose:
+        log.set_verbose()
+
+    outpath = args.outpath or "%s.osm.pbf"%(args.id)
+    if os.path.exists(outpath):
+        log.warning("Warning: output path %s already exists."%outpath)
+
+    downloader = OsmExtractDownloader()
+    downloader.download(
+        outpath,
+        osm_extract_id=args.id,
+        api_token=args.api_token or os.getenv('INTERLINE_API_TOKEN')
+    )
+
+if __name__ == '__main__':
+    main()

planetutils/osm_extract_downloader.py

@@ -0,0 +1,29 @@
+import urllib
+import urlparse
+import subprocess
+import json
+import urllib2
+
+import log
+import download
+
+class OsmExtractDownloader(object):
+    HOST = 'https://app.interline.io'
+    def download(self, outpath, osm_extract_id, osm_extract_version='latest', api_token=None):
+        # Endpoint
+        url = '%s/osm_extracts/%s/download'%(self.HOST, osm_extract_version)
+        if osm_extract_version == 'latest':
+            url = '%s/osm_extracts/download_latest'%(self.HOST)
+
+        # Make url
+        u = list(urlparse.urlsplit(url))
+        q = urlparse.parse_qs(u[3])
+        if osm_extract_version == "latest":
+            q['string_id'] = osm_extract_id
+        if api_token:
+            q['api_token'] = api_token
+        u[3] = urllib.urlencode(q)
+        url = urlparse.urlunsplit(u)
+
+        # Download
+        download.download_curl(url, outpath)

planetutils/tilepack_download.py

@@ -5,6 +5,7 @@
 import log
 import tilepack
 from bbox import bbox_string, load_bboxes_csv
+from tilepack_downloader import TilepackDownloader
 
 def main():
     parser = argparse.ArgumentParser(usage="Valhalla Tilepack Download tool. If no Tilepack ID is provided, the latest Tilepack is used.")
@@ -25,12 +26,10 @@ def main():
     else:
         if not outpath.endswith('.tar'):
             log.warning("Warning: decompressed output path %s does not end in .tar"%outpath)
-    if os.path.exists(outpath):
-        log.warning("Warning: output path %s already exists."%outpath)
 
-    tp = tilepack.Tilepack()
-    tp.download(
-        args.outpath,
+    downloader = TilepackDownloader()
+    downloader.download(
+        outpath,
         version=args.id,
         compressed=args.compressed,
         api_token=args.api_token or os.getenv('INTERLINE_API_TOKEN')

planetutils/tilepack.py → planetutils/tilepack_downloader.py

@@ -1,12 +1,14 @@
+import os
 import urllib
 import urlparse
 import subprocess
 import json
 import urllib2
 
 import log
+import download
 
-class Tilepack(object):
+class TilepackDownloader(object):
     HOST = 'https://app.interline.io'
     def download(self, outpath, version='latest', api_token=None, compressed=False):
         # Endpoint
@@ -21,19 +23,8 @@ def download(self, outpath, version='latest', api_token=None, compressed=False):
         u[3] = urllib.urlencode(q)
         url = urlparse.urlunsplit(u)
         # Download
-        args = ['curl', '-L', '--fail', '-o', outpath, url]
-        if not compressed:
-            args.append('--compressed')
-        log.info("Downloading to %s"%outpath)
-        log.debug(' '.join(args))
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out, err = p.communicate()
-        e = p.wait()
-        if e != 0:
-            raise Exception("Error downloading: %s"%err.split("curl:")[-1])
-        else:
-            log.info("Done")
-
+        download.download_curl(url, outpath, compressed=compressed)
+
     def list(self):
         url = "%s/valhalla_planet_tilepacks.json"%(self.HOST)
         contents = urllib2.urlopen(url).read()
@@ -59,7 +50,7 @@ def list(self):
             """%(
                 tilepack['id'],
                 a['osm_planet_datetime'],
-                a['bucket_key'],
+                os.path.basename(a['bucket_key']),
                 a['bucket_provider'],
                 ", ".join(a.get('data_contents', [])),
                 tilepack.get('links',{}).get('self'),

planetutils/tilepack_list.py

@@ -2,14 +2,14 @@
 import os
 import argparse
 
-import tilepack
 from bbox import bbox_string, load_bboxes_csv
+from tilepack_downloader import TilepackDownloader
 
 def main():
     parser = argparse.ArgumentParser(usage="List Valhalla Tilepacks.")
     args = parser.parse_args()
-    tp = tilepack.Tilepack()
-    tp.list()
+    downloader = TilepackDownloader()
+    downloader.list()
 
 if __name__ == '__main__':
     main()

setup.py

@@ -9,26 +9,26 @@
     long_description = f.read()
 
 setup(name='interline-planetutils',
-    version='0.2.8-rc1',
+    version='0.3.0',
     description='Interline PlanetUtils',
     long_description=long_description,
     url='https://github.com/interline-io/planetutils',
     author='Ian Rees',
     author_email='[email protected]',
     license='MIT',
     packages=find_packages(exclude=['contrib', 'docs', 'tests']),
-    install_requires=['boto3'],
+    install_requires=['requests','boto3'],
     tests_require=['nose'],
     test_suite = 'nose.collector',
     entry_points={
         'console_scripts': [
             'osm_planet_update=planetutils.osm_planet_update:main',
             'osm_planet_extract=planetutils.osm_planet_extract:main',
             'osm_planet_get_timestamp=planetutils.osm_planet_get_timestamp:main',
+            'osm_extract_download=planetutils.osm_extract_download:main',
             'elevation_tile_download=planetutils.elevation_tile_download:main',
             'valhalla_tilepack_download=planetutils.tilepack_download:main',
             'valhalla_tilepack_list=planetutils.tilepack_list:main'
-
         ],
     },
     classifiers=[

tests/test_elevation.py → tests/test_elevation_tile_downloader.py

@@ -3,23 +3,23 @@
 import types
 import unittest
 
-import planetutils.elevation as elevation
+from planetutils.elevation_tile_downloader import ElevationTileDownloader
 
 CA = [-126.386719,32.157012,-113.532715,42.244785]
 
-class TestElevationDownloader(unittest.TestCase):
+class TestElevationTileDownloader(unittest.TestCase):
     def test_download_bboxes(self):
         pass
 
     def test_hgtpath(self):
-        e = elevation.ElevationDownloader('.')
+        e = ElevationTileDownloader('.')
         expect = ('N122', 'N122E037.hgt')
         hgtpath = e.hgtpath(37, 122)
         self.assertEquals(hgtpath[0], expect[0])
         self.assertEquals(hgtpath[1], expect[1])
 
     def test_get_bbox_tiles(self):
-        e = elevation.ElevationDownloader('.')
+        e = ElevationTileDownloader('.')
         tiles = e.get_bbox_tiles(CA)
         self.assertEquals(len(tiles), 154)
         tiles = e.get_bbox_tiles([-180,-90,180,90])
@@ -29,21 +29,21 @@ def download_bbox(self, e, method, args, expect):
         COUNT = []
         def c(self, bucket, prefix, x, y):
             COUNT.append([x,y])
-        e.download_hgt = types.MethodType(c, elevation.ElevationDownloader)
+        e.download_hgt = types.MethodType(c, ElevationTileDownloader)
         method(*args)
         self.assertEquals(len(COUNT), expect)
 
     def test_download_planet(self):
-        e = elevation.ElevationDownloader('.')
+        e = ElevationTileDownloader('.')
         self.download_bbox(e, e.download_planet, [], 64800)
 
     def test_download_bbox(self):
-        e = elevation.ElevationDownloader('.')
+        e = ElevationTileDownloader('.')
         self.download_bbox(e, e.download_bbox, [CA], 154)
 
     def test_download_bbox_found(self):
         d = tempfile.mkdtemp()
-        e = elevation.ElevationDownloader(d)
+        e = ElevationTileDownloader(d)
         # correct size
         path = e.hgtpath(-119, 37)
         os.makedirs(os.path.join(d, path[0]))