From e0c5cc202e13c879f6723cbeb5df13817778c265 Mon Sep 17 00:00:00 2001
From: Ian Rees <ian@ianrees.net>
Date: Mon, 14 May 2018 15:36:59 -0700
Subject: [PATCH 1/2] Refactoring and osm_extract_download

---
 .vscode/settings.json                         |  3 ++
 planetutils/download.py                       | 22 +++++++++++++
 planetutils/elevation_tile_download.py        |  4 +--
 ...vation.py => elevation_tile_downloader.py} |  2 +-
 planetutils/osm_extract_download.py           | 33 +++++++++++++++++++
 planetutils/osm_extract_downloader.py         | 29 ++++++++++++++++
 planetutils/tilepack_download.py              |  5 +--
 .../{tilepack.py => tilepack_downloader.py}   | 18 +++-------
 planetutils/tilepack_list.py                  |  6 ++--
 setup.py                                      |  4 +--
 ...n.py => test_elevation_tile_downloader.py} | 16 ++++-----
 11 files changed, 110 insertions(+), 32 deletions(-)
 create mode 100644 .vscode/settings.json
 create mode 100644 planetutils/download.py
 rename planetutils/{elevation.py => elevation_tile_downloader.py} (98%)
 create mode 100644 planetutils/osm_extract_download.py
 create mode 100644 planetutils/osm_extract_downloader.py
 rename planetutils/{tilepack.py => tilepack_downloader.py} (78%)
 rename tests/{test_elevation.py => test_elevation_tile_downloader.py} (81%)

diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 0000000..6b8e32a
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,3 @@
+{
+    "python.pythonPath": "${workspaceFolder}/virtualenv/bin/python"
+}
\ No newline at end of file
diff --git a/planetutils/download.py b/planetutils/download.py
new file mode 100644
index 0000000..a2d72b0
--- /dev/null
+++ b/planetutils/download.py
@@ -0,0 +1,22 @@
+import subprocess
+import log
+
+def download(url, outpath):
+    pass
+
+def download_curl(url, outpath, compressed=False):
+    args = ['curl', '-L', '--fail', '-o', outpath, url]
+    if not compressed:
+        args.append('--compressed')
+
+    log.info("Downloading to %s"%outpath)
+    log.debug(url)
+    log.debug(' '.join(args))
+    p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    out, err = p.communicate()
+    e = p.wait()
+    if e != 0:
+        raise Exception("Error downloading: %s"%err.split("curl:")[-1])
+    else:
+        log.info("Done")
+        
\ No newline at end of file
diff --git a/planetutils/elevation_tile_download.py b/planetutils/elevation_tile_download.py
index 655403a..3b8a1c2 100644
--- a/planetutils/elevation_tile_download.py
+++ b/planetutils/elevation_tile_download.py
@@ -1,8 +1,8 @@
 #!/usr/bin/env python
 import argparse
-from elevation import *
 
 from bbox import load_bboxes_csv, bbox_string
+from elevation_tile_downloader import ElevationTileDownloader
 
 def main():
     parser = argparse.ArgumentParser()
@@ -15,7 +15,7 @@ def main():
     if args.verbose:
         log.set_verbose()
 
-    p = ElevationDownloader(args.outpath)
+    p = ElevationTileDownloader(args.outpath)
     if args.csv:
         p.download_bboxes(load_bboxes_csv(args.csv))
     elif args.bbox:
diff --git a/planetutils/elevation.py b/planetutils/elevation_tile_downloader.py
similarity index 98%
rename from planetutils/elevation.py
rename to planetutils/elevation_tile_downloader.py
index 34f4f48..6d5c0a3 100644
--- a/planetutils/elevation.py
+++ b/planetutils/elevation_tile_downloader.py
@@ -18,7 +18,7 @@ def download_gzip(url, path):
         ps2 = subprocess.Popen(['gzip', '-d'], stdin=ps1.stdout, stdout=f)
         ps2.wait()    
 
-class ElevationDownloader(object):
+class ElevationTileDownloader(object):
     HGT_SIZE = (3601 * 3601 * 2)
     
     def __init__(self, outpath='.'):
diff --git a/planetutils/osm_extract_download.py b/planetutils/osm_extract_download.py
new file mode 100644
index 0000000..3daca50
--- /dev/null
+++ b/planetutils/osm_extract_download.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+import os
+import argparse
+
+import log
+from bbox import bbox_string, load_bboxes_csv
+from osm_extract_downloader import OsmExtractDownloader
+
+def main():
+    parser = argparse.ArgumentParser(usage="OSM Extract Download tool.")
+    parser.add_argument('id', help='Extract ID')
+    # parser.add_argument('--osm-extract-version', help='OSM Extract version', default='latest')
+    parser.add_argument('--outpath', help='Output path for Extract; default is <name>.osm.pbf')
+    parser.add_argument('--api-token', help='Interline Auth Token; default is read from $INTERLINE_API_TOKEN')
+    parser.add_argument('--verbose', help="Verbose output", action='store_true')
+    args = parser.parse_args()
+
+    if args.verbose:
+        log.set_verbose()
+
+    outpath = args.outpath or "%s.osm.pbf"%(args.id)
+    if os.path.exists(outpath):
+        log.warning("Warning: output path %s already exists."%outpath)
+
+    downloader = OsmExtractDownloader()
+    downloader.download(
+        outpath,
+        osm_extract_id=args.id,
+        api_token=args.api_token or os.getenv('INTERLINE_API_TOKEN')
+    )
+
+if __name__ == '__main__':
+    main()
diff --git a/planetutils/osm_extract_downloader.py b/planetutils/osm_extract_downloader.py
new file mode 100644
index 0000000..abb06f9
--- /dev/null
+++ b/planetutils/osm_extract_downloader.py
@@ -0,0 +1,29 @@
+import urllib
+import urlparse
+import subprocess
+import json
+import urllib2
+
+import log
+import download
+
+class OsmExtractDownloader(object):
+    HOST = 'https://app.interline.io'
+    def download(self, outpath, osm_extract_id, osm_extract_version='latest', api_token=None):
+        # Endpoint
+        url = '%s/osm_extracts/%s/download'%(self.HOST, osm_extract_version)
+        if osm_extract_version == 'latest':
+            url = '%s/osm_extracts/download_latest'%(self.HOST)
+
+        # Make url
+        u = list(urlparse.urlsplit(url))
+        q = urlparse.parse_qs(u[3])
+        if osm_extract_version == "latest":
+            q['string_id'] = osm_extract_id
+        if api_token:
+            q['api_token'] = api_token
+        u[3] = urllib.urlencode(q)
+        url = urlparse.urlunsplit(u)
+
+        # Download
+        download.download_curl(url, outpath)
\ No newline at end of file
diff --git a/planetutils/tilepack_download.py b/planetutils/tilepack_download.py
index deee1c9..e6e37ce 100644
--- a/planetutils/tilepack_download.py
+++ b/planetutils/tilepack_download.py
@@ -5,6 +5,7 @@
 import log
 import tilepack
 from bbox import bbox_string, load_bboxes_csv
+from tilepack_downloader import TilepackDownloader
 
 def main():
     parser = argparse.ArgumentParser(usage="Valhalla Tilepack Download tool. If no Tilepack ID is provided, the latest Tilepack is used.")
@@ -28,8 +29,8 @@ def main():
     if os.path.exists(outpath):
         log.warning("Warning: output path %s already exists."%outpath)
 
-    tp = tilepack.Tilepack()
-    tp.download(
+    downloader = TilepackDownloader()
+    downloader.download(
         args.outpath,
         version=args.id,
         compressed=args.compressed,
diff --git a/planetutils/tilepack.py b/planetutils/tilepack_downloader.py
similarity index 78%
rename from planetutils/tilepack.py
rename to planetutils/tilepack_downloader.py
index 3f65616..cec6a83 100644
--- a/planetutils/tilepack.py
+++ b/planetutils/tilepack_downloader.py
@@ -5,8 +5,9 @@
 import urllib2
 
 import log
+import download
 
-class Tilepack(object):
+class TilepackDownloader(object):
     HOST = 'https://app.interline.io'
     def download(self, outpath, version='latest', api_token=None, compressed=False):
         # Endpoint
@@ -21,19 +22,8 @@ def download(self, outpath, version='latest', api_token=None, compressed=False):
         u[3] = urllib.urlencode(q)
         url = urlparse.urlunsplit(u)
         # Download
-        args = ['curl', '-L', '--fail', '-o', outpath, url]
-        if not compressed:
-            args.append('--compressed')
-        log.info("Downloading to %s"%outpath)
-        log.debug(' '.join(args))
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out, err = p.communicate()
-        e = p.wait()
-        if e != 0:
-            raise Exception("Error downloading: %s"%err.split("curl:")[-1])
-        else:
-            log.info("Done")
-    
+        download.download_curl(url, outpath, compressed=compressed)
+   
     def list(self):
         url = "%s/valhalla_planet_tilepacks.json"%(self.HOST)
         contents = urllib2.urlopen(url).read()
diff --git a/planetutils/tilepack_list.py b/planetutils/tilepack_list.py
index 362e596..dff060c 100644
--- a/planetutils/tilepack_list.py
+++ b/planetutils/tilepack_list.py
@@ -2,14 +2,14 @@
 import os
 import argparse
 
-import tilepack
 from bbox import bbox_string, load_bboxes_csv
+from tilepack_downloader import TilepackDownloader
 
 def main():
     parser = argparse.ArgumentParser(usage="List Valhalla Tilepacks.")
     args = parser.parse_args()
-    tp = tilepack.Tilepack()
-    tp.list()
+    downloader = TilepackDownloader()
+    downloader.list()
 
 if __name__ == '__main__':
     main()
diff --git a/setup.py b/setup.py
index 33c01e2..bba3b45 100644
--- a/setup.py
+++ b/setup.py
@@ -17,7 +17,7 @@
     author_email='ian@interline.io',
     license='MIT',
     packages=find_packages(exclude=['contrib', 'docs', 'tests']),
-    install_requires=['boto3'],
+    install_requires=['requests','boto3'],
     tests_require=['nose'],
     test_suite = 'nose.collector',
     entry_points={
@@ -25,10 +25,10 @@
             'osm_planet_update=planetutils.osm_planet_update:main',
             'osm_planet_extract=planetutils.osm_planet_extract:main',
             'osm_planet_get_timestamp=planetutils.osm_planet_get_timestamp:main',
+            'osm_extract_download=planetutils.osm_extract_download:main',
             'elevation_tile_download=planetutils.elevation_tile_download:main',
             'valhalla_tilepack_download=planetutils.tilepack_download:main',
             'valhalla_tilepack_list=planetutils.tilepack_list:main'
-
         ],
     },
     classifiers=[
diff --git a/tests/test_elevation.py b/tests/test_elevation_tile_downloader.py
similarity index 81%
rename from tests/test_elevation.py
rename to tests/test_elevation_tile_downloader.py
index 280ec34..1b3ebcb 100644
--- a/tests/test_elevation.py
+++ b/tests/test_elevation_tile_downloader.py
@@ -3,23 +3,23 @@
 import types
 import unittest
 
-import planetutils.elevation as elevation
+from planetutils.elevation_tile_downloader import ElevationTileDownloader
 
 CA = [-126.386719,32.157012,-113.532715,42.244785]
 
-class TestElevationDownloader(unittest.TestCase):
+class TestElevationTileDownloader(unittest.TestCase):
     def test_download_bboxes(self):
         pass
         
     def test_hgtpath(self):
-        e = elevation.ElevationDownloader('.')
+        e = ElevationTileDownloader('.')
         expect = ('N122', 'N122E037.hgt')
         hgtpath = e.hgtpath(37, 122)
         self.assertEquals(hgtpath[0], expect[0])
         self.assertEquals(hgtpath[1], expect[1])
     
     def test_get_bbox_tiles(self):
-        e = elevation.ElevationDownloader('.')
+        e = ElevationTileDownloader('.')
         tiles = e.get_bbox_tiles(CA)
         self.assertEquals(len(tiles), 154)
         tiles = e.get_bbox_tiles([-180,-90,180,90])
@@ -29,21 +29,21 @@ def download_bbox(self, e, method, args, expect):
         COUNT = []
         def c(self, bucket, prefix, x, y):
             COUNT.append([x,y])
-        e.download_hgt = types.MethodType(c, elevation.ElevationDownloader)
+        e.download_hgt = types.MethodType(c, ElevationTileDownloader)
         method(*args)
         self.assertEquals(len(COUNT), expect)
     
     def test_download_planet(self):
-        e = elevation.ElevationDownloader('.')
+        e = ElevationTileDownloader('.')
         self.download_bbox(e, e.download_planet, [], 64800)
     
     def test_download_bbox(self):
-        e = elevation.ElevationDownloader('.')
+        e = ElevationTileDownloader('.')
         self.download_bbox(e, e.download_bbox, [CA], 154)
     
     def test_download_bbox_found(self):
         d = tempfile.mkdtemp()
-        e = elevation.ElevationDownloader(d)
+        e = ElevationTileDownloader(d)
         # correct size
         path = e.hgtpath(-119, 37)
         os.makedirs(os.path.join(d, path[0]))

From c69207b112aa1aeb7d4755ca37444ab39440746e Mon Sep 17 00:00:00 2001
From: Ian Rees <ian@ianrees.net>
Date: Mon, 14 May 2018 15:57:12 -0700
Subject: [PATCH 2/2] Fixes

---
 planetutils/download.py                  | 9 +++++++++
 planetutils/elevation_tile_download.py   | 1 +
 planetutils/elevation_tile_downloader.py | 9 ++-------
 planetutils/tilepack_download.py         | 4 +---
 planetutils/tilepack_downloader.py       | 3 ++-
 setup.py                                 | 2 +-
 6 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/planetutils/download.py b/planetutils/download.py
index a2d72b0..5711f9a 100644
--- a/planetutils/download.py
+++ b/planetutils/download.py
@@ -1,10 +1,19 @@
+import os
 import subprocess
 import log
 
 def download(url, outpath):
     pass
 
+def download_gzip(url, outpath):
+    with open(outpath, 'wb') as f:
+        ps1 = subprocess.Popen(['curl', '-L', '--fail', '-s', url], stdout=subprocess.PIPE)
+        ps2 = subprocess.Popen(['gzip', '-d'], stdin=ps1.stdout, stdout=f)
+        ps2.wait()
+
 def download_curl(url, outpath, compressed=False):
+    if os.path.exists(outpath):
+        log.warning("Warning: output path %s already exists."%outpath)
     args = ['curl', '-L', '--fail', '-o', outpath, url]
     if not compressed:
         args.append('--compressed')
diff --git a/planetutils/elevation_tile_download.py b/planetutils/elevation_tile_download.py
index 3b8a1c2..8b24d8e 100644
--- a/planetutils/elevation_tile_download.py
+++ b/planetutils/elevation_tile_download.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 import argparse
 
+import log
 from bbox import load_bboxes_csv, bbox_string
 from elevation_tile_downloader import ElevationTileDownloader
 
diff --git a/planetutils/elevation_tile_downloader.py b/planetutils/elevation_tile_downloader.py
index 6d5c0a3..c6a5b0d 100644
--- a/planetutils/elevation_tile_downloader.py
+++ b/planetutils/elevation_tile_downloader.py
@@ -3,6 +3,7 @@
 import subprocess
 import math
 
+import download
 import log
 from bbox import validate_bbox
 
@@ -12,12 +13,6 @@ def makedirs(path):
     except OSError, e:
         pass
 
-def download_gzip(url, path):
-    with open(path, 'wb') as f:
-        ps1 = subprocess.Popen(['curl', '-s', url], stdout=subprocess.PIPE)
-        ps2 = subprocess.Popen(['gzip', '-d'], stdin=ps1.stdout, stdout=f)
-        ps2.wait()    
-
 class ElevationTileDownloader(object):
     HGT_SIZE = (3601 * 3601 * 2)
     
@@ -75,5 +70,5 @@ def download_hgt(self, bucket, prefix, x, y):
         makedirs(os.path.join(self.outpath, od))
         url = 'http://s3.amazonaws.com/%s/%s/%s/%s.gz'%(bucket, prefix, od, key)
         log.info("downloading %s to %s"%(url, op))
-        download_gzip(url, op)
+        download.download_gzip(url, op)
         
diff --git a/planetutils/tilepack_download.py b/planetutils/tilepack_download.py
index e6e37ce..54c6eac 100644
--- a/planetutils/tilepack_download.py
+++ b/planetutils/tilepack_download.py
@@ -26,12 +26,10 @@ def main():
     else:
         if not outpath.endswith('.tar'):
             log.warning("Warning: decompressed output path %s does not end in .tar"%outpath)
-    if os.path.exists(outpath):
-        log.warning("Warning: output path %s already exists."%outpath)
 
     downloader = TilepackDownloader()
     downloader.download(
-        args.outpath,
+        outpath,
         version=args.id,
         compressed=args.compressed,
         api_token=args.api_token or os.getenv('INTERLINE_API_TOKEN')
diff --git a/planetutils/tilepack_downloader.py b/planetutils/tilepack_downloader.py
index cec6a83..10f097d 100644
--- a/planetutils/tilepack_downloader.py
+++ b/planetutils/tilepack_downloader.py
@@ -1,3 +1,4 @@
+import os
 import urllib
 import urlparse
 import subprocess
@@ -49,7 +50,7 @@ def list(self):
             """%(
                 tilepack['id'],
                 a['osm_planet_datetime'],
-                a['bucket_key'],
+                os.path.basename(a['bucket_key']),
                 a['bucket_provider'],
                 ", ".join(a.get('data_contents', [])),
                 tilepack.get('links',{}).get('self'),
diff --git a/setup.py b/setup.py
index bba3b45..e98d47a 100644
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@
     long_description = f.read()
 
 setup(name='interline-planetutils',
-    version='0.2.8-rc1',
+    version='0.3.0',
     description='Interline PlanetUtils',
     long_description=long_description,
     url='https://github.com/interline-io/planetutils',