diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..6b8e32a --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.pythonPath": "${workspaceFolder}/virtualenv/bin/python" +} \ No newline at end of file diff --git a/planetutils/download.py b/planetutils/download.py new file mode 100644 index 0000000..5711f9a --- /dev/null +++ b/planetutils/download.py @@ -0,0 +1,31 @@ +import os +import subprocess +import log + +def download(url, outpath): + pass + +def download_gzip(url, outpath): + with open(outpath, 'wb') as f: + ps1 = subprocess.Popen(['curl', '-L', '--fail', '-s', url], stdout=subprocess.PIPE) + ps2 = subprocess.Popen(['gzip', '-d'], stdin=ps1.stdout, stdout=f) + ps2.wait() + +def download_curl(url, outpath, compressed=False): + if os.path.exists(outpath): + log.warning("Warning: output path %s already exists."%outpath) + args = ['curl', '-L', '--fail', '-o', outpath, url] + if not compressed: + args.append('--compressed') + + log.info("Downloading to %s"%outpath) + log.debug(url) + log.debug(' '.join(args)) + p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + e = p.wait() + if e != 0: + raise Exception("Error downloading: %s"%err.split("curl:")[-1]) + else: + log.info("Done") + \ No newline at end of file diff --git a/planetutils/elevation_tile_download.py b/planetutils/elevation_tile_download.py index 655403a..8b24d8e 100644 --- a/planetutils/elevation_tile_download.py +++ b/planetutils/elevation_tile_download.py @@ -1,8 +1,9 @@ #!/usr/bin/env python import argparse -from elevation import * +import log from bbox import load_bboxes_csv, bbox_string +from elevation_tile_downloader import ElevationTileDownloader def main(): parser = argparse.ArgumentParser() @@ -15,7 +16,7 @@ def main(): if args.verbose: log.set_verbose() - p = ElevationDownloader(args.outpath) + p = ElevationTileDownloader(args.outpath) if args.csv: p.download_bboxes(load_bboxes_csv(args.csv)) elif args.bbox: diff --git a/planetutils/elevation.py b/planetutils/elevation_tile_downloader.py similarity index 88% rename from planetutils/elevation.py rename to planetutils/elevation_tile_downloader.py index 34f4f48..c6a5b0d 100644 --- a/planetutils/elevation.py +++ b/planetutils/elevation_tile_downloader.py @@ -3,6 +3,7 @@ import subprocess import math +import download import log from bbox import validate_bbox @@ -12,13 +13,7 @@ def makedirs(path): except OSError, e: pass -def download_gzip(url, path): - with open(path, 'wb') as f: - ps1 = subprocess.Popen(['curl', '-s', url], stdout=subprocess.PIPE) - ps2 = subprocess.Popen(['gzip', '-d'], stdin=ps1.stdout, stdout=f) - ps2.wait() - -class ElevationDownloader(object): +class ElevationTileDownloader(object): HGT_SIZE = (3601 * 3601 * 2) def __init__(self, outpath='.'): @@ -75,5 +70,5 @@ def download_hgt(self, bucket, prefix, x, y): makedirs(os.path.join(self.outpath, od)) url = 'http://s3.amazonaws.com/%s/%s/%s/%s.gz'%(bucket, prefix, od, key) log.info("downloading %s to %s"%(url, op)) - download_gzip(url, op) + download.download_gzip(url, op) diff --git a/planetutils/osm_extract_download.py b/planetutils/osm_extract_download.py new file mode 100644 index 0000000..3daca50 --- /dev/null +++ b/planetutils/osm_extract_download.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +import os +import argparse + +import log +from bbox import bbox_string, load_bboxes_csv +from osm_extract_downloader import OsmExtractDownloader + +def main(): + parser = argparse.ArgumentParser(usage="OSM Extract Download tool.") + parser.add_argument('id', help='Extract ID') + # parser.add_argument('--osm-extract-version', help='OSM Extract version', default='latest') + parser.add_argument('--outpath', help='Output path for Extract; default is .osm.pbf') + parser.add_argument('--api-token', help='Interline Auth Token; default is read from $INTERLINE_API_TOKEN') + parser.add_argument('--verbose', help="Verbose output", action='store_true') + args = parser.parse_args() + + if args.verbose: + log.set_verbose() + + outpath = args.outpath or "%s.osm.pbf"%(args.id) + if os.path.exists(outpath): + log.warning("Warning: output path %s already exists."%outpath) + + downloader = OsmExtractDownloader() + downloader.download( + outpath, + osm_extract_id=args.id, + api_token=args.api_token or os.getenv('INTERLINE_API_TOKEN') + ) + +if __name__ == '__main__': + main() diff --git a/planetutils/osm_extract_downloader.py b/planetutils/osm_extract_downloader.py new file mode 100644 index 0000000..abb06f9 --- /dev/null +++ b/planetutils/osm_extract_downloader.py @@ -0,0 +1,29 @@ +import urllib +import urlparse +import subprocess +import json +import urllib2 + +import log +import download + +class OsmExtractDownloader(object): + HOST = 'https://app.interline.io' + def download(self, outpath, osm_extract_id, osm_extract_version='latest', api_token=None): + # Endpoint + url = '%s/osm_extracts/%s/download'%(self.HOST, osm_extract_version) + if osm_extract_version == 'latest': + url = '%s/osm_extracts/download_latest'%(self.HOST) + + # Make url + u = list(urlparse.urlsplit(url)) + q = urlparse.parse_qs(u[3]) + if osm_extract_version == "latest": + q['string_id'] = osm_extract_id + if api_token: + q['api_token'] = api_token + u[3] = urllib.urlencode(q) + url = urlparse.urlunsplit(u) + + # Download + download.download_curl(url, outpath) \ No newline at end of file diff --git a/planetutils/tilepack_download.py b/planetutils/tilepack_download.py index deee1c9..54c6eac 100644 --- a/planetutils/tilepack_download.py +++ b/planetutils/tilepack_download.py @@ -5,6 +5,7 @@ import log import tilepack from bbox import bbox_string, load_bboxes_csv +from tilepack_downloader import TilepackDownloader def main(): parser = argparse.ArgumentParser(usage="Valhalla Tilepack Download tool. If no Tilepack ID is provided, the latest Tilepack is used.") @@ -25,12 +26,10 @@ def main(): else: if not outpath.endswith('.tar'): log.warning("Warning: decompressed output path %s does not end in .tar"%outpath) - if os.path.exists(outpath): - log.warning("Warning: output path %s already exists."%outpath) - tp = tilepack.Tilepack() - tp.download( - args.outpath, + downloader = TilepackDownloader() + downloader.download( + outpath, version=args.id, compressed=args.compressed, api_token=args.api_token or os.getenv('INTERLINE_API_TOKEN') diff --git a/planetutils/tilepack.py b/planetutils/tilepack_downloader.py similarity index 77% rename from planetutils/tilepack.py rename to planetutils/tilepack_downloader.py index 3f65616..10f097d 100644 --- a/planetutils/tilepack.py +++ b/planetutils/tilepack_downloader.py @@ -1,3 +1,4 @@ +import os import urllib import urlparse import subprocess @@ -5,8 +6,9 @@ import urllib2 import log +import download -class Tilepack(object): +class TilepackDownloader(object): HOST = 'https://app.interline.io' def download(self, outpath, version='latest', api_token=None, compressed=False): # Endpoint @@ -21,19 +23,8 @@ def download(self, outpath, version='latest', api_token=None, compressed=False): u[3] = urllib.urlencode(q) url = urlparse.urlunsplit(u) # Download - args = ['curl', '-L', '--fail', '-o', outpath, url] - if not compressed: - args.append('--compressed') - log.info("Downloading to %s"%outpath) - log.debug(' '.join(args)) - p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - e = p.wait() - if e != 0: - raise Exception("Error downloading: %s"%err.split("curl:")[-1]) - else: - log.info("Done") - + download.download_curl(url, outpath, compressed=compressed) + def list(self): url = "%s/valhalla_planet_tilepacks.json"%(self.HOST) contents = urllib2.urlopen(url).read() @@ -59,7 +50,7 @@ def list(self): """%( tilepack['id'], a['osm_planet_datetime'], - a['bucket_key'], + os.path.basename(a['bucket_key']), a['bucket_provider'], ", ".join(a.get('data_contents', [])), tilepack.get('links',{}).get('self'), diff --git a/planetutils/tilepack_list.py b/planetutils/tilepack_list.py index 362e596..dff060c 100644 --- a/planetutils/tilepack_list.py +++ b/planetutils/tilepack_list.py @@ -2,14 +2,14 @@ import os import argparse -import tilepack from bbox import bbox_string, load_bboxes_csv +from tilepack_downloader import TilepackDownloader def main(): parser = argparse.ArgumentParser(usage="List Valhalla Tilepacks.") args = parser.parse_args() - tp = tilepack.Tilepack() - tp.list() + downloader = TilepackDownloader() + downloader.list() if __name__ == '__main__': main() diff --git a/setup.py b/setup.py index 33c01e2..e98d47a 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ long_description = f.read() setup(name='interline-planetutils', - version='0.2.8-rc1', + version='0.3.0', description='Interline PlanetUtils', long_description=long_description, url='https://github.com/interline-io/planetutils', @@ -17,7 +17,7 @@ author_email='ian@interline.io', license='MIT', packages=find_packages(exclude=['contrib', 'docs', 'tests']), - install_requires=['boto3'], + install_requires=['requests','boto3'], tests_require=['nose'], test_suite = 'nose.collector', entry_points={ @@ -25,10 +25,10 @@ 'osm_planet_update=planetutils.osm_planet_update:main', 'osm_planet_extract=planetutils.osm_planet_extract:main', 'osm_planet_get_timestamp=planetutils.osm_planet_get_timestamp:main', + 'osm_extract_download=planetutils.osm_extract_download:main', 'elevation_tile_download=planetutils.elevation_tile_download:main', 'valhalla_tilepack_download=planetutils.tilepack_download:main', 'valhalla_tilepack_list=planetutils.tilepack_list:main' - ], }, classifiers=[ diff --git a/tests/test_elevation.py b/tests/test_elevation_tile_downloader.py similarity index 81% rename from tests/test_elevation.py rename to tests/test_elevation_tile_downloader.py index 280ec34..1b3ebcb 100644 --- a/tests/test_elevation.py +++ b/tests/test_elevation_tile_downloader.py @@ -3,23 +3,23 @@ import types import unittest -import planetutils.elevation as elevation +from planetutils.elevation_tile_downloader import ElevationTileDownloader CA = [-126.386719,32.157012,-113.532715,42.244785] -class TestElevationDownloader(unittest.TestCase): +class TestElevationTileDownloader(unittest.TestCase): def test_download_bboxes(self): pass def test_hgtpath(self): - e = elevation.ElevationDownloader('.') + e = ElevationTileDownloader('.') expect = ('N122', 'N122E037.hgt') hgtpath = e.hgtpath(37, 122) self.assertEquals(hgtpath[0], expect[0]) self.assertEquals(hgtpath[1], expect[1]) def test_get_bbox_tiles(self): - e = elevation.ElevationDownloader('.') + e = ElevationTileDownloader('.') tiles = e.get_bbox_tiles(CA) self.assertEquals(len(tiles), 154) tiles = e.get_bbox_tiles([-180,-90,180,90]) @@ -29,21 +29,21 @@ def download_bbox(self, e, method, args, expect): COUNT = [] def c(self, bucket, prefix, x, y): COUNT.append([x,y]) - e.download_hgt = types.MethodType(c, elevation.ElevationDownloader) + e.download_hgt = types.MethodType(c, ElevationTileDownloader) method(*args) self.assertEquals(len(COUNT), expect) def test_download_planet(self): - e = elevation.ElevationDownloader('.') + e = ElevationTileDownloader('.') self.download_bbox(e, e.download_planet, [], 64800) def test_download_bbox(self): - e = elevation.ElevationDownloader('.') + e = ElevationTileDownloader('.') self.download_bbox(e, e.download_bbox, [CA], 154) def test_download_bbox_found(self): d = tempfile.mkdtemp() - e = elevation.ElevationDownloader(d) + e = ElevationTileDownloader(d) # correct size path = e.hgtpath(-119, 37) os.makedirs(os.path.join(d, path[0]))