diff --git a/README.md b/README.md index 1b6d423..cde7333 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,14 @@ For complete help on command-line arguments: osm_planet_extract -h ``` +### osm_planet_get_timestamp + +A simple utitlity to print the timestamp of an OSM pbf file. + +```sh +osm_planet_get_timestamp planet-latest.osm.pbf +``` + ### elevation_tile_download Download elevation tiles from the [Terrain Tiles in the AWS Public Datasets program](https://aws.amazon.com/public-datasets/terrain/). Download for the entire planet, only tiles within a single bounding box, or within multiple bounding boxes. diff --git a/planetutils/elevation.py b/planetutils/elevation.py index a1015ff..34f4f48 100644 --- a/planetutils/elevation.py +++ b/planetutils/elevation.py @@ -3,6 +3,7 @@ import subprocess import math +import log from bbox import validate_bbox def makedirs(path): @@ -54,12 +55,12 @@ def download_bbox(self, bbox, bucket='elevation-tiles-prod', prefix='skadi'): found.add((x,y)) else: download.add((x,y)) - print "found %s tiles; %s to download"%(len(found), len(download)) + log.info("found %s tiles; %s to download"%(len(found), len(download))) if len(download) > 100: - print " warning: downloading %s tiles will take an additional %0.2f GiB disk space"%( + log.warning(" warning: downloading %s tiles will take an additional %0.2f GiB disk space"%( len(download), (len(download) * self.HGT_SIZE) / (1024.0**3) - ) + )) for x,y in sorted(download): self.download_hgt(bucket, prefix, x, y) @@ -73,6 +74,6 @@ def download_hgt(self, bucket, prefix, x, y): op = os.path.join(self.outpath, od, key) makedirs(os.path.join(self.outpath, od)) url = 'http://s3.amazonaws.com/%s/%s/%s/%s.gz'%(bucket, prefix, od, key) - print "downloading %s to %s"%(url, op) + log.info("downloading %s to %s"%(url, op)) download_gzip(url, op) diff --git a/planetutils/elevation_tile_download.py b/planetutils/elevation_tile_download.py index 41b63d5..c0757d8 100644 --- a/planetutils/elevation_tile_download.py +++ b/planetutils/elevation_tile_download.py @@ -9,7 +9,12 @@ def main(): parser.add_argument('--outpath', help='Output path for elevation tiles.', default='.') parser.add_argument('--csv', help='Path to CSV file with bounding box definitions.') parser.add_argument('--bbox', help='Bounding box for extract file. Format for coordinates: left,bottom,right,top') + parser.add_argument('--verbose', help="Verbose output", action='store_true') args = parser.parse_args() + + if args.verbose: + log.set_verbose() + p = ElevationDownloader(args.outpath) if args.csv: p.download_bboxes(load_bboxes_csv(args.csv)) diff --git a/planetutils/log.py b/planetutils/log.py new file mode 100644 index 0000000..89e7bde --- /dev/null +++ b/planetutils/log.py @@ -0,0 +1,19 @@ +import logging +logging.basicConfig(format='[%(levelname)s] %(message)s') +logger = logging.getLogger(__name__) + +def set_quiet(): + logger.setLevel(logging.ERROR) + +def set_verbose(): + logger.setLevel(logging.DEBUG) + +def set_default(): + logger.setLevel(logging.INFO) + +set_default() + +info = logger.info +debug = logger.debug +warning = logger.warning +error = logger.error \ No newline at end of file diff --git a/planetutils/osm_planet_extract.py b/planetutils/osm_planet_extract.py index 2547bd7..f5b06e0 100644 --- a/planetutils/osm_planet_extract.py +++ b/planetutils/osm_planet_extract.py @@ -12,7 +12,12 @@ def main(): parser.add_argument('--geojson', help='Path to GeoJSON file: bbox for each feature is extracted.') parser.add_argument('--name', help='Name to give to extract file.') parser.add_argument('--bbox', help='Bounding box for extract file. Format for coordinates: left,bottom,right,top') + parser.add_argument('--verbose', help="Verbose output", action='store_true') args = parser.parse_args() + + if args.verbose: + log.set_verbose() + p = Planet(args.osmpath) bboxes = {} @@ -24,7 +29,6 @@ def main(): bboxes[args.name] = bbox.bbox_string(args.bbox) else: parser.error('must specify --csv, --geojson, or --bbox and --name') - print bboxes p.extract_bboxes(bboxes, outpath=args.outpath) diff --git a/planetutils/osm_planet_get_timestamp.py b/planetutils/osm_planet_get_timestamp.py new file mode 100644 index 0000000..11f16b6 --- /dev/null +++ b/planetutils/osm_planet_get_timestamp.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +import argparse +from planet import * +import log + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('osmpath', help='OSM file') + args = parser.parse_args() + p = Planet(args.osmpath) + log.set_quiet() + print p.get_timestamp() + +if __name__ == '__main__': + main() diff --git a/planetutils/osm_planet_update.py b/planetutils/osm_planet_update.py index 24453af..37a7a92 100644 --- a/planetutils/osm_planet_update.py +++ b/planetutils/osm_planet_update.py @@ -1,5 +1,7 @@ #!/usr/bin/env python import argparse + +import log from planet import * def main(): @@ -8,9 +10,14 @@ def main(): parser.add_argument('outpath', help='Name or path to where updated output file should be placed.') parser.add_argument('--s3', action='store_true', help='Download using S3 client from AWS Public Datasets program. AWS credentials required.') parser.add_argument('--workdir', help="Osmosis replication workingDirectory.", default='.') + parser.add_argument('--verbose', help="Verbose output", action='store_true') args = parser.parse_args() + + if args.verbose: + log.set_verbose() + if not os.path.exists(args.osmpath): - print "planet does not exist; downloading" + log.info("planet does not exist; downloading") if args.s3: d = PlanetDownloaderS3(args.osmpath) else: diff --git a/planetutils/planet.py b/planetutils/planet.py index 96cb5a4..005c174 100644 --- a/planetutils/planet.py +++ b/planetutils/planet.py @@ -6,6 +6,7 @@ import boto3 +import log from bbox import validate_bbox class PlanetBase(object): @@ -16,7 +17,7 @@ def __init__(self, osmpath=None, grain='hour', changeset_url=None, osmosis_workd def osmosis(self, *args): cmd = ['osmosis'] + list(args) - print ' '.join(cmd) + log.debug(' '.join(cmd)) return subprocess.check_output( cmd, shell=False @@ -24,7 +25,7 @@ def osmosis(self, *args): def osmconvert(self, *args): cmd = ['osmconvert'] + list(args) - print ' '.join(cmd) + log.debug(' '.join(cmd)) return subprocess.check_output( cmd, shell=False @@ -36,6 +37,7 @@ def get_timestamp(self): '--out-timestamp' ) if 'invalid' in timestamp: + log.debug('no timestamp; falling back to osmconvert --out-statistics') statistics = self.osmconvert( self.osmpath, '--out-statistics' @@ -103,9 +105,9 @@ def download_planet_latest(self, bucket=None, prefix=None, match=None): objs = self._get_planets(bucket, prefix, match) objs = sorted(objs, key=lambda x:x.key) for i in objs: - print 'found planet: s3://%s/%s'%(i.bucket_name, i.key) + log.info('found planet: s3://%s/%s'%(i.bucket_name, i.key)) planet = objs[-1] - print 'downloading: s3://%s/%s to %s'%(planet.bucket_name, planet.key, self.osmpath) + log.info('downloading: s3://%s/%s to %s'%(planet.bucket_name, planet.key, self.osmpath)) self._download(planet.bucket_name, planet.key) def _download(self, bucket_name, key): diff --git a/planetutils/tilepack.py b/planetutils/tilepack.py index 8b5d33e..3f65616 100644 --- a/planetutils/tilepack.py +++ b/planetutils/tilepack.py @@ -4,6 +4,8 @@ import json import urllib2 +import log + class Tilepack(object): HOST = 'https://app.interline.io' def download(self, outpath, version='latest', api_token=None, compressed=False): @@ -22,14 +24,15 @@ def download(self, outpath, version='latest', api_token=None, compressed=False): args = ['curl', '-L', '--fail', '-o', outpath, url] if not compressed: args.append('--compressed') - print "Downloading to %s"%outpath + log.info("Downloading to %s"%outpath) + log.debug(' '.join(args)) p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() e = p.wait() if e != 0: - print "Error downloading: %s"%err.split("curl:")[-1] + raise Exception("Error downloading: %s"%err.split("curl:")[-1]) else: - print "Done" + log.info("Done") def list(self): url = "%s/valhalla_planet_tilepacks.json"%(self.HOST) @@ -42,7 +45,7 @@ def list(self): bucket = 'gs://%s/%s'%(a['bucket_name'], a['bucket_key']) elif a.get('bucket_provider') == 's3': bucket = 's3://%s/%s'%(a['bucket_name'], a['bucket_key']) - print """ + print(""" Tilepack ID: %s Timestamp: %s Filename: %s @@ -63,4 +66,4 @@ def list(self): a['valhalla_version'], a['interline_planetutils_version'], a['interline_valhalla_tile_cutter_version'] - ) \ No newline at end of file + )) \ No newline at end of file diff --git a/planetutils/tilepack_download.py b/planetutils/tilepack_download.py index 525e2c5..deee1c9 100644 --- a/planetutils/tilepack_download.py +++ b/planetutils/tilepack_download.py @@ -2,6 +2,7 @@ import os import argparse +import log import tilepack from bbox import bbox_string, load_bboxes_csv @@ -11,17 +12,21 @@ def main(): parser.add_argument('--outpath', help='Output path for Valhalla Tilepack; default is tiles.tar', default='tiles.tar') parser.add_argument('--api-token', help='Interline Auth Token; default is read from $INTERLINE_API_TOKEN') parser.add_argument('--compressed', help='Do not decompress Tilepack', action='store_true') + parser.add_argument('--verbose', help="Verbose output", action='store_true') args = parser.parse_args() + if args.verbose: + log.set_verbose() + outpath = args.outpath if args.compressed: if not (outpath.endswith('.tar') or outpath.endswith('.tgz')): - print "Warning: compressed output path %s does not in end in .tar.gz or .tgz"%outpath + log.warning("Warning: compressed output path %s does not in end in .tar.gz or .tgz"%outpath) else: if not outpath.endswith('.tar'): - print "Warning: decompressed output path %s does not end in .tar"%outpath + log.warning("Warning: decompressed output path %s does not end in .tar"%outpath) if os.path.exists(outpath): - print "Warning: output path %s already exists."%outpath + log.warning("Warning: output path %s already exists."%outpath) tp = tilepack.Tilepack() tp.download( diff --git a/setup.py b/setup.py index 0aca9fc..5a8b1be 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ 'console_scripts': [ 'osm_planet_update=planetutils.osm_planet_update:main', 'osm_planet_extract=planetutils.osm_planet_extract:main', + 'osm_planet_get_timestamp=planetutils.osm_planet_get_timestamp:main', 'elevation_tile_download=planetutils.elevation_tile_download:main', 'valhalla_tilepack_download=planetutils.tilepack_download:main', 'valhalla_tilepack_list=planetutils.tilepack_list:main' diff --git a/tests/test_bbox.py b/tests/test_bbox.py index c268d4f..64e649c 100644 --- a/tests/test_bbox.py +++ b/tests/test_bbox.py @@ -46,7 +46,6 @@ def test_load(self): bboxes = bbox.load_bboxes_geojson(TESTGEOJSON) union = (-122.42400169372557, 37.7860125252054, -122.40559101104735, 37.7985943621788) pentagon = (-122.39975452423094, 37.78370618798191, -122.38949775695801, 37.791879793952084) - print bboxes for a,b in zip(bboxes['union'], union): self.assertAlmostEqual(a,b) for a,b in zip(bboxes['pentagon'], pentagon):