Merge pull request #10 from interline-io/logging

Improved stdout

README.md

@@ -132,6 +132,14 @@ For complete help on command-line arguments:
 osm_planet_extract -h
 ```
 
+### osm_planet_get_timestamp
+
+A simple utitlity to print the timestamp of an OSM pbf file.
+
+```sh
+osm_planet_get_timestamp planet-latest.osm.pbf
+```
+
 ### elevation_tile_download
 
 Download elevation tiles from the [Terrain Tiles in the AWS Public Datasets program](https://aws.amazon.com/public-datasets/terrain/). Download for the entire planet, only tiles within a single bounding box, or within multiple bounding boxes.

planetutils/elevation.py

@@ -3,6 +3,7 @@
 import subprocess
 import math
 
+import log
 from bbox import validate_bbox
 
 def makedirs(path):
@@ -54,12 +55,12 @@ def download_bbox(self, bbox, bucket='elevation-tiles-prod', prefix='skadi'):
                 found.add((x,y))
             else:
                 download.add((x,y))
-        print "found %s tiles; %s to download"%(len(found), len(download))
+        log.info("found %s tiles; %s to download"%(len(found), len(download)))
         if len(download) > 100:
-            print "  warning: downloading %s tiles will take an additional %0.2f GiB disk space"%(
+            log.warning("  warning: downloading %s tiles will take an additional %0.2f GiB disk space"%(
                 len(download),
                 (len(download) * self.HGT_SIZE) / (1024.0**3)
-            )
+            ))
         for x,y in sorted(download):
             self.download_hgt(bucket, prefix, x, y)
 
@@ -73,6 +74,6 @@ def download_hgt(self, bucket, prefix, x, y):
         op = os.path.join(self.outpath, od, key)
         makedirs(os.path.join(self.outpath, od))
         url = 'http://s3.amazonaws.com/%s/%s/%s/%s.gz'%(bucket, prefix, od, key)
-        print "downloading %s to %s"%(url, op)
+        log.info("downloading %s to %s"%(url, op))
         download_gzip(url, op)
 

planetutils/elevation_tile_download.py

@@ -9,7 +9,12 @@ def main():
     parser.add_argument('--outpath', help='Output path for elevation tiles.', default='.')
     parser.add_argument('--csv', help='Path to CSV file with bounding box definitions.')
     parser.add_argument('--bbox', help='Bounding box for extract file. Format for coordinates: left,bottom,right,top')
+    parser.add_argument('--verbose', help="Verbose output", action='store_true')
     args = parser.parse_args()
+
+   if args.verbose:
+        log.set_verbose()
+
     p = ElevationDownloader(args.outpath)
     if args.csv:
         p.download_bboxes(load_bboxes_csv(args.csv))

planetutils/log.py

@@ -0,0 +1,19 @@
+import logging
+logging.basicConfig(format='[%(levelname)s] %(message)s')
+logger = logging.getLogger(__name__)
+
+def set_quiet():
+    logger.setLevel(logging.ERROR)
+
+def set_verbose():
+    logger.setLevel(logging.DEBUG)
+
+def set_default():
+    logger.setLevel(logging.INFO)
+
+set_default()
+
+info = logger.info
+debug = logger.debug
+warning = logger.warning
+error = logger.error

planetutils/osm_planet_extract.py

@@ -12,7 +12,12 @@ def main():
     parser.add_argument('--geojson', help='Path to GeoJSON file: bbox for each feature is extracted.')
     parser.add_argument('--name', help='Name to give to extract file.')
     parser.add_argument('--bbox', help='Bounding box for extract file. Format for coordinates: left,bottom,right,top')
+    parser.add_argument('--verbose', help="Verbose output", action='store_true')
     args = parser.parse_args()
+
+    if args.verbose:
+        log.set_verbose()
+
     p = Planet(args.osmpath)
 
     bboxes = {}
@@ -24,7 +29,6 @@ def main():
         bboxes[args.name] = bbox.bbox_string(args.bbox)
     else:
         parser.error('must specify --csv, --geojson, or --bbox and --name')
-    print bboxes
 
     p.extract_bboxes(bboxes, outpath=args.outpath)
 

planetutils/osm_planet_get_timestamp.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+import argparse
+from planet import *
+import log
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('osmpath', help='OSM file')
+    args = parser.parse_args()
+    p = Planet(args.osmpath)
+    log.set_quiet()
+    print p.get_timestamp()
+
+if __name__ == '__main__':
+    main()

planetutils/osm_planet_update.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 import argparse
+
+import log
 from planet import *
 
 def main():
@@ -8,9 +10,14 @@ def main():
     parser.add_argument('outpath', help='Name or path to where updated output file should be placed.')
     parser.add_argument('--s3', action='store_true', help='Download using S3 client from AWS Public Datasets program. AWS credentials required.')
     parser.add_argument('--workdir', help="Osmosis replication workingDirectory.", default='.')
+    parser.add_argument('--verbose', help="Verbose output", action='store_true')
     args = parser.parse_args()
+
+    if args.verbose:
+        log.set_verbose()
+
     if not os.path.exists(args.osmpath):
-        print "planet does not exist; downloading"
+        log.info("planet does not exist; downloading")
         if args.s3:
             d = PlanetDownloaderS3(args.osmpath)
         else:

planetutils/planet.py

@@ -6,6 +6,7 @@
 
 import boto3
 
+import log
 from bbox import validate_bbox
 
 class PlanetBase(object):
@@ -16,15 +17,15 @@ def __init__(self, osmpath=None, grain='hour', changeset_url=None, osmosis_workd
 
     def osmosis(self, *args):
         cmd = ['osmosis'] + list(args)
-        print ' '.join(cmd)
+        log.debug(' '.join(cmd))
         return subprocess.check_output(
             cmd,
             shell=False
         )
 
     def osmconvert(self, *args):
         cmd = ['osmconvert'] + list(args)
-        print ' '.join(cmd)
+        log.debug(' '.join(cmd))
         return subprocess.check_output(
             cmd,
             shell=False
@@ -36,6 +37,7 @@ def get_timestamp(self):
             '--out-timestamp'
         )
         if 'invalid' in timestamp:
+            log.debug('no timestamp; falling back to osmconvert --out-statistics')
             statistics = self.osmconvert(
                 self.osmpath,
                 '--out-statistics'
@@ -103,9 +105,9 @@ def download_planet_latest(self, bucket=None, prefix=None, match=None):
         objs = self._get_planets(bucket, prefix, match)
         objs = sorted(objs, key=lambda x:x.key)
         for i in objs:
-            print 'found planet: s3://%s/%s'%(i.bucket_name, i.key)
+            log.info('found planet: s3://%s/%s'%(i.bucket_name, i.key))
         planet = objs[-1]
-        print 'downloading: s3://%s/%s to %s'%(planet.bucket_name, planet.key, self.osmpath)
+        log.info('downloading: s3://%s/%s to %s'%(planet.bucket_name, planet.key, self.osmpath))
         self._download(planet.bucket_name, planet.key)
 
     def _download(self, bucket_name, key):

planetutils/tilepack.py

@@ -4,6 +4,8 @@
 import json
 import urllib2
 
+import log
+
 class Tilepack(object):
     HOST = 'https://app.interline.io'
     def download(self, outpath, version='latest', api_token=None, compressed=False):
@@ -22,14 +24,15 @@ def download(self, outpath, version='latest', api_token=None, compressed=False):
         args = ['curl', '-L', '--fail', '-o', outpath, url]
         if not compressed:
             args.append('--compressed')
-        print "Downloading to %s"%outpath
+        log.info("Downloading to %s"%outpath)
+        log.debug(' '.join(args))
         p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         out, err = p.communicate()
         e = p.wait()
         if e != 0:
-            print "Error downloading: %s"%err.split("curl:")[-1]
+            raise Exception("Error downloading: %s"%err.split("curl:")[-1])
         else:
-            print "Done"
+            log.info("Done")
 
     def list(self):
         url = "%s/valhalla_planet_tilepacks.json"%(self.HOST)
@@ -42,7 +45,7 @@ def list(self):
                 bucket = 'gs://%s/%s'%(a['bucket_name'], a['bucket_key'])
             elif a.get('bucket_provider') == 's3':
                 bucket = 's3://%s/%s'%(a['bucket_name'], a['bucket_key'])
-            print """
+            print("""
 Tilepack ID: %s
     Timestamp: %s
     Filename: %s
@@ -63,4 +66,4 @@ def list(self):
                 a['valhalla_version'],
                 a['interline_planetutils_version'],
                 a['interline_valhalla_tile_cutter_version']
-            )
+            ))

planetutils/tilepack_download.py

@@ -2,6 +2,7 @@
 import os
 import argparse
 
+import log
 import tilepack
 from bbox import bbox_string, load_bboxes_csv
 
@@ -11,17 +12,21 @@ def main():
     parser.add_argument('--outpath', help='Output path for Valhalla Tilepack; default is tiles.tar', default='tiles.tar')
     parser.add_argument('--api-token', help='Interline Auth Token; default is read from $INTERLINE_API_TOKEN')
     parser.add_argument('--compressed', help='Do not decompress Tilepack', action='store_true')
+    parser.add_argument('--verbose', help="Verbose output", action='store_true')
     args = parser.parse_args()
 
+    if args.verbose:
+        log.set_verbose()
+
     outpath = args.outpath
     if args.compressed:
         if not (outpath.endswith('.tar') or outpath.endswith('.tgz')):
-            print "Warning: compressed output path %s does not in end in .tar.gz or .tgz"%outpath
+            log.warning("Warning: compressed output path %s does not in end in .tar.gz or .tgz"%outpath)
     else:
         if not outpath.endswith('.tar'):
-            print "Warning: decompressed output path %s does not end in .tar"%outpath
+            log.warning("Warning: decompressed output path %s does not end in .tar"%outpath)
     if os.path.exists(outpath):
-        print "Warning: output path %s already exists."%outpath
+        log.warning("Warning: output path %s already exists."%outpath)
 
     tp = tilepack.Tilepack()
     tp.download(

setup.py

@@ -24,6 +24,7 @@
         'console_scripts': [
             'osm_planet_update=planetutils.osm_planet_update:main',
             'osm_planet_extract=planetutils.osm_planet_extract:main',
+            'osm_planet_get_timestamp=planetutils.osm_planet_get_timestamp:main',
             'elevation_tile_download=planetutils.elevation_tile_download:main',
             'valhalla_tilepack_download=planetutils.tilepack_download:main',
             'valhalla_tilepack_list=planetutils.tilepack_list:main'

tests/test_bbox.py

@@ -46,7 +46,6 @@ def test_load(self):
         bboxes = bbox.load_bboxes_geojson(TESTGEOJSON)
         union = (-122.42400169372557, 37.7860125252054, -122.40559101104735, 37.7985943621788)
         pentagon = (-122.39975452423094, 37.78370618798191, -122.38949775695801, 37.791879793952084)
-        print bboxes
         for a,b in zip(bboxes['union'], union):
             self.assertAlmostEqual(a,b)
         for a,b in zip(bboxes['pentagon'], pentagon):