Merge pull request #72 from sat-utils/develop

publish 0.2.1

.circleci/config.yml

@@ -44,7 +44,6 @@ jobs:
             pwd
             pip install -r requirements.txt
             pip install -r requirements-dev.txt
-            export SATUTILS_API_URL=https://n34f767n91.execute-api.us-east-1.amazonaws.com/prod
             pytest --cov satsearch test/
       - save_cache:
           key: v1-dependencies-{{ checksum "requirements.txt"}}

CHANGELOG.md

@@ -7,9 +7,26 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## [Unreleased]
 
 
+## [v0.2.1] - 2019-02-14
+
+### Fixed
+- Fix number found reported when using .found() function and searching by IDs
+- Fixed URL paths in windows by using urljoin instead of os.path.join
+
+### Changed
+- update default API URL to sat-api.developmentseed.org
+- update default save path from ${eo:platform}/${date} to ${collection}/${date}
+- Default limit to search.items() changed from 1000 to 10000
+- Changed internal page size from 1000 to 500 (page size of queries to endpoint)
+
+### Added
+- Warning issued when number of items found greater than limit
+- requestor-pays option to acknowledge paying of egress costs when downloading (defaults to False)
+
+
 ## [v0.2.0] - 2019-01-31
 
-## Changed
+### Changed
 - Works with version 0.2.0 of sat-api (STAC 0.6.x)
 - Major refactor, uses sat-stac library
 
@@ -19,5 +36,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 Initial Release
 
 [Unreleased]: https://github.com/sat-utils/sat-search/compare/master...develop
+[v0.2.1]: https://github.com/sat-utils/sat-search/compare/0.2.0...v0.2.1
 [v0.2.0]: https://github.com/sat-utils/sat-search/compare/0.1.0...v0.2.0
-[v0.1.0]: https://github.com/sat-utils/sat-search/tree/0.1.0
+[v0.1.0]: https://github.com/sat-utils/sat-search/tree/0.1.0

requirements.txt

@@ -1 +1 @@
-sat-stac~=0.1.1
+sat-stac~=0.1.2

satsearch/__init__.py

@@ -1,4 +1,5 @@
 from satsearch.search import Search
+from satsearch.version import __version__
 
 import logging
 

satsearch/config.py

@@ -1,10 +1,10 @@
 import os
 
 # API URL
-API_URL = os.getenv('SATUTILS_API_URL', 'https://sat-api-dev.developmentseed.org')
+API_URL = os.getenv('SATUTILS_API_URL', 'https://sat-api.developmentseed.org')
 
 # data directory to store downloaded imagery
-DATADIR = os.getenv('SATUTILS_DATADIR', './${eo:platform}/${date}')
+DATADIR = os.getenv('SATUTILS_DATADIR', '${collection}/${date}')
 
 # filename pattern for saving files
 FILENAME = os.getenv('SATUTILS_FILENAME', '${id}')

satsearch/main.py

@@ -10,7 +10,7 @@
 
 
 def main(items=None, printmd=None, printcal=False, found=False,
-         save=None, download=None, **kwargs):
+         save=None, download=None, requestor_pays=False, **kwargs):
     """ Main function for performing a search """
 
     if items is None:
@@ -45,7 +45,7 @@ def main(items=None, printmd=None, printcal=False, found=False,
             # get complete set of assets
             download = set([k for i in items for k in i.assets])
         for key in download:
-            items.download(key=key, path=config.DATADIR, filename=config.FILENAME)
+            items.download(key=key, path=config.DATADIR, filename=config.FILENAME, requestor_pays=requestor_pays)
 
     return items
 

satsearch/parser.py

@@ -28,6 +28,8 @@ def __init__(self, *args, **kwargs):
         self.download_group.add_argument('--filename', default=config.FILENAME,
                            help='Save assets with this filename pattern based on metadata keys')
         self.download_group.add_argument('--download', help='Download assets', default=None, nargs='*')
+        h = 'Acknowledge paying egress costs for downloads (if in request pays bucket)'
+        self.download_group.add_argument('--requestor-pays', help=h, default=False, action='store_true', dest='requestor_pays')
 
         self.output_parser = argparse.ArgumentParser(add_help=False)
         self.output_group = self.output_parser.add_argument_group('output options')

satsearch/search.py

@@ -3,11 +3,11 @@
 import logging
 import requests
 
-import os.path as op
 import satsearch.config as config
 
 from satstac import Collection, Item, Items
 from satstac.utils import dict_merge
+from urllib.parse import urljoin
 
 
 logger = logging.getLogger(__name__)
@@ -64,7 +64,8 @@ def search(cls, **kwargs):
     def found(self):
         """ Small query to determine total number of hits """
         if 'ids' in self.kwargs:
-            return len(self.kwargs['ids'])
+            cid = self.kwargs['query']['collection']['eq']
+            return len(self.items_by_id(self.kwargs['ids'], cid))
         kwargs = {
             'page': 1,
             'limit': 0
@@ -74,7 +75,7 @@ def found(self):
         return results['meta']['found']
 
     @classmethod
-    def query(cls, url=op.join(config.API_URL, 'stac/search'), **kwargs):
+    def query(cls, url=urljoin(config.API_URL, 'stac/search'), **kwargs):
         """ Get request """
         logger.debug('Query URL: %s, Body: %s' % (url, json.dumps(kwargs)))
         response = requests.post(url, data=json.dumps(kwargs))
@@ -86,22 +87,25 @@ def query(cls, url=op.join(config.API_URL, 'stac/search'), **kwargs):
     @classmethod
     def collection(cls, cid):
         """ Get a Collection record """
-        url = op.join(config.API_URL, 'collections', cid)
+        url = urljoin(config.API_URL, 'collections/%s' % cid)
         return Collection(cls.query(url=url))
 
     @classmethod
     def items_by_id(cls, ids, collection):
         """ Return Items from collection with matching ids """
         col = cls.collection(collection)
         items = []
-        base_url = op.join(config.API_URL, 'collections', collection, 'items')
+        base_url = urljoin(config.API_URL, 'collections/%s/items' % collection)
         for id in ids:
-            items.append(Item(cls.query(op.join(base_url, id))))
+            try:
+                items.append(Item(cls.query(urljoin(base_url, id))))
+            except SatSearchError as err:
+                pass
         return Items(items, collections=[col])
 
-    def items(self, limit=1000):
+    def items(self, limit=10000):
         """ Return all of the Items and Collections for this search """
-        _limit = 1000
+        _limit = 500
         if 'ids' in self.kwargs:
             col = self.kwargs.get('query', {}).get('collection', {}).get('eq', None)
             if col is None:
@@ -110,6 +114,8 @@ def items(self, limit=1000):
 
         items = []
         found = self.found()
+        if found > limit:
+            logger.warning('There are more items found (%s) than the limit (%s) provided.' % (found, limit))
         maxitems = min(found, limit)
         kwargs = {
             'page': 1,

satsearch/version.py

@@ -1 +1 @@
-__version__ = '0.2.0'
+__version__ = '0.2.1'